from sklearn.svm import SVC

import numpy as np
import tensorflow as tf
import argparse
import pickle
import json
import collections
import os

class DocumentSpace:
    class_sess = tf.Session()

    def __init__(self, language_model, lang_mod_order, span):
        self.train_vectors = list()  # structure: [(type, vector)]
        self.test_vectors = list()  # structure: [(type, vector)]
        self.span = span
        self.lang_mod_order = lang_mod_order  # list of words of the language model (to know which
        # word an embedding represents)

        with DocumentSpace.class_sess.as_default(), \
                    tf.device('/cpu:0'):

            self.language_model = tf.nn.l2_normalize(language_model,
                                                     1).eval()  # Word embeddings used to get vectors from text.

    def get_abs_vectors(self, papers):

        """ Generates a vector for every abstract to be classified.
        :parameter papers: JSON array (python list) containing
        the processed papers, represented as dicts."""

        labels = [paper["classification"] for paper in papers]  # ground truth

        print("Calculating abstracts' vectors...")
        parsed = 0
        n_abstracts = len(papers)
        print("started vector build")
        rel_freqs = []
        pooled_vectors = list()
        append_rel_freqs = rel_freqs.append
        lmo = self.lang_mod_order
        span = self.span
        asarr = np.asarray
        for paper in papers:  # 10000
            word_mtx = list()
            words = paper["abstract"].split()
            abs_count = collections.Counter(words)
            word_count = 0
            i = 0
            while word_count < self.span and i < len(words):
                word = words[i]
                if word in lmo:
                    index = self.lang_mod_order.index(word)
                else:
                    index = 0
                word_mtx.append(self.language_model[index])
                word_count += 1
                i += 1
            try:
                dim_mtx = np.transpose(word_mtx) #transposed : rows:value for every word at a given dimension
                pooled_vector = [max(dimension) for dimension in dim_mtx]
            except TypeError:
                pooled_vector = [0] * 10
            pooled_vectors.append(pooled_vector)
            if not parsed % 1000:
                print("{}/{}".format(parsed, n_abstracts))
            parsed += 1
        assert n_abstracts == parsed
        print("Finished calculating abstracts' vectors.")
        return list(zip(labels, pooled_vectors))

    @staticmethod
    def slice(vectors):
        data = [t[1] for t in vectors]
        labels = [t[0] for t in vectors]

        return data, labels

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Classify papers.")
    parser.add_argument("--model_path", help="Path to a folder containing everything related to the model, namely "
                                             "embeddings and vocab files.",
                        required=True)
    parser.add_argument("--span", help="Number of words used for classification, counting from "
                                       "the start of the abstract", type=int, default=10)
    parser.add_argument("--KNN_papers_set", help="Path to the KNN paper set.",
                        required=True)

    if os.path.exists("test_data"):
        print("opening previous data")
        with open("train_data", "rb") as trd, \
                open("train_labels", "rb") as trl, \
                open("test_data", "rb") as ted, \
                open("test_labels", "rb") as tel:
            train_data = pickle.load(trd)
            train_labels = pickle.load(trl)
            test_data = pickle.load(ted)
            test_labels = pickle.load(tel)

        args = parser.parse_args()

    else:
        args = parser.parse_args()

        path_to_model = args.model_path
        path_to_papers = args.KNN_papers_set
        join_path = os.path.join

        with open(join_path(path_to_model, "embeddings"), 'rb') as embeddings, \
                open(join_path(path_to_model, "vocab.txt")) as vocab, \
                open(join_path(path_to_papers, "train_papers")) as train_set, \
                open(join_path(path_to_papers, "test_papers")) as test_set:

            model = pickle.load(embeddings)
            model_order = [line.split()[0].strip("b'") for line in vocab]

            train = json.load(train_set)
            test = json.load(test_set)

        fthousand = []
        ps = 0
        sr = 0
        i = 0
        while ps < 5000 or sr < 5000:
            if test[i]["classification"] == "primary-study" and ps < 5000:
                fthousand.append(test[i])
                ps += 1
            elif test[i]["classification"] == "systematic-review" and sr < 5000:
                fthousand.append(test[i])
                sr += 1
            i += 1
        print(len(fthousand))

        Space = DocumentSpace(model, model_order, args.span)
        Space.train_vectors = Space.get_abs_vectors(train)
        Space.test_vectors = Space.get_abs_vectors(fthousand)
        train_data, train_labels = Space.slice(Space.train_vectors)
        test_data, test_labels = Space.slice(Space.test_vectors)

        with open("train_data", "wb") as trd, \
                open("train_labels", "wb") as trl, \
                open("test_data", "wb") as ted, \
                open("test_labels", "wb") as tel:
            pickle.dump(train_data, trd)
            pickle.dump(train_labels, trl)
            pickle.dump(test_data, ted)
            pickle.dump(test_labels, tel)

    classifier = SVC()
    print("fitting")
    classifier.fit(np.asarray(train_data), np.asarray(train_labels))
    print("predicting")
    predictions = classifier.predict(np.asarray(test_data))
    classes = [ "primary-study",
                 "systematic-review"]

    if len(test_labels) != len(predictions):
        print("dimensions error. labels: {}, predictions: {}".format(len(test_labels),
                                                                      len(predictions)))

    class_dimension = len(classes)
    conf_mtx = np.zeros([class_dimension, class_dimension])
    for i in range(0, len(predictions)):
        predicted_class = classes.index(predictions[i])
        actual_class = classes.index(test_labels[i])
        conf_mtx[actual_class][predicted_class] += 1
    np.set_printoptions(suppress=True)
    print(conf_mtx)

    hits = 0
    for l, p in zip(test_labels, predictions):
        if l == p:
            hits += 1
    accuracy = hits / len(test_labels) #saved for output
    print(accuracy)

    recall = lambda i: (conf_mtx[i][i]/sum(conf_mtx[i][j] for j in range(0,class_dimension)))
    recall_sum = 0
    recall_list = []
    for i in range(0,class_dimension):
        rcl = recall(i)
        if not np.isnan(rcl):
            recall_sum += rcl
        recall_list.append((i, rcl))
        print('Recall {}: {:.5f}'.format(i, rcl))
    print()
    recall_mean = recall_sum/class_dimension
    print('Recall mean: {:.5f}'.format(recall_mean))

    precision = lambda i: (conf_mtx[i][i]/sum(conf_mtx[j][i] for j in range(0,class_dimension)))
    precision_sum = 0
    precision_list = list()
    for i in range(0,class_dimension):
        label_precision = precision(i)
        if not np.isnan(label_precision):
            precision_sum += label_precision
        precision_list.append((i, label_precision))
        print('Precision {}: {:.5f}'.format(i, label_precision))
    print()
    precision_mean = precision_sum/class_dimension
    print('Precision mean: {:.5f}'.format(precision_mean))

    output = ''
    output += 'Model: {}\n'.format(args.model_path)
    output += 'span = {}\n'.format(args.span)
    output += 'Set: {}\n'.format(args.KNN_papers_set)
    output += 'Accuracy : {}\n'.format(accuracy)
    output += "RECALL\n"
    for rcl in recall_list:
        output += 'Recall {}: {:.5f}\n'.format(rcl[0], rcl[1])
    output += 'Recall mean: {:.5f}\n'.format(recall_mean)
    output += "PRECISION\n"
    for pcsn in precision_list:
        output += 'Precision {}: {:.5f}\n'.format(pcsn[0], pcsn[1])
    output += 'Precision mean: {:.5f}\n'.format(precision_mean)
    output += 'CONFUSSION MATRIX\n'
    output += str(conf_mtx)

    with open("output{}.txt".format(args.model_path.split('/')[-2]), "w") as out_file:
        out_file.write(output)


from django.urls import path, include
from .views import memo_list, memo_write, memo_detail, memo_edit

app_name = "notepad"
urlpatterns = [
    path('', memo_list),
    path('write/', memo_write),
    path('<int:post_id>/', memo_detail, name="detail"),
    path('edit/<int:post_id>', memo_edit, name="edit")
]


import pygame

from configs import get_value, add_from_dict, get_config_dict, add_value
from default_configs import fonts as default_fonts

fonts = {}
break_chars = get_value("DISPLAY", "Text", "break-chars",
                        r"-.!?+=_,:;()[]{}`~@\\$%^&*|<>")
default_color = get_value("DISPLAY", "Textboxes", "color", (255, 255, 255))
default_border_size = get_value("DISPLAY", "Textboxes", "border-size", -1)
default_scroll_amount = get_value("DISPLAY", "Textboxes", "scroll-amount", .5)
default_zoom_scale = get_value("DISPLAY", "Textboxes", "zoom-scale", 1)
min_font_size = get_value("DISPLAY", "Text", "min-font-size", 3)
max_font_size = get_value("DISPLAY", "Text", "max-font-size", 500)

def reload_fonts():
    """Pulls the latest fonts from the config"""
    global fonts
    fonts = {}

def get_font_dict(font_name, font_resize=0):
    """Updates font dict to include font object and returns dict"""
    global fonts

    actual_size = None

    if isinstance(font_name, type(None)):
        font_name = "default"
    if font_name not in fonts:
        # Update fonts if not in list
        add_from_dict(
            default_fonts)  # Update config to include all default fonts
        fonts = get_config_dict()[
            "FONTS"]  # Update fonts list to include conf fonts
        if font_name not in fonts:
            # Font missing from default)_fonts/config
            raise Exception("Font: '" + font_name + "' not found in fonts list")
    # font_name is in fonts
    if "font object" not in fonts[font_name] or font_resize != 0:
        # Need to add pygame font object to font's dict.
        # has not been created yet

        # Get/add each necessary variable to fonts dict
        if "bold" not in fonts[font_name]:
            fonts[font_name]["bold"] = False
        font_bold = fonts[font_name]["bold"]

        if "italic" not in fonts[font_name]:
            fonts[font_name]["italic"] = False
        font_italic = fonts[font_name]["italic"]

        if "size" not in fonts[font_name]:
            fonts[font_name]["size"] = fonts["Default"]["size"]
        font_size = fonts[font_name]["size"]

        if "color" not in fonts[font_name]:
            fonts[font_name]["color"] = fonts["Default"]["color"]

        if "highlight color" not in fonts[font_name]:
            fonts[font_name]["highlight color"] = None

        try:
            # Create the font
            new_font_size = font_size + int(font_resize)
            actual_size = min(max(new_font_size, min_font_size), max_font_size)
            font = pygame.font.SysFont(fonts[font_name]["name"], actual_size,
                                       font_bold,
                                       font_italic)
        except OSError:
            if font_name == "Default":
                raise Exception("Default font could not be loaded. "
                                "Change in config.")
            # Font is not default, use default instead
            print("Could not load font: '" + font_name + ",' using Default "
                                                         "font instead.")
            font, actual_size = get_font_dict("Default")
            font = font["font object"]
        fonts[font_name]["font object"] = font

    return fonts[font_name], actual_size

class Text(object):
    """Stores variables and methods for displaying pygame text"""

    def __init__(self, text="\n", font_name=None, hover_text_list=None,
                 animations=None):
        self.text = text
        self.whole = text
        self.font_name = font_name
        self.hover_text_list = hover_text_list
        self.pos = None
        self.label = None
        self.width = None
        self.height = None

        self.color_override = None
        self.font_resize = 0

        self.animations = animations
        if isinstance(animations, type(None)):
            self.animations = []

    def set_color(self, color, perma=False):
        if perma:
            fonts[self.font_name]["color"] = color
        self.color_override = color
        self.label = None

    def resize(self, size, perma=False):
        font_dict, actual_size = get_font_dict(self.font_name)
        old_size = font_dict["size"]
        self.font_resize = size - old_size
        self.label = None
        if perma:
            fonts[self.font_name]["size"] = size

    def get_color(self):
        font_dict, actual_size = get_font_dict(self.font_name)
        return font_dict["color"]

    def get_text(self):
        return self.text

    def copy(self, text=None):
        """Returns a copy of the text object with new text"""
        if isinstance(text, type(None)):
            text = self.text
        new_text = Text(text, self.font_name, self.hover_text_list)
        new_text.whole = self.whole
        new_text.animations = self.animations
        new_text.font_resize = self.font_resize
        new_text.color_override = self.color_override
        return new_text

    def break_line(self, remaining_width, font_resize=0):
        """Breaks into two texts where overflows remaining_width"""
        word = ""
        dash = ""
        for char in self.text:
            word += char
            if self.copy(word).get_dimensions(font_resize)[0] > remaining_width:
                if len(word) >= 3 and word[-3] != " ":
                    dash = "-"
                    word = word[:-2]
                else:
                    word = word[:-1]
                break
        word2 = self.text[len(word):]
        return self.copy(word + dash), self.copy(word2)

    def break_chars(self, break_char="\n"):
        """Breaks text into separate words at break_char"""
        words = []
        word = ""
        i = 0
        for char in self.text:
            i += 1
            word += char
            if char == break_char or len(self.text) == i:
                words.append(self.copy(word))
                word = ""
        return words

    def get_dimensions(self, font_resize=0):
        """Returns dimensions of text, generates new dimensions if necessary"""
        if isinstance(self.width, type(None)):
            font, actual_size = get_font_dict(self.font_name, font_resize)
            font = font["font object"]
            self.width, self.height = font.size(self.text)
        return self.width, self.height

    def get_label(self, font_resize=0):
        """Generates if necessary and returns a label

        :param font_resize: amount to resize fonts by for zooming
        """
        actual_size = None

        if isinstance(self.label, type(None)) or font_resize != 0:
            font_dict, actual_size = get_font_dict(self.font_name, font_resize)
            font = font_dict["font object"]
            color = font_dict["color"]
            if not isinstance(self.color_override, type(None)):
                color = self.color_override
            self.label = font.render(self.text.strip().replace("\n", ""), 1,
                                     color, font_dict["highlight color"])
        return self.label, actual_size

    def clear_for_pickle(self):
        """Clears the label in order to not mess with the pickling process"""
        self.label = None

    def render(self, display, pos, font_resize):
        """Renders text to given display"""
        self.pos = pos
        _font_resize = font_resize
        if self.font_resize != 0:
            _font_resize = self.font_resize
        label, actual_size = self.get_label(_font_resize)
        display.blit(label, pos)
        return actual_size

    def is_hovered(self, mouse_pos):
        """Checks whether text is being hovered"""
        if isinstance(self.pos, type(None)):
            print("Warning: Text not yet rendered when click attempted")
            # TODO: Does this ever occur and is it a problem?
            return False
        width, height = self.get_dimensions()
        if self.pos[0] <= mouse_pos[0] <= self.pos[0] + width:
            if self.pos[1] <= mouse_pos[1] <= self.pos[1] + height:
                return True
        return False

    def get_hover_text(self):
        return self.hover_text_list

class Textbox:

    def __init__(self, x_pin, y_pin, x2_pin, y2_pin, text_list=None, name=None,
                 color=None, border_size=None, scroll_amount=None,
                 zoom_scale=None, allow_zoom=False):
        self.name = name
        self.x_pin = x_pin
        self.y_pin = y_pin
        self.x2_pin = x2_pin
        self.y2_pin = y2_pin

        self.color = color
        if isinstance(color, type(None)):
            self.color = default_color
        self.border_size = border_size
        if isinstance(border_size, type(None)):
            self.border_size = default_border_size
        self.scroll_amount = scroll_amount  # Number of lines to scroll
        if isinstance(scroll_amount, type(None)):
            self.scroll_amount = default_scroll_amount
        self.zoom_scale = zoom_scale  # Multiply zoom amount by this
        if isinstance(zoom_scale, type(None)):
            self.zoom_scale = default_zoom_scale

        self.text_list = []
        self.whole_text_list = []
        if not isinstance(text_list, type(None)):
            self.add_text_list(text_list)
        self.lines = []
        self.line_num = 0  # Current line scroll is on
        self.current_scroll = 0
        # ^ Must reach 1/self.scroll_amount if self.scroll_amount < 0 to scroll

        self.width = None
        self.height = None
        self.screen_width = None
        self.screen_height = None

        self.font_resize = 0
        self.zoom_amount = 0

        self.zoom_switch = None
        self.zoom_limit = None

        self.allow_zoom = allow_zoom

        self.rect = None

        # TODO: Allow saving of multiple self.lines for easily redisplaying tooltips?

    def get_all_contained_texts(self):
        texts = []
        texts += self.text_list
        texts += self.whole_text_list
        for line in self.lines:
            texts += line
        return texts

    def get_color(self):
        for text in self.whole_text_list:
            if "color" in text.animations:
                return text.get_color()
        return self.whole_text_list[0].get_color()

    def set_color(self, color, perma=False):
        for text in self.get_all_contained_texts():
            if "color" in text.animations:
                text.set_color(color, perma)

    def resize(self, size, perma=False):
        for text in self.get_all_contained_texts():
            if "size" in text.animations:
                text.resize(size, perma)
        self.rewrap()

    def get_pos(self):
        if isinstance(self.rect, type(None)):
            return None
        return list(int(val) for val in self.rect)

    def set_pos(self, point):
        """Moves upper left corner to point"""
        self.x_pin = point[0] / self.screen_width
        self.y_pin = point[1] / self.screen_height
        self.x2_pin = (point[0] + self.rect[1]) / self.screen_width
        self.y2_pin = (point[1] + self.rect[3]) / self.screen_height

    def within_bounds(self, pos, display):
        width = display.get_width()
        height = display.get_height()
        bounds = [int(self.x_pin * width),
                  int(self.y_pin * height),
                  int(self.x2_pin * width),
                  int(self.y2_pin * height)]

        if bounds[2] >= pos[0] >= bounds[0]:
            if bounds[3] >= pos[1] >= bounds[1]:
                return True
        return False

    def handle_event(self, event, display):
        cursor_pos = pygame.mouse.get_pos()
        if self.within_bounds(cursor_pos, display):
            if event.type == pygame.MOUSEBUTTONDOWN:
                if event.button == 4:
                    # Scroll down in box
                    self.scroll(False)
                elif event.button == 5:
                    # Scroll up in box
                    self.scroll()

    def scroll(self, up=True):
        # Allow for scroll amounts of less than zero which scroll every interval
        ctrl_held = pygame.key.get_mods() & pygame.KMOD_CTRL
        if ctrl_held and self.allow_zoom:
            if not up:
                if self.zoom_limit != 1:
                    # Zoom instead of scroll
                    if self.zoom_limit == -1:
                        self.zoom_limit = None
                    if self.zoom_switch == -1:
                        self.zoom_amount = 0
                    self.zoom_switch = 1
                    if self.zoom_amount == 0:
                        self.zoom_amount = self.zoom_scale
                    self.zoom_amount *= self.zoom_scale
                    self.zoom_amount *= self.zoom_scale
                    self.font_resize += self.zoom_amount
                    self.rewrap()
            else:
                if self.zoom_limit != -1:
                    if self.zoom_limit == 1:
                        self.zoom_limit = None
                    if self.zoom_switch == 1:
                        self.zoom_amount = 0
                    self.zoom_switch = -1
                    if self.zoom_amount == 0:
                        self.zoom_amount = self.zoom_scale
                    self.zoom_amount *= self.zoom_scale
                    self.font_resize -= self.zoom_amount
                    self.rewrap()
        else:
            scroll_int = 0
            scroll_amount = self.scroll_amount
            if self.scroll_amount < 1:
                scroll_int = 1 / self.scroll_amount
                scroll_amount = 1
            self.current_scroll += 1
            if self.current_scroll >= scroll_int:
                self.current_scroll = 0
                if not up:
                    self.line_num -= int(scroll_amount)
                else:
                    self.line_num += int(scroll_amount)
                self.check_scroll()

    def add_text_list(self, text_list):
        self.text_list += text_list
        self.whole_text_list += text_list

    def get_dimensions(self):
        if isinstance(self.width, type(None)):
            raise Exception("Access to dimensions attempted before rendering "
                            "for textbox: '" + self.name + "'.")
        return self.width, self.height

    def update_dimensions(self, display):
        # Update dimensions if necessary
        width = display.get_width()
        height = display.get_height()
        changed = False
        if width != self.screen_width:
            # Width has been changed
            self.screen_width = width
            self.width = (self.x2_pin * width) - (self.x_pin * width)
            changed = True
        if height != self.screen_height:
            # Width has been changed
            self.screen_height = height
            self.height = (self.y2_pin * height) - (self.y_pin * height)
            changed = True
        if changed:
            self.rewrap()

    def render(self, display):
        self.update_dimensions(display)  # Will always rewrap on first render
        self.wrap()

        x = self.x_pin * display.get_width()
        y = self.y_pin * display.get_height()
        x2 = self.x2_pin * display.get_width()
        y2 = self.y2_pin * display.get_height()
        rect = [x, y, x2 - x, y2 - y]

        self.rect = rect

        pygame.draw.rect(display, self.color, rect, self.border_size)
        self.check_scroll()

        _y = y
        for line in self.lines[self.line_num:]:
            line_height = self.get_line_dimensions(line)[1]
            if _y + line_height > y2:
                break
            else:
                _x = x
                for word in line:
                    word_size = word.render(display, (_x, _y), self.font_resize)
                    if not isinstance(word_size, type(None)):
                        if word_size == max_font_size:
                            self.zoom_limit = 1
                        elif word_size == min_font_size:
                            self.zoom_limit = -1
                    _x += word.get_dimensions(self.font_resize)[0]
            _y += line_height
        if not isinstance(self.zoom_limit, type(None)):
            self.zoom_amount = 0

    def get_line_dimensions(self, line=None):
        if isinstance(line, type(None)):
            line = self.lines[0]
        height = 0
        width = 0
        for word in line:
            height = max(height, word.get_dimensions(self.font_resize)[1])
            width += word.get_dimensions(self.font_resize)[0]
        return width, height

    def check_scroll(self):
        if self.line_num >= len(self.lines):
            self.line_num = len(self.lines) - 1
        if self.line_num < 0:
            self.line_num = 0

    def rewrap(self):
        """Rewraps all lines in the event of a resolution change."""
        self.lines = []
        self.text_list = self.whole_text_list[:]
        self.wrap()

    def check_line_can_fit(self, text):
        """Checks whether given text can fit in last line"""
        if len(self.lines[-1]) > 0 and self.lines[-1][-1].get_text().endswith(
                "\n"):
            return False  # Last line ended with new line char
        width = 0
        for _text in self.lines[-1]:
            width += _text.get_dimensions(self.font_resize)[0]
        if width + text.get_dimensions(self.font_resize)[0] <= \
                self.get_dimensions()[0]:
            return True
        return False

    def wrap(self):
        """Wraps all text left in text_list, does not affect previous lines
        except where new text fits on last line.
        """
        break_char_ind = 0
        while len(self.text_list) > 0:
            text = self.text_list.pop(0)  # Get first text in queue
            words = text.break_chars()  # Break at newlines
            while len(words) > 0:
                word = words[0]

                # Add initial line
                if len(self.lines) == 0:
                    self.lines.append([])
                    continue

                # Check if fits on previous line
                elif self.check_line_can_fit(word):
                    # It fits, don't continue
                    self.lines[-1].append(word)

                # Try breaking at spaces
                elif " " in word.get_text()[1:-1]:
                    # Test whether space is within word (not at ends)
                    words.pop(0)
                    for w in word.break_chars(" ")[::-1]:
                        words.insert(0, w)
                    continue

                # Try making new line
                elif len(self.lines[-1]) > 0:
                    self.lines.append([])
                    continue

                # Try breaking at break chars
                elif break_char_ind < len(break_chars):
                    break_char = break_chars[break_char_ind]
                    if break_char in word.get_text()[1:-1]:
                        # Test whether char is within word (not at ends)
                        words.pop(0)
                        for w in word.break_chars(break_char)[::-1]:
                            words.insert(0, w)
                    break_char_ind += 1
                    continue

                # Try breaking at char closest to edge
                else:
                    char_width = word.copy("W").get_dimensions(
                        self.font_resize)[0]

                    if char_width <= self.width:
                        # Single char can fit on screen
                        if len(self.lines[-1]) == 0 and len(self.lines) > 1:
                            # Remove previously added empty line
                            self.lines.pop()
                        previous_width = self.get_line_dimensions(self.lines[
                                                                      -1])[0]
                        char_line_width = previous_width + char_width
                        can_fit_char = char_line_width < self.width
                        previous_line_exists = len(self.lines[-1]) > 0
                        previous_newline = False
                        if previous_line_exists:
                            previous_newline = self.lines[-1][-1].get_text(). \
                                endswith("\n")
                        if previous_line_exists and can_fit_char \
                                and not previous_newline:
                            # Previous line has room for chars,
                            # not newline char
                            words.pop(0)
                            for w in word.break_line(
                                    self.width - previous_width,
                                    self.font_resize)[::-1]:
                                words.insert(0, w)
                            continue
                        else:
                            self.lines.append([])
                            words.pop(0)
                            for w in word.break_line(self.width,
                                                     self.font_resize)[::-1]:
                                words.insert(0, w)
                            continue
                    else:
                        # Skip word, font too large
                        pass

                # TODO: Wrap at wrap points

                # Word was added, did not continue
                break_char_ind = 0
                words.pop(0)
                if word.get_text().endswith("\n"):
                    self.lines.append([])


from datetime import datetime
from airflow import DAG
from airflow.operators.python import PythonOperator

from helper import transformed_to_parquet, upload_to_bucket

args = {
        "owner": "airflow",
        "start_date": datetime(2009, 1, 1),
        "end_date": datetime(2020, 1, 1),
        "depends_on_past": True,
        "retries": 1,
}

dag = DAG(
    dag_id='2_transform_and_upload_GCS',
    default_args=args,
    description='transform data from CSC to parquet and upload to GCS yearly',
    schedule_interval="@yearly",
)

transformed_to_parquet = PythonOperator(
    task_id='transformed_to_parquet',
    python_callable=transformed_to_parquet,
    dag=dag,
    op_kwargs={
            "year": f"{{{{ execution_date.strftime(\'%Y\') }}}}",
        },
)

upload_to_bucket = PythonOperator(
    task_id='upload_to_bucket_S3',
    python_callable=upload_to_bucket,
    dag=dag,
    op_kwargs={
            "year": f"{{{{ execution_date.strftime(\'%Y\') }}}}",
        },
)
transformed_to_parquet>>upload_to_bucket


import unittest
from conformity import *
import networkx as nx
import numpy as np
import json
import random

class ConformityTestCase(unittest.TestCase):

    def test_attribute_conformity(self):
        g = nx.karate_club_graph()
        nodes = list(g.nodes())
        for n in nodes:
            g.add_node(n, pippo="si", topolino="op")

        res = attribute_conformity(g, list(np.arange(1, 4, 0.2)), ['club', 'pippo', 'topolino'], profile_size=3)

        with open(f"conformity.json", "w") as o:
            json.dump(res, o)

        for k, v in res.items():
            for z, t in v.items():
                for _, val in t.items():
                    self.assertTrue(-1 <= val <= 1)

    def test_hierarchical_attribute_conformity(self):

        labels = ['one', 'two', 'three', 'four']
        age = ["A", "B", "C"]
        hierarchy = {'labels': {'one': 1, 'two': 2, 'three': 3, 'four': 4}}

        g = nx.barabasi_albert_graph(100, 5)

        for node in g.nodes():
            g.add_node(node, labels=random.choice(labels), age=random.choice(age))

        res = attribute_conformity(g, list(np.arange(1, 4, 0.2)), ['labels', 'age'], profile_size=2,
                                   hierarchies=hierarchy)

        with open(f"conformity_hierarchy.json", "w") as o:
            json.dump(res, o)

        for k, v in res.items():
            for z, t in v.items():
                for _, val in t.items():
                    self.assertTrue(-1 <= val <= 1)

if __name__ == '__main__':
    unittest.main()


import asyncio

from assume import World, load_scenario_folder

def test_world():
    scenario = "example_01a"
    study_case = "base"
    world = World(database_uri=None, export_csv_path=None)
    load_scenario_folder(
        world, inputs_path="examples/inputs", scenario=scenario, study_case=study_case
    )

    assert world.db is None
    assert world.export_csv_path is None

    assert isinstance(world.loop, asyncio.BaseEventLoop)
    assert world.market_operators.keys()
    assert world.markets.keys()
    assert world.unit_operators.keys()


import torch
from torchvision.transforms.functional import to_pil_image,resize
import src.mvtec_loader as loader

import re
import json
import numpy as np
from lavis.models import load_model_and_preprocess
from src.util import result
import argparse

parser = argparse.ArgumentParser()
parser.add_argument('--model_type', type=str,default='1',choices=['1','2','3','4'])
parser.add_argument('--eval_way', type=str,default='1',choices=['1','2'])
parser.add_argument('--question_way', type=str,default='1',choices=['1','2'])
args = parser.parse_args()

device=torch.device('cuda')
if args.model_type=='1': model, vis_processors, _ = load_model_and_preprocess(name="blip2_opt", model_type="pretrain_opt2.7b", is_eval=True, device=device)
elif args.model_type=='2': model, vis_processors, _ = load_model_and_preprocess(name="blip2_opt", model_type="pretrain_opt6.7b", is_eval=True, device=device)
elif args.model_type=='3': model, vis_processors, _ = load_model_and_preprocess(name="blip2_t5", model_type="pretrain_flant5xl", is_eval=True, device=device)
elif args.model_type=='4': model, vis_processors, _ = load_model_and_preprocess(name="blip2_t5", model_type="pretrain_flant5xxl", is_eval=True, device=device)

gpu_num = torch.cuda.device_count()
if(torch.cuda.device_count()>1):
    model = torch.nn.DataParallel(model,device_ids=np.arange(0,gpu_num)[range(0,gpu_num,1)].tolist())
"""DataParallel objectにmodelもmoduleを移す"""
model = model.module

mvtec = ['bottle','cable','carpet','hazelnut','pill','screw','toothbrush','wood','capsule','grid','leather','metal_nut','tile','transistor','zipper']
# mvtec = ['carpet','hazelnut']
result["Question"] = "Does this "+ "○○" +"hava any defect in this image? Answer:"
for breed in mvtec:
    _,test_data = loader.get_mvtec_loader(breed)
    data_num=0
    acc_num = 0
    all_path=[]
    all_label=[]
    all_pred=[]
    for idx,(data,label,_,path) in enumerate(test_data):
        pred = []
        data,label=data.to(device),label.to(device)
        if args.question_way=='1':
            Question = "Does this "+ breed +"hava any defect in this image? Answer:"
        elif args.question_way=='2':
            Question = "Does this boject hava any defect in this image? Answer:"
        # Question = "Is this wood perfect in this image? Answer:"
        text = model.generate({"image": resize(data, size=(224, 224)), "prompt": Question})
        # text = model.generate({"image": resize(data, size=(364,364)), "prompt": Question})

        result[breed]['path'].extend(path)
        result[breed]['pred'].extend(text)
        result[breed]['gt'].extend(list(label.cpu().numpy()))

        if args.eval_way=='1':
            for t in text:
                pred.append((np.array(re.split('[, ]',text[0]))=='yes').sum())
            pred = np.array(pred)
            pred[pred>0] = 1
            label[label>0] = 1
            pred = list(pred)
            data_num+= len(label)
            acc_num += (pred==label.cpu().numpy()).sum()
            print(text)
            print(label)
            print('')
        elif args.eval_way=='2':
            for t in text:
                pred.append((np.array(re.split('[, ]',text[0]))=='no').sum())
                pred.append((np.array(re.split('[, ]',text[0]))=='No').sum())
            pred = np.array(pred)
            pred[pred==0] = -1; pred[pred>0] = 0
            label[label>0] = -1; label[label==0] = 0
            pred = list(pred)
            data_num+= len(label)
            acc_num += (pred==label.cpu().numpy()).sum()
            print(text)
            print(label)
            print('')
    print(f'accuracy:{(acc_num/data_num)*100}%')
    result[breed]['test_acc'] = round((acc_num/data_num)*100,3)

import pandas as pd
df = pd.DataFrame.from_dict(result, orient='index', columns=['data'])
df.to_json('Qtype:'+args.question_way+'_type:'+args.eval_way+'_data:'+args.model_type+'.json')


import numpy as np
from scipy import sparse
from collections import deque
from collections import defaultdict
import networkx as nx

class MultiCom(object):
    # https://github.com/ahollocou/multicom
    #
    # Hollocou, Alexandre and Bonald, Thomas and Lelarge, Marc
    # "Multiple Local Community Detection"
    # International Symposium on Computer Performance, Modeling, Measurements and Evaluation 2017

    def __init__(self, g):
        self.g = g

    def __load_graph(self):
        """
        Load an undirected and unweighted graph from an edge-list file.
        :param edgelist_filename: string or unicode
            Path to the edge-list file.
            Id of nodes are assumed to be non-negative integers.
        :param delimiter: str, default '\t'
        :param comment: str, default '#'
        :return: Compressed Sparse Row Matrix
            Adjacency matrix of the graph
        """
        edge_df = nx.to_pandas_edgelist(self.g)
        edge_list = edge_df.values
        n_nodes = int(np.max(edge_list) + 1)
        adj_matrix = sparse.coo_matrix(
            (np.ones(edge_list.shape[0]), (edge_list[:, 0], edge_list[:, 1])),
            shape=tuple([n_nodes, n_nodes]),
            dtype=edge_list.dtype,
        )
        adj_matrix = adj_matrix.tocsr()
        adj_matrix = adj_matrix + adj_matrix.T
        return adj_matrix

    def __convert_adj_matrix(self, adj_matrix):
        """
        Convert an adjacency matrix to the Compressed Sparse Row type.
        :param adj_matrix: An adjacency matrix.
        :return: Compressed Sparse Row Matrix
            Adjacency matrix with the expected type.
        """
        if type(adj_matrix) == sparse.csr_matrix:
            return adj_matrix
        elif type(adj_matrix) == np.ndarray:
            return sparse.csr_matrix(adj_matrix)
        else:
            raise TypeError(
                "The argument should be a Numpy Array or a Compressed Sparse Row Matrix."
            )

    def __approximate_ppr(self, adj_matrix, seed_set, alpha=0.85, epsilon=1e-5):
        """
        Compute the approximate Personalized PageRank (PPR) from a set set of seed node.
        This function implements the push method introduced by Andersen et al.
        in "Local graph partitioning using pagerank vectors", FOCS 2006.
        :param adj_matrix: compressed sparse row matrix or numpy array
            Adjacency matrix of the graph
        :param seed_set: list or set of int
            Set of seed nodes.
        :param alpha: float, default 0.85
            1 - alpha corresponds to the probability for the random walk to restarts from the seed set.
        :param epsilon: float, default 1e-3
            Precision parameter for the approximation
        :return: numpy 1D array
            Vector containing the approximate PPR for each node of the graph.
        """
        adj_matrix = self.__convert_adj_matrix(adj_matrix)
        degree = np.array(np.sum(adj_matrix, axis=0))[0]
        n_nodes = adj_matrix.shape[0]

        prob = np.zeros(n_nodes)
        res = np.zeros(n_nodes)
        res[list(seed_set)] = 1.0 / len(seed_set)

        next_nodes = deque(seed_set)

        while len(next_nodes) > 0:
            node = next_nodes.pop()
            push_val = res[node] - 0.5 * epsilon * degree[node]
            res[node] = 0.5 * epsilon * degree[node]
            prob[node] += (1.0 - alpha) * push_val
            put_val = alpha * push_val
            for neighbor in adj_matrix[node].indices:
                old_res = res[neighbor]
                res[neighbor] += put_val * adj_matrix[node, neighbor] / degree[node]
                threshold = epsilon * degree[neighbor]
                if res[neighbor] >= threshold > old_res:
                    next_nodes.appendleft(neighbor)
        return prob

    def __conductance_sweep_cut(self, adj_matrix, score, window=10):
        """
        Return the sweep cut for conductance based on a given score.
        During the sweep process, we detect a local minimum of conductance using a given window.
        The sweep process is described by Andersen et al. in
        "Communities from seed sets", 2006.
        :param adj_matrix: compressed sparse row matrix or numpy array
            Adjacency matrix of the graph.
        :param score: numpy vector
            Score used to order the nodes in the sweep process.
        :param window: int, default 10
            Window parameter used for the detection of a local minimum of conductance.
        :return: set of int
             Set of nodes corresponding to the sweep cut.
        """
        adj_matrix = self.__convert_adj_matrix(adj_matrix)
        n_nodes = adj_matrix.shape[0]
        degree = np.array(np.sum(adj_matrix, axis=0))[0]
        total_volume = np.sum(degree)
        sorted_nodes = [node for node in range(n_nodes) if score[node] > 0]
        sorted_nodes = sorted(sorted_nodes, key=lambda node: score[node], reverse=True)
        sweep_set = set()
        volume = 0.0
        cut = 0.0
        best_conductance = 1.0
        best_sweep_set = {sorted_nodes[0]}
        inc_count = 0
        for node in sorted_nodes:
            volume += degree[node]
            for neighbor in adj_matrix[node].indices:
                if neighbor in sweep_set:
                    cut -= 1
                else:
                    cut += 1
            sweep_set.add(node)

            if volume == total_volume:
                break
            conductance = cut / min(volume, total_volume - volume)
            if conductance < best_conductance:
                best_conductance = conductance
                # Make a copy of the set
                best_sweep_set = set(sweep_set)
                inc_count = 0
            else:
                inc_count += 1
                if inc_count >= window:
                    break
        return best_sweep_set

    def execute(self, seed_node, clustering=None, n_steps=5, explored_ratio=0.8):
        """
        Algorithm for multiple local algorithms detection from a seed node.
        It implements the algorithm presented by Hollocou, Bonald and Lelarge in
        "Multiple Local Community Detection".
        :param g: networkx graph
        :param seed_node: int
            Id of the seed node around which we want to detect communities.
        :param clustering: Scikit-Learn Cluster Estimator
            Algorithm used to cluster nodes in the local embedding space.
            Example: sklearn.cluster.DBSCAN()
        :param n_steps: int, default 5
            Parameter used to control the number of detected communities.
        :param explored_ratio: float, default 0.8
            Parameter used to control the number of new seeds at each step.
        :return:
        seeds: list of int
            Seeds used to detect communities around the initial seed (including this original seed).
        communities: list of set
            Communities detected around the seed node.
        """
        seeds = dict()
        scores = dict()
        communities = list()
        explored = set()

        if clustering is None:
            from sklearn.cluster import DBSCAN

            clustering = DBSCAN()

        adj_matrix = self.__load_graph()
        adj_matrix = self.__convert_adj_matrix(adj_matrix)
        n_nodes = adj_matrix.shape[0]
        degree = np.array(np.sum(adj_matrix, axis=0))[0]

        new_seeds = [seed_node]
        step = -1
        n_iter = 0

        while step < n_steps and len(new_seeds) > 0:
            n_iter += 1

            for new_seed in new_seeds:
                step += 1
                seeds[step] = new_seed
                scores[step] = self.__approximate_ppr(adj_matrix, [seeds[step]])
                community = self.__conductance_sweep_cut(adj_matrix, scores[step])
                communities.append(community)
                # We add the algorithms to the explored nodes
                explored |= set(community)

            new_seeds = list()

            # Clustering of the nodes in the space (scores[seed1], scores[seed2],...,)
            embedding = np.zeros((n_nodes, step + 1))
            for i in range(step + 1):
                embedding[:, i] = scores[i][:]
            indices = np.where(np.sum(embedding, axis=1))[0]
            y = clustering.fit_predict(embedding[indices, :])
            clusters = defaultdict(set)
            for i in range(y.shape[0]):
                if y[i] != -1:
                    clusters[y[i]].add(indices[i])

            # Pick new seeds in unexplored clusters
            for c in range(len(clusters)):
                cluster_size = 0
                cluster_explored = 0
                for node in clusters[c]:
                    cluster_size += 1
                    if node in explored:
                        cluster_explored += 1
                if float(cluster_explored) / float(cluster_size) < explored_ratio:
                    candidates = list(set(clusters[c]) - explored)
                    candidate_degrees = np.array([degree[node] for node in candidates])
                    new_seeds.append(candidates[np.argmax(candidate_degrees)])

        return list(communities)

    @staticmethod
    def __get_node_membership(communities):
        """
        Get the algorithms membership for each node given a list of communities.
        :param communities: list of list of int
            List of communities.
        :return: membership: dict (defaultdict) of set of int
            Dictionary such that, for each node,
            membership[node] is the set of algorithms ids to which the node belongs.
        """
        membership = defaultdict(set)
        for i, community in enumerate(communities):
            for node in community:
                membership[node].add(i)
        return membership


import os
import sys
import cv2
import argparse
import numpy as np

import matplotlib
import matplotlib.pyplot as plt
from matplotlib.patches import Rectangle
import seaborn as sns
import SimpleITK as sitk
import json
from mean_average_precision.detection_map import DetectionMAP

sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'lib'))

import torch
import torch.utils.data

from nets.hourglass import get_hourglass
from utils.utils import load_model
from utils.image import transform_preds, get_affine_transform
from utils.post_process import ctdet_decode

# Training settings
parser = argparse.ArgumentParser(description='centernet')

parser.add_argument('--root_dir', type=str, default='./')
parser.add_argument('--ckpt_dir', type=str, default='./ckpt/hourglass/checkpoint.t7')
parser.add_argument('--arch', type=str, default='large_hourglass')

cfg = parser.parse_args()

os.chdir(cfg.root_dir)

def evaluation(model,validation_folder,visualization = True,map_save_name='map.png'):
  frames = []
  n_class = 1
  mAP = DetectionMAP(n_class)
  max_per_image = 100

  validation_label = r'L:\TB_Project\src\Detector\data\coco\annotations\instances_val2017.json'
  with open(validation_label,'r') as load_f:
    load_dict = json.load(load_f)
  file_dict = {}
  for file in load_dict['images']:
      file_dict[file['file_name']] = file['id']
  label_dict = {}
  for label in load_dict['annotations']:
      label_dict[label['image_id']] = []
  for label in load_dict['annotations']:
      label_dict[label['image_id']].append(label['bbox'])

  for img_file in os.listdir(validation_folder):
    img_file_path = os.path.join(validation_folder,img_file)
    image = np.load(img_file_path)

    gt_boxes = []
    for (x_0,y_0,bbox_width,bbox_height) in label_dict[file_dict[img_file]]:
      x_1 = x_0 + bbox_width
      y_1 = y_0 + bbox_height
      gt_boxes.append((x_0,y_0,x_1,y_1))

    height, width = image.shape[0:2]
    padding = 127 if 'hourglass' in cfg.arch else 31
    imgs = {}

    for scale in [1]:
      new_height = int(height * scale)
      new_width = int(width * scale)

      img_height = 512
      img_width = 512
      center = np.array([new_width / 2., new_height / 2.], dtype=np.float32)
      scaled_size = max(height, width) * 1.0
      scaled_size = np.array([scaled_size, scaled_size], dtype=np.float32)

      img = cv2.resize(image, (new_width, new_height))
      trans_img = get_affine_transform(center, scaled_size, 0, [img_width, img_height])
      img = cv2.warpAffine(img, trans_img, (img_width, img_height))

      img = (img.astype(np.float32)+1024.) / 4096.
      img -= 0.09102
      img /= 0.3735
      img = np.stack((img,img,img),axis=2)
      img = img.transpose(2, 0, 1)[None, :, :, :]  # from [H, W, C] to [1, C, H, W]

      imgs[scale] = {'image': torch.from_numpy(img).float(),
                    'center': np.array(center),
                    'scale': np.array(scaled_size),
                    'fmap_h': np.array(img_height // 4),
                    'fmap_w': np.array(img_width // 4)}

    with torch.no_grad():
      detections = []
      for scale in imgs:
        imgs[scale]['image'] = imgs[scale]['image'].cuda()

        output = model(imgs[scale]['image'])[-1]
        dets = ctdet_decode(*output, K=100) #test_topk
        dets = dets.detach().cpu().numpy().reshape(1, -1, dets.shape[2])[0]

        top_preds = {}
        dets[:, :2] = transform_preds(dets[:, 0:2],
                                      imgs[scale]['center'],
                                      imgs[scale]['scale'],
                                      (imgs[scale]['fmap_w'], imgs[scale]['fmap_h']))
        dets[:, 2:4] = transform_preds(dets[:, 2:4],
                                      imgs[scale]['center'],
                                      imgs[scale]['scale'],
                                      (imgs[scale]['fmap_w'], imgs[scale]['fmap_h']))
        cls = dets[:, -1]
        for j in range(1):
          inds = (cls == j)
          top_preds[j + 1] = dets[inds, :5].astype(np.float32)
          top_preds[j + 1][:, :4] /= scale

        detections.append(top_preds)

      bbox_and_scores = {}
      for j in range(1):
        temp= np.concatenate([d[j+1] for d in detections], axis=0)
        bbox_and_scores[j] = temp

      scores = np.hstack([bbox_and_scores[j][:, 4] for j in range(1)])

      if len(scores) > max_per_image:
        kth = len(scores) - max_per_image
        thresh = np.partition(scores, kth)[kth]
        for j in range(1):
          keep_inds = (bbox_and_scores[j][:, 4] >= thresh)
          bbox_and_scores[j] = bbox_and_scores[j][keep_inds]

      fig = plt.figure(0)
      matplotlib.use('QT5Agg')
      plt.imshow(image)
      count = 0

      pred_bb = []
      pred_cls = []
      pred_conf = []
      gt_bb = []
      gt_cls = []
      for lab in bbox_and_scores:
        for boxes in bbox_and_scores[lab]:
          x1, y1, x2, y2, score = boxes
          if score > 0.15:
            pred_bb.append([x1, y1, x2, y2])
            pred_cls.append(0)
            pred_conf.append(score)
            count = count+1
            plt.gca().add_patch(Rectangle((x1, y1), x2 - x1, y2 - y1,
                                          linewidth=2, edgecolor='r', facecolor='none'))
            plt.text(x1 + 3, y1 + 3, '%.2f' % score,
                    bbox=dict(facecolor='r', alpha=0.3), fontsize=7, color='k')
      for boxes in gt_boxes:
        x1, y1, x2, y2 = boxes
        gt_bb.append(boxes)
        gt_cls.append(0)
        plt.gca().add_patch(Rectangle((x1, y1), x2 - x1, y2 - y1,
                                          linewidth=2, edgecolor='r', facecolor='none'))

      frames.append((np.array(pred_bb), np.array(pred_cls), np.array(pred_conf), np.array(gt_bb), np.array(gt_cls)))
      fig.patch.set_visible(False)
      plt.axis('off')
      if visualization:
        save_path='./result/'
        save_name=save_path+img_file[:-3]+'png'
        plt.savefig(save_name, dpi=300, transparent=True)
      plt.close('all')

  for frame in frames:
      mAP.evaluate(*frame)
  mAP.plot()
  plt.savefig(map_save_name)

def main():
  cfg.device = torch.device('cuda')
  torch.backends.cudnn.benchmark = False
  validation_folder=r'L:\TB_Project\src\Detector\data\coco\val2017'
  print('Creating model...')
  if 'hourglass' in cfg.arch:
    model = get_hourglass[cfg.arch]
  else:
    raise NotImplementedError

  model = load_model(model, cfg.ckpt_dir)
  model = model.to(cfg.device)
  model.eval()
  evaluation(model,validation_folder)

if __name__ == '__main__':
  main()


from django.db import models
from django.utils.translation import gettext_lazy as _

# AWX
from awx.api.versioning import reverse
from awx.main.models.base import CommonModelNameNotUnique
from awx.main.models.unified_jobs import UnifiedJobTemplate, UnifiedJob
from awx.main.models.inventory import Inventory
from awx.main.models.schedules import Schedule
from awx.main.models.workflow import WorkflowJobTemplateNode, WorkflowJobNode

__all__ = ('Label',)

class Label(CommonModelNameNotUnique):
    """
    Generic Tag. Designed for tagging Job Templates, but expandable to other models.
    """

    class Meta:
        app_label = 'main'
        unique_together = (("name", "organization"),)
        ordering = ('organization', 'name')

    organization = models.ForeignKey(
        'Organization',
        related_name='labels',
        help_text=_('Organization this label belongs to.'),
        on_delete=models.CASCADE,
    )

    def get_absolute_url(self, request=None):
        return reverse('api:label_detail', kwargs={'pk': self.pk}, request=request)

    def is_detached(self):
        return Label.objects.filter(
            id=self.id,
            unifiedjob_labels__isnull=True,
            unifiedjobtemplate_labels__isnull=True,
            inventory_labels__isnull=True,
            schedule_labels__isnull=True,
            workflowjobtemplatenode_labels__isnull=True,
            workflowjobnode_labels__isnull=True,
        ).exists()

    def is_candidate_for_detach(self):
        count = UnifiedJob.objects.filter(labels__in=[self.id]).count()  # Both Jobs and WFJobs
        count += UnifiedJobTemplate.objects.filter(labels__in=[self.id]).count()  # Both JTs and WFJT
        count += Inventory.objects.filter(labels__in=[self.id]).count()
        count += Schedule.objects.filter(labels__in=[self.id]).count()
        count += WorkflowJobTemplateNode.objects.filter(labels__in=[self.id]).count()
        count += WorkflowJobNode.objects.filter(labels__in=[self.id]).count()
        return (count - 1) == 0


import cPickle
import networkx as nx

def make_tree(sequences):

	G = nx.Graph()

	for sequence in sequences:
		for i in xrange(len(sequence)-1):
			G.add_edge(code_2_index[sequence[i]], code_2_index[sequence[i+1]])

	return G

mesh_to_seq = cPickle.load(open('mesh_to_seq.pkl','r'))
sequences = mesh_to_seq.values()
sequences = [['SSOS','SOS']+sequence.split(".")+['EOS'] for sequence in sequences]

code_2_index = cPickle.load(open('code_2_index.pkl', 'r'))
G = make_tree(sequences)
nx.write_adjlist(G, './mesh_codes_tree.adj')


'''
给定一个整数数组 nums 和一个目标值 target，
请你在该数组中找出和为目标值的那 两个 整数，并返回他们的数组下标。
你可以假设每种输入只会对应一个答案。但是，你不能重复利用这个数组中同样的元素。
'''
import numpy as np
import logging

def two_sum(nums, target):
    # nums是数组，遍历nums，找target-nums【i】，
    if len(nums)>2:
        result = []
        for idx,value in enumerate(nums):
            for i in range(len(nums)-idx):
                if nums[idx+i] == target - value:
                    result.append((idx, idx+i))
        return result
    else:
        logging.error("nums len less than 2")
        return 0

if __name__=="__main__":
    nums = [2, 7, 11, 15, 4, 3, 5, 6, 12, 9]
    target = 15
    print(two_sum(nums, target))


import bpy
from . import VGU_Operators
from . import VGU_Preferences
from . import utility_function

def draw_ui_list(self, context, layout, data, item, icon, active_data, active_propname, emboss_name=False):

    ob = data
    vgroup = item

    row = layout.row(align=True)

    preferences = utility_function.get_addon_preferences(context)

    if preferences.SoloIcon:
        row.operator("vgu.panel_solo", text="", icon="SOLO_ON").index = item.index

    if preferences.VisibilityIcon:

        row.operator("vgu.panel_visibility_toogle", text="", icon="HIDE_OFF").index = item.index

    if context.mode == "EDIT_MESH":
        if preferences.SelectionIcon:
            row.operator("vgu.panel_selection_toogle", text="", icon="RESTRICT_SELECT_OFF").index = item.index

    if preferences.SetPivotIcon:
        row.operator("vgu.panel_set_origin", text="", icon="OBJECT_ORIGIN").index = item.index

    row.prop(vgroup, "name", text="", emboss=emboss_name, icon_value=icon)

    if preferences.LockWeightIcon:
        icon = 'LOCKED' if vgroup.lock_weight else 'UNLOCKED'
        row.prop(vgroup, "lock_weight", text="", icon=icon, emboss=False)

    if preferences.LockAllButThis:
        row.operator("vgu.lock_all_weight_but_this", text="",icon="MARKER_HLT").index = item.index

    if preferences.AddModifierIcon:
        row.operator("vgu.panel_addmodifier", text="", icon="MODIFIER").index = item.index

    if preferences.SeperateIcon:
        row.operator("vgu.panel_seperate", text="", icon="MOD_BOOLEAN").index = item.index

    if context.mode == "EDIT_MESH":
        if preferences.AssignIcon:
            row.operator("vgu.panel_assign", text="", icon="ADD").index = item.index

    if context.mode == "EDIT_MESH":
        if preferences.UnassignIcon:
            row.operator("vgu.panel_unassign", text="", icon="REMOVE").index = item.index

    if preferences.RemoveIcon:
        row.operator("vgu.panel_remove", text="", icon="TRASH").index = item.index

class VGU_UL_VertexGroup_List(bpy.types.UIList):

    def draw_item(self, context, layout, data, item, icon, active_data, active_propname):

        draw_ui_list(self, context, layout, data, item, icon, active_data, active_propname, emboss_name=False)

class VGU_UL_VertexGroup_Pie_List(bpy.types.UIList):

    def draw_item(self, context, layout, data, item, icon, active_data, active_propname):

        draw_ui_list(self, context, layout, data, item, icon, active_data, active_propname, emboss_name=True)

def draw_panels(layout, context, emboss_name):

    obj = context.object

    preferences = utility_function.get_addon_preferences(context)

    row=layout.row()

    if obj:
        if context.object.type == "MESH":

            if emboss_name:
                row.template_list("VGU_UL_VertexGroup_Pie_List", "", obj, "vertex_groups", obj.vertex_groups, "active_index")
            else:
                row.template_list("VGU_UL_VertexGroup_List", "", obj, "vertex_groups", obj.vertex_groups, "active_index")

            col = row.column(align=True)
            col.operator("object.vertex_group_add", icon='ADD', text="")
            props = col.operator("object.vertex_group_remove", icon='REMOVE', text="")
            props.all_unlocked = props.all = False

            col.separator()

            col.menu("MESH_MT_vertex_group_context_menu", icon='DOWNARROW_HLT', text="")

            col.menu("VGU_MT_icon_expose", text="",icon="VIS_SEL_11")

            col.separator()

            col.operator("object.vertex_group_move", icon='TRIA_UP', text="").direction = 'UP'
            col.operator("object.vertex_group_move", icon='TRIA_DOWN', text="").direction = 'DOWN'

            row = layout.row()

            # row.prop(context.space_data.overlay,"show_weight" )

            row = layout.row(align=True)
            row.prop(context.scene.tool_settings, "vertex_group_weight", text="Weight")

            row = layout.row(align=True)
            row.operator("vgu.panel_new_vertexgroup", text="New", icon="PLUS").weight = context.scene.tool_settings.vertex_group_weight
            row.operator("object.vertex_group_remove", text="Remove", icon="TRASH")

            row = layout.row(align=True)
            if len(obj.vertex_groups) > 0:
                if obj.vertex_groups.active_index >= 0 and obj.vertex_groups.active_index < len(obj.vertex_groups):
                    if context.mode == "EDIT_MESH":
                        reassign = row.operator("vgu.panel_assign", text="Assign", icon="ADD")
                        reassign.weight = context.scene.tool_settings.vertex_group_weight
                        reassign.index = obj.vertex_groups.active_index

                        row.operator("object.vertex_group_remove_from", text="Unassign", icon="REMOVE")

                        layout.separator()
                        layout.operator("object.vertex_group_remove_from", text="Unassign from all", icon="REMOVE").use_all_groups=True
                    layout.operator("vgu.unhide_all_visibility", text="Unhide All", icon="HIDE_OFF")

class VGU_PT_Vertex_Group_List_SIDE(bpy.types.Panel):

    bl_label = "Vertex Group List"
    bl_idname = "VERTEXGROUPUTILS_PT_list_panel_SIDE"
    bl_space_type = 'VIEW_3D'
    bl_region_type = 'UI'
    bl_category = "Vertex Group Utils"

    @classmethod
    def poll(cls, context):
        preferences = utility_function.get_addon_preferences(context)
        if preferences.Vertex_Group_List_Side_Panel:
            if context.object:
                if context.object.type == "MESH":
	                return True

    def draw(self, context):

        layout = self.layout
        emboss_name = False
        draw_panels(layout, context, emboss_name)

        layout.prop(context.space_data.overlay,"show_weight" )

class VGU_PT_Vertex_Group_List_DATA(bpy.types.Panel):

    bl_label = "Vertex Group List"
    bl_idname = "VERTEXGROUPUTILS_PT_list_panel_DATA"
    bl_space_type = 'PROPERTIES'
    bl_region_type = 'WINDOW'
    bl_context = "data"

    @classmethod
    def poll(cls, context):
        preferences = utility_function.get_addon_preferences(context)
        if preferences.Vertex_Group_List_Data_Panel:
            if context.object:
                if context.object.type == "MESH":
	                return True

    def draw(self, context):

        layout = self.layout
        emboss_name = True
        draw_panels(layout, context, emboss_name)

def draw_tools(layout, context):

    if context.object:
        if context.object.type == "MESH":

            layout.label(text="Clean Up")
            layout.operator("vgu.panel_remove_empty_vertexgroup", text="Empty Group", icon="TRASH")
            layout.operator("vgu.panel_unassign_zero_weight_from_all_group", text="Remove Zero Weights from Group", icon="REMOVE").index = context.object.vertex_groups.active_index

            layout.separator()
            layout.label(text="Groups")

            row = layout.row()
            if context.mode == "OBJECT":
                row.operator("vgu.panel_join_as_vertex_group", text="Join as Group", icon="OUTLINER_DATA_MESH")

            row.operator("vgu.panel_split_loose_to_vertex_group", text="Loose Mesh to Group", icon="SNAP_VERTEX")

            row = layout.row()
            row.operator("vgu.panel_vertex_group_from_side", text="Group From Side", icon="SNAP_VERTEX")
            row.operator("vgu.panel_vertex_group_from_material", text="Group From Material", icon="SNAP_VERTEX")

            layout.operator("vgu.panel_separate_all_vertexgroup", text="Separate All", icon="MOD_EXPLODE")

            layout.separator()
            layout.label(text="Utility")
            layout.operator("vgu.panel_batch_rename_group", text="Batch Rename Vertex Groups", icon="ZOOM_ALL")
            layout.operator("vgu.panel_merge_vertex_group", text="Merge Group", icon="AUTOMERGE_ON")
            layout.operator("vgu.panel_merge_multiple_vertex_groups", text="Merge Multiple Groups", icon="AUTOMERGE_ON")
            layout.operator("vgu.round_vertex_weight", text="Round All Vertex Weight", icon="AUTOMERGE_ON")

            row = layout.row(align=True)
            row.operator("vgu.select_unlocked_vertex_group", text="Select Unlocked", icon="RESTRICT_SELECT_OFF").mode = "SELECT"
            row.operator("vgu.select_unlocked_vertex_group", text="Deselect Unlocked", icon="RESTRICT_SELECT_ON").mode = "DESELECT"

            layout.separator()
            layout.label(text="Armature")
            bone = layout.operator("vgu.panel_create_bone", text="Bone from Vertex Group", icon="BONE_DATA")
            bone.index = context.object.vertex_groups.active_index
            bone.from_loose_mesh = False

            bone = layout.operator("vgu.panel_create_bone", text="Bone from Loose Mesh", icon="BONE_DATA")
            bone.index = context.object.vertex_groups.active_index
            bone.from_loose_mesh = True

            layout.separator()

            layout.operator("vgu.smooth_all_vertex_groups_weights", text="Smooth All Vertex Groups Weight")

            layout.separator()
            layout.label(text="Selected Objects")
            # layout.operator("vgu.panel_batch_rename_selected_objects_vertex_group", text="Find And Replace", icon="ZOOM_ALL")
            layout.operator("vgu.duplicate_selected_objects_mesh_and_rename_vertex_groups", text="Duplicate Mesh And Rename Vertex Group (Experimental)", icon="DUPLICATE")

            layout.separator()
            layout.label(text="Vertex Colors")
            layout.operator("vgu.vertex_color_from_vertex_group", text="Vertex Color From Vertex Group", icon="VPAINT_HLT")
            layout.operator("vgu.random_vertex_color_from_all_vertex_groups", text="Random Vertex Color From All Vertex Groups", icon="VPAINT_HLT")

            layout.separator()
            layout.label(text="Active Object")
            layout.label(text=context.object.name, icon="OBJECT_DATA")
            layout.prop(context.object, "VGU_Object_Seed_Offset", text="Seed")

class VGU_PT_Vertex_Group_Tools(bpy.types.Panel):

    bl_label = "Vertex Group Tools"
    bl_space_type = 'VIEW_3D'
    bl_region_type = 'UI'
    bl_category = "Vertex Group Utils"
    bl_options = {'DEFAULT_CLOSED'}

    @classmethod
    def poll(cls, context):
        preferences = utility_function.get_addon_preferences(context)
        if preferences.Vertex_Group_Tools:
            return True

    def draw(self, context):

        layout = self.layout

        notice = True

        if context.object:
            if context.object.type == "MESH":
                notice = False
                draw_tools(layout, context)

        if notice:
            layout.label(text="Please Select A Mesh Object", icon="INFO")

classes = (
        VGU_UL_VertexGroup_List,
        VGU_UL_VertexGroup_Pie_List,
        # VIEW3D_PT_SIDE_VGU_Panel_SIDE,
        # VIEW3D_PT_SIDE_VGU_Panel_DATA,
        # VIEW3D_PT_VGU_Vertex_Group_Utils_Tools
        )

def register():

    for cls in classes:
        bpy.utils.register_class(cls)

def unregister():

    for cls in classes:
        bpy.utils.unregister_class(cls)

if __name__ == "__main__":
    register()


#!/usr/bin/env python
import sys, os
import numpy as np
import synergia
import pytest

macroparticles=16
realparticles=4.0e10

nturns=1

mp = synergia.foundation.pconstants.mp

# prop_fixture is a propagator
@pytest.fixture
def prop_fixture():

    channel_madx = """
beam, particle=proton,energy=8.0+pmass;
rfc: rfcavity, l=0.0, volt=1.0, harmon=84;
channel: sequence, l=3200.0, refer=entry;
rfc, at=3200.0;
endsequence;
"""
    reader = synergia.lattice.MadX_reader()
    reader.parse(channel_madx)
    lattice = reader.get_lattice('channel')
    lattice.set_all_string_attribute('extractor_type', 'libff')
    synergia.simulation.Lattice_simulator.tune_linear_lattice(lattice)
    stepper = synergia.simulation.Independent_stepper_elements(1)
    propagator = synergia.simulation.Propagator(lattice, stepper)

    return propagator

# two trains start out with the same reference particle and design reference momentum
# because they are propagating in the
# the same accelerator, but the secondary bunch will have it bunch energy increase
# by 100 MeV to mimic slip stacking. The bunches propagate in the same pipe but one
# secondary moves faster than the primary so its cdt will go negative compared to
# primary.
def create_simulator(ref_part_pri, ref_part_sec):
    sim = synergia.simulation.Bunch_simulator.create_two_trains_simulator(ref_part_pri, ref_part_sec, macroparticles, realparticles)
    bunch_pri = sim.get_bunch(0, 0)
    bunch_pri.checkout_particles()
    lp = bunch_pri.get_particles_numpy()
    lp[:, 0:6] = 0.0
    bunch_pri.checkin_particles()

    design_energy = ref_part_pri.get_total_energy()
    bunch_sec = sim.get_bunch(1, 0)
    bunch_sec.get_reference_particle().set_total_energy(design_energy + 0.100)
    bunch_sec.checkout_particles()
    lp = bunch_sec.get_particles_numpy()
    lp[:, 0:6] = 0.0
    bunch_sec.checkin_particles()

    return sim

def test_co_propagate(prop_fixture):

    #print('lattice: ')
    #print(prop_fixture.get_lattice())

    refpart = prop_fixture.get_lattice().get_reference_particle()

    sim = create_simulator(refpart, refpart)

    simlog = synergia.utils.parallel_utils.Logger(0, synergia.utils.parallel_utils.LoggerV.INFO_TURN, False)
    prop_fixture.propagate(sim, simlog, nturns)

    bunch_pri = sim.get_bunch(0, 0)
    bunch_sec = sim.get_bunch(1, 0)

    # primary bunch should be locked to the main energy so its cdt will be 0
    # the secondary bunch will slip compared to primary so its particle's cdt
    # will be negative.

    lattice = prop_fixture.get_lattice()
    L = lattice.get_length()

    beta_pri = bunch_pri.get_reference_particle().get_beta()
    beta_sec = bunch_sec.get_reference_particle().get_beta()

    print('beta_pri: ', beta_pri)
    print('beta_sec: ', beta_sec)

    bunch_pri.checkout_particles()
    lp_pri = bunch_pri.get_particles_numpy()
    bunch_sec.checkout_particles()
    lp_sec = bunch_sec.get_particles_numpy()

    assert lp_pri[0, 4] == pytest.approx(0.0)
    tdiff = L * (1/beta_sec - 1/beta_pri)
    print('tdiff: ', tdiff)

    assert lp_sec[0, 4] == pytest.approx(tdiff)

    print('lp_pri[0, 4]: ', lp_pri[0, 4])
    print('lp_sec[0, 4]: ', lp_sec[0, 4])

def main():
    pf = prop_fixture()
    test_accel1(pf)

if __name__ == "__main__":
    main()


import h5py
import numpy as np
from numpy.lib.stride_tricks import as_strided
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split

if __name__ == '__main__':
    with h5py.File('temp_train.h5', 'w') as trainfile, \
    h5py.File('temp_valid.h5', 'w') as validfile, \
    h5py.File('temp_input.h5', 'w') as input_file:

        trial_size = 380
        sin_ang_freq = 2*np.pi/100 #angular frequency of sine wave used for test data
        n_co = 2
        n_factor = 40
        n_outputs = 100
        n_trials = 500
        chunk_size = 60
        overlap = 20

        def create_data(n_dim, n_trials, trial_size=trial_size, sin_ang_freq=sin_ang_freq,
                        chunk_size=chunk_size, overlap=overlap):

            data = np.array([np.array([np.sin(np.random.rand(1)*2*np.pi +
                                        sin_ang_freq*np.arange(trial_size)) for j in
                                        range(n_dim)]).T for i in range(n_trials)])
            float_size = 8 #float size in bytes
            step = chunk_size - overlap
            n_chunks = np.round((trial_size - overlap)/step).astype(int)
            # dimensions -- trial, chunk, time, data dimension
            chunked_data = as_strided(data, (n_trials, n_chunks, chunk_size, n_dim),
                                            (trial_size*n_dim*float_size, step*n_dim*float_size, float_size*n_dim, float_size))
            #stacking chunks across trials
            chunked_data = np.vstack(chunked_data)
            return chunked_data, data

        n_chunks = np.round((trial_size - overlap)/(chunk_size-overlap)).astype(int)
        inds = np.arange(n_chunks*n_trials)
        train_inds, valid_inds = train_test_split(inds, test_size=0.2)
        input_file['train_inds'] = train_inds
        input_file['valid_inds'] = valid_inds
        trials = np.concatenate([[i]*n_chunks for i in range(n_trials)])
        input_file['train_trials'] = trials[train_inds]
        input_file['valid_trials'] = trials[train_inds]
        input_file['all_trials'] = trials

        keys = ['controller_outputs', 'factors', 'output_dist_params']
        n_dims = [n_co, n_factor, n_outputs] #dimensions of data for each input above

        for key, n_dim in zip(keys, n_dims):
            data,td = create_data(n_dim, n_trials)
            train_data = data[train_inds,:,:]
            valid_data = data[valid_inds,:,:]
            trainfile[key] = train_data
            validfile[key] = valid_data


"""Defines URL patterns for pages."""
from django.urls import path

from . import views

app_name = 'pages'
urlpatterns = [
    #Home page
    path('', views.home, name='home'),
    path('home/', views.home, name='home'),
    # Page that show services section.
    path('services/', views.services, name='services'),
    #Page that show skills section.
    path('skills/', views.skills, name='skills'),
    #Page that show teams section.
    path('projects/', views.projects, name='projects'),
    #Page that show contact section.
    path('contact/', views.contact, name='contact')
]


# 27. Remove Element
from typing import List

class Solution:
    def removeElement(self, nums: List[int], val: int) -> int:
        if len(nums)==0:
            return 0
        i=0
        j = len(nums)-1
        while (i<=j):
            if nums[i] != val:
                i = i+1
            elif nums[j]== val:
                j = j-1
            else:
                nums[i] = nums[j]
                i = i+1
                j = j-1
        return i

s = Solution
nums = [0,1,2,2,3,0,4,2]
val = 2
nums = [3,2,2,3]; val = 3
nums = [0]; val = 0
# nums = [3,3,3,3]; val = 3
# nums = [2]; val = 3
# nums = [4,5]; val = 4
print(s.removeElement(s, nums, val))
print(nums)


class Vertex(object):
    def __init__(self, value):
        self.value = value
        self.visited = False

class Graph(object):
    def __init__(self):
        from collections import deque
        self.vertices = []
        self.edges = []
        self.stack = []
        self.queue = deque()

    def add_vertex(self, value):
        self.vertices.append(Vertex(value))
        self.edges = [x + [0] for x in self.edges]
        self.edges.append([0] * len(self.vertices))

    def add_edge(self, start, end, weight=1):
        l = len(self.edges)
        if 0 < start >= l or 0 < end >= l:
            print("Add edge error")
        else:
            self.edges[start][end] = self.edges[end][start] = weight

    def find_next_adjacent(self, index):
        for vert_index, value in enumerate(self.edges[index]):
            if (value and not self.vertices[vert_index].visited):
                return vert_index

        return None

    def display_vertex(self, index):
        return f"{self.vertices[index].value}"

    def dfs(self):
        result = []
        if not self.vertices:
            return result

        index = 0
        self.vertices[index].visited = True
        result.append(self.display_vertex(index))
        self.stack.append(index)

        while (self.stack):
            index = self.find_next_adjacent(self.stack[-1])
            if index:
                self.vertices[index].visited = True
                result.append(self.display_vertex(index))
                self.stack.append(index)
            else:
                index = self.stack.pop()

        return result

    def bfs(self):
        result = []
        if not self.vertices:
            return result

        index = 0
        self.vertices[index].visited = True
        result.append(self.display_vertex(index))
        self.queue.appendleft(index)

        while(self.queue):
            index = self.find_next_adjacent(self.queue[-1])
            if index:
                self.vertices[index].visited = True
                result.append(self.display_vertex(index))
                self.queue.appendleft(index)
            else:
                index = self.queue.pop()

        return result


from datetime import datetime
from retrofix import c43
from retrofix.exception import RetrofixException
from trytond.pool import Pool, PoolMeta
from trytond.model import fields
from trytond.i18n import gettext
from trytond.exceptions import UserError

__all__ = ['Configuration', 'ConfigurationDefaultAccount', 'Import',
    'ImportStart']

csb43_date = fields.Selection([
    ('operation_date', 'Operation Date'),
    ('value_date', 'Value Date'),
    ], 'CSB43 Date',
    help='Set date line from CSB43 file')

class Configuration(metaclass=PoolMeta):
    __name__ = 'account.configuration'
    csb43_date = fields.MultiValue(csb43_date)

    @classmethod
    def multivalue_model(cls, field):
        pool = Pool()
        if field == 'csb43_date':
            return pool.get('account.configuration.default_account')
        return super(Configuration, cls).multivalue_model(field)

    @classmethod
    def default_csb43_date(cls, **pattern):
        return cls.multivalue_model('csb43_date').default_csb43_date()

class ConfigurationDefaultAccount(metaclass=PoolMeta):
    __name__ = 'account.configuration.default_account'
    csb43_date = csb43_date

    @classmethod
    def default_csb43_date(cls):
        return 'operation_date'

class ImportStart(metaclass=PoolMeta):
    __name__ = 'account.bank.statement.import.start'

    @classmethod
    def __setup__(cls):
        super(ImportStart, cls).__setup__()
        cls.type.selection += [('csb43', 'CSB 43')]

class Import(metaclass=PoolMeta):
    __name__ = 'account.bank.statement.import'

    def process(self, statement):
        super(Import, self).process(statement)
        if self.start.type != 'csb43':
            return

        pool = Pool()
        BankStatement = pool.get('account.bank.statement')
        BankStatementLine = pool.get('account.bank.statement.line')

        try:
            data = self.start.import_file.decode('latin1').replace('\r', '')
        except UnicodeDecodeError as e:
            raise UserError(gettext('account_bank_statement.format_error',
                error=str(e)))

        try:
            records = c43.read(data)
        except RetrofixException as e:
            raise UserError(gettext('account_bank_statement.format_error',
                error=str(e)))

        description = []
        lines = []
        line = {}
        start_date = records[0].start_date
        if isinstance(start_date, datetime):
            start_date = start_date.date()
        end_date = records[0].end_date
        if isinstance(end_date, datetime):
            end_date = end_date.date()

        BankStatement.write([statement], {
            'start_date': start_date,
            'end_date': end_date,
            'start_balance': records[0].initial_balance,
            'end_balance': records[-1].final_balance,
            })

        for record in records[1:-1]:
            if record.record_code == '23':
                description.append(record.concept_1)
                description.append(record.concept_2)
            elif record.record_code == '22':
                if line:
                    description = [x.strip() for x in description if x != '']
                    line['description'] = " ".join(description)
                    lines.append(line.copy())
                line = self.get_line_vals_from_record(record, statement)
                description = [record.reference_1]
                description.append(record.reference_2)

        if line:
            description = [x.strip() for x in description if x != '']
            line['description'] = " ".join(description)
            lines.append(line.copy())
        BankStatementLine.create(lines)
        return 'end'

    def get_line_vals_from_record(self, record, statement):
        Config = Pool().get('account.configuration')
        config = Config(1)

        return {
            'statement': statement.id,
            'date': getattr(record, config.csb43_date or 'value_date'),
            'amount': record.amount,
            }


import pickle
import streamlit as st
import requests
import pandas as pd

def fetch_poster(movie_id):
    url = "https://api.themoviedb.org/3/movie/{}?api_key=d4b65d015870427bde13e7261ad70435&language=en-US".format(movie_id)
    data = requests.get(url)
    data = data.json()
    poster_path = data['poster_path']
    full_path = "https://image.tmdb.org/t/p/w500/" + poster_path
    return full_path

def recommend(movie):
    movie_index = movies[movies['title'] == movie].index[0]
    distances = similarity[movie_index]
    movie_list=sorted(list(enumerate(distances)),reverse=True,key=lambda x:x[1])[1:6]
    recommended_movies = []
    recommended_movie_posters = []
    for i in movie_list:
        # fetch the movie poster
        movie_id = movies.iloc[i[0]].movie_id
        recommended_movie_posters.append(fetch_poster(movie_id))
        recommended_movies.append(movies.iloc[i[0]].title)

    return recommended_movies,recommended_movie_posters

st.header('Movie Recommender System')
movies = pickle.load(open('movie_.pkl','rb'))
similarity = pickle.load(open('similarity.pkl','rb'))

movie_list = movies['title'].values
selected_movie = st.selectbox(
    "Type or select a movie from the dropdown to get recommendations",
    movies['title'].values)

if st.button('Show Recommendation'):
    names,posters = recommend(selected_movie)
    col1, col2, col3, col4, col5 = st.columns(5)
    with col1:
        st.text(names[0])
        st.image(posters[0])
    with col2:
        st.text(names[1])
        st.image(posters[1])

    with col3:
        st.text(names[2])
        st.image(posters[2])
    with col4:
        st.text(names[3])
        st.image(posters[3])
    with col5:
        st.text(names[4])
        st.image(posters[4])


import numpy as np

import torch
import torch.nn as nn
from torch.nn.parameter import Parameter

class TransformerEmbedding(nn.Module):
    """Embeddings matrix mixed with positional encoding for sequences."""

    def __init__(self, vocab_size, num_units, use_positional_encoding):
        super(TransformerEmbedding, self).__init__()
        self.num_units = num_units
        self.use_positional_encoding = use_positional_encoding
        self.embed = nn.Embedding(vocab_size, num_units)

    def _sin_cos_enc(self, from_length, to_length, embedding_size):
        position_enc = np.array(
            [[
                pos / np.power(10000, 2 * i / embedding_size)
                for i in range(embedding_size)
            ] for pos in range(from_length, to_length)],
            dtype=np.float32)

        # put sinusodial on even position
        position_enc[:, 0::2] = np.sin(position_enc[:, 0::2])
        # put cosine on odd position
        position_enc[:, 1::2] = np.cos(position_enc[:, 1::2])

        result = Parameter(torch.from_numpy(position_enc))  # Why is this a parameter?
        if next(self.embed.parameters()).is_cuda:
            result = result.cuda()
        return result

    def forward(self, ids):
        embed = self.embed(ids)
        if self.use_positional_encoding:
            embed += self._sin_cos_enc(0, embed.size(1), self.num_units)
        return embed

    def sample(self, ids, position):
        embed = self.embed(ids)
        if self.use_positional_encoding:
            embed += self._sin_cos_enc(position, position + 1, self.num_units)
        return embed


import folium
from folium import plugins
import random

def createstart(startloc):
    map = folium.Map(location=startloc, zoom_start=8, tiles='openstreetmap')
    #display map
    folium.Marker(startloc, popup='Start Location',icon=folium.Icon(color='green')).add_to(map)
    map.save('templates/map.html')

def createdest(startloc, destloc):
    map = folium.Map(location=startloc, zoom_start=8, tiles='openstreetmap')
    #display map
    folium.Marker(startloc, popup='Start Location',icon=folium.Icon(color='green')).add_to(map)
    folium.Marker(destloc, popup='Destination Location',icon=folium.Icon(color='red')).add_to(map)
    map.save('templates/map.html')

def plotroute(route1,danger1,route2,danger2,route3,danger3):
    if route1!= None:
        route1 = route1["coordinates"]
        for item in route1:
            temp = item[0]
            item[0]=item[1]
            item[1]=temp
    if route2!= None:
        route2 = route2["coordinates"]
        for item in route2:
            temp = item[0]
            item[0]=item[1]
            item[1]=temp

    if route3 != None:
        route3 = route3["coordinates"]
        for item in route3:
            temp = item[0]
            item[0]=item[1]
            item[1]=temp

    if route2 != None:
        map_plot_antroute = folium.Map(location=route2[int((len(route2)/2)-1):int(len(route2)/2)][0], zoom_start=15)
    else:
        map_plot_antroute = folium.Map(location=route1[int((len(route1)/2)-1):int(len(route1)/2)][0], zoom_start=15)

    folium.Marker(route1[0], popup='Start Location',icon=folium.Icon(color='green')).add_to(map_plot_antroute)
    folium.Marker(route1[-1], popup='Destination Location',icon=folium.Icon(color='red')).add_to(map_plot_antroute)

    if route1 != None:
        # added lat long to route
        route_lats_longs = route1
        if danger1 in [0,1]:
            color = 'green'
        if danger1 in [2,3]:
            color = 'orange'
        if danger1 in [4,5]:
            color = 'red'
        # Ploting ant-route
        plugins.AntPath(route_lats_longs,color='green').add_to(map_plot_antroute)

    if route2 != None:
        # added lat long to route
        route_lats_longs = route2
        if danger2 in [0,1]:
            color = 'green'
        if danger2 in [2,3]:
            color = 'orange'
        if danger2 in [4,5]:
            color = 'red'
        # Ploting ant-route
        plugins.AntPath(route_lats_longs,color='orange').add_to(map_plot_antroute)

    if route3 != None:
        # added lat long to route
        route_lats_longs = route3
        if danger3 in [0,1]:
            color = 'green'
        if danger3 in [2,3]:
            color = 'orange'
        if danger3 in [4,5]:
            color = 'red'
        # Ploting ant-route
        plugins.AntPath(route_lats_longs,color='red').add_to(map_plot_antroute)

    map_plot_antroute.save('templates/map.html')


import requests
import json
from statistics import mean, stdev
from os import environ

# get_quarter(int) -> string
# takes a month and returns the quarter
# it belongs to as string to use as key
# for gas_per_quarter dictionary
def get_quarter(month):
	if 1 <= month <= 3:
		return "Q1"
	elif 4 <= month <= 6:
		return "Q2"
	elif 7 <= month <= 9:
		return "Q3"
	elif 10 <= month <= 12:
		return "Q4"

# series_id specifies which data set you want to get
# in particular, ELEC.CONS_TOT.NG-TX-99.M has the
# total consumption of natural gas in thousand McF
# per month in Texas
parameters = {
	"api_key": environ['API_KEY'],
	"series_id": "ELEC.CONS_TOT.NG-TX-99.M",
	"start": 201801,
	"end": 201812
}

# try to fetch from endpoint
try:
	response = requests.get("https://api.eia.gov/series", params=parameters)
	response.raise_for_status()

# if connection unsuccessful raise and print error info
except requests.exceptions.HTTPError as err:
	print(f"HTTP error: {err.response.status_code}")

else:
	# retrieve the data we care about
	data = response.json()['series'][0]['data']
	# retrieve unit of data for later printing of statistics
	units = response.json()['series'][0]['units']

	# dict to store gas consumption per quarter
	gas_per_quarter = {"Q1":[], "Q2":[], "Q3":[], "Q4":[]}

	# since the date format given is YYYYMM, splice the
	# last two chars from date, cast to int and save as
	# month variable. then append consumption value into
	# corresponding quarter in gas_per_quarter
	for date, val in data:
		month = int(date[-2:])
		gas_per_quarter[get_quarter(month)].append(float(val))

	# retrieve data per quarter and output statistics
	print("Total consumption of natural gas in Texas in 2018")
	print("Quarterly statistics:\n")
	for quarter in gas_per_quarter.keys():
		print(f"{quarter}:")
		print(f"average: {mean(gas_per_quarter[quarter])}")
		print(f"standard deviation: {stdev(gas_per_quarter[quarter])}")
		print(f"max value: {max(gas_per_quarter[quarter])}")
		print(f"min value: {min(gas_per_quarter[quarter])}")
		print("")


"""Word Drop Module."""
from .OneDim import OneDim

import copy
import math
import numpy as np
import pandas as pd
import random
import spacy

from progress.bar import ShadyBar
from checklist.perturb import Perturb

class DropWordsOneDim(OneDim):
    """Class for word drop task."""

    __slots__ = [
        "texts", "results", "dmgd_texts",
        "combined_results", "step_arr", "path",
        "name", "df", "descr"]

    def __init__(self, params: dict):
        """Initialize."""
        super(DropWordsOneDim, self).__init__(params=params)
        self.name = "dropped_words"
        self.descr = "Dropped words"

    @staticmethod
    def drop_single(
            sentence: str,
            doc: spacy.tokens.doc.Doc,
            step: float) -> tuple:
        """Drop words.

        Params
        ------
        sentence : str
            sentence
        doc : spacy.tokens.doc.Doc
            spacy document containing linguistic information
            about the sentence
        step : float
            fraction of how many tokens are to be dropped

        Returns
        -------
        tuple
            Tuple including sentence and success of modification
        """
        bound: float = 1 - 1 / len(doc)
        if len(doc) == 0:
            return sentence, False

        if step == 0:
            return sentence, True

        candidates: list = []
        for i in range(len(doc)):
            if doc[i].pos_ != "PUNCT":
                candidates.append(i)
            else:
                continue

        # one word must be in the sentence at least
        if step > bound:
            step = bound

        prop: float = int(math.floor(step * len(candidates)))
        drop_list: list = random.sample(candidates, k=prop)

        sent: str = ""

        for i in range(len(doc)):

            # exclude words to be dropped
            if i in drop_list:
                continue

            # TODO annotate
            token = doc[i]

            word = ""
            if i == 0:
                word = token.text
            else:
                if token.pos_ == "PUNCT":
                    word = token.text
                else:
                    word = " " + token.text
            sent += word

        if len(sent.strip()) == 0:
            print("Sentence empty! Word drop")
            return sent, False

        return sent, True

    def perturbate(self) -> None:
        """Perturbate sentences."""
        bar: ShadyBar = ShadyBar(
            message="Perturbating " + self.name + " ",
            max=len(self.step_arr) * len(self.texts))

        for step in self.step_arr:
            ret_tuple: tuple = ([], [])
            for _, (sentences, doc) in enumerate(self.texts):

                sentences: list = copy.deepcopy(sentences)
                indices: list = []

                for i in range(len(sentences)):

                    if len(doc[i]) < 2:
                        continue

                    new_sentence = sentences[i]
                    new_sentence, success = self.drop_single(
                        sentence=sentences[i],
                        doc=doc[i],
                        step=step)

                    if success:
                        indices.append(i)
                        sentences[i] = new_sentence

                ret_tuple[0].append(sentences)
                ret_tuple[1].append(indices)
                bar.next()

            self.dmgd_texts.append(ret_tuple)

        bar.finish()