import utils.data as datautils

import re
from unidecode import unidecode
from keras import Input
from keras.callbacks import EarlyStopping, ReduceLROnPlateau, LambdaCallback
from keras.engine import Model
from keras.layers import Embedding, Bidirectional, LSTM, TimeDistributed, Dense, Dropout, Activation
from keras.models import load_model
from keras.optimizers import RMSprop
from keras.preprocessing.sequence import pad_sequences
from sklearn.model_selection import train_test_split

import constants as c
import numpy as np

from models.relation_classifier import LSTM_SIZE, EMBEDDING_SIZE, DROPOUT, BATCH_SIZE, MAX_EPOCHS
from utils.dependency_parser import get_spacy_parser
import pickle

class RelationClassifierRNNBased():
    def __init__(self,
                 lstm_size=LSTM_SIZE,
                 embedding_size=EMBEDDING_SIZE,
                 dropout=DROPOUT,
                 batch_size=BATCH_SIZE,
                 max_epochs=MAX_EPOCHS
                 ):
        self.w2idx = {c.PAD_TAG: 0, c.UNK_TAG: 1}
        self.idx2w = {0: c.PAD_TAG, 1: c.UNK_TAG}

        self.rel2idx = {c.NIL_TAG:0}
        self.idx2rel = {0:c.NIL_TAG}
        self.MAX_LENGTH = 35

        self.lstm_size = lstm_size
        self.embedding_size = embedding_size
        self.dropout = dropout
        self.batch_size = batch_size
        self.max_epochs = max_epochs

    def make_vocab(self, sents):
        parser = get_spacy_parser()
        for s in sents:
            cleaned_s = datautils.clean_question(s)
            doc = parser.tokenizer(cleaned_s)
            for t in doc:
                if not t.text in self.w2idx:
                    new_idx = len(self.w2idx)
                    self.w2idx[t.text] = new_idx
                    self.idx2w[new_idx] = t.text
                if not t.text.lower() in self.w2idx:
                    new_idx = len(self.w2idx)
                    self.w2idx[t.text.lower()] = new_idx
                    self.idx2w[new_idx] = t.text.lower()

    def to_word_list(self, vect):
        words = list(map(lambda i: self.idx2w[i], vect))
        return words

    def to_tags(self, vect):
        tags = c.entity_tags[vect.argmax()]
        return tags

    def fit(self, X,Y, dev=None):
        # early stopping callback for stop the training when needed
        earlyStopping = EarlyStopping(monitor='val_loss', patience=2, verbose=1, mode='auto')

        # reduce learning rate callback for optimize on plateau
        reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=0, verbose=1, mode='auto',
                                      epsilon=0.0001, cooldown=0, min_lr=0)

        def printer(x):
            print(x)

        print_callback = LambdaCallback(
            on_epoch_end=lambda epoch, logs: printer(logs),
            on_train_end=lambda logs: printer(logs)
        )

        self.model.fit([X], Y, epochs=self.max_epochs, batch_size=self.batch_size,
                       shuffle=True,
                       validation_data=dev,
                       verbose=1,
                       callbacks=[print_callback, reduce_lr, earlyStopping],
                       )

    def train(self, sents, concepts, dev=None):
        X, Y = self.make_XY(sents, concepts)
        if dev:
            X_dev, Y_dev = self.make_XY(dev[0], dev[1])
        else:
            (X_dev, Y_dev) = (None, None)
        self.model = self.get_model()
        self.fit(X,Y,dev=(X_dev, Y_dev))

    def test(self, sents, relations):
        X,Y = self.make_XY(sents, relations)
        Y_pred = self.model.predict(X)

        total = len(X)
        correct = 0
        prediction_per_relation = {k:[0,0] for k in c.RELATIONS}

        for x,yp,yt, sent, r in zip(X, Y_pred, Y, sents, relations):
            no_pads = [t for t in x if t!=0]
            true_len = len(no_pads)
            pred_tags = self.idx2rel[yp.argmax()]
            true_tags = self.idx2rel[yt.argmax()]

            prediction_per_relation[true_tags][1] += 1
            if true_tags==pred_tags:
                correct+=1
                prediction_per_relation[true_tags][0] += 1
            else:
                print("="*50)
                print(sent)
                print(r)
                print(self.to_word_list(x)[:true_len])
                print(pred_tags)
                print(true_tags)
                print("=" * 50)

        print(total, correct)
        print(correct/total)
        print("accuracy per relation: ")

        print("Relation\tSupport\tCorrect\tAcc.")
        for k, (cor, tot) in prediction_per_relation.items():
            print("\t".join([k, str(tot), str(cor), str(cor/tot)]))

        print("\t".join(["Total", str(total), str(correct), str(correct/total)]))

    def get_model(self):
        main_input = Input(shape=(self.MAX_LENGTH,), name="main_input")

        embedded = Embedding(len(self.w2idx),self.embedding_size,mask_zero=True)(main_input)

        blstm_l1 = Bidirectional(LSTM(
            self.lstm_size,
            return_sequences=True)
        )(embedded)

        blstm_l2 = Bidirectional(LSTM(
            self.lstm_size,
            return_sequences=False)
        )(blstm_l1)

        output_l = Dense(len(self.rel2idx), use_bias=True)(blstm_l2)
        dropout_out_l = Dropout(0.1)(output_l)
        activation = Activation(activation="softmax")(dropout_out_l)

        model = Model(inputs=[main_input], outputs=[activation])

        # learning rate = 0.001
        opt = RMSprop()
        model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])
        model.summary()
        return model

    def make_x(self, sent):
        parser = get_spacy_parser()
        question = datautils.clean_question(sent)

        # tokenize the question string
        q_parsed = parser(question)
        q_vect = list(map(lambda t: self.w2idx[t.text] if t.text in self.w2idx else self.w2idx[c.UNK_TAG], q_parsed))
        pad_length = self.MAX_LENGTH - len(q_vect)
        nil_X = 0
        x = (q_vect + (pad_length) * [nil_X])
        return np.array(x)

    def make_XY(self, sents, relations):
        """

        :param sents: a list of strings
        :param relations: a list of relation strings
        :return: X, Y, where:
            - X is a list of one-hot encoded sentences (eventually padded), but not vectorized
                i.e. a np.Array of shape (len(sents), MAX_LENGTH)
            - Y is the list of one-hot encoded relations.
        """
        X, Y = [], []
        for sent, rel in zip(sents, relations):

            x = self.make_x(sent)

            if not rel in self.rel2idx:
                new_idx = len(self.rel2idx)
                self.rel2idx[rel]=new_idx
                self.idx2rel[new_idx]=rel

            y = datautils.to_ohenc(self.rel2idx[rel], 17)

            X.append(np.array(x))
            Y.append(np.array(y))
        return np.array(X),np.array(Y)

    def predict(self, query, candidate_relations=[]):
        x = self.make_x(query)

        Y_pred = self.model.predict(np.array([x]))

        Y_pred = enumerate(Y_pred[0])
        Y_pred = sorted(Y_pred, key=lambda x: -x[1])

        relation = self.idx2rel[Y_pred[0][0]]

        # Actually, the mapping relation 2 domain are not so good.
        # For example, for "Geography and places" there missing the "PLACE" relation, which I think it is necessary.
        # However, the implementation is below.

        # if candidate_relations and relation not in candidate_relations:
        #     Y_pred = list(filter(lambda x: x[0]!=self.rel2idx[c.NIL_TAG]  and self.idx2rel[x[0]] in candidate_relations, Y_pred))
        #     Y_pred = sorted(Y_pred, key=lambda x: -x[1])
        #     if len(Y_pred)==0:
        #         relation =""
        #     else:
        #         relation = self.idx2rel[Y_pred[0][0]]

        return relation

    def save(self, dirpath):
        import os
        os.system("rm -rf " + dirpath)
        os.mkdir(dirpath)
        temp = self.get_model()
        self.model.save(dirpath+"/model")
        self.model = None
        import pickle
        pickle.dump(self, open(dirpath+"/obj", "wb"))
        self.model = temp

    @staticmethod
    def load(dirpath):
        obj = pickle.load(open(dirpath+"/obj", "rb"))
        model = load_model(dirpath+"/model")
        obj.model = model
        return obj


from transformers import AutoTokenizer, AutoModelForCausalLM
# from auto_gptq import AutoGPTQForCausalLM
from evaluators.evaluator import Evaluator
import torch
import pdb

class LLama2_FlagAlpha_Evaluator(Evaluator):
    def __init__(self, model_path):
        super().__init__(model_path)
        # self.model = AutoGPTQForCausalLM.from_quantized(model_path)
        self.model = AutoModelForCausalLM.from_pretrained(model_path, torch_dtype=torch.float16, trust_remote_code=True).cuda()
        self.tokenizer = AutoTokenizer.from_pretrained(model_path,use_fast=False)
        self.device = 'cuda'

    def format_prompt(self, query, history, memory_limit=10, input=None):
        prompt = ""
        for i, (old_query, response) in enumerate(history):
            prompt += f'<s>Human: {old_query}\n</s><s>Assistant: {response}'

        prompt += f'<s>Human: {query}\n</s><s>Assistant: '
        return prompt

    def generate(self, q, history):
        # pdb.set_trace()
        prompt = self.format_prompt(q,history)
        # input_ids = self.tokenizer(prompt, return_tensors="pt",add_special_tokens=False).to(self.device)
        inputs = self.tokenizer(prompt, return_tensors="pt").to(self.device)
        generate_input = {
            "input_ids":inputs.input_ids,
            "max_new_tokens":2048,
            "do_sample":True,
            "top_k":50,
            "top_p":0.95,
            "temperature":0.3,
            "repetition_penalty":1.3,
            "eos_token_id":self.tokenizer.eos_token_id,
            "bos_token_id":self.tokenizer.bos_token_id,
            "pad_token_id":self.tokenizer.pad_token_id
        }
        generate_ids  = self.model.generate(**generate_input)
        response = self.tokenizer.decode(generate_ids[0])
        if '<s>Assistant:' in response:
            response = response.split("<s>Assistant:")[-1]
        elif '<s> Assistant:' in response:
            response = response.split("<s> Assistant:")[-1]
        else:
            response = response.replace(prompt,"")
        print(f"response:{response}")
        return response


import pytest
from scipy.stats import uniform
import numpy as np
import matplotlib.pyplot as plt
import pint

from SOSAT import StressState
from SOSAT.constraints import StressMeasurement, FaultConstraint

units = pint.UnitRegistry()

# depth in feet
depth = 8520.0 * units('ft')
# density in kg/m^3
avg_overburden_density = 2601.27615 * units('kg/m^3')
# pore pressure gradient in MPa/km
pore_pressure_grad = 9.72686 * units('MPa/km')

pore_pressure = pore_pressure_grad * depth

ss = StressState(depth=depth.to('ft').magnitude,
                 avg_overburden_density=avg_overburden_density
                               .to('lb/ft^3').magnitude,
                 pore_pressure=pore_pressure.to('psi').magnitude,
                 depth_unit='ft',
                 density_unit='lb/ft^3',
                 pressure_unit='psi',
                 stress_unit='psi')

meas_shmin_ul = (0.62 * units('psi/ft')
                        * depth) \
                        .to('psi').magnitude
meas_shmin_ll = meas_shmin_ul - 500.0
meas_shmin_dist = uniform(meas_shmin_ll,
                          scale=(meas_shmin_ul
                                 - meas_shmin_ll))
ss.add_constraint(
         StressMeasurement(meas_shmin_dist))

# ss.add_constraint(FaultConstraint())
fig = ss.plot_posterior()
plt.savefig("stress_measurement_constraint_posterior.png")

shmin_ll, shmin_ul = ss.get_shmin_confidence_intervals(0.99)
print("smin_ll= ", shmin_ll)
print("smin_ul= ", shmin_ul)


# -*- coding: utf-8 -*-
"""Utilities for project configs"""
from typing import Any, Optional, Type

from django.conf import settings

class RegionAwareConfig:
    """A region aware config wrapper tool class

    :param user_settings: User settings, when it's a dict-like type and includes a key '_lookup_field' with value
        'region', current configs will be marked as region aware.
    """

    def __init__(self, user_settings: Any):
        try:
            lookup_field = user_settings['_lookup_field']
        except (TypeError, KeyError):
            lookup_field = None

        if lookup_field == 'region':
            self.lookup_with_region = True
            self.data = user_settings['data']
        else:
            self.lookup_with_region = False
            self.data = user_settings

    def get(self, region: str, use_default_value: bool = False) -> Any:
        """Return the config value by region

        :param use_default_value: If True, when region was not found, it will return a default config instead
        """
        if not self.lookup_with_region:
            return self.data

        try:
            return self.data[region]
        except KeyError:
            if use_default_value:
                return next(iter(self.data.values()), {})
            raise

    def __bool__(self):
        return bool(self.data)

def get_region_aware(name: str, region: str, result_cls: Optional[Type] = None) -> Any:
    """Shortcut function for getting a region aware config from settings

    :param name: Name of settings
    :param region: Name of region
    :param result_cls: Optional, when given, will try initializing the config data with `data_cls`
    :raises: KeyError, when no config can be found for region
    """
    config = getattr(settings, name)
    result = RegionAwareConfig(config).get(region)
    if result_cls is not None:
        return result_cls(**result)
    else:
        return result


from django.urls import path
from . import views

app_name = 'portfolio_app'
urlpatterns = [
    path('', views.index, name='index'),
    path('contact/', views.contact, name='contact'),
    path('about_me/', views.about_me, name='about_me'),
    path('hobbies/', views.hobbies, name='hobbies'),
    path('work/', views.work, name='work'),
    path('contact_me/', views.contact_me, name='contact_me'),
]


import math
import pickle
from pathlib import Path

import PySpice.Spice.NgSpice.Shared
import pygame
import os
import uuid

from tkinter import filedialog as fd
from tkinter import simpledialog as sd
from tkinter import messagebox as mb
from pygame.locals import SCALED
from PySpice.Unit import *
from PySpice.Spice.Netlist import Circuit
from PySpice.Logging.Logging import setup_logging

from runtime.environment import Environment

from ui.text import TextHandler
from ui.colours import *
from ui.visualiser import Visualiser
from ui.button import Button, ElementManager
from ui.interface import TabbedMenu

from protosim.project import Project, Occupier
from logic.electronics import Wire, ICSpiceSubCircuit, Sink, Node
from logic.parts import PartManager, Part, parse, PowerSupply, Breadboard, IntegratedCircuit, LED, PluginPart, Switch

# Versioning
version = "1.0.0"

# PySpice Logger
setup_logging(logging_level='INFO')

# Enable smart scaling
flags = SCALED

# Size constants
WIDTH, HEIGHT = 1300, 800
SIDEBAR_WIDTH = 320
ACTION_BAR_HEIGHT = 60

# Determine execution environment
ENV = Environment()

# Frames per second constant
HOME_FPS = 60
PROTOSIM_FPS = 120

# Enum values for code readability
HOME, PROTOSIM = 0, 1
TOP, LEFT, BOTTOM, RIGHT = 0, 1, 2, 3

# Pygame custom events
NEW_PROJECT_EVENT = pygame.USEREVENT + 1
OPEN_PROJECT_EVENT = pygame.USEREVENT + 2
EXIT_EVENT = pygame.USEREVENT + 3
UNDO_EVENT = pygame.USEREVENT + 4
REDO_EVENT = pygame.USEREVENT + 5
SAVE_EVENT = pygame.USEREVENT + 6
HOME_EVENT = pygame.USEREVENT + 7
MENU_EVENT = pygame.USEREVENT + 8
EDIT_EVENT = pygame.USEREVENT + 9
PROJECT_CHANGE_EVENT = pygame.USEREVENT + 10

def draw_homepage(win, homepage_title, homepage_version, visualiser, buttons):
    """Draw the home page of de:volt"""

    # DVD Circuit Graphic
    draw_circuit_graphic(win, visualiser)

    # Title / Version
    title_text, title_shadow = homepage_title
    title_coords = ((WIDTH/2) - title_text.get_width()/2, HEIGHT/2 - 140)
    win.blit(title_shadow, tuple(x + 2 for x in title_coords))
    win.blit(title_text, title_coords)
    version_text, version_shadow = homepage_version
    version_x = WIDTH/2 + title_text.get_width()/2 - version_text.get_width()
    version_y = HEIGHT/2 + title_text.get_height() - 150
    win.blit(version_shadow, tuple(x + 1 for x in (version_x, version_y)))
    win.blit(version_text, (version_x, version_y))

    # Buttons
    buttons.draw(win)

def draw_circuit_graphic(win, visualiser):
    """Draw the DVD-like circuit visualiser"""
    win.fill(COL_HOME_BKG)
    visualiser.draw(win)

def draw_sim(win, sidebar_width, project, buttons, title, sidebar, show_datasheet):
    """Draw the simulator space"""

    # Background
    win.fill(COL_HOME_BKG)

    # Set the project size if sidebar is open
    if project.width != WIDTH-sidebar_width:
        project.set_size(width=WIDTH-sidebar_width)

    # Draw action bar text elements
    action_text, action_shadow = title
    action_coords = (WIDTH/2 - action_text.get_width()/2, ACTION_BAR_HEIGHT/2 - action_text.get_height()/2)
    win.blit(action_shadow, tuple(x + 1 for x in action_coords))
    win.blit(action_text, action_coords)

    # Draw project
    project.pos = (sidebar_width, ACTION_BAR_HEIGHT)
    win.blit(project.surface(), project.pos)

    # Draw action bar buttons
    buttons.draw(win)

    # Draw datasheet
    if show_datasheet[0]:
        win.blit(show_datasheet[1], project.pos)
        pygame.draw.rect(win, COL_BLACK, show_datasheet[1].get_rect(topleft=project.pos), width=2)

    # Draw sidebar
    if sidebar_width > 0:
        win.blit(sidebar.surface(), (0, ACTION_BAR_HEIGHT))
        sidebar.listen()

def open_dev():
    """Open a window to prompt the user to select a .dev file"""
    filetypes = (("de:volt Project", "*.dev"),)
    return fd.askopenfile(title="Open de:volt Project", initialdir=ENV.get_main_path(), filetypes=filetypes)

def save_dev(project):
    """Open a window to prompt the user to save a .dev file"""
    filetypes = (("de:volt Project", "*.dev"),)
    file = fd.asksaveasfile(title="Save de:volt Project", initialdir=ENV.get_main_path(), filetypes=filetypes,
                            defaultextension=".dev", initialfile=project.display_name)
    return file

def unionise_nodes(nodes):
    """Takes a list of lists and returns a list with all common elements amongst original lists paired"""
    out = []
    while len(nodes) > 0:
        first, *rest = nodes
        first = set(first)

        lf = -1
        while len(first) > lf:
            lf = len(first)

            rest2 = []
            for r in rest:
                if len(first.intersection(set(r))) > 0:
                    first |= set(r)
                else:
                    rest2.append(r)
            rest = rest2

        out.append(first)
        nodes = rest
    return out

def main():

    # Initialise pygame modules
    pygame.font.init()
    pygame.display.init()

    # Create an opaque window surface with defined width and height, and set a title
    win = pygame.display.set_mode((WIDTH, HEIGHT), flags)
    win.set_alpha(None)
    pygame.display.set_caption("de:volt")

    # Set the icon of the window
    icon = pygame.image.load(os.path.join(ENV.get_main_path(), 'assets', 'textures', 'logo.png'))
    pygame.display.set_icon(icon)

    # Initialise pygame's clock and start the game loop
    clock = pygame.time.Clock()
    running = True
    fps = HOME_FPS

    # Set the initial state to the title screen
    current_state = HOME

    # Create text handlers
    title_handler = TextHandler(ENV, 'Play-Bold.ttf', 75)
    version_handler = TextHandler(ENV, 'Play-Regular.ttf', 15)
    button_text_handler = TextHandler(ENV, 'Play-Regular.ttf', 25)
    action_text_handler = TextHandler(ENV, 'Play-Regular.ttf', 30)

    # HOMEPAGE ELEMENTS
    # Buttons
    new_button = Button((220, 60), ((WIDTH/2)-110, (HEIGHT/2)-20), 'plus.png', "New Project", NEW_PROJECT_EVENT, ENV)
    open_button = Button((220, 60), ((WIDTH/2)-110, (HEIGHT/2)+60), 'open.png', "Open Project", OPEN_PROJECT_EVENT, ENV)
    exit_button = Button((220, 60), ((WIDTH/2)-110, (HEIGHT/2)+140), 'exit.png', "Exit", EXIT_EVENT, ENV)
    home_buttons = [new_button, open_button, exit_button]
    home_button_manager = ElementManager(home_buttons, button_text_handler)

    # Pre-rendered text
    home_title = title_handler.render_shadow("de:volt", shadow_colour=COL_HOME_TITLE, colour=COL_HOME_SHADOW)
    home_version = version_handler.render_shadow(version, shadow_colour=COL_HOME_TITLE, colour=COL_HOME_SHADOW)

    # Circuit visualiser for homepage
    visualiser = Visualiser(WIDTH, HEIGHT)

    # SIMULATOR ELEMENTS
    # Project
    sidebar_width = SIDEBAR_WIDTH

    project = Project(WIDTH - sidebar_width, HEIGHT - ACTION_BAR_HEIGHT, ENV)

    # Parts
    default_parts = parse(os.path.join(ENV.get_main_path(), 'assets', 'parts.xml'))
    boards = PartManager("Boards", Part.BOARD_DESC, default_parts[0], project)
    ics = PartManager("Integrated Circuits", Part.IC_DESC, default_parts[1], project, small_title="ICs")
    electronics = PartManager("Electronics", Part.ELECTRONICS_DESC, default_parts[2], project)

    # Sidebar
    sidebar_tabs = [boards, ics, electronics]
    sidebar = TabbedMenu((sidebar_width, HEIGHT - ACTION_BAR_HEIGHT), sidebar_tabs, 30, (0, ACTION_BAR_HEIGHT), ENV)

    # Pre-rendered text
    action_bar_title = action_text_handler.render_shadow(project.display_name)
    edit_title = WIDTH/2 + action_bar_title[0].get_width()/2 + 10

    # Buttons
    button_dimensions = ACTION_BAR_HEIGHT - 16
    button_size = (button_dimensions, button_dimensions)
    button_y = 8
    home_button = Button(button_size, (20 + button_dimensions, button_y), 'home.png', 'Home', HOME_EVENT, ENV)
    menu_button = Button(button_size, (10, button_y), 'menu.png', 'Menu', MENU_EVENT, ENV)
    edit_button = Button(tuple(x - 6 for x in button_size), (edit_title + 5, 10), 'edit.png', 'Edit', EDIT_EVENT, ENV)
    redo_button = Button(button_size, (WIDTH - 20 - 2*button_dimensions, button_y), 'redo.png', 'Redo', REDO_EVENT, ENV)
    undo_button = Button(button_size, (WIDTH - 30 - 3*button_dimensions, button_y), 'undo.png', 'Undo', UNDO_EVENT, ENV)
    save_button = Button(button_size, (WIDTH - 10 - button_dimensions, button_y), 'save.png', 'Save', SAVE_EVENT, ENV)
    sim_elements = [project, sidebar, undo_button, redo_button, save_button, home_button, menu_button, edit_button]
    sim_manager = ElementManager(sim_elements, version_handler)

    # Global environment
    pygame.env = ENV

    # Store if the datasheet should be shown
    show_datasheet = (False, None)

    while running:

        # Check for invalid SPICE netlists, dead LEDs, post warning if so
        warning = ""

        # Limit the loop to run at the frame tick rate
        clock.tick(fps)

        # Create a virtual SPICE circuit
        circuit = Circuit("dev", ground="gnd")

        # IS = Saturated Current, RS = Ohmic Parasitic Resistance, N = Emission Coefficient
        circuit.model('LED', 'D', IS=1e-19, N=1.6, RS=2.5, EG=2.1)

        # Find all power supplies and breadboards in the project
        supplies = [supply for supply in project.boards.values() if isinstance(supply, PowerSupply)]
        boards = [board for board in project.boards.values() if isinstance(board, Breadboard)]

        # Reset all nodes
        for node in Node.instances:
            node.temp = node.uuid

        # Unionise wire sets to create common nodes in virtual circuit
        connected = []
        for wire in project.wires:
            a = wire.point_a.common
            b = wire.point_b.common
            if wire.resistance != 0:
                continue
            connected.append([a, b])

        # Connect switch nodes together
        for board in boards:
            for plugin in board.plugins:
                plugin_object = board.plugins[plugin]
                if isinstance(plugin_object, Switch):
                    pins_to_nodes = plugin_object.pins_to_nodes
                    connected.append([pins_to_nodes[0], pins_to_nodes[5]])
                    connected.append([pins_to_nodes[1], pins_to_nodes[4]])
                    connected.append([pins_to_nodes[2], pins_to_nodes[3]])
                    if plugin_object.state:
                        connected.append([pins_to_nodes[1], pins_to_nodes[2]])
                    else:
                        connected.append([pins_to_nodes[1], pins_to_nodes[0]])

        connected = unionise_nodes(connected)

        # Give all unionised nodes a new unique identifier
        for common_node in connected:
            new_uuid = str(uuid.uuid4()) if not len([i for i in common_node if isinstance(i, Sink)]) else "gnd"
            for child_node in common_node:
                child_node.temp = new_uuid

        displays = []
        goes_to_gnd = []

        # Create voltage sources in virtual circuit
        for index, supply in enumerate(supplies):
            circuit.V(index, supply.points[0].common.temp, supply.points[1].common.temp, supply.voltage)

        # Create ICs and Electronics in virtual circuit
        for index, board in enumerate(boards):
            for jndex, plugin in enumerate(board.plugins):
                plugin_object = board.plugins[plugin]
                if isinstance(plugin_object, IntegratedCircuit) and not isinstance(plugin_object, Switch):
                    name, raw, nodes = plugin_object.name, plugin_object.raw_spice, plugin_object.spice_nodes
                    pins_to_nodes = [i.temp for i in plugin_object.pins_to_nodes.values()]
                    circuit.subcircuit(ICSpiceSubCircuit(f'{plugin_object.name}-{index}{jndex}', raw, nodes))
                    circuit.X(f'{index}{jndex}', f'{plugin_object.name}-{index}{jndex}', *pins_to_nodes)
                if isinstance(plugin_object, LED):
                    if not plugin_object.cathode_connecting:
                        point_a, point_b = plugin_object.anode_point.common.temp, plugin_object.cathode_point.common.temp
                        circuit.Diode(f'{index}{jndex}', point_a, point_b, model='LED')
                        displays.append(plugin_object)

        # Create resistors in virtual circuit
        for index, wire in enumerate(project.wires):
            if wire.resistance != 0:
                circuit.R(index, wire.point_a.common.temp, wire.point_b.common.temp, int(wire.resistance))
                if wire.point_a.common.temp == "gnd":
                    goes_to_gnd.append(wire.point_b.common.temp)
                if wire.point_b.common.temp == "gnd":
                    goes_to_gnd.append(wire.point_a.common.temp)

        # Simulate the circuit
        simulator = circuit.simulator()

        # If there are elements present in the circuit, evaluate their logic state
        try:
            if len(circuit.elements):
                analysis = simulator.operating_point()
                node_analysis = analysis.nodes
            else:
                node_analysis = {}
        except PySpice.Spice.NgSpice.Shared.NgSpiceCommandError:
            warning += "Oops! Looks like one of your electrical components has floating input/s " \
                       "(not connected). Check to make sure ALL inputs are plugged in, even if they are" \
                       " not in use! You may have also shorted your power supply."
            node_analysis = {}

        # For every LED in the project, evaluate if it is receiving the correct power
        # If so, turn it on or kill it
        for display in displays:
            if display.alive:
                point_a, point_b = display.anode_point.common.temp, display.cathode_point.common.temp
                if point_a in node_analysis:
                    if 2 >= float(node_analysis[point_a]) >= 1.2 and point_b == circuit.gnd:
                        display.state = 1
                    else:
                        if float(node_analysis[point_a]) > 2:
                            if point_b == circuit.gnd:
                                display.alive = False
                                display.state = 0
                            elif point_b in node_analysis:
                                if 2 >= (float(node_analysis[point_a]) - float(node_analysis[point_b])) >= 1.2:
                                    display.state = 1
                                else:
                                    if (float(node_analysis[point_a]) - float(node_analysis[point_b])) > 2:
                                        display.alive = False
                                    display.state = 0
                            else:
                                display.state = 0
                        elif float(node_analysis[point_a]) < 1.2:
                            display.state = 0
                        else:
                            if point_b in goes_to_gnd:
                                display.state = 1
                            else:
                                display.state = 0
                else:
                    display.state = 0
            else:
                warning += "An LED has received too much voltage/current and has died " \
                           "(indicated by an X on the LED). Please replace it and use a resistor to limit the current."

        # Obliterate stupid memory leak
        if len(circuit.elements):
            try:
                ngspice = simulator.factory(circuit).ngspice
                ngspice.remove_circuit()
                ngspice.destroy()
            except PySpice.Spice.NgSpice.Shared.NgSpiceCommandError:
                pass

        # Check if the project was saved
        saved = "" if project.saved[0] else "*"

        mouse_button_down = False

        # Check for new events
        for event in pygame.event.get():

            # Exit the program if the user quit
            if event.type == pygame.QUIT:
                pygame.quit()
                sys.exit()

            # Listen for button events on the homepage
            if current_state == HOME:

                if event.type == NEW_PROJECT_EVENT:
                    fps = PROTOSIM_FPS
                    current_state = PROTOSIM

                if event.type == OPEN_PROJECT_EVENT:
                    selected_file = open_dev()
                    if selected_file is not None:
                        project.saved = (True, selected_file)
                        selected_file = open(selected_file.name, 'rb')
                        project.load_save_state(selected_file.read())
                        fps = PROTOSIM_FPS
                        current_state = PROTOSIM
                        action_bar_title = action_text_handler.render_shadow(saved + project.display_name)
                        edit_button.pos = (WIDTH / 2 + action_bar_title[0].get_width() / 2 + 10 + 5, edit_button.pos[1])

                if event.type == EXIT_EVENT:
                    pygame.quit()
                    sys.exit()

            # Listen for button events in the simulator
            if current_state == PROTOSIM:

                # Change made in the project
                if event.type == PROJECT_CHANGE_EVENT:
                    action_bar_title = action_text_handler.render_shadow(saved + project.display_name)
                    edit_button.pos = (WIDTH / 2 + action_bar_title[0].get_width() / 2 + 10 + 5, edit_button.pos[1])

                # Project is being zoomed or list is being scrolled
                if event.type == pygame.MOUSEWHEEL:
                    if event.y:
                        mouse_pos = pygame.mouse.get_pos()
                        if project.last_surface.get_rect(topleft=project.pos).collidepoint(mouse_pos):
                            project.scale(event.y*2)
                        element_list = sidebar.lists[sidebar.selected]
                        if element_list.surface().get_rect(topleft=element_list.real_pos).collidepoint(mouse_pos):
                            element_list.scroll(-event.y*20)

                # Menu is opened/closed
                if event.type == MENU_EVENT:
                    if sidebar_width > 0:
                        sidebar_width = 0
                    else:
                        sidebar_width = SIDEBAR_WIDTH

                # The user wants to undo
                if event.type == UNDO_EVENT:
                    if project.in_hand is None and project.incomplete_wire is None:
                        if len(ENV.undo_states):
                            ENV.redo_states.append(project.make_save_state())
                            if len(ENV.redo_states) > 10:
                                ENV.redo_states.pop(0)
                            project.load_save_state(ENV.undo_states[-1])
                            ENV.undo_states.pop()
                            action_bar_title = action_text_handler.render_shadow(saved + project.display_name)
                            edit_button.pos = (WIDTH / 2 + action_bar_title[0].get_width() / 2 + 10 + 5, edit_button.pos[1])
                            continue

                # The user wants to redo
                if event.type == REDO_EVENT:
                    if project.in_hand is None and project.incomplete_wire is None:
                        if len(ENV.redo_states):
                            ENV.undo_states.append(project.make_save_state())
                            if len(ENV.undo_states) > 10:
                                ENV.undo_states.pop(0)
                            project.load_save_state(ENV.redo_states[-1])
                            ENV.redo_states.pop()
                            action_bar_title = action_text_handler.render_shadow(saved + project.display_name)
                            edit_button.pos = (WIDTH / 2 + action_bar_title[0].get_width() / 2 + 10 + 5, edit_button.pos[1])
                            continue

                # The name of the file is queued to be edited
                if event.type == EDIT_EVENT:
                    new_name = sd.askstring("Edit Project Name", "Enter your new project name below.",
                                            initialvalue=project.display_name)
                    if new_name is not None and new_name != "":
                        if not new_name.endswith(".dev"):
                            new_name += ".dev"
                        project.change_made()
                        project.display_name = new_name
                        action_bar_title = action_text_handler.render_shadow(saved + project.display_name)
                        edit_button.pos = (WIDTH / 2 + action_bar_title[0].get_width() / 2 + 10 + 5, edit_button.pos[1])
                    else:
                        mb.showerror("Error", "You did not provide a name.")

                # Return to the homepage
                if event.type == HOME_EVENT:
                    if not project.saved[0]:
                        save = mb.askquestion("Unsaved Warning", "You have unsaved work! "
                                                                 "Do you want to save your project?", icon='warning')
                        if save == 'yes':
                            pygame.event.post(pygame.event.Event(SAVE_EVENT))
                        else:
                            project.reset()
                            current_state = HOME
                            fps = HOME_FPS
                            continue
                    else:
                        project.reset()
                        current_state = HOME
                        fps = HOME_FPS
                        continue

                # Save the current project
                if event.type == SAVE_EVENT:
                    if project.saved[1] is not None and Path(project.saved[1].name).name == project.display_name:
                        file = project.saved[1]
                    else:
                        file = save_dev(project)
                        if file is None:
                            continue
                    file = open(file.name, 'wb')
                    pickle.dump(project.save(file), file)
                    action_bar_title = action_text_handler.render_shadow(saved + project.display_name)
                    edit_button.pos = (WIDTH / 2 + action_bar_title[0].get_width() / 2 + 10 + 5, edit_button.pos[1])

                # Check if the mouse button was depressed but not lifted
                if event.type == pygame.MOUSEBUTTONDOWN:

                    mouse_button_down = True

                    if pygame.mouse.get_pressed()[0]:

                        if project.in_hand is None:

                            if project.point_hovered is not None:

                                mouse = pygame.mouse.get_pos()

                                # If a wire was initiated, place it
                                if project.last_surface.get_rect(topleft=project.pos).collidepoint(mouse):

                                    if project.incomplete_wire is None:
                                        project.incomplete_wire = project.point_hovered
                                        ENV.selected = None
                                    else:
                                        if project.point_hovered != project.incomplete_wire:
                                            project.change_made()
                                            project.wires.append(Wire(project.incomplete_wire, project.point_hovered))
                                            if project.incomplete_wire in ENV.query_disable:
                                                ENV.query_disable.remove(project.incomplete_wire)
                                            project.incomplete_wire = None
                                            project.point_hovered = None

                        else:

                            # Check if an LED was queued to be placed
                            if isinstance(project.in_hand, LED):
                                if project.point_hovered is not None:
                                    if not project.in_hand.cathode_connecting:
                                        project.change_made()
                                        project.in_hand.anode_point = project.point_hovered
                                        project.in_hand.cathode_connecting = True
                                        project.point_hovered.parent.plugins[project.point_hovered] = project.in_hand
                                    else:
                                        project.in_hand.cathode_point = project.point_hovered
                                        project.in_hand.cathode_connecting = False
                                        if project.in_hand in ENV.query_disable:
                                            ENV.query_disable.remove(project.in_hand)
                                        project.in_hand = None

                            # Check if an Integrated Circuit was placed
                            if isinstance(project.in_hand, IntegratedCircuit):
                                if project.point_hovered is not None:
                                    parent = project.point_hovered.parent
                                    discriminator = project.point_hovered.discriminator
                                    if parent.ic_allowed(project.in_hand, project.point_hovered):
                                        project.change_made()
                                        parent.plugins[project.point_hovered] = project.in_hand
                                        req = parent.ic_requirements(discriminator, project.in_hand.dip_count)
                                        project.in_hand.pins_to_nodes = req
                                        if project.in_hand in ENV.query_disable:
                                            ENV.query_disable.remove(project.in_hand)
                                        project.in_hand = None

                            # Check if a large component was placed
                            if isinstance(project.in_hand, Breadboard) or isinstance(project.in_hand, PowerSupply):
                                relative_mouse = project.relative_mouse()
                                point = (math.floor(relative_mouse[0] / project.zoom), math.floor(relative_mouse[1] / project.zoom))
                                occupying_points = []
                                allowed = True
                                for row in range(project.in_hand.size[0]):
                                    for column in range(project.in_hand.size[1]):
                                        occupying_point = tuple(map(sum, zip(point, (row, column))))
                                        if occupying_point in project.boards:
                                            allowed = False
                                            break
                                        occupying_points.append(occupying_point)
                                # If there are no collisions, place it
                                if allowed:
                                    project.boards[point] = project.in_hand
                                    occupier = Occupier(point)
                                    for occupying_point in occupying_points:
                                        if occupying_point == point:
                                            continue
                                        project.boards[occupying_point] = occupier
                                    if project.in_hand in ENV.query_disable:
                                        ENV.query_disable.remove(project.in_hand)
                                    project.in_hand = None
                                    project.change_made()

                if event.type == pygame.KEYDOWN:

                    keys_pressed = pygame.key.get_pressed()

                    # Undo, redo, save and reset key-binds
                    if keys_pressed[pygame.K_LCTRL]:
                        if event.key == pygame.K_z:
                            pygame.event.post(pygame.event.Event(UNDO_EVENT))
                        if event.key == pygame.K_y:
                            pygame.event.post(pygame.event.Event(REDO_EVENT))
                        if event.key == pygame.K_s:
                            pygame.event.post(pygame.event.Event(SAVE_EVENT))
                        if event.key == pygame.K_r:
                            project.offset_x, project.offset_y = 0, 0

                    # Force remove any restrictions if ESCAPE is pressed
                    if event.key == pygame.K_ESCAPE:

                        if project.in_hand is not None:
                            if isinstance(project.in_hand, LED) and project.in_hand.cathode_connecting:
                                del project.in_hand.anode_point.parent.plugins[project.in_hand.anode_point]
                            project.in_hand = None
                        if project.incomplete_wire is not None:
                            project.incomplete_wire = None
                        if ENV.selected is not None:
                            ENV.selected = None

                        ENV.query_disable.clear()

                    # Show a datasheet if an Integrated Circuit was selected
                    if event.key == pygame.K_d:

                        if not show_datasheet[0]:
                            if ENV.selected is not None:
                                if isinstance(ENV.selected, IntegratedCircuit):
                                    scaled_image = pygame.transform.scale(ENV.selected.datasheet_image, (300, 300))
                                    show_datasheet = (True, scaled_image)
                        else:
                            show_datasheet = (False, None)

                    # Delete a component if it was selected
                    if event.key == pygame.K_DELETE:
                        if ENV.selected is not None:
                            if isinstance(ENV.selected, Breadboard):
                                coord = [i for i in project.boards if project.boards[i] == ENV.selected][0]
                                project.delete(coord)
                            if isinstance(ENV.selected, PowerSupply):
                                coord = [i for i in project.boards if project.boards[i] == ENV.selected][0]
                                project.delete(coord)
                            if isinstance(ENV.selected, PluginPart):
                                project.change_made()
                                board, coord = ENV.selected.deletion_key
                                del board.plugins[coord]
                            if isinstance(ENV.selected, Wire):
                                project.change_made()
                                project.wires.remove(ENV.selected)
                            ENV.selected = None
                            ENV.query_disable.clear()

        # Display the page corresponding to the program state
        if current_state == HOME:

            pygame.display.set_caption(f"Home • de:volt")
            draw_homepage(win, home_title, home_version, visualiser, home_button_manager)

            # Change the cursor back to an arrow
            if not home_button_manager.hovered:
                if pygame.mouse.get_cursor() != pygame.SYSTEM_CURSOR_ARROW:
                    pygame.mouse.set_cursor(pygame.SYSTEM_CURSOR_ARROW)

        if current_state == PROTOSIM:

            pygame.display.set_caption(f"{saved}{project.display_name} • de:volt")
            draw_sim(win, sidebar_width, project, sim_manager, action_bar_title, sidebar, show_datasheet)

            # If there is a pending warning, show it
            if warning != "":
                warning_rect = pygame.Rect(WIDTH - 300, ACTION_BAR_HEIGHT, 300, 0)
                warning_label, height = version_handler.render_multiline(warning, width=warning_rect.w - 20, colour=COL_BLACK)
                warning_rect.h = height + 40
                pygame.draw.rect(win, COL_WARNING, warning_rect)
                acc = 0
                for label in warning_label:
                    win.blit(label, (WIDTH - 300 + 20, ACTION_BAR_HEIGHT + 20 + acc))
                    acc += label.get_height() + 5

            # Change the cursor back to an arrow
            if not sim_manager.hovered:
                if pygame.mouse.get_cursor() != pygame.SYSTEM_CURSOR_ARROW:
                    pygame.mouse.set_cursor(pygame.SYSTEM_CURSOR_ARROW)

            # Check if a switch was pressed
            if mouse_button_down and pygame.mouse.get_pressed()[0]:
                if project.last_surface.get_rect(topleft=project.pos).collidepoint(pygame.mouse.get_pos()):
                    if ENV.selected is not None:
                        if isinstance(ENV.selected, Switch):
                            ENV.selected.state = 1 - ENV.selected.state

        pygame.display.update()

if __name__ == '__main__':
    main()


"""Spatiotemporal generator"""
from math import ceil
import random
import pandas as pd
import numpy as np
from keras.utils import Sequence
from Misc.misc import get_image, get_data_paths
from Misc.preprocessing import filter_input_based_on_steering, filter_input_based_on_speed_and_tl, filter_corrupt_input

class BatchGenerator(Sequence):
    """ Generator that yields batches of training data concisting of multiple inputs"""
    #pylint: disable=too-many-instance-attributes
    def __init__(self, conf, data="Training_data"):
        self.conf = conf
        self.batch_size = self.conf.train_conf.batch_size
        self.seq_len = self.conf.input_size_data["Sequence_length"]
        self.data = None
        self.data_paths = get_data_paths(data)
        self.input_measures = [
            key for key in self.conf.available_columns if self.conf.input_data[key]
            ]
        self.output_measures = [
            key for key in self.conf.available_columns if self.conf.output_data[key]
            ]
        self.get_measurements_recordings(data)

    def __len__(self):
        return np.ceil(len(self.data.index)/self.batch_size)

    def __getitem__(self, idx):
        images = []
        measurments =  []

        for i in range(self.seq_len):
            images.append([])
            measurments.append([])
            for j in range(len(self.input_measures)):
                measurments[i].append([])

        y = []
        for i in range(len(self.output_measures)):
            y.append([])

        cur_idx = idx*self.batch_size
        shift = 0
        for _ in range(self.batch_size):
            #print("new_sample")
            # Add the current sequence to the batch
            initial_row = self.data.iloc[cur_idx, :]
            last_row = self.data.iloc[cur_idx + (self.seq_len-1), :]
            if initial_row["Recording"] != last_row["Recording"]:
                shift = self.seq_len  - 1

            for j in range(self.seq_len):
                row = self.data.iloc[cur_idx + shift + j, :]
                #print(self.data_paths[row["Recording"]])
                #print(str(row["frame"]))
                images[j].append(self.get_image(row))
                input_measurements = row[self.input_measures]
                output_measurements = row[self.output_measures]

                for i, measure in enumerate(self.input_measures):
                    measurments[j][i].append(input_measurements[measure])

            for i, measure in enumerate(self.output_measures):
                y[i].append(output_measurements[measure])
            cur_idx += 1

        # Convert x to dict to allow for multiple inputs
        X = {}
        for j in range(self.seq_len):
            X["input_Image"+str(j)] = np.array(images[j])
            for i, measure in enumerate(self.input_measures):
                X["input_" + measure + str(j)] = np.array(measurments[j][i])

        Y = {}
        for i, measure in enumerate(self.output_measures):
            Y["output_" + measure] = np.array(y[i])
        return X, Y

    def get_measurements_recordings(self, data):
        dfs = []
        for i, path in enumerate(self.data_paths):
            df = pd.read_csv(path + self.conf.recordings_path)
            df["Recording"] = i
            dfs.append(df)
        self.data = pd.concat(dfs, ignore_index=True)

        #Filter
        if self.conf.filter_input and data != "Validation_data":
            self.data = filter_input_based_on_steering(self.data, self.conf, temporal=False)
            self.data = filter_input_based_on_speed_and_tl(self.data, self.conf, temporal=False)
            self.data = filter_corrupt_input(self.data, self.conf, temporal=False)

        for index, row in self.data.iterrows():
            if self.conf.input_data["Direction"]:
                self.data.at[index, "Direction"] = [int(x) for x in row["Direction"].strip("][").split(".")[:-1]]
            if self.conf.input_data["TL_state"]:
                self.data.at[index, "TL_state"] = [int(x) for x in row["TL_state"].strip("][").split(".")[:-1]]
            if self.conf.input_data["ohe_speed_limit"]:
                self.data.at[index, "ohe_speed_limit"] = [int(x) for x in row["ohe_speed_limit"].strip("][").split(".")[:-1]]

    def get_image(self, row):
        """" Returns the image sequence"""
        path = self.data_paths[row["Recording"]] + "/Images/"
        frame = str(row["frame"])
        img = get_image(path, frame)

        img = img[..., ::-1]
        img = img[self.conf.top_crop:, :, :]
        return img


# -*- coding: utf-8 -*-
"""
Created on Fri May 28 15:28:03 2021

@author: jm980
"""
import cv2 as cv
import numpy as np
import datetime

#동적인 물체가 없으면 캡쳐본을 남기는 프로그램
cap = cv.VideoCapture('360_0273_Stitch_XHC.mp4')
bgs = cv.createBackgroundSubtractorKNN(dist2Threshold =500,detectShadows=False)

while cap.isOpened():
    now = datetime.datetime.now().strftime("%d_%H-%M-%S")
    ret, frame = cap.read()

    if not ret:
        break

    frame = cv.resize(frame,None,fx=0.3,fy=0.3,interpolation = cv.INTER_CUBIC)
    fgmask = bgs.apply(frame)

    cv.imshow('video', frame)
    #B&W 영상 추출
    cv.imshow('moving', fgmask)
    #cv.imshow('fm', fmask)

    blur = cv.blur(fgmask,(10,10))
    #cv.imshow('blured' , blur)

    verticalStructure = cv.getStructuringElement(cv.MORPH_ELLIPSE,(6, 6))
    #img = cv.erode(blur, verticalStructure)
    img = cv.dilate(blur, verticalStructure)

    img[img > 180] = 255
    img[img <= 180] = 0

    cv.imshow("imgToFindContours" , img)

    contours, hirachy = cv.findContours(img, cv.RETR_CCOMP, cv.CHAIN_APPROX_NONE)
    print("contours : " + str(len(contours)))

    if len(contours) < 1 :
        saved = cv.imwrite( now + ".jpg" , frame)

    if cv.waitKey(1) == ord('q'):
        break

cap.release()
cv.destroyAllWindows()


from typing import List
from typing import Tuple
from solution import Solution
import random
import numpy
from collections import Counter

class PopulationElement:

    def __init__(self, workerId: str, solId: int, sol: Solution) -> None:
        self.solution = sol
        self.workerId = workerId
        self.solId = solId

    def getSolution(self) -> Solution:
        return self.solution

    def getWorkerId(self) -> str:
        return self.workerId

    def getSolutionId(self) -> int:
        return self.solId

class FrontElement:

    def __init__(self, popElement: PopulationElement, crowdingDistance: float) -> None:
        self.populationElement = popElement
        self.crowding = crowdingDistance
        #self.objectivesNumber = numberOfObjectives

    def setCrowding(self, crowVal: float) -> None:
        self.crowding = crowVal

    def getCrowding(self) -> float:
        return self.crowding

    def getPopulationElement(self) -> PopulationElement:
        return self.populationElement

    def getSolution(self) -> Solution:
        popEl = self.getPopulationElement()
        return popEl.getSolution()

    def getSolutionId(self) -> int:
        return self.getPopulationElement().getSolutionId()

    def getWorkerId(self) -> str:
        return self.getPopulationElement().getWorkerId()

class SolutionSpace:

    def __init__(self, numObj: int, rng: numpy.random.mtrand.RandomState) -> None:
        self.fronts = list()
        self.population = list()
        self.solutionId=0
        self.objectivesNumber = numObj
        self.randomNG = rng

    def getFrontCrowdingBySolId(self,idWorker: str, idSol:int ) -> Tuple[int,float]:

        for i,front in enumerate(self.fronts):
            for j,element in enumerate(front):
                if element.getWorkerId()==idWorker and element.getSolutionId()==idSol:
                    return i, element.getCrowding()
        return 0 #ERROR: solution is not found

    '''
    This function is executed when the execution of the worker starts. The initial population of the
    worker has the size of the configuration file and the solution template.
    The population is stored in self.population and the elements of this list are dictionaries with
    the workerId the solutionId and the solution (instance of the solution object)
    The structure to store the fronts self.fronts is also created (empty). For each new solution
    (one loop of the for) the new solution is included in the self.population and in the corresponding
    front (calling function self.incorporateSolutionIntoFront). The identificator of the solution is
    also increased (solutionId)
    INPUTS: None
    '''
    def createRandomPopulationAndJoin(self,solTemplate: dict, infrastructure: dict, workerId: str, numberOfSolutions: int) -> None:
        # we create the population with the specified solution template
#        self.population = list()
#        self.fronts = list()
        solutionsForParetos = list()
        for i in range(numberOfSolutions):
            oneSolution = Solution(self.randomNG,solTemplate, infrastructure)
            newPopulationElement = PopulationElement(workerId,self.solutionId,oneSolution)
            #newSolutionElement['workerId'] = optCnf.selfId4Worker
            #newSolutionElement['solutionId'] = self.solutionId
            #newSolutionElement['solution'] = oneSolution

            self.population.append(newPopulationElement)
            solutionsForParetos.append(newPopulationElement)
            # self.incorporateSolutionIntoFront(newSolutionElement) #recalculate pareto for each single solution
            self.solutionId = self.solutionId + 1

        self.shiftSolutionsToHigherFronts(solutionsForParetos,
                                          0)  # recalculate pareto fronts for all the new solutions

    '''
    When a new worker is started, it locally create a set of N solutions. Those N new solutions in the worker,
    are included in the coordinator population. The size of the population of the coordinator is increased by N,
    i.e., no solutions are removed front coordinator population
    '''
    def createSolutionWithInputAndJoinWithCurrentPopulation(self, workerId: str, solSet: List[dict]) -> None:
        solutionsForParetos = list()
        for sol in solSet:
            newSol = Solution(self.randomNG)
            newSol.setFitness(sol['fitness'])
            newPopulationElement = PopulationElement(workerId,sol['id'],newSol)
            #newSolutionElement['workerId'] = workerId
            #newSolutionElement['solutionId'] = sol['id']
            #newSolutionElement['solution'] = newSol
            self.population.append(newPopulationElement)
            solutionsForParetos.append(newPopulationElement)
            # self.incorporateSolutionIntoFront(newSolutionElement)  # recalculate pareto fronts

        self.shiftSolutionsToHigherFronts(solutionsForParetos,
                                          0)  # recalculate pareto fronts for all the new solutions

    '''
    Created for the case of the FullMigration scenario (population distributed along workers). It is required because
    a central copy of the solutions in the workers is required for analysis of the experiments porpouse. Not really
    need to implement the framework, just for experimentation
    '''
    def createSolutionWithInputAndAggregateWithCurrentPopulationWithoutClassifyingFronts(self, workerId: str, solSet: List[dict]) -> None:
        for sol in solSet:
            newSol = Solution(self.randomNG)
            newSol.setFitness(sol['fitness'])
            newPopulationElement = PopulationElement(workerId,sol['id'],newSol)
            #newSolutionElement['workerId'] = workerId
            #newSolutionElement['solutionId'] = sol['id']
            #newSolutionElement['solution'] = newSol
            self.population.append(newPopulationElement)
            if len(self.fronts)==0:
                front0 = list()
                self.fronts.append(front0)
            self.fronts[0].append(newPopulationElement)

    def joinToThePopulation(self,offspring: List[PopulationElement]) -> None:
        #decide what it is explained in def crossoverWithRemoteSolution join the two new children or replace two old solutions.
        ##ANSWER: we add it without removing. The role to remove solutions is in charge of the coordinator.
        self.population = self.population + offspring
        #for sol in offspring:
        #    self.incorporateSolutionIntoFront(sol) #call to recalculate pareto fronts for only one solution
        self.shiftSolutionsToHigherFronts(offspring, 0)  # call to recalculate pareto fronts can be done with both of the solutions in the same set

    '''
    This function is executed when the execution of the coordinator starts. The initial population of the
    coordinator is empty, because the random initial populations are created in the workers.
    INPUTS: None
    '''
    def createRandomPopulationCoordinator(self, solTemplate: dict, infrastructure: dict, workerId: str) -> None:
        pass

    '''
    This function is call from the worker, when a solution has to be removed as the action of a topic of removesolution
    This topic indicates the solutions to remove that have been discarded from the coordinator, and this is always the
    worst solution in the pareto fronts of the coordinator. The worst solution in the coordinator fronts, is also the
    worst solution in the worker fronts.
    '''
    def removeSolution(self, idSol: str, idWorker: str) -> bool:
        for pos, popElement in enumerate(self.population):
            if popElement.getSolutionId() == idSol and popElement.getWorkerId() == idWorker:
                self.removeWorstSolutionFromFront(popElement)
                self.population.pop(pos)
                return True
        return False

    '''
    This funciton is usually call after a crossover operation and the following incorporation of the children in the
    population. A set of the input size is removed from the population. The removed solutions are the worst ones in
    the fronts, ie, the solutions with the smallest crowding distance in the last front
    '''
    def removeSetOfWorstSolutions(self, solNumber: int) -> List[PopulationElement]:
        # The worst solutions from the fronts are selected. The worst solutions are the ones in the last front
        #with the smallest crowding distance (closest to other solutions).
        toRemove = list()
        #        self.log.print("COORDINATOR:::choosing to remove")
        for i in range(solNumber):
            fe,posFe = self.getFrontElementWithSmallestCrowdingInFront(-1)
            self.removeSolutionFromFrontByFrontIdAndPos(-1,posFe)
            #frontPos = random.randrange(len(self.fronts[
            #                                    -1]))    no hay que elmininar una del ultimo front aleatoriamente, hay que eliminar la que tenga menor crowding distance

            #sol2Remove = dict()
            #sol2Remove['solutionId'] = self.fronts[-1][frontPos][0]['solutionId']
            #sol2Remove['workerId'] = self.fronts[-1][frontPos][0]['workerId']
            # self.log.print("COORDINATOR:::removing["+str(sol2Remove['solutionId'])+ " from " + str(sol2Remove['workerId']))

            popElementToRemove, posPopElement = self.getPopulationElementBySolutionId(fe.getSolutionId(),fe.getWorkerId())
            self.population.pop(posPopElement)

            toRemove.append(fe.getPopulationElement())
            #self.removeWorstSolutionFromFront(popElementToRemove)

        #        self.log.print("COORDINATOR:::all to remove" + str(toRemove))
        return (toRemove)

    def removeSolutionFromFrontByFrontIdAndPos(self, front:int, pos:int) -> None:
        self.fronts[front].pop(pos)
        if len(self.fronts[front])==0:
            self.fronts.pop(front)

    '''
    Remove one solution of the fronts. The removing process is performed under the assumption that the element to
    remove is the worst one, and consequently, it is placed in the last front. Thus, a reorganization of the fronts
    is not required.
    INPUTS: one the population Element from the list self.population. This popElement is a dictionary with the
    workerId the solutionId and the solution (instance of the solution)
    '''
    def removeWorstSolutionFromFront(self,popElement: PopulationElement) -> None:
        for i,front in enumerate(self.fronts):  #go through the fronts
            for j,frontElement in enumerate(front): #go through the solutions in a front. those front elements ara a duple of the populationElement and the croding distance
                if frontElement.getSolutionId()==popElement.getSolutionId() and frontElement.getWorkerId()==popElement.getWorkerId():
                   #when the solution to remove is found, the front element is removed
                   front.pop(j)
                   if len(front)==0:
                       #if the removed solution is the last one in the front, the front must be removed.
                       self.fronts.pop(i)
                   return
        #self.log.print("ERRORRRR: solution to remove from the front is not found")
        print("ERRORRRR: solution to remove from the front is not found")

    def calculateCuboid(self, sortedElements: List[dict], pos: int, front: List[FrontElement]) -> float:
        solMiddle = sortedElements[pos]
        if pos != 0: #check if the middle solution has the same values for all the objectives of the previous lower solution
            solPrev = sortedElements[pos - 1]
            if solMiddle['fitness'] == solPrev['fitness']:
                if front[solPrev['posInFront']].getSolution().fitness == front[solMiddle['posInFront']].getSolution().fitness:
                    return 0.0
        if pos != -1 and pos != (len(sortedElements)-1): #check if the middle solution has the same values for all the objectives of the next higher solution
            solNext = sortedElements[pos + 1]
            if solMiddle['fitness'] == solNext['fitness']:
                if front[solNext['posInFront']].getSolution().fitness == front[solMiddle['posInFront']].getSolution().fitness:
                    return 0.0
        if pos != 0 and pos != -1:
            return solNext['fitness'] - solPrev['fitness']
        elif pos == 0:   #the smallest and biggest elements have a crawding distance of the double distance with the only neighbourd they have
            return 2.0*(solNext['fitness'] - solMiddle['fitness'])
        elif pos == -1 or pos == (len(sortedElements)-1):   #the smallest and biggest elements have a crawding distance of the double distance with the only neighbourd they have
            return 2.0*(solMiddle['fitness'] - solPrev['fitness'])

    def calculateCrawdingDistance(self, front: List[FrontElement], numObjectives: int) -> None:
        for frontElement in front:
            frontElement.setCrowding(0.0) #all the crowding distances are initialized to 0
            #frontElement[1]=0.0
        if len(front)<3:
            return #if there are only 2, 1 or 0 elements in the front, the crowding distance is 0
        for objPos in range(numObjectives):
            listOfObjectivePosPair =list()
            for pos, frontElement in enumerate(front): #we create a structure to know the value of the fitness and the position in the front for the solution with this value
                fitnessValue =  frontElement.getSolution().fitness[objPos]
                objectivePosElement = dict()
                objectivePosElement['fitness']=fitnessValue
                objectivePosElement['posInFront']=pos
                listOfObjectivePosPair.append(objectivePosElement)
            sortedElements = sorted(listOfObjectivePosPair,key=lambda x:x['fitness'])
            minValue = sortedElements[0]['fitness']
            maxValue = sortedElements[-1]['fitness']
            normalizedDenominator=float(maxValue-minValue)
            if normalizedDenominator>0.0: #check normalizedDenominator bigger than 0.0
                #if the max and minimum value are equal, the crowding distance of all the elements is 0.0
                #if normalizeddenominator == 0.0 do nothing FOR THIS OBJECTIVE because all the crowding distances were initialized to 0.0 at the beginning of the function
                for thePair in sortedElements:
                    thePair['fitness']=float(float(thePair['fitness'])/float(normalizedDenominator))
                #assumption: there are more than 2 elements in the list, because it was checked at the beginning of the function
                for posPair in range(1,len(sortedElements)-1):
                    #crawdingIncrement = sortedElements[posPair + 1][0] - sortedElements[posPair - 1][0]
                    crawdingIncrement = self.calculateCuboid(sortedElements,posPair,front)
                    frontPos = sortedElements[posPair]['posInFront']
                    front[frontPos].setCrowding(front[frontPos].getCrowding() + crawdingIncrement)

                for posPair in [0,-1]:  #calculation of the crowding distances of the two extrem elements (the smallest and the highest)
                    #crawdingIncrement = 2*(sortedElements[posPair + 1][0] - sortedElements[posPair][0])
                    crawdingIncrement = self.calculateCuboid(sortedElements, posPair, front)
                    frontPos = sortedElements[posPair]['posInFront']
                    front[frontPos].setCrowding(front[frontPos].getCrowding() + crawdingIncrement)

    '''
    Recursive functions that,for a set of new solutions, checks if the are in the current pareto front. Each
    solution in the set of moving solutions is isolated analyzed to check if it dominates solutions in the current
    front, or if it is dominated by solutions in the fronts.
    If the the solution dominates solutions in the fronts, the set of dominated solution is included in the set of
    solutions to include in the next recursive call to the function. I,e, they are moved to higher fronts.
    If the solution is not dominated by any solution in the front, the new solution is included in this front. If not,
    the new solution is included in the in the set of solutions to include in the next recursive call to the function,
    i.e. the search of its front will continue in higher fronts.
    INPUTS: listOfMovingSolutions: the solutions that need to be checked if the are included in the front. The elements
    of this list are population elements, which are classes of PopulationElement with workerId, solutionId
    and PopuliatonElement (object instance).
    frontNumber: the current front to analyize the inclusion of the set of solutions
    '''
    def shiftSolutionsToHigherFronts(self, listOfMovingSolutions: List[PopulationElement] ,frontNumber: int) -> None:
        if self.getNumberOfFronts()<=frontNumber: #if we have already analized all the previous fronts, we need to create a new front to include the current moving/shifted solutions. Those solutions are the ones that are not included in previous fronts.
            newFront = self.addFront(listOfMovingSolutions)
            self.calculateCrawdingDistance(newFront, self.objectivesNumber)
            return #no more fronts require to be analyzed.
        currentFront = self.getFrontByNumber(frontNumber)
        newShiftedSolutions = list() # the set of solutions that will be moved/shifted to higher fronts
        recalculateCrawding = False
        for newSolution in listOfMovingSolutions: #for all the solutions in the parameter of solution lists to evaluate where to be placed in the fronts
            solInstanceInMovingList = newSolution.getSolution()
            dominatedSolutions = list()
            for i,frontElement in enumerate(currentFront):  #searching the solutions that are dominated by the newsolution in the current from
                solInstanceInFront = frontElement.getSolution()
                if solInstanceInMovingList.dominatesTo(solInstanceInFront):
                    dominatedSolutions.append(i)
            if len(dominatedSolutions) > 0:
                recalculateCrawding = True
            dominatedSolutions.sort(reverse=True)#the elements in a list has to be removed in descending order, if not, the indexes change as the elements are removed
            for pos in dominatedSolutions: #all the dominated solutions are removed from the front and shifted to the higher front
                newShiftedSolutions.append(currentFront[pos].getPopulationElement())
                currentFront.pop(pos)
            isNewSolutionDominatedbySomeOne = False
            for i,frontElement in enumerate(currentFront): #we check if the new solution is dominated by someone
                solInstanceInFront = frontElement.getSolution()
                if solInstanceInFront.dominatesTo(solInstanceInMovingList):
                    isNewSolutionDominatedbySomeOne = True #we found that at least one solutin in the front dominates the new solution
                    break
            if isNewSolutionDominatedbySomeOne:
                newShiftedSolutions.append(newSolution) #if it is dominated by some solution in the front, the new solutions is pass through to a higher front
            else:
                self.addElement2Front(newSolution,frontNumber) #if no other solution dominates the new one, it is included in the front.
                recalculateCrawding = True

        if recalculateCrawding: #if some solutions are removed from this front and shifted to higher fronts, or if any of the new solution is included in the front we need to recalculate crawding distances
            self.calculateCrawdingDistance(currentFront, self.objectivesNumber)

        if len(newShiftedSolutions)>0: #if there is still solutions to place into the fronts, the function is recursevilly called to the next higher front
            self.shiftSolutionsToHigherFronts(newShiftedSolutions,frontNumber+1)

    def getRandomSolutions(self,numberOfSolutions: int) -> List[PopulationElement]:
        #idsList = random.sample(range(0, len(self.population)),
        #                                               numberOfSolutions)  # two solution are randomly selected
        if (numberOfSolutions>len(self.population)):
            idsList = self.randomNG.choice(len(self.population), numberOfSolutions, replace=True).tolist()
            #we need to select more solutions than the number of solutions in the population, thus
            # replace is equal True in order to allow the repetition of a same solution in the list of selected solutions
        else:
            idsList = self.randomNG.choice(len(self.population), numberOfSolutions, replace=False).tolist()
        solutionList = list()
        for i in idsList:
            solutionList.append(self.population[i])
        return solutionList

    def chooseBetter(self, pe1: PopulationElement, pe2: PopulationElement) -> PopulationElement:
        frontNumber1, crowding1 = self.getFrontCrowdingBySolId(pe1.getWorkerId(),
                                                       pe1.getSolutionId())
        frontNumber2, crowding2 = self.getFrontCrowdingBySolId(pe2.getWorkerId(),
                                                       pe2.getSolutionId())
        # the best solution is returned. The best is the one in the lowest front, and if they are in the same front, the one with the highest crowding distance
        if frontNumber1 > frontNumber2:
            return pe1
        elif frontNumber2 > frontNumber1:
            return pe2
        elif frontNumber1 == frontNumber2:
            if crowding1 > crowding2:
                return pe1
            else:
                return pe2
        else:
            return 0  # error, not comparable fronts

    def getFrontByNumber(self,frontNumber: int) -> List[FrontElement]:
        return self.fronts[frontNumber]

    def getNumberOfFronts(self) -> int:
        return len(self.fronts)

    def getSolutionByFrontAndPos(self, frontId: int, frontElementId: int) -> Solution:
        return self.fronts[frontId][frontElementId].getSolution()

    def addFront(self, listOfSolutions: List[PopulationElement]) -> List[FrontElement]:
        newFront = list()
        for newSolution in listOfSolutions:
            fe = FrontElement(newSolution, 0.0)  # the front element is a class with the population element and the crowding distance
            newFront.append(fe)
        self.fronts.append(newFront)
        return newFront

    def addElement2Front(self, newSolution: PopulationElement, frontNumber: int) -> List[FrontElement]:
        currentFront = self.getFrontByNumber(frontNumber)
        fe = FrontElement(newSolution, 0.0)
        currentFront.append(fe)
        return currentFront

    def getFrontElementWithSmallestCrowdingInFront(self, frontNumber: int) -> Tuple[FrontElement,int]:
        feSmallestCrowding = None
        pos_fe = -1
        valueSmallest = float('inf')
        #if two elements solutions have the same crowding, we select the first in the list, that is the one
        #that was generated before. Consequently, we increase the probability to replace old solutions with newest
        #solutions, in the case with the same crowding.
        for i,fe in enumerate(self.fronts[frontNumber]):
            if fe.getCrowding()<valueSmallest:
                valueSmallest = fe.getCrowding()
                feSmallestCrowding = fe
                pos_fe = i
        return feSmallestCrowding, pos_fe

    def getPopulationElementBySolutionId(self, solId:int, workerId: str)-> Tuple[PopulationElement, int]:
        for i, pe in enumerate(self.population):
            if pe.getSolutionId() == solId and pe.getWorkerId() == workerId:
                return pe, i
        return None, None

    '''
    This function returns a list/dictionary structure with the fitnesses of the solutions. This structure
    can be serialized into a json file
    INPUTS: If a population is included as parameter, we serialized the input populaiton, if not,
    self.populuation is serialized
    '''
    def fitness2json(self, pop: List[PopulationElement] =None) -> List[dict]:
        # If a population is included as parameter, we serialized the input populaiton, if not, self.populuation is serialized
        if pop==None:
            population = self.population
        else:
            population = pop

        #a dictionary structure is create where id is the solutionId and fitness is the array with all the fitness values of the objectives
        setOfSolutions = list()
        for sol in population:
            oneSolution = dict()
            oneSolution['id']=sol.getSolutionId()
            oneSolution['fitness']=list(sol.getSolution().getFitness())
            setOfSolutions.append(oneSolution)
        return setOfSolutions

    def getFitnessInFronts2List(self) -> list:
        fitFronts = list()
        for i, front in enumerate(self.fronts):
            oneFront = list()
            for fe in front:
                oneFront.append(fe.getSolution().getFitness())
            fitFronts.append(oneFront)
        return fitFronts

    def getSolDistributionInWorkers(self) -> dict:
        #TODO  check if correct
        #devolver una distribucion de soluciones entre los workers. num de  workers con 1 solution, num de workers con
        #dos soluciones, etc... y lo mismo con el pareto. Asi veremos la distribucion de las soluciones entre los workers

        #calculate the density function of number of solutions in workers
        theWorkersInAll = list()
        for i, front in enumerate(self.fronts):
            for fe in front:
                theWorkersInAll.append(fe.getWorkerId())
        countInEachWorker = Counter(theWorkersInAll) #count how many solutions in each workerId
        countsValues=list(countInEachWorker.values()) #get the list of the number of solutions
        distributionOfCountValues = Counter(countsValues) #count how many workers has the same quantity of solutions

        #calculate the density functino of number of pareto's solutions in workers
        theWorkersInPareto = list()
        front=self.fronts[0]
        for fe in front:
            theWorkersInPareto.append(fe.getWorkerId())
        countInEachWorkerPareto = Counter(theWorkersInPareto)  # count how many solutions in each workerId
        countsValuesPareto = list(countInEachWorkerPareto.values())  # get the list of the number of solutions
        distributionOfCountValuesPareto = Counter(countsValuesPareto)  # count how many workers has the same quantity of solutions

        totalValues = dict()
        totalValues['front']=dict(distributionOfCountValues)
        totalValues['pareto']=dict(distributionOfCountValuesPareto)

        #print(totalValues)
        return totalValues

    def serializeFrontsWithoutCrowding(self) -> str:
        returnedStr  = ''
        for i, front in enumerate(self.fronts):
            returnedStr += 'FRONT NUMBER: ' + str(i) + '\n'
            for fe in front:
                theFitness = fe.getSolution().getFitness()
                #theCrowdingDistance = fe.getCrowding()
                theCrowdingDistance = None
                theSolId = fe.getSolutionId()
                theWorkerId = fe.getWorkerId()
                returnedStr  += '---- fitness('+str(theWorkerId)+'-'+str(theSolId)+'): ' + str(theFitness) + '\t\tcrowding distance: ' + str(theCrowdingDistance) + '\n'
        return returnedStr

    def serializeFronts(self) -> str:
        returnedStr  = ''
        for i, front in enumerate(self.fronts):
            returnedStr += 'FRONT NUMBER: ' + str(i) + '\n'
            for fe in front:
                theFitness = fe.getSolution().getFitness()
                theCrowdingDistance = fe.getCrowding()
                theSolId = fe.getSolutionId()
                theWorkerId = fe.getWorkerId()
                returnedStr  += '---- fitness('+str(theWorkerId)+'-'+str(theSolId)+'): ' + str(theFitness) + '\t\tcrowding distance: ' + str(theCrowdingDistance) + '\n'
        return returnedStr

    '''
    The input is a lists of solutions that has to be pre-included in the self.population by only generating the
    populationElement instance and by adding the solutionID
    '''
    def createSubPopulation(self, solutions: List[Solution], workerId: str):
        offspring = list()
        for sol in solutions:
            pe = PopulationElement(workerId, self.solutionId, sol)
            self.solutionId = self.solutionId + 1
            offspring.append(pe)

        return offspring


import sqlalchemy
from sqlalchemy.orm import Session

from models import Base, Gracze, Gry, Konta

login = 'root'
password = '****'
host = '127.0.0.1'
port = '3306'
db_name = 'zabawa'

connection_string = f'mysql+mysqlconnector://{login}:{password}@{host}:{port}/{db_name}'

print(connection_string)

silnik = sqlalchemy.create_engine(connection_string)
Base.metadata.create_all(silnik)

# with sqlalchemy.orm.Session(silnik) as moja_sesja:
#     autorek = Autor(autor_id = 10, imie = 'Andrzej', nazwisko ='Sapkowski')
#     autorek = Autor(autor_id = 11, imie='Jan', nazwisko='Kowal')
#
#     moja_sesja.add(autorek)
#
#     ksiazeczka = Ksiazka(ksiazka_id=1, tytul ='Wiedzmin: Krew Elfow', autor_id = 10)
#     moja_sesja.add(ksiazeczka)
#
#     ksiazeczka = Ksiazka(tytul='K :)')
#     moja_sesja.add(ksiazeczka)
#     moja_sesja.commit()

with sqlalchemy.orm.Session(silnik) as moja_sesja:
    gracze = [
        ["Jola", "A"],
        ["Kasia", "B"],
        ["Marcin", "J"],
        ["Zofia", "K"],
        ["Marian", "Z"],
    ]

    gry = [
        ["League of Legends"],
        ["Szachy"],
        ["Fortnite"],
        ["Rocket League"],
        ["DOOM"],
        ["Pasjans Pajak"],
    ]

    konta = [
        [1, 3, "Drzewo"],
        [4, 2, "Dewastator"],
        [2, 1, "Krzeslo"],
        [5, 1, "Quake"],
        [2, 4, "Admin"],
    ]

    for imie, nazwisko in gracze:
        item = Gracze(imie=imie, nazwisko=nazwisko)
        moja_sesja.add(item)

    for gra in gry:
        item = Gry(nazwa=gra[0])
        moja_sesja.add(item)

    for gracz, gra, login in konta:
        item = Konta(id_gracz=gracz, id_gra=gra, login=login)
        moja_sesja.add(item)

    moja_sesja.commit()


# Building the NMT (Neural Machine Translation) model

from keras.models import Sequential
from keras.layers import Dense, LSTM, Embedding, RepeatVector, TimeDistributed

def Seq2seqModel(input_vocab_size, output_vocab_size, input_maxwords, output_maxwords, hidden_units):

    model = Sequential()

    # Encoder Part...
    embedding_size = 512
    model.add(Embedding(input_vocab_size, embedding_size, input_length=input_maxwords, mask_zero=True))
    model.add(LSTM(hidden_units))

    # Fitting the encoder and decoder parts together by adjusting dimensions...
    model.add(RepeatVector(output_maxwords))

    # Decoder Part...
    model.add(LSTM(hidden_units, return_sequences=True))
    model.add(Dense(output_vocab_size, activation='softmax'))

    return model

2.1. Language Translation System/models_seq2seq/seq2seq_model.py


#!/usr/bin/env python3

import time
import json
import os

from modules import contract_interactions
from modules import startup
from modules.mint import load_env_vars

waiting_contracts = [] # Contract's without a start time
countdown_contracts = [] #Contracts with a start time

def main_loop():
    while True:
        contract_interactions.check_timestamps(waiting_contracts, countdown_contracts)
        if len(countdown_contracts) == 0:
            time.sleep(30)
        else:
            contract_interactions.check_starting(countdown_contracts)
            time.sleep(1)
            contract_interactions.check_starting(countdown_contracts)

def restart_contracts():
    startup.setup_contracts(waiting_contracts)

load_env_vars()
startup.setup_contracts(waiting_contracts)
main_loop()


from aiogram import Bot,types
from aiogram.dispatcher import Dispatcher, FSMContext
from aiogram.utils import executor
from aiogram.types import ReplyKeyboardMarkup, KeyboardButton,BotCommand, BotCommandScopeDefault
from aiogram.dispatcher.filters.state import State, StatesGroup
from aiogram.contrib.fsm_storage.memory import MemoryStorage
from parse_excel import excel_to_json
import json

import datetime
import sqlite3
storage = MemoryStorage()
admin_id = 612579531
token = '5579241270:AAEXkjIVCrDcSLLKjfR6SFH8C72JYgOPzz4'
bot = Bot(token=token)
dp = Dispatcher(bot,storage=storage)
PERSON='00'

db = sqlite3.connect('id.db')
cursor = db.cursor()
db.commit()

btn_zam = KeyboardButton('Замены')
btn_notice = KeyboardButton('Включить уведомления')
zamen = ReplyKeyboardMarkup(resize_keyboard=True).add(btn_zam,btn_notice)

btn_today = KeyboardButton('На сегодня')
btn_tomorrow = KeyboardButton('На завтра')
den = ReplyKeyboardMarkup(resize_keyboard=True).add(btn_today,btn_tomorrow)

async def set_commands(bot: Bot):
    commands = [
        BotCommand(
            command='start',
            description='Начало работы'
        ),

    ]

    await bot.set_my_commands(commands, BotCommandScopeDefault())

class Test(StatesGroup):
    test1 = State()

class Client(StatesGroup):
    client = State()
class Clas(StatesGroup):
    clas = State()

@dp.message_handler(commands=['start','help'],state=None)
async def command_start(message: types.Message):
    await set_commands(bot)
    global zamen
    if message.from_user.id != admin_id:
        if len(cursor.execute("SELECT id FROM ids WHERE notice='T'").fetchall()) != 0 and str(message.from_user.id) in \
                cursor.execute("SELECT id FROM ids WHERE notice='T'").fetchall()[0]:
            zamen = ReplyKeyboardMarkup(resize_keyboard=True).add(btn_zam)
        else:
            zamen = ReplyKeyboardMarkup(resize_keyboard=True).add(btn_zam, btn_notice)
        await Clas.clas.set()
        await bot.send_message(message.from_user.id, '''Приветствую, этот бот создан для просматривания замен в 9 Лицее.\nУкажите ваш класс в формате цБ (пример - 3Л)'''
                                                 )
    else:
        zamen = ReplyKeyboardMarkup(resize_keyboard=True).add(btn_zam)
        await bot.send_message(message.from_user.id,'Что вы хотите добавить/изменить',reply_markup=zamen)
        await main_dialog(message)
@dp.message_handler(state=Clas.clas)
async def clas1(message: types.Message,state: FSMContext):
    global PERSON
    PERSON = message.text
    if str(message.from_user.id) in [i[0] for i in cursor.execute("SELECT id FROM ids").fetchall()]:
        cursor.execute('''UPDATE ids SET class = ? WHERE id = ?''', (PERSON, str(message.from_user.id)))
        db.commit()
        print('312')
    else:
        cursor.execute(f"INSERT INTO ids VALUES ('{message.from_user.id}','{PERSON}','F')")
        db.commit()
    print(cursor.execute("SELECT id,class FROM ids").fetchall())
    await bot.send_message(message.from_user.id,'Приняли',reply_markup=zamen)
    await main_dialog(message)
    await state.finish()

@dp.message_handler()
async def main_dialog(message: types.Message):
    global cursor,zamen,PERSON,db
    if message.text == 'Замены':
        if message.from_user.id == admin_id:
            await admin_service(message)
        else:
            await client(message)
    if message.text == 'Включить уведомления':
        cursor.execute('''UPDATE ids SET notice = ? WHERE id = ?''', ('T', str(message.from_user.id)))
        db.commit()
        zamen = ReplyKeyboardMarkup(resize_keyboard=True).add(btn_zam)
        await bot.send_message(message.from_user.id,'Уведомления включены',reply_markup=zamen)

@dp.message_handler(state=None)
async def client(message: types.Message):
    await Client.client.set()
    await bot.send_message(message.from_user.id, 'На какой дату вы хотите посмотреть замены? (формат - дд.мм.гг) или воспользуйтесь кнопками',reply_markup=den)

@dp.message_handler(state=Client.client)
async def state1(message: types.Message, state: FSMContext):
    if message.text == 'На сегодня' or message.text == 'На завтра':
        if message.text == 'На сегодня':
            d = 0
        else:
            d=1
        print(datetime.datetime.now())
        datezamen = (datetime.date.today() + datetime.timedelta(days=d)).strftime('%d.%m.%y')
    else:
        datezamen = message.text
    with open('zam.json', encoding='utf-8') as f:
        zam_d = json.load(f)
    print(cursor.execute(f"SELECT class FROM ids WHERE id='{message.from_user.id}'").fetchone()[0])
    try:
        s = zam_d[datezamen]
        for i in s:
            if i[0] == cursor.execute(f"SELECT class FROM ids WHERE id='{message.from_user.id}'").fetchone()[0]:
                print(message.from_user.id,cursor.execute(f"SELECT class FROM ids WHERE id='{message.from_user.id}'").fetchone()[0],i[0])
                await bot.send_message(message.from_user.id,f'{i[0]} - {i[1]} урок - {i[2]}',reply_markup=zamen)

    except:
        await bot.send_message(message.from_user.id,'На такую дату нет замен',reply_markup=zamen)
        await state.finish()
    await state.finish()

@dp.message_handler(state=None)
async def admin_service(message: types.Message):
    await Test.test1.set()
    await bot.send_message(admin_id, 'Пришлите файл с заменами')

@dp.message_handler(state=Test.test1,content_types=[types.ContentType.DOCUMENT])
async def state1(message: types.Message, state: FSMContext):
    file_id = message.document.file_id
    file = await bot.get_file(file_id)
    file_path = file.file_path
    await bot.download_file(file_path, "zam.xlsx")
    await state.finish()
    datezamen = excel_to_json('zam.xlsx')
    for row in cursor.execute("SELECT id,class,notice FROM ids").fetchall():
        print(row)
        if row[2] == 'T':
            t=False
            with open('zam.json', encoding='utf-8') as f:
                zam_d = json.load(f)
            try:
                s = zam_d[datezamen]
                if row[1] != '00':
                    for i in s:
                        if i[0] == row[1]:
                            if t == False:
                                await bot.send_message(int(row[0]), f'Замены на {datezamen}')
                                t=True
                            await bot.send_message(int(row[0]), f'{row[1]} - {i[1]} урок - {i[2]}', reply_markup=zamen)
                else:
                    for i in s:
                        await bot.send_message(int(row[0]), f'{i[0]} - {i[1]} урок - {i[2]}', reply_markup=zamen)

            except:
                print('43')

    await bot.send_message(message.from_user.id,'Замены перемещены в базу данных')
#except:
#     await bot.send_message(message.from_user.id, 'Неправильный файл')
#     await main_dialog(message)

executor.start_polling(dp, skip_updates=True)
db.close()