import glob
import matplotlib.pyplot as plt
import numpy as np
from sklearn import cluster
from sklearn import metrics

# vj_file = np.load("/home/ligia/Desktop/Emerson2017/trial/TCRBV01-01TCRBJ01-01.npy")
# vj_file_ix = np.load("/home/ligia/Desktop/files/TCRBV01-01TCRBJ01-01_index.npy")

class Kmeans_VJ:
    def __init__(self, pathdirvjcombs):
        '''
        :param pathdirvjcombs: directory specified in VJ matrix class (outbound loc for files)
        '''
        self.pathdirvjcombs = pathdirvjcombs

    def getsilscores(self):
        '''
        :return: generate kmeans clustering and find optimal cluster number with silhouette score
        '''
        all_sil = []
        all_files =[]
        # for i in glob.glob('/home/ligia/Desktop/Emerson2017/kmeans_sil/*npy'):
        for i in sorted(glob.glob('{}*.npy'.format(self.pathdirvjcombs))):
            file = np.load(i)
            all_files.append(i)
            notwork=[]
            check=[]
        # for i in f:
            silhouette_values = []
            k_values = range(2, 10)
            for k in k_values:
                try:
                    kmeans = cluster.KMeans(n_clusters=k)
                    kmeans.fit(file)
                    labels = kmeans.labels_
                    # print(labels)
                    centroids = kmeans.cluster_centers_
                    # print(centroids)
                    silhouette_score = metrics.silhouette_score(file, labels, metric='euclidean')
                    silhouette_score = silhouette_score.mean()
                    #print(silhouette_score)
                    silhouette_values.append(silhouette_score)
                except:
                    notwork.append(file)
            all_sil.append(silhouette_values)
        return all_sil, all_files, k_values

    def plot_silscores(self):
        '''
        :return: plots silhouette score per vj combination
        '''
        all_sil, all_files, k_values = self.getsilscores()
        check=[]
        ix_notwork = []
        for v, f in zip(all_sil, all_files):
            try:
                plt.plot(k_values, v, 'b-')
                plt.xlabel('k')
                plt.ylabel('silhouette score')
                plt.ylim([0, 1])
                plt.show()
                #n =+1
                plt.savefig('{0}.png'.format(str(f[43:-4])))
                plt.close()
            except:
                ix_notwork.append(all_sil.index(v))
                check.append(f[43:-4])

    def get_clust_ix(self):
        '''
        :return: indexes for which combination has n clusters
        '''
        all_sil, all_files, k_values = self.getsilscores()
        twoclst = []
        threeclst = []
        fourclst = []
        fiveclst = []
        sixclst = []
        sevenclst = []
        eightclst = []
        nineclst = []
        check=[]
        for i in all_sil:
            try:
                if i[0] == max(i):
                    twoclst.append(all_sil.index(i))
                if i[1] == max(i):
                    threeclst.append(all_sil.index(i))
                if i[2] == max(i):
                    fourclst.append(all_sil.index(i))
                if i[4] == max(i):
                    sixclst.append(all_sil.index(i))
                if i[3] == max(i):
                    fiveclst.append(all_sil.index(i))
                if i[5] == max(i):
                    sevenclst.append(all_sil.index(i))
                if i[6] == max(i):
                    eightclst.append(all_sil.index(i))
                if i[7] == max(i):
                    nineclst.append(all_sil.index(i))
            except:
                check.append(i)
        return twoclst, threeclst, fourclst, fiveclst, sixclst, sevenclst, eightclst, nineclst, check

    def run_all(self):
        print('Generating silhouette scores...')
        all_sil, all_files, k_values = self.getsilscores()
        print('Generating plots...')
        self.plot_silscores()
        print('Generating indexes per cluster type...')
        self.get_clust_ix()
        print('Done.')


from math import sqrt
import matplotlib.pyplot as plt
class SquareFunction:
    def __init__(self, a, b, c):
        self.a = a
        self.b = b
        self.c = c

    def __str__(self):
        if self.b > 0:
            sign_b = '+'
        else:
            sign_b = ''

        if self.c > 0:
            sign_c = '+'
        else:
            sign_c = ''
        return f'{self.a}x^2 {sign_b} {self.b}x {sign_c} {self.c}'

    def delta(self):
        return self.b * self.b - 4 * self.a * self.c

    def solve(self):
        fun_delta = self.delta()
        if fun_delta >= 0:
            x1 = (-1 * self.b - sqrt(fun_delta)) / 2
            x2 = (-1 * self.b + sqrt(fun_delta)) / 2
            return x1, x2
        else:
            return None

    def count_y(self, x):
        return x * x * self.a + x * self.b + self.c

    def generate_y(self):
        tab_y = []
        tab_x = []
        tab_0 = []
        for x in range(-10, 11):
            tab_y.append(self.count_y(x))
            tab_x.append(x)
            tab_0.append(0)
        return tab_x, tab_y, tab_0

    def draw_fun(self):
        plt.plot(self.generate_y()[0], self.generate_y()[1])
        plt.plot([-10, 10], [0, 0])
        plt.show()

my_func = SquareFunction(1, 0, 0)
print(my_func)
print(my_func.draw_fun())


import click
from click_shell import shell
import time

from DHTNodeDataValue import DHTNodeDataValue
from DHTServer import DHTServer
from DHTServerManager import dhtServerManager
from DHTMessage import DHTMessage, DHTMessageType
from DHTSocket import DHTSocketAddress

KeySpaceRange = None

# ------------------------------------------------------------------------------- private functions

def getDHTNodeDataValueListFromRange(range):
    datalist = []
    for i in range:
        datalist.append(DHTNodeDataValue(i))
    return datalist

def initialize(file):
    ## Key space
    key_space = "#key-space"
    key_space_range = None

    ## Nodes
    nodes = "#nodes"
    node_array_str = None

    ## Shortcuts
    shortcuts = "#shortcuts"
    shortcuts_array = []

    file_name = file
    file_object = open(file_name , "r")
    file_list = file_object.read().splitlines()

    my_list = list(filter(lambda x: x is not None and x != " " and x != "", file_list))
    for index, line in enumerate(my_list):
        if(line.lower() == key_space):
            key = my_list[index + 1]
            key_array = key.split(',')
            if len(key_array) > 0:
                key_space_range = range(int(key_array[0]), int(key_array[1]) + 1)

        if(line.lower() == nodes):
            key = my_list[index + 1]
            node_array_str = key.split(',')

        if(line.lower() == shortcuts):
            key = my_list[index + 1]
            if key.startswith("#") == False:
                shortcuts_array = key.split(',')

    node_array = []
    for i in node_array_str:
        if i != " " and i != "":
            value = int(i)
            if value <= key_space_range[-1]:
                node_array.append(value)

    node_array.sort()
    file_dictionary = dict()
    for i, value in enumerate(node_array):
        dht_server = DHTServer()
        dhtServerManager.AssignHostAndPort(dht_server)
        socket_address = dht_server.GetDHTSocketAddress()
        file_dictionary[value] = (dht_server, socket_address)
        servers = [dht_server]

    lastIndex = len(node_array) - 1

    for i, value in enumerate(node_array):

        prevIndex = (i - 1) if i != 0 else lastIndex
        nextIndex = 0 if i == lastIndex else i + 1

        if i == lastIndex - 1:
            nextNextIndex = 0
        elif i == lastIndex:
            nextNextIndex = 1
        else:
            nextNextIndex = i + 2

        if lastIndex == 0:
            nextNextIndex = 0

        prevServerAddress = file_dictionary[node_array[prevIndex]][1]
        successorServerAddress = file_dictionary[node_array[nextIndex]][1]
        nextSuccessorServerAddress = file_dictionary[node_array[nextNextIndex]][1]

        shortcutsArray = []
        for shortcut in shortcuts_array:
            shortcutInfo = shortcut.split(':')
            if int(shortcutInfo[0]) == node_array[i]:
                if int(shortcutInfo[1]) in file_dictionary:
                    shortcutsArray.append(file_dictionary[int(shortcutInfo[1])][1])

        dataList = None
        if i == 0:
            dataList = getDHTNodeDataValueListFromRange(range(node_array[lastIndex] + 1, key_space_range[-1] + 1))
            dataList.extend(getDHTNodeDataValueListFromRange(range(key_space_range[0], node_array[i] + 1)))
        else:
            firstData = node_array[i - 1] + 1
            lastData = node_array[i]
            dataList = getDHTNodeDataValueListFromRange(range(firstData, lastData + 1))

        server = file_dictionary[node_array[i]][0]
        server.Upload(DHTNodeDataValue(node_array[i]), dataList, prevServerAddress, successorServerAddress, nextSuccessorServerAddress, shortcutsArray)

    servers = []
    for key in file_dictionary:
        servers.append(file_dictionary[key][0])

    return (servers, key_space_range)

def runDHTServers(dhtServers):
    servers = []
    for dhtServer in dhtServers:
        dhtServer = dhtServerManager.StartServer(dhtServer)
        servers.append(dhtServer)

    return servers

def AddShortcutToNode(nodeValue, shortcutNodeValue):

        if (not nodeValue) or nodeValue == '':
            return "the node to add the shortcut to, is not valid"

        if (not shortcutNodeValue) or shortcutNodeValue == '':
            return "the shortcut node to add the shortcut to, is not valid"

        try:
            nodeValue = int(nodeValue)
        except:
            return "the node data value for the node to add the shortcut to, is not valid"

        try:
            shortcutNodeValue = int(shortcutNodeValue)
        except:
            return "the shortcut node data value, is not valid"

        server = dhtServerManager.GetServerByDataValue(nodeValue)

        if server==None:
            return "the node to add the shortcut to, was not found"

        shortcutserver = dhtServerManager.GetServerByDataValue(shortcutNodeValue)
        if shortcutserver == None:
            return "the node is supposed to be added as a shortcut, was not found"

        msg=DHTMessage(DHTMessageType.Shortcut)
        msg.Argument=shortcutserver.Host
        msg.Tag=str(shortcutserver.Port)
        return dhtServerManager.SendRequestTo(server,msg)

def joinTheData(param):
    if param =='':
        return "the node is not valid"

    try:
        param=int(param)
    except:
        return "the join node data value, is not valid"

    server = dhtServerManager.GetServerByDataValue(param)

    if server ==None:

        dhtServer4= DHTServer()
        dhtServer4.DHTNodeDataValue=DHTNodeDataValue(param)
        dhtServerManager.AssignHostAndPort(dhtServer4)
        socketAddress4 = dhtServer4.GetDHTSocketAddress()

        serverLovestValue= dhtServerManager.GetTheServerWithTheLowestDataValue()

        msg=DHTMessage(DHTMessageType.Join)
        msg.Argument=dhtServer4.Host
        msg.Tag=str(dhtServer4.Port)
        msg.Nodevalue=str(param)

        return dhtServerManager.SendRequestTo(serverLovestValue,msg)

    else:
        return 'chord table already has this node'
#def initialize(file):

#    dhtServer5 = DHTServer()
#    dhtServerManager.AssignHostAndPort(dhtServer5)
#    socketAddress5 = dhtServer5.GetDHTSocketAddress()

#    dhtServer17 = DHTServer()
#    dhtServerManager.AssignHostAndPort(dhtServer17)
#    socketAddress17 = dhtServer17.GetDHTSocketAddress()

#    datalist5 = getDHTNodeDataValueListFromRange(range(93, 101))
#    datalist5.extend(getDHTNodeDataValueListFromRange(range(1, 6)))
#    dhtServer5.Upload(DHTNodeDataValue(5), datalist5, socketAddress17, socketAddress17, socketAddress5, [])

#    dhtServer17.Upload(DHTNodeDataValue(17), getDHTNodeDataValueListFromRange(range(6, 18)), socketAddress5, socketAddress5, socketAddress17, [])

#    servers = [dhtServer5, dhtServer17]
#    keySpace = range(1, 101)

#    return (servers, keySpace)

def validateLeaveParam(param):
    result = False

    # TODO result = ????

    return result

# ------------------------------------------------------------------------------- private functions

@shell(prompt='dht > ')
def main():
    pass

@main.command()
@click.argument('file')
def init(file):

    (servers, keySpace) = initialize(file)
    KeySpaceRange = keySpace
    dhtServers = runDHTServers(servers)

    click.echo("Initialized Done... Server(s) created and you can ping them. Server(s) is/are as follow:")
    for dhtServer in dhtServers:
         click.echo("Host: " + dhtServer.Host + " , Port: " + str(dhtServer.Port))

@main.command()
@click.argument('host')
@click.argument('port')
def ping(host, port):
    result = dhtServerManager.PingTheHostAndPort(host, int(port), DHTMessage(DHTMessageType.Ping))
    click.echo(result)

@main.command()
@click.argument('param')

def join(param):
    result = ""
    param=param

    result= joinTheData(param)

    click.echo(result)

@main.command()
@click.argument('param')
def lookup(param):
    #msg = DHTMessage(DHTMessageType.Ping)
    #msg.DataValue = 7
    result = ""
    click.echo(result)

@main.command()
@click.argument('shortcut')

def shortcut(shortcut):
    result = ""
    nodes = shortcut.split(':')
    if len(nodes) != 2:
        result="the input format is not satisfied, e.g. <Node_Value>:<ShortcutNode_Value>"
    else:
        result= AddShortcutToNode(nodes[0], nodes[1])

    click.echo(result)
    List()

@main.command()
def List():
    server = dhtServerManager.GetTheServerWithTheLowestDataValue()
    if server == None:
        click.echo("the Ring is not initialized!!")
        return

    result = dhtServerManager.SendRequestTo(server, DHTMessage(DHTMessageType.List))
    click.echo(result)

@main.command()
@click.argument('param')
def leave(param):

    #if validateLeaveParam(param):
    #    dataValue = int(param)
    #    dhtServer = dhtServerManager.ShutdownDHTServerByKey(dataValue)

    #result = ""
    #click.echo(result)
    pass

if __name__ == '__main__':
    main()


'''
NAME
       [argumentos_AT].py

VERSION
        [1.0]

AUTHOR
        Mateo Maya Martínez <[matteo.mayam@outlook.com>

DESCRIPTION
        [Este programa calcula el contenido de AT de ciertas secuencias que se encuentra en un archivo .txt usando comandos en terminal.]

CATEGORY
       [Sequence.]

USAGE
       [De uso bioinformático. Corre en python.]

OPTIONS
       [ -h, --help     Muestra la ayuda del script

        -i, --input     Es el archivo del que se leeran las secuencias.

        -o, --output    Archivo en el que se guardarán los porcentajes ]

EXAMPLES
       [Input file: 4_dna_sequence.txt
        Output: Archivo .txt con los porcentajes]

LIGA DE GITHUB: [https://github.com/MateoMayaM/python_class/blob/master/src/argumentos_AT.py]
'''

#Importar la librería para escribir los argumentos desde terminal.
import argparse

#Definir el parser.
parser = argparse.ArgumentParser(description= "Este programa calcula el porcentaje de AT de una serie de secuencias encontradas en un archivo" )

#Definir los argumentos del programa.
parser.add_argument("-i", "--input",
                    help="Archivo con las secuencias",
                    required=True)

parser.add_argument("-o", "--output",
                    help="Archivo en el que se guardarán los porcentajes",
                    default="contenido_AT_out.txt", required=False)

#Ejecutar el método parse_args
args = parser.parse_args()

#Función que calcula el contenido de AT
def get_AT_content (sequence, decimales=2):

    sequence = sequence.upper()
    if sequence.count("N") > 0:
        raise ValueError(f'Your sequence has {sequence.count("N")} N´s')
    A = sequence.count("A")
    T = sequence.count("T")
    length = len(sequence)
    AT_percentage = (A + T)/length
    return round (AT_percentage, decimales)

#Abrir el archivo que se pase por línea de comandos
try:
    file = open(args.input)
#Si no se encuentra el archivo imprimir un mensaje y solicitar un archivo válido.
except IOError as ex:
    print ("Couldn´t find the file. " + ex.strerror)
    new_file = input("Please enter a valid file: \n")
    file = open (new_file)

#Leer el archivo línea por línea y cerrarlo
finally:
    all_lines = file.readlines()
    file.close()

#Crear el archivo de salida.
output_file = open(args.output, "w")

#i será un iterador que nos ayudará con el formato del archivo output.
i = 0

#Dar el formato que necesitamos a cada línea del archivo autput.
for line in all_lines:
    line = line.split(" = ")
    sequence = line[1]
    sequence = str(sequence).replace('"', "")
    sequence = str(sequence).replace("-", "")
    i+=1

#Escribimos el archivo output
    try:
        output_file.write("Seq" + str(i) + " has " + str(get_AT_content(sequence)) + "% AT content \n")
#Si las secuencias en el archivo input tienen un error, se muestra el error en pantalla y no se agregan al archivo output.
    except ValueError as ex:
        print ("Skipping sequence " + str(i)+ ". " + ex.args[0])

#Cerrar el archivo output
output_file.close


#!/usr/bin/env python3

import requests
import argparse
import re
import os

def main():
  parser = argparse.ArgumentParser(
    description='Test Codeforces sample tests')
  parser.add_argument('prob', type=str,
      help='Codeforces problem ID (Ex: 514A)')
  args = parser.parse_args()

  if os.path.exists("sample-%s-000.in" % args.prob) == False:
    os.system("cf-fetch %s" % args.prob)

  if os.path.exists(args.prob) == False:
    print("Please compile your code first!")
    return

  cid, pid = [re.search(r'(\d+)(\w+)', args.prob).group(x) for x in [1, 2]]

  i = 0
  while os.path.exists("sample-%s-%03d.in" % (args.prob, i)):
    infile = "sample-%s-%03d.in" % (args.prob, i)
    userout = "sample-%s-%03d.out" % (args.prob, i)
    outfile = "sample-%s-%03d.ans" % (args.prob, i)
    print("======= Sample #%d =======" % i)
    os.system("./%s < %s > %s" % (args.prob, infile, userout))
    os.system("cat %s" % userout)
    os.system("cat %s" % outfile)
    i += 1

if __name__ == "__main__":
  main()


from django.conf.urls import url

from . import views

urlpatterns = [
    url(r'^$', views.index, name='index'),
    url(r'^(?P<item_id>[0-9]+)/$', views.detail, name='detail'),
    url(r'^new/', views.new, name="new"),
    url(r'^newpilot/', views.newpilot, name="newpilot")
]


from app import app
from flask import session
from pymongo import MongoClient
from bson.objectid import ObjectId
import requests, json
from datetime import datetime
import pandas as pd

ip='127.0.0.1'
port=27017
DB_NAME = 'scotchbox'
DATABASE = MongoClient(ip,port)[DB_NAME]

def guestData():
    user_obj = session.get('user', None)
    name = user_obj['hotel']
    place = user_obj['Place']
    data = []
    customerData = list(DATABASE.customers.find({"name":name}))
    if customerData:
        customer_id = customerData[0]['_id']
        con = list(DATABASE.connections.find({"customer_id":str(customer_id),"status":1}))
        if con:
            deviceId = []
            for connections in con:
                deviceId.append(connections["device_id"])
            for device in deviceId:
                deviceData = list(DATABASE.devices.find({"_id":ObjectId(device)}))
                if deviceData:
                    phoneNo = deviceData[0]["phone"]
                    socialData = list(DATABASE.social_accounts.find({"device_id":device}))
                    if socialData:
                        userName = socialData[0]["name"]
                        userPic = "http://graph.facebook.com/" + socialData[0]["provider_user_id"] + "/picture?type=large"
                        userEmail = socialData[0]["email"]
                        userGender = socialData[0]["gender"]
                        userHometown = "Unknown"
                    else:
                        userName = "Anonymous"
                        userPic = "/static/images/anonymous-user.jpeg"
                        userEmail = "Unknown"
                        userGender = "Unknown"
                        userHometown = "Unknown"
                    if user_obj['propType'] == 'Hotel':
                        surveyData = list(DATABASE.hotel_surveys.find({"device_id": device}))
                    else:
                        surveyData = list(DATABASE.restaurant_surveys.find({"device_id":device}))
                    sug_data = []
                    if surveyData:
                        for survey in surveyData:
                            suggestions = survey["suggestion"]
                            sug_date = survey["created_at"]
                            sug_data.append({"suggestions":suggestions, "sug_date":sug_date})
                    else:
                        sug_data.append({"suggestions":"No notes availabe", "sug_date": ""})
                    historyData = list(DATABASE.connections.find({"device_id":device, "status":2}))
                    his_data = []
                    df = pd.DataFrame(columns=["last_visit", "property", "placeType"])
                    for past in historyData:
                        pastDate = past["created_at"]
                        d = datetime.strptime(str(pastDate), '%Y-%m-%d %H:%M:%S.%f')
                        last_visit = d.strftime('%Y-%m-%d')
                        ind = next(index for (index,d) in enumerate(customerData) if d["_id"] == customer_id)
                        property = customerData[ind]["name"]
                        customer_category_id = customerData[0]["customer_category_id"]
                        customer_category_data = list(DATABASE.customer_categories.find({"_id": ObjectId(customer_category_id)}))
                        placeType = customer_category_data[0]["name"]
                        his_data.append({"last_visit":last_visit,"property":property,"placeType":placeType})
                    for i in range(len(his_data)):
                        df.loc[i] = [ his_data[i]["last_visit"],his_data[i]["property"],his_data[i]["placeType"] ]
                    series1 = df.groupby(['property']).size()
                    df2 = pd.DataFrame({"property": series1.index, "Count": series1.values})
                    series2 = df.groupby(['property'])["last_visit"].max()
                    df3 = pd.DataFrame({"property": series2.index, "last_visit": series2.values})
                    df4 = pd.merge(df2, df3, on="property")
                    df4 = pd.merge(df4, df[['property', 'placeType']], on='property', how="left")
                    df4 = df4.drop_duplicates()
                    data.append({"phoneNo":phoneNo,"userName":userName,"userPic":userPic,"userEmail":userEmail,"userGender":userGender, "userHometown":userHometown ,"notesData":sug_data,"historyData":df4})
        else:
            data.append({"phoneNo": "None", "userName": "None", "userPic": "/static/images/anonymous-user.jpeg",
                         "userEmail": "None",
                         "userGender": "None", "userHometown": "None", "notesData": "", "historyData": pd.DataFrame()})
    else:
        data.append({"phoneNo": "None", "userName": "None", "userPic": "/static/images/anonymous-user.jpeg" , "userEmail": "None",
                     "userGender": "None", "userHometown": "None", "notesData": "", "historyData": pd.DataFrame()})
    return data


from os import makedirs, listdir, path, walk
from shutil import copyfile
from zipfile import ZipFile
from utils import read_parameters, load_last_config
from json import dump, load

IN_VOLUME  = "/job_data"
OUT_VOLUME = "/job_results"
PARAMS = read_parameters()
ARGS = PARAMS["args"]

LAST_FOLDER = "{0}/{1}".format(IN_VOLUME, PARAMS["last"])
LAST_CONFIG= load_last_config(LAST_FOLDER)

def copy_data():
    ''' Copies the data into the job result folder '''

    out_dir = "{0}/{1}".format(OUT_VOLUME, ARGS["job_id"])

    if not path.exists(out_dir):
        makedirs(out_dir)

    for file_path in LAST_CONFIG["file_paths"]:
        file_path = "{0}/{1}".format(IN_VOLUME, file_path)
        file_name = file_path.split("/")[-1]
        copyfile(file_path, "{0}/{1}".format(out_dir, file_name))
        print(" -> Copied file: " + file_path)

if __name__ == "__main__":
    print("Start result data copy process...")
    copy_data()
    print("Finish result data copy process.")


import pandas as pd
import numpy as np
from sklearn.preprocessing import StandardScaler

def _normalize_helper(df, cols, scaler=None):

    # Normalizing train set
    if(scaler is None):
      scaler = StandardScaler()
      normalized_features = pd.DataFrame(scaler.fit_transform(df[cols]))

    # Normalizing test set (with the values based on the training set)
    else:
      normalized_features = pd.DataFrame(scaler.transform(df[cols]))

    normalized_features.columns = cols
    normalized_df = pd.concat([df.drop(columns=cols).reset_index(drop=True), normalized_features], axis=1)

    return normalized_df, scaler

def normalize(train, test, cols):
    """
    Normalize training and test data
    """
    training_normalized, scaler = _normalize_helper(train, cols)
    test_normalized, _ = _normalize_helper(test, cols, scaler)

    return training_normalized, test_normalized

def one_hot_encode(df, var_list):
    """
    Helper function for standardize_encoding
    """
    for var in var_list:
        dummies = pd.get_dummies(df[var], prefix=var)
        df = pd.concat([df.reset_index(drop=True), dummies.reset_index(drop=True)], axis=1)

    return df.drop(columns=var_list)


import pygame
import simpy
from environment import (PyGameEnvironment, FrameRenderer)
import time
import itertools
import random
import json
import sys

import time

AIRPLANE_ARRIVING_TIME = [3000, 7000]  # Прибытие самолета каждые [min, max] секунд
# Define constants for the screen width and height
SCREEN_WIDTH = 1000
SCREEN_HEIGHT = 700
SIMULATION_SPEED = 0.0001  # 0.001 - NORMAL_speed, 0.0001 - FAST_speed, 0.0000000001 - MAX_speed
# Airplane_Settings####################
AIRPLANE_SPEED_X = 3
AIRPLANE_SPEED_Y = 10
# Loader_Settings######################
LOADER_SPEED_X = 3
LOADER_SPEED_Y = 2
NUMBER_OF_LOADERS = 3
# Truck_Settings######################
TRUCK_SPEED_X = 2
TRUCK_SPEED_Y = 3

"""Вспомогательные переменные для отображения количества агентов, координат"""
############################################################
stoyanka_counts = 0  # Кол-во самолетов на стоянке
warehouse_loaders = 0  # Кол-во команд грузчиков на складе

WAREHOSE_STATION_SIZE = 50  # Максимальное(изначальное) количество деталей на складе
THRESHOLD = 20  # Порог имеющихся деталей для заказа новых запчастей (в %)
number_station = 2

iteration = 0
WAREHOSE_MAX = 50
WAREHOSE_STATION_SIZE2 = WAREHOSE_MAX  # Максимальное(изначальное) количество деталей на складе

class Station:
    def __init__(self, number_of_station):
        self.number_of_station = number_of_station

        if self.number_of_station == 1:
            self.x = 285
            self.y = 280
            self.x_loaders = 0
            self.y_loaders = 230
        elif self.number_of_station == 2:
            self.x = 285
            self.y = 465
            self.x_loaders = 0
            self.y_loaders = 410
        elif self.number_of_station == 3:
            self.x = 0
            self.y = 0
            self.x_loaders = 0
            self.y_loaders = 0

        self.station_status = 0  # Занята или свободна(самолетом)
        self.stoyanka_to_station = 0  # Находится ли какой-нибудь самолет в пути от стоянки к станции
        self.station_repair = ""  # Статус ремонта самолета на станции - "repair"/"ready"
        self.repairing = ""  # Ведется ли сейчас ремонт на 1 станции? (now/done)-ожидание грузчиком и самолетом ремонта
        self.details_required = 0  # Сколько деталей требуются для починки самолета на станции
        self.loader_to_station = 0  # Находится ли кто-то из грузчиков в пути от склада к станции
        self.loaders_count_on_station = 0

    def change_station_status_to_busy(self, station_number, airplane_number):
        """
        При прибытии самолета на станцию обслуживания
        изменить ее статус на занятый ремонтом
        """
        global event, event_time

        self.station_status = 1
        if station_number == 1:
            self.stoyanka_to_station = 0
        elif station_number == 2:
            self.stoyanka_to_station = 0
        self.station_repair = "repair"
        self.details_required = random.randint(1, 30)
        event = f"Самолет{airplane_number} на ремонте(1)"
        event_time = round(env.now / 1000)

class Airplane:
    """
    Самолет - либо на стоянке, либо на станции тех.обслуживания(№1 или №2), либо покаидает аэропорт.
    Случайным образом выбирается количество запчастей, которые необходимо заменить.
    Методы перемещения и перерисовки самолета вызываются внутри метода RUN с бесконечным циклом.

    Описание методов самолета:
    1)перемещение на стоянку
    2)перемещение со стоянки на станцию тех.обслуживания
    3)перемещение со станции тех.обслуживания на взлетную полосу и вылет с аэропорта
    """

    def __init__(self, env, name):
        IMG_size = 60
        self.image = pygame.image.load("airplane4.png")  # Загрузка в pygame картинки
        self.image = pygame.transform.scale(self.image, (IMG_size, IMG_size))  # Изменение размера картинки
        self.x = 0  # Изначальное положение центра картинки
        self.y = 0
        self.env = env
        self.status_now = ""  # "on_parking", "on_station{№}", "to_station{№}", "from_station{№}"
        self.status = "arrived_on_airport"  # "on_parking", "on_service_station", "moving"
        self.name = name
        self.time_parking = 0  # Время прибытия на стоянку
        self.time_station = 0  # Время прибытия на станцию на ремонт
        self.time_leave = 0  # Самолет готов, покидает аэропорт
        self.time_result = 0  # Итоговое время пребывания самолета в аэропорту
        self.monitoring = ""  # Текущее событие самолета
        self.time_now = 0  # Время текущего события

        self.selected_station = 0  # Станция выбранная для обслуживания

    def arriving(self):
        """Прибытие самолета в аэропорт и перемещение на стоянку"""
        current_x_stoyanka = SCREEN_HEIGHT - 80  # Текущая координата Y для стоянки
        global stoyanka_counts
        global event, event_time
        # Движение по вертикали(Конечная координата зависит от того, сколько самолетов уже на стоянке):
        self.y += 5
        if self.y >= current_x_stoyanka - 60 * stoyanka_counts:
            self.y = current_x_stoyanka - 60 * stoyanka_counts
            # Движение по горизонтиали:
            self.x += 10
            # Удержание игрока в рамках окна
            if self.x > 56:
                self.x = 56
                self.status = "on_parking"
        # Увеличиваем количество самолетов на стоянке
        if self.status == "on_parking":
            stoyanka_counts = stoyanka_counts + 1
            self.time_parking = round(env.now / 1000)
            event = f"Самолет{self.name} на парковке"
            event_time = round(env.now / 1000)

    def go_to_free_station(self, selected_station):
        """Перемещение самолета от стоянки на СВОБОДНУЮ станцию обслуживания"""
        global stoyanka_counts
        global event_time, event

        station_number = selected_station.number_of_station
        # Движение по горизонтиали:
        self.x += AIRPLANE_SPEED_X
        if 150 < self.x < 154:
            self.x = 150  # Промежуточная координата м/у станциями - для смены движения
        #############################
        # Движение по вертикали(до станции обслуживания №2):
        self.y -= AIRPLANE_SPEED_Y
        if self.y <= selected_station.y:
            self.y = selected_station.y
            ###############################
            # Движение по горизонтиали:
            if self.x == 150:
                self.x = self.x + 5
            if self.x >= selected_station.x:
                self.x = selected_station.x
                self.status = "on_service_station"
            # Увеличиваем количество самолетов на стоянке
            if self.status == "on_service_station":
                selected_station.change_station_status_to_busy(station_number=int(station_number),
                                                               airplane_number=self.name)
                self.status_now = "on_station" + str(station_number)

    def checking_stations(self):
        """
        Выбор свободной станции обслуживания
        Уменьшаем кол-во самолетов на стоянке на 1 при отправлении к свободной станции обслжуивания
        """
        global stoyanka_counts
        for station_object in stations_objects:
            if self.status == "on_parking" and not station_object.station_status and not station_object.stoyanka_to_station:
                if "to_station" not in self.status_now:
                    self.status_now = "to_station" + str(station_object.number_of_station)
                    station_object.stoyanka_to_station = 1
                if stoyanka_counts != 0:
                    stoyanka_counts -= 1
                    return station_object

    def counting_required_details(self):
        """
        При нахождении самолета на станции обслуживания происходит подсчет необходимых для ремонта деталей
        """
        for station_object in stations_objects:
            station_number = str(station_object.number_of_station)
            if self.status_now == ("on_station" + station_number) and station_object.station_repair == "ready":
                self.status_now = "from_station" + station_number
                station_object.station_status = 0
                station_object.details_required = 0

    def leaving_airport(self):
        """Самолеты покидают аэропорт со станций обслуживания"""
        # Движение по горизонтиали:
        self.x += 2
        if self.x > 440 and self.y != 600:
            self.x = 440
            # Движение по вертикали(до станции обслуживания №1):
            self.y += 3
            if self.y > 600:
                self.y = 600
        if self.x == 440 and self.y == 600:
            self.x += 3

    def stopping_simulation(self):
        """
        Останавливаем программу, когда последний самолет покидает АТБ
        """
        global event_time, event
        # self.monitoring = "Самолет1 улетает"
        # self.time_now = round(env.now / 1000)
        self.time_result = self.time_leave - self.time_parking
        print(f"Общее время нахождения самолета{self.name} в аэропорту ={self.time_result}")
        results.append(self.time_result)
        event = f"Самолет{self.name} улетает"
        event_time = round(env.now / 1000)
        if len(results) == 10:  # Когда покинет последний самолет
            with open('file.txt', 'w+') as fw:
                # записываем
                json.dump(results, fw)
            raise SystemExit

    @staticmethod
    def stopping_simulation_by_time():
        """
        Остановить симуляцию по истечению отведенного времени
        """
        environment_time = str(round(env.now / 1000))
        if int(environment_time) > 160:
            print(environment_time)
            raise SystemExit

    def run(self):
        simulation_run = True  # Переменная для остановки симуляции
        while True:
            # selected_station = 0
            # Прибытие самолета в аэропорт:
            if self.status == "arrived_on_airport":
                self.arriving()

            # Станция обслуживания выбирается только один раз для каждого из самолетов
            if not self.selected_station:
                self.selected_station = self.checking_stations()  # Выбор свободной станции обслуживания

            # # Перемещение самолета со стоянки на свободную станцию тех.обслуживания:
            if self.status == "on_parking":
                if "to_station" in self.status_now and self.selected_station:
                    yield self.env.timeout(100)
                    self.go_to_free_station(selected_station=self.selected_station)

            if self.status == "on_service_station":
                self.counting_required_details()
                # После ремонта(станция тех.обслуживания) самолет покидает аэропорт:
                if "from_station" in self.status_now:
                    self.leaving_airport()
                    if simulation_run:
                        self.time_leave = round(env.now / 1000)
                        simulation_run = False
                        self.stopping_simulation()
            yield self.env.timeout(20)

    def __call__(self, screen):
        # screen.blit(self.image, (self.x, self.y))  # Расположить картинку по координатам
        self.image1 = self.image.get_rect(topleft=(self.x, self.y))
        screen.blit(self.image, self.image1)

class Loader:
    """Команда грузчиков"""

    def __init__(self, env, number):
        self.number = number
        self.IMG_size = 45
        self.image = pygame.image.load("loader.png")  # Загрузка в pygame картинки
        self.image = pygame.transform.scale(self.image, (self.IMG_size, self.IMG_size))  # Изменение размера картинки
        self.x = 650  # Изначальное положение центра картинки(На складе)
        self.y = 380
        self.env = env
        self.status_now = ""  # "on_parking", "on_service_station", "moving"
        self.status = "in_warehouse"  # "on_service_station", "moving"
        self.repair_time = random.randint(1200, 2000)  # Среднее время ремонта на станции тех.обслуживания

        # self.search_time = random.randint(150, 300)  # Среднее время поиска 1 детали на складе(стандартные технологии)
        self.search_time = random.randint(33, 67)  # Среднее время поиска 1 детали на складе(технологии IoT)

        self.search_status = "search"
        self.warehose_status = "empty"  # "empty", "full"
        self.loader_details = 0  # Запчасти которые несет с собой грузчик от склада к станции

        self.requesting_station = 0  # Выбор станции, к которой нужно перенести детали со склада

    def go_to_requesting_details_station(self, requesting_station):
        """
        Перемещение грузчиков от склада к станции тех.обслуживания,
        которой нужны запчасти со склада
        """
        number_of_station = requesting_station.number_of_station
        self.x -= LOADER_SPEED_X
        if self.x < 450 and self.y != 410 and self.y != 230:
            self.x = 450
            """К станции тех.обслуживания №2:"""
            self.y += LOADER_SPEED_Y
            if self.y > requesting_station.y_loaders:
                self.y = requesting_station.y_loaders

        if self.y == 230 or self.y == 410:
            self.x -= (LOADER_SPEED_X - 1)
            if self.x < 400:
                self.x = 400
                self.status = "on_service_station" + str(number_of_station)
                self.image = pygame.image.load("loader_empty1.png")  # Загрузка в pygame картинки
                self.image = pygame.transform.scale(self.image,
                                                    (self.IMG_size, self.IMG_size))  # Изменение размера картинки
                requesting_station.loaders_count_on_station = 1
                requesting_station.loader_to_station = 0
                requesting_station.repairing = "now"
                self.requesting_station = 0

        yield self.env.timeout(10)  # Для того чтобы можно было вызвать как генератор

    def to_warehouse(self):
        """Перемещение грузчиков от станции тех.обслуживания к складу"""
        global warehouse_loaders
        self.x += LOADER_SPEED_X
        if self.x > 450 and self.y != 380:
            self.x = 450
            # От станции тех.обслуживания
            self.y += LOADER_SPEED_Y
            if self.y > 380:
                self.y = 380
        if self.y == 380:
            self.x += LOADER_SPEED_X
            if self.x > 650:
                self.x = 650
                self.status = "in_warehouse"
                self.search_status = "search"
                self.image = pygame.image.load("loader.png")  # Загрузка в pygame картинки
                self.image = pygame.transform.scale(self.image,
                                                    (self.IMG_size, self.IMG_size))  # Изменение размера картинки
                warehouse_loaders += 1
                self.warehose_status = "empty"

    def checking_stations(self):
        """
        Проверка станций - выбор станции, к которой нужно перенсти детали со склада
        """
        for station_object in stations_objects:
            if self.status == "in_warehouse" and not station_object.loaders_count_on_station:
                if station_object.station_status and not station_object.loader_to_station:
                    station_object.loader_to_station = 1
                    return station_object

    def departure_from_warehouse_to_station(self, WAREHOSE_STATION_SIZE2, station_object):
        """
        Если грузчиком получен запрос станции о ремонте, он проверяет:
         1.находится ли кто-то из грузчиков уже в пути
         2.есть ли на складе нужные запчасти
        Забирает детали со склада и отправляется к нужной станции
        """
        global warehouse_loaders
        station_number = station_object.number_of_station
        if "to_station" not in self.status_now:
            self.status_now = "to_station" + str(station_object.number_of_station)
        if warehouse_loaders != 0:
            warehouse_loaders -= 1
        if station_object.loader_to_station and self.status_now == f"to_station{station_number}":
            if WAREHOSE_STATION_SIZE2 > station_object.details_required:
                self.take_details_from_warehouse(details_required=station_object.details_required)
                if self.loader_details != 0:
                    if self.search_status == "search":
                        yield self.env.timeout(
                            self.search_time * station_object.details_required)  # Время поиска запчастей на складе
                        self.search_status = "done"
                    if self.search_status == "done":
                        yield env.process(self.go_to_requesting_details_station(requesting_station=station_object))

    def take_details_from_warehouse(self, details_required):
        """
        Забрать необходимое кол-во деталей(для станции тех.обслуживания №1 или №2) со склада:
        Уменьшить кол-во деталей на складе
        """
        global WAREHOSE_STATION_SIZE2
        if self.warehose_status == "empty":
            WAREHOSE_STATION_SIZE2 -= details_required
            self.loader_details = details_required
            self.warehose_status = "full"

    @staticmethod
    def ordering_new_details():
        """
        Грузчики ищут на складе нужные для ремонта детали.
        Если деталей недостаточно, они сообщают грузовику отправится на производство за новыми запчастями
        """
        global WAREHOSE_STATION_SIZE2
        for station_object in stations_objects:
            if WAREHOSE_STATION_SIZE2 < station_object.details_required:
                # Ждем грузовик с новыми запчастями и пополняем запас склада
                if truck.status == "in_warehouse":
                    yield env.process(truck.to_production())
                # Грузовик находится на производстве:
                if truck.status == "on_production":
                    if truck.loading_status == "now":
                        yield env.process(truck.loading())
                        truck.loading_status = "done"
                    if truck.loading_status == "done":
                        yield env.process(truck.to_warehouse())
                        if truck.y == 240 and truck.x == 600:
                            WAREHOSE_STATION_SIZE2 = WAREHOSE_MAX

    def return_to_warehouse(self):
        """
        Если грузчик на станции обслуживания и там находится самолет, требующий ремонта -
        происходит ремонт, по завершению которого грузчик возвращается на склад
        """
        if "on_service_station" in self.status:
            self.status_now = "to_warehouse"
            for station_object in stations_objects:
                if station_object.repairing == "now" and self.status == "on_service_station" + str(
                        station_object.number_of_station):
                    # Ремонт на текущей станции:
                    yield self.env.timeout(self.repair_time)  # Время ремонта
                    self.loader_details = 0
                    if self.status == "on_service_station" + str(station_object.number_of_station):
                        station_object.repairing = "done"
                        station_object.station_repair = "ready"
                if self.status == "on_service_station" + str(station_object.number_of_station):
                    station_object.loaders_count_on_station = 0
            # Отправление грузчика на склад:
            self.to_warehouse()

    def run(self):
        while True:
            # Станция обслуживания выбирается только один раз для каждого из самолетов
            if self.requesting_station:
                print(f"Грузчик №{self.number}, его requesting_station={self.requesting_station.number_of_station}")
            else:
                print(f"Грузчик №{self.number}, его requesting_station={self.requesting_station}")
            if not self.requesting_station:
                self.requesting_station = self.checking_stations()  # Выбор свободной станции обслуживания
            station_object = self.requesting_station

            # Грузчик отпраляется на запросившую ремонт станцию обслуживания
            if station_object:
                yield from self.departure_from_warehouse_to_station(WAREHOSE_STATION_SIZE2, station_object)

            # Говорим грузовику отпрваится на склад за новыми запчастями:
            yield from self.ordering_new_details()

            # Отправление грузчика при завершении ремонта со станции обратно на склад:
            yield from self.return_to_warehouse()
            yield self.env.timeout(50)

    def __call__(self, screen):
        self.image1 = self.image.get_rect(topleft=(self.x, self.y))
        screen.blit(self.image, self.image1)  # Расположить картинку по координатам

class Truck:
    """
    Грузовик:
        При количестве деталей на складе меньше допустимого предела - перемещается на производство,
        загружает детали и везет обратно на склад
    """

    def __init__(self, env):
        self.IMG_size = 70
        self.image = pygame.image.load("truck.png")  # Загрузка в pygame картинки
        self.image = pygame.transform.scale(self.image,
                                            (self.IMG_size + 20, self.IMG_size))  # Изменение размера картинки
        self.x = 600  # Изначальное положение центра картинки(Склад)
        self.y = 240
        # self.x = 660  # На производстве
        # self.y = 50
        self.env = env
        self.status_prev = ""  # "on_parking", "on_service_station", "moving"
        self.status = "in_warehouse"  # "on_production"
        # self.status = "on_production"
        self.loading_status = ""  # Загружается ли сейчас грузовик? ("now"/"done")

    def loading(self):
        yield self.env.timeout(1500)

    def to_production(self):
        """Перемещение грузовика от склада к производству/заводу"""
        global iteration
        global event_time, event
        self.y -= TRUCK_SPEED_Y
        if self.y < 50:
            self.y = 50
            self.x += TRUCK_SPEED_X
            if self.x > 660:
                self.x = 660
                self.status = "on_production"
                self.image = pygame.image.load("truck.png")  # Загрузка в pygame картинки
                self.image = pygame.transform.scale(self.image,
                                                    (self.IMG_size + 20, self.IMG_size))  # Изменение размера картинки
                iteration += 1
                self.loading_status = "now"
                event = "Грузовик на заводе"
                event_time = round(env.now / 1000)
        yield self.env.timeout(10)  # # Для того чтобы можно было вызвать как генератор

    def to_warehouse(self):
        """Перемещение грузовика от завода к складу"""
        global iteration
        global WAREHOSE_STATION_SIZE2
        global event, event_time
        self.x -= TRUCK_SPEED_X
        if self.x < 600:
            self.x = 600
            self.y += TRUCK_SPEED_Y
            if self.y > 240:
                self.y = 240
                self.status = "in_warehouse"
                self.image = pygame.image.load("truck.png")  # Загрузка в pygame картинки
                self.image = pygame.transform.scale(self.image,
                                                    (self.IMG_size + 20, self.IMG_size))  # Изменение размера картинки
                iteration += 1
                event = "Грузовик на складе"
                event_time = round(env.now / 1000)
        yield self.env.timeout(10)  # Для того чтобы можно было вызвать как генератор

    def __call__(self, screen):
        self.image1 = self.image.get_rect(topleft=(self.x, self.y))
        screen.blit(self.image, self.image1)  # Расположить картинку по координатам

class Monitoring:
    """Класс для отображения текста и состояния переменных"""

    def __init__(self, stoyanka_counts):
        self.stoyanka_counts = stoyanka_counts
        self.font = pygame.font.Font(None, 20)

    def parameter_displaying(self, text: str, parameter, x, y, indent=0):
        """
        Отображение текста и параметра на экране по переданным координатам
        params:
            indent - Отступ параметра от текста
        """
        display_text = self.font.render(text, True, (0, 0, 255))
        screen.blit(display_text, [x, y])
        display_parametr = self.font.render(str(parameter), True, (0, 0, 255))
        screen.blit(display_parametr, [x + 130 + indent, y])

    def __call__(self, screen):
        # Отображение текста на экране:
        airplanes_counts_text = "Кол-во самолетов:"
        loaders_counts_text = "Кол-во грузчиков:"
        details_required_text = "Требуется деталей:"

        # ################### Отображение кол-ва самолетов на стоянке: ###################
        self.parameter_displaying(text=airplanes_counts_text, parameter=stoyanka_counts, x=10, y=230)

        # ################### Отображение кол-ва самолетов на станции тех.обслуживания №1: ###################
        self.parameter_displaying(text=airplanes_counts_text, parameter=station1.station_status, x=250, y=215)
        # ################### Отображение кол-ва деталей на станции тех.обслуживания №1: ###################
        self.parameter_displaying(text=details_required_text, parameter=station1.details_required, x=240, y=165)
        # ################### Отображение кол-ва команд механиков на станции тех.обслуживания №1: ###################
        self.parameter_displaying(text=loaders_counts_text, parameter=station1.loaders_count_on_station, x=250, y=190)

        # ################### Отображение кол-ва самолетов на станции тех.обслуживания №2: ###################
        self.parameter_displaying(text=airplanes_counts_text, parameter=station2.station_status, x=250, y=390)
        # ################### Отображение кол-ва деталей на станции тех.обслуживания №2: ###################
        self.parameter_displaying(text=details_required_text, parameter=station2.details_required, x=240, y=340)
        # ################### Отображение кол-ва команд механиков на станции тех.обслуживания №2: ###################
        self.parameter_displaying(text=loaders_counts_text, parameter=station2.loaders_count_on_station, x=250, y=365)

        # ################### Отображение кол-ва команд грузчиков на складе: ###################
        self.parameter_displaying(text=loaders_counts_text, parameter=warehouse_loaders, x=778, y=300)
        # ################### Отображение кол-ва деталей на складе(2): ###################
        self.parameter_displaying(text="Кол-во деталей на складе:", parameter=WAREHOSE_STATION_SIZE2, x=778, y=325,
                                  indent=50)
        # From Monitoring1(Для отладки)
        # Отображение события:
        self.parameter_displaying(text="Событие:", parameter=event, x=550, y=440, indent=-50)
        # Время события:
        self.parameter_displaying(text="Время:", parameter=event_time, x=550, y=470, indent=-70)
        # Время в симуляции:
        self.parameter_displaying(text="Время симуляции:", parameter=round(env.now / 1000), x=550, y=620)

###########################################################
def airplane_generator(env, cars):
    """Генерируем новые самолеты, которые прибывают на обслуживание"""
    for i in itertools.count():
        yield env.timeout(random.randint(AIRPLANE_ARRIVING_TIME[0], AIRPLANE_ARRIVING_TIME[1]))
        if len(cars) != 0:
            a = cars.pop(0)
            renderer.add(a)
            env.process(a.run())

###########################################################

"""Описание игры"""
pygame.init()

event = ""  # Глобальное событие в системе
event_time = 0  # Время глобального события

##############################################################
screen = pygame.display.set_mode((SCREEN_WIDTH, SCREEN_HEIGHT))
renderer = FrameRenderer(screen)
env = PyGameEnvironment(renderer, factor=SIMULATION_SPEED, strict=False)  # factor - Для скорости воспроизведения модели
renderer.add(Monitoring(stoyanka_counts))

station1 = Station(number_of_station=1)
station2 = Station(number_of_station=2)
stations_objects = [station1, station2]

airplanes = [Airplane(env, i) for i in range(1, 11)]

mechanics = [Loader(env, number=i) for i in range(1, NUMBER_OF_LOADERS + 1)]
for mechanic in mechanics:
    renderer.add(mechanic)
    env.process(mechanic.run())

truck = Truck(env)
renderer.add(truck)

service_station = simpy.Resource(env, number_station)  # Общий ресурс - станции обслуживания
warehose = simpy.Container(env, WAREHOSE_STATION_SIZE, init=WAREHOSE_STATION_SIZE)  # Склад(контейнер) - механики /
# забирают/ детали, грузовик привозит новые

env.process(airplane_generator(env, airplanes))

results = []
print(results)

env.run()
print(results)


# coding: utf-8
'''
This script reads data from various sources to process and store in MongoDB.
'''
import pyexcel
import logging
import json

import models

logging.basicConfig(filename='logs/altmetrics.info.txt', level=logging.INFO)
logger = logging.getLogger(__name__)

# Add Access count for journals
def scielogaaccess(filename, sheetname, year):
    altmetrics = pyexcel.get_sheet(
        file_name=filename,
        sheet_name=sheetname,
        name_columns_by_row=0)

    # Edit labels
    labels = []
    for t in altmetrics.colnames:
        labels.append(
            t.strip().lower().
            replace(", ", "_").
            replace(' ', '_').
            replace("'", "").
            replace("(", "").
            replace(")", "")
        )

    for i, k in enumerate(labels):
        altmetrics.colnames[i] = k

    altmetrics_json = altmetrics.to_records()

    for rec in altmetrics_json:
        # remove empty keys
        rec = {k: v for k, v in rec.items() if v or v == 0}

        # rec['issns'] = rec['issns'].split(';')
        rec['issn_list'] = []
        if 'issn1' in rec:
            rec['issn_list'].append(rec['issn1'].strip())
        if 'issn2' in rec:
            rec['issn_list'].append(rec['issn2'].strip())

        print(rec['issn_list'])

        for issn in rec['issn_list']:

            flag = 0

            query = models.Scielo.objects.filter(issn_scielo=issn)

            if len(query) == 1 and flag == 0:
                doc = query[0]
                print(query[0]['issn_scielo'])

                data = {}
                if 'altmetrics' in doc:
                    data['altmetrics'] = json.loads(
                        query[0].to_json())['altmetrics']
                    data['altmetrics'][year] = dict(rec)
                else:
                    data['altmetrics'] = {}
                    data['altmetrics'][year] = dict(rec)
                if data:
                    doc.modify(**data)
                    flag = 1

def main():
    # Altmetrics
    scielogaaccess(
        'data/scielo/altmetrics/Altmetric-SciELO journals - 20170601-20183005.xlsx',
        'import', '2018')

if __name__ == "__main__":
    main()


# Import modules
import numpy as np
import random
import pandas as pd
import matplotlib.pyplot as plt
import time
import NN_2

class PSO:

    def __init__(self, n_networks):
        """
        The PSO object contains an input n_networks which is the number of neural networks
        that are to be initialised.
        networks: is a list to store the initialised networks
        global_best_value: is initialised as infinity
        global_best_position: gets its shape from the Neural Network's getParams function
        yHat: is initialised at floating point 0. It is needed to plot a graph
        yHat_l: is a list to store the yHat values that is needed to plot a graph
        """
        self.neurons = int(input("Inform the number of neurons in hidden layer of NN: "))
        self.n_networks = n_networks
        self.networks = [NN_2.NeuralNetwork(NN_2.x,NN_2.y, self.neurons) for i in range(self.n_networks)]
        self.global_best_value = float("inf")
        self.global_best_position = NN_2.NeuralNetwork(NN_2.x,NN_2.y, self.neurons).getParams.shape
        self.global_best_yHat = 0

    def set_personal_best(self):
        """
        The set_personal_best method loops through a list of networks, assisigns a
        fitness_candidate which is the network's fitness. If the networks'
        personal_best_value is greater that fitness_candidate; it then assigns the
        personal_best_value as the fitness_candidate. It then updates the network's
        personal_best_position as the network's position.
        """
        for network in self.networks:
            if(network.personal_best_value > network.fitness):
                network.personal_best_value = network.fitness
                network.personal_best_position = network.position

    def get_personal_best(self):
        particles_position = []
        for network in self.networks:
            particles_position.append(network.position)
        return

    # The variable informants is in each network, here I just create informants for each of them.
    def set_informants(self):
        for network in self.networks:
            informants = random.choices(self.networks, k=3) # 3 informants for each particle
            network.informants = informants

    # In this funcion I am instantiating the best_value of each informant in
    def set_informants_best(self):
        for network in self.networks:
            for informant in network.informants:
                if(informant.personal_best_value > informant.fitness):
                    informant.informants_best_value = informant.fitness
                    informant.informants_best_position = informant.position

    def set_global_best(self):
        """
        The set_global_best method loops through a list of networks and assigns the
        best_fitness_candidate to the network's fitness. If the global_best_value
        is greater than the best_fitness_candidate the global_best_value is assigned as
        best_fitness_candidate and the global_best_position becomes the network's position
        """
        for network in self.networks:
             if(self.global_best_value > network.personal_best_value):
                self.global_best_value = network.personal_best_value
                self.global_best_position = network.position
                self.global_best_yHat = network.yHat

    def get_global_best(self):
        print (f"Value:{self.global_best_value}, Position: {self.global_best_position}")

    def move_particles(self):
        """
        The move_particles method contains: the Intertia weight(a),
        Cognitive(b), Social (c) and Informants (d) weights of the PSO algorithm which can be adjusted
        and affect directly the value of the velocity.
        There is an extra weight value (e) that is called the Jump and is used over the whole velocity.

        This method loops through a list of neural networks and stores the product of
        of interia weight multiplied by network's velocity plus a random number multiplied
        by the cognitive weight multiplied by the difference of the personal_best_position
        of the network and network's position plus the social weight into a random number
        multiplied by the difference of global_best_position of the networks and network's
        position plus the weighted value of the informants best position minus the network position
        in a variable called new_velocity.

        This will be weighted by the jump value and then it ssigns the network's velocity
        to this variable and calls the move function from the NeuralNetwork class.
        """
        a = 0.5 # Intertia: proportion of velocity to be retained
        b = 0.8 # Cognitive/personal velocity: proportion of personal best to be retained
        c = 1   # Social velocity: proportion of the informants' best to be retained
        d = 0.9 # Global: proportion of global best to be retained
        e = 1   # Jump size of a particle

        for network in self.networks:
            new_velocity = (a*network.velocity) + (b*random.random())*\
            (network.personal_best_position - network.position) +\
            (c*random.random())*(network.informants_best_position - network.position) +\
            (d*random.random())*(self.global_best_position - network.position)

            network.velocity = e*new_velocity
            network.move()

        # I added the Jump (the value is 1 by the pseudocode of the book they suggest, so does not affect)
        # but I think we do need to put it.

    def optimise(self):
        """
        The optimise method loops through a list of neural networks and:
        w1: takes the first three numbers from network's position array which is then
        reshaped to the dimensions of the NeuralNetwork object's W1 parameter
        w2: takes the next three numbers from network's position array which is then
        reshaped to the dimensions of the NeuralNetwork object's W2 parameter
        b1: takes the 7th item from the array
        b2: takes the 8th item from the array

        and uses these variables to forward propagate the neural network with these values.
        z2: is the dot product of input(x) and w1 plus bias(b1)
        a2: is the activation of the z2 using the activation function in NeuralNetwork class
        z3: is the dot product of a2 and W2 plus bias(b2)
        yHat: is the activation of the z3 using the activation function in NeuralNetwork class
        yHat_l: the yHat values are stored in a list for plotting graphs
        error: is calculated by using the Mean Square Error(mse) method using the target value(y)
        and predicted value(yHat). The network's fitness is updated using the error.
        """
        for network in self.networks:
         # by calling the methods here, the optimization is automatic and I do not need to call them outside.
         # just by calling PSO(num_NN) it is done.

            network.forward()
            network.mse()
            self.set_personal_best()
            self.set_informants()
            self.set_informants_best()
            self.set_global_best()
            self.move_particles()

            # Update of weights
            W1 = network.position[0:(len(network.pw1))]
            W2 = network.position[(len(network.pw1)):(len(network.pw1)+len(network.pw2))]
            network.W1 = np.reshape(W1,network.W1.shape)
            network.W2 = np.reshape(W2,network.W2.shape)
            network.b1 = network.position[len(network.position)-2]
            network.b2 = network.position[len(network.position)-1]

"""
pso1.optimise()
pso1.get_global_best()

plt.figure()
yHat1 = pso1.global_best_yHat
plt.plot(NN.y,"red",yHat1,"blue")
plt.title("y,yHat")
# plt.xlabel("Iterations")
# plt.ylabel("Errors")
plt.show()

"""
if __name__ == "__main__":
    pso = PSO(10)
    n_iterations = 100
    error_list = []
    yHat = 0
    # The start time to calculate how long the algorithm takes.
    start = time.process_time()
    # Sets the number of starting iterations/epochs

    iterations = 0
    while(iterations < n_iterations):
        pso.optimise()
        error_list.append(pso.global_best_value)
        yHat = pso.global_best_yHat

        iterations +=1

    #the global_best_value and the time taken to execute the algorithm
    print(f"GlobalBest: {pso.global_best_position} iters: {iterations} GlobalBestVal: {pso.global_best_value}")
    print(f"------------------------ total time taken: {time.process_time() - start} seconds")

    # Show the graph
    yHat = pso.global_best_yHat
    plt.figure()

    plt.plot(NN_2.y,"red",yHat,"blue")
    plt.xlabel("Input values")
    plt.ylabel("Output values")
    plt.title("Desired vs Predicted output")
    plt.show()

    fitness = error_list
    plt.figure()
    plt.plot(fitness)
    plt.xlabel("Number of iterations")
    plt.ylabel("Mean square error")
    plt.title("Mean Square Error per iteration")
    plt.show()


#!/usr/bin/env python
"""
Merges the intermediate localization files into a single
localization file.

Hazen 08/17
"""

import glob
import os
from xml.etree import ElementTree

import storm_analysis.sa_library.readinsight3 as readinsight3
import storm_analysis.sa_library.writeinsight3 as writeinsight3

def mergeAnalysis(dir_name, bin_base_name, extensions = [".bin"]):

    # Create Insight3 file writers.
    i3_out = []
    for ext in extensions:
        i3_out.append(writeinsight3.I3Writer(bin_base_name + ext))

    # Find all the job*.xml files.
    job_xml_files = glob.glob(dir_name + "job*.xml")

    # Sort job files.
    job_xml_files = sorted(job_xml_files, key = lambda x: int(os.path.splitext(os.path.basename(x))[0].split("_")[1]))

    # Check for corresponding mlist.bin files.
    metadata = None
    last_frame = 0
    for i in range(len(job_xml_files)):

        job_complete = True
        for j, ext in enumerate(extensions):
            mlist_name = dir_name + "p_" + str(i+1) + "_mlist" + ext

            if os.path.exists(mlist_name) and readinsight3.checkStatus(mlist_name):

                # Load metadata from the first file.
                if (i == 0) and (j == 0):
                    metadata = readinsight3.loadI3Metadata(mlist_name)

                # Read localizations.
                i3_data = readinsight3.loadI3File(mlist_name, verbose = False)

                # Check for empty file.
                if (i3_data.size == 0):
                    print("No localizations found in", mlist_name)

                else:
                    # Print frame range covered.
                    if (j == 0):
                        last_frame = i3_data["fr"][-1]
                        print(i3_data["fr"][0], last_frame, mlist_name)

                    # Add localizations to the output file.
                    i3_out[j].addMolecules(i3_data)

            else:
                job_complete = False
                break

        if not job_complete:
            print("Merge failed because", job_xml_files[i], "is incomplete.")
            for j, ext in enumerate(extensions):
                i3_out[j].close()
                os.remove(bin_base_name + ext)
            assert(False)

    if metadata is None:
        print("No metadata found.")
        for i3w in i3_out:
            i3w.close()
    else:

        # Fix movie length node based on the last frame of the last molecule.
        metadata.find("movie").find("movie_l").text = str(last_frame)

        # Also need to fix analysis end points. We are assuming that the
        # entire movie was analyzed.
        metadata.find("settings").find("start_frame").text = "-1"
        metadata.find("settings").find("max_frame").text = "-1"

        for i3w in i3_out:
            i3w.closeWithMetadata(ElementTree.tostring(metadata, 'ISO-8859-1'))

if (__name__ == "__main__"):

    import argparse

    parser = argparse.ArgumentParser(description = 'Merge analysis results from parallel analysis.')

    parser.add_argument('--working_dir', dest='wdir', type=str, required=True,
                        help = "The name of the analysis working directory.")
    parser.add_argument('--bin_base_name', dest='merged', type=str, required=True,
                        help = "The base name of the merged localization file (i.e. without .bin extension)")
    parser.add_argument('--ext', dest='ext', type=str, required=False, default=[".bin"], nargs = "*",
                        help = "The name of the extensions, if any.")

    args = parser.parse_args()

    mergeAnalysis(args.wdir, args.merged, args.ext)


from __future__ import absolute_import, unicode_literals

import functools

import newrelic.agent

from lymph.core import trace
from lymph.core.plugins import Plugin
from lymph.core.container import ServiceContainer
from lymph.web.interfaces import WebServiceInterface

def with_trace_id(func):
    @functools.wraps(func)
    def wrapped(*args, **kwargs):
        newrelic.agent.add_custom_parameter('trace_id', trace.get_id())
        return func(*args, **kwargs)
    return wrapped

class NewrelicPlugin(Plugin):
    def __init__(self, container, config_file=None, environment=None, **kwargs):
        super(NewrelicPlugin, self).__init__()
        self.container = container
        self.container.error_hook.install(self.on_error)
        self.container.http_request_hook.install(self.on_http_request)
        newrelic.agent.initialize(config_file, environment)
        for method in ('send_request', 'emit_event', 'lookup'):
            setattr(ServiceContainer, method, newrelic.agent.function_trace()(getattr(ServiceContainer, method)))

    def on_interface_installation(self, interface):
        self._wrap_methods(interface.methods)
        self._wrap_methods(interface.event_handlers)
        if isinstance(interface, WebServiceInterface):
            interface.application = newrelic.agent.wsgi_application()(interface.application)

    def _wrap_methods(self, methods):
        for name, method in methods.items():
            method.decorate(with_trace_id)
            method.decorate(newrelic.agent.background_task())

    def on_error(self, exc_info, **kwargs):
        newrelic.agent.add_custom_parameter('trace_id', trace.get_id())
        newrelic.agent.record_exception(exc_info)

    def on_http_request(self, request, rule, kwargs):
        newrelic.agent.set_transaction_name("%s %s" % (request.method, rule))
        newrelic.agent.add_custom_parameter('trace_id', trace.get_id())


import discord
import boiler
from discord.ext import commands

class moderate():
    def __init__(self, bot: commands.Bot):
        self.bot = bot

    @commands.command(description="Enables a user with Manage Messages to bulk delete the last `amount` messages.")
    async def purge(self, ctx: commands.Context, amount: int):
        '''Bulk remove messages.'''
        if (ctx.channel.permissions_for(ctx.author).manage_messages):
            i = 0
            await ctx.message.delete()
            async with ctx.typing():
                async for m in ctx.history(limit=amount):
                    await m.delete()
                    i += 1
            em = boiler.embed_template("Purged {} messages".format(i))
            if (ctx.author.nick is not None):
                em.set_footer(text="Requested by {}".format(
                    ctx.author.nick), icon_url="https://i.imgur.com/2VepakW.png")
            else:
                em.set_footer(text="Requested by {}".format(
                    ctx.author.name), icon_url="https://i.imgur.com/2VepakW.png")
            await ctx.send(None, embed=em, delete_after=5 if i <= 5 else None)
        else:
            await ctx.send("Purge not performed because you ain't got the power.")

def setup(bot):
    bot.add_cog(moderate(bot))


import numpy as np
import pandas as pd
import random
import matplotlib.pyplot as ploty
import time

import numpy as np
from sklearn.naive_bayes import GaussianNB

word=[]
word_test=[]

lines = [line.rstrip('\n') for line in open("traindata.txt","r")]
label = [line.rstrip('\n') for line in open("trainlabels.txt","r")]
lines_test = [line.rstrip('\n') for line in open("testdata.txt","r")]
test_label = [line.rstrip('\n') for line in open("testlabels.txt","r")]
stopword = [line.rstrip('\n') for line in open("stoplist.txt","r")]
#####################################################################
for i in range(0,(len(lines))):
	lines_sep=lines[i].split(" ")
	word.append(lines_sep)
line_words = word
word = [y for x in word for y in x]
# print(len(word))
word = sorted(list(set(word)-set(stopword)))
# print(len(word))
#print (word)
# print(word)

row=[]
X_train=[]
X_train_temp=[]
X_test=[]
X_test_temp=[]
for i in range(0,len(line_words)):
	row=[0]*(len(word)+1)
	row_temp = [0]*(len(word))
	for j in range(len(line_words[i])):
		if(line_words[i][j] in word):
			k=word.index(line_words[i][j])
			row_temp[k]=1
			row[k]=1
	row[len(word)]=int(label[i])
	X_train.append(row_temp)
	#print(len(row))
	X_train_temp.append(row)
with open("preprocess1.txt","w+") as f:
    f.write(",".join((map(str, word))))
    f.write("\n".join(",".join(map(str, x)) for x in X_train_temp))
################################################################
for i in range(0,(len(lines_test))):
	lines_sep_test=lines[i].split(" ")
	word_test.append(lines_sep_test)
#print(len(word_test))
line_words_test=word_test
for i in range(0,len(line_words_test)):
	row=[0]*(len(word)+1)
	row_temp = [0]*(len(word))
	for j in range(len(line_words_test[i])):
		if(line_words_test[i][j] in word):
			k=word.index(line_words_test[i][j])
			row_temp[k]=1
			row[k]=1
	row[len(word)]=(test_label[i])
	X_test.append(row_temp)
	#print(len(row))
	X_test_temp.append(row)

###############################################################

clf = GaussianNB()
clf.fit(X_train,label)

X_test= np.array(X_test)
predicts=[]
counter=0;
# print(test_label[0])
z=clf.predict(X_test)
for i in range(len(X_test)):
	if(test_label[i]==z[i]):
		counter+=1
avg=counter/len(X_test)
# print(avg)
z=[]
z.append(clf.score(X_test, test_label, sample_weight=None))
print(z)
with open("output1.txt","w+") as l:
    l.write(",".join((map(str, z))))


'''
Introductory GAN, taking in 64x64 color images for training from various datasets, and generating more examples from them.
Code HEAVILY inspired by: https://github.com/pytorch/examples/blob/master/dcgan/main.py
More info: https://pytorch.org/tutorials/beginner/dcgan_faces_tutorial.html
'''

import os
import random
import torch
import torch.nn as nn
#import torch.backends.cudnn as cudnn
import torch.optim as optim
import torch.utils.data
import torchvision.datasets as dset
import torchvision.transforms as transforms
import torchvision.utils as vutils

SIZE_Z = 100
G_FEATURE_SIZE = 64
D_FEATURE_SIZE = 64
IMAGE_SIZE = 64

device = "cpu"

def weights_init(m):
    classname = m.__class__.__name__
    if classname.find("Conv") != -1:
        torch.nn.init.normal_(m.weight, 0.0, 0.02)
    elif classname.find("BatchNorm") != -1:
        torch.nn.init.normal_(m.weight, 1.0, 0.02)
        torch.nn.init.zeros_(m.bias)

class Generator(nn.Module):
    def __init__(self, num_colors):
        super(Generator, self).__init__()
        self.num_colors = num_colors
        self.main = nn.Sequential(
            #Z is latent vector of noise
            nn.ConvTranspose2d(            SIZE_Z, G_FEATURE_SIZE * 8, 4, 1, 0, bias=False),
            nn.BatchNorm2d(G_FEATURE_SIZE * 8),
            nn.ReLU(True),
            #size: 512 * 4 * 4
            nn.ConvTranspose2d(G_FEATURE_SIZE * 8, G_FEATURE_SIZE * 4, 4, 2, 1, bias=False),
            nn.BatchNorm2d(G_FEATURE_SIZE * 4),
            nn.ReLU(True),
            #size: 256 * 8 * 8
            nn.ConvTranspose2d(G_FEATURE_SIZE * 4, G_FEATURE_SIZE * 2, 4, 2, 1, bias=False),
            nn.BatchNorm2d(G_FEATURE_SIZE * 2),
            nn.ReLU(True),
            #size: 128 * 16 * 16
            nn.ConvTranspose2d(G_FEATURE_SIZE * 2,     G_FEATURE_SIZE, 4, 2, 1, bias=False),
            nn.BatchNorm2d(G_FEATURE_SIZE),
            nn.ReLU(True),
            #size: 64 * 32 * 32
            nn.ConvTranspose2d(    G_FEATURE_SIZE,         num_colors, 4, 2, 1, bias=False),
            nn.Tanh()
            #size: num_colors x 64 x 64
        )

    def forward(self, input):
        output = self.main(input)
        return output

class Discriminator(nn.Module):
    def __init__(self, num_colors):
        super(Discriminator, self).__init__()
        self.num_colors = num_colors
        self.main = nn.Sequential(
            #input is num_colors x 64 x 64
            nn.Conv2d(    num_colors,       D_FEATURE_SIZE, 4, 2, 1, bias=False),
            nn.LeakyReLU(0.2, inplace=True),
            #size: 64 x 32 x 32
            nn.Conv2d(D_FEATURE_SIZE,   D_FEATURE_SIZE * 2, 4, 2, 1, bias=False),
            nn.BatchNorm2d(D_FEATURE_SIZE * 2),
            nn.LeakyReLU(0.2, inplace=True),
            #size: 128 x 16 x 16
            nn.Conv2d(D_FEATURE_SIZE * 2, D_FEATURE_SIZE * 4, 4, 2, 1, bias=False),
            nn.BatchNorm2d(D_FEATURE_SIZE * 4),
            nn.LeakyReLU(0.2, inplace=True),
            #size: 256 x 8 x 8
            nn.Conv2d(D_FEATURE_SIZE * 4, D_FEATURE_SIZE * 8, 4, 2, 1, bias=False),
            nn.BatchNorm2d(D_FEATURE_SIZE * 8),
            nn.LeakyReLU(0.2, inplace=True),
            #size: 512 x 4 x 4
            nn.Conv2d(D_FEATURE_SIZE * 8,                  1, 4, 1, 0, bias=False),
            nn.Sigmoid()
            #size: 1 x 2 x 2
        )

    def forward(self, input):
        output = self.main(input)
        return output.view(-1, 1).squeeze(1)

def run_nn(dataset, dataroot=None, workers=2, batch_size=64, niter=25, lr=0.0002, beta1=0.5,
           cuda=True,
           dry_run=False,
           existing_G="",
           existing_D="",
           outf=".",
           manualSeed=None,
           classes="bedroom"):
    try:
        os.makedirs(outf)
    except OSError:
        pass

    if manualSeed == None:
        manualSeed = random.randint(1, 10000)
    print("Seed: ", manualSeed)
    random.seed(manualSeed)
    torch.manual_seed(manualSeed)

    '''
    if cuda:
        cudnn.benchmark = True
    '''

    if torch.cuda.is_available() and not cuda:
        print("WARNING: You have a CUDA device, so you should probably run with cuda=True")

    if dataroot == None and str(dataset).lower() != "fake":
        raise  ValueError(f"dataroot parameter is required for dataset {dataset}")

    if dataset in ["imagenet", "folder", "lfw"]:
        #folder dataset
        dataset = dset.ImageFolder(root=dataroot,
                                   transform=transforms.Compose([
                                       transforms.Resize(IMAGE_SIZE),
                                       transforms.CenterCrop(IMAGE_SIZE),
                                       transforms.ToTensor(),
                                       transforms.Normalize((0.5, 0.5 ,0.5), (0.5, 0.5, 0.5))
                                   ]))
        num_colors = 3
    elif dataset == "lsun":
        classes = [ c + '_train' for c in classes.split(',')]
        dataset = dset.LSUN(root=dataroot, classes=classes,
                           transform=transforms.Compose([
                               transforms.Resize(IMAGE_SIZE),
                               transforms.CenterCrop(IMAGE_SIZE),
                               transforms.ToTensor(),
                               transforms.Normalize((0.5, 0.5 ,0.5), (0.5, 0.5, 0.5))
                           ]))
        num_colors = 3
    elif dataset == "cifar10":
        dataset = dset.CIFAR10(root=dataroot, download=True,
                           transform=transforms.Compose([
                               transforms.Resize(IMAGE_SIZE),
                               transforms.ToTensor(),
                               transforms.Normalize((0.5, 0.5 ,0.5), (0.5, 0.5, 0.5))
                           ]))
        num_colors=3
    elif dataset == "mnist":
        dataset = dset.MNIST(root=dataroot, download=True,
                           transform=transforms.Compose([
                               transforms.Resize(IMAGE_SIZE),
                               transforms.ToTensor(),
                               transforms.Normalize((0.5,), (0.5,))
                           ]))
        num_colors = 1
    elif dataset == "fake":
        dataset = dset.FakeData(image_size=(3, IMAGE_SIZE, IMAGE_SIZE),
                                transform=transforms.ToTensor())
        num_colors = 3

    assert dataset
    dataloader = torch.utils.data.DataLoader(dataset, batch_size, shuffle=True, num_workers=int(workers))

    device = torch.device("cuda:0" if cuda else "cpu")

    netG = Generator(num_colors).to(device)
    netG.apply(weights_init)
    previous_epoch = 0
    if existing_G != "":
        netG.load_state_dict(torch.load(existing_G))
        previous_epoch = int(existing_G[-5])

    netD = Discriminator(num_colors).to(device)
    netD.apply(weights_init)
    if existing_D != "":
        netD.load_state_dict(torch.load(existing_D))
        previous_epoch = int(existing_D[-5])

    criterion = nn.BCELoss()

    fixed_noise = torch.randn(batch_size, SIZE_Z, 1, 1, device=device)
    real_label = 1
    fake_label = 0

    optimizerD = optim.Adam(netD.parameters(), lr=lr, betas=(beta1, 0.999))
    optimizerG = optim.Adam(netG.parameters(), lr=lr, betas=(beta1, 0.999))

    if dry_run:
        niter = 1

    for epoch in range(previous_epoch+1,previous_epoch+niter+1):
        running_errD = 0
        running_errG = 0
        running_D_x = 0
        running_D_G_z1 = 0
        running_D_G_z2 = 0
        for i, data in enumerate(dataloader, 1):
            #train with real image
            netD.zero_grad()
            real_cpu = data[0].to(device)
            batch_size = real_cpu.size(0)
            label = torch.full((batch_size,), real_label, dtype=real_cpu.dtype, device=device)

            output = netD(real_cpu)
            errD_real = criterion(output, label)
            errD_real.backward()
            running_D_x += output.mean().item()

            #train with fake image
            noise = torch.randn(batch_size, SIZE_Z, 1, 1, device=device)
            fake = netG(noise)
            label.fill_(fake_label)
            output = netD(fake.detach())
            errD_fake = criterion(output, label)
            errD_fake.backward()
            running_D_G_z1 += output.mean().item()
            running_errD += errD_real.item() + errD_fake.item()
            optimizerD.step()

            netG.zero_grad()
            label.fill_(real_label)
            output = netD(fake,)
            errG = criterion(output, label)
            errG.backward()
            running_errG += errG.item()
            running_D_G_z2 += output.mean().item()
            optimizerG.step()
            if i == 1:
                print(f"Starting Epoch {epoch}...\n")
            elif i % 100 == 0:
                print(f"[{epoch}/{previous_epoch+niter}] [{i}/{len(dataloader)}]")
            elif i == len(dataloader):
                print(f'''
Completed Epoch {epoch}
Loss_D: {running_errD/i:.4f}
Loss_G: {running_errG/i:.4f}
D(x): {running_D_x/i:.4f}
D(G(z)): {running_D_G_z1/i:.4f}/{running_D_G_z2/i:.4f}
                ''')
                vutils.save_image(real_cpu, f"{outf}/real_samples.png", normalize=True)
                fake = netG(fixed_noise)
                vutils.save_image(fake.detach(), f"{outf}/fake_samples_epoch_{epoch}.png", normalize=True)
            if dry_run:
                break

    print("Run completed, ending execution")
    torch.save(netG.state_dict(), f"{outf}/netG_epoch_{epoch}.pth")
    torch.save(netD.state_dict(), f"{outf}/netD_epoch_{epoch}.pth")

if __name__ == '__main__':
    run_nn(dataset="mnist",
           dataroot="./StartingWithGANs/data",
           outf="./StartingWithGANs/MNIST",
           cuda=True)


import torch
import torch.nn as nn
from DataLoader import PerReviewDataLoader, DataLoader
import numpy as np
from helper_functions import training_loop, cross_validation_metrics
from models import BERTClassifier
import pathlib
import os
import matplotlib.pyplot as plt
import yaml

# read configuration
with open('src/config/BERT_classifier_per_review.yaml') as f:
    config = yaml.load(f, Loader=yaml.FullLoader)

print(config)

# read cude idx
device_idx = config['CUDA']
GPU = True
if GPU:
    device = torch.device("cuda:" + device_idx if torch.cuda.is_available() else "cpu")
else:
    device = torch.device("cpu")
print(device)

cross_validation = config['cross_validation']
folds = config['folds']
causal_layer = config['causal_layer']
aspect = config['aspect']

data_loader = PerReviewDataLoader(device=device,
                                  final_decision='exclude',
                                  allow_empty=False,
                                  truncate_policy='right',
                                #   pretrained_weights='bert-base-uncased',
                                  pretrained_weights='scibert_scivocab_uncased',
                                  remove_duplicates=True,
                                  remove_stopwords=False)

try:
    embeddings_input = data_loader.read_embeddigns_from_file()
    data_loader.read_reviews_only_text()
    scores = data_loader.read_scores_from_file()
    labels = (scores > 5).float()
except FileNotFoundError:
    # create file with embeddings if it does not exist
    embeddings_input = data_loader.get_embeddings_from_reviews()
    data_loader.write_embeddings_to_file()
    scores = data_loader.read_scores()
    data_loader.write_scores_to_file()
print(embeddings_input.shape)
print(scores.shape)

_, embedding_dimension = embeddings_input.shape

if causal_layer:
    if aspect == 'abstract':
        confounders = data_loader.read_abstract_embeddings()
        confounders = torch.tensor(data_loader.copy_to_peer_review(confounders), dtype=torch.float)

if causal_layer == 'residual':
    nn_conf = config[causal_layer]
else:
    nn_conf = config['not_residual']

epochs = nn_conf['epochs'] # 200 # 500
batch_size = nn_conf['batch_size'] # 300 # 120
lr = nn_conf['lr'] # 0.0001
hidden_dimensions = nn_conf['hidden_dimensions'] #[128, 64] # [1500, 700, 300] # [700, 300]
causal_hidden_dimensions = nn_conf['causal_hidden_dimensions'] # [64]
BERT_hidden_dimensions = nn_conf['BERT_hidden_dimensions'] # [64]
dropout = nn_conf['dropout']
dropout2 = nn_conf['dropout2']
activation = nn_conf['activation']
activation2 = nn_conf['activation2']

if cross_validation:
    network = BERTClassifier
    network_params = {
        'device': device,
        'input_size': embedding_dimension,
        'hidden_dimensions': hidden_dimensions,
        'causal_hidden_dimensions': causal_hidden_dimensions,
        'BERT_hidden_dimensions': BERT_hidden_dimensions,
        'dropout': dropout,
        'dropout2': dropout2,
        'activation': activation,
        'activation2': activation2,
        'causal_layer': causal_layer
    }
    optimizer = torch.optim.Adam
    lr = lr
    loss_fn = nn.BCELoss
    if not causal_layer:
        data = [embeddings_input, labels, labels]
        cross_validation_metrics(network, network_params, optimizer, loss_fn, lr,
                                 epochs, batch_size, device, data, k=folds, shuffle=True)
    else:
        data = [embeddings_input, labels, labels, confounders]
        confounding_loss_fn = nn.BCELoss
        cross_validation_metrics(network, network_params, optimizer, loss_fn, lr,
                                 epochs, batch_size, device, data, causal_layer=causal_layer,
                                 confounding_loss_fn=confounding_loss_fn, k=folds,
                                 shuffle=True, loss2_mult=config['loss2_mult'])

else:
    # hold-one-out split
    model = BERTClassifier(device,
                           input_size=embedding_dimension,
                           hidden_dimensions=hidden_dimensions,
                           causal_hidden_dimensions=causal_hidden_dimensions,
                           BERT_hidden_dimensions=BERT_hidden_dimensions,
                           dropout=dropout,
                           dropout2=dropout2,
                           activation=activation,
                           activation2=activation2,
                           causal_layer=causal_layer)
    shuffle = False
    valid_size = 0.1
    print(embeddings_input.shape)

    num_train = embeddings_input.shape[0]
    indices = list(range(num_train))
    split = int(np.floor(valid_size * num_train))

    if shuffle:
        np.random.shuffle(indices)

    if config['on_validation_set']:
        # will test on validation set and train on train set
        train_idx, test_idx = indices[2*split:], indices[split:2*split]
    else:
        # will test on test set and train on train and vaildation set
        train_idx, test_idx = indices[split:], indices[:split]

    test_embeddings_input = embeddings_input[test_idx, :]
    test_labels = labels[test_idx]

    if causal_layer:
        test_confounders = confounders[test_idx, :]

    embeddings_input = embeddings_input[train_idx, :]
    labels = labels[train_idx]

    if causal_layer:
        confounders = confounders[train_idx, :]

    optimizer = torch.optim.Adam(model.parameters(), lr=lr)
    loss_fn = nn.BCELoss()
    if not causal_layer:
        confounding_loss_fn = None
        data = [embeddings_input, labels, labels]
        test_data = [test_embeddings_input, test_labels, test_labels]
    else:
        confounding_loss_fn = nn.BCELoss()
        data = [embeddings_input, labels, labels, confounders]
        test_data = [test_embeddings_input, test_labels, test_labels, test_confounders]

    model.to(device)

    losses = training_loop(data,
                           test_data,
                           model,
                           device,
                           optimizer,
                           loss_fn,
                           confounder_loss_fn=confounding_loss_fn,
                           verbose=True,
                           causal_layer=causal_layer,
                           epochs=epochs,
                           batch_size=batch_size,
                           return_losses=True,
                           loss2_mult=config['loss2_mult']
                         )

    if not causal_layer:
        train_losses, test_losses = losses
        plt.plot(train_losses, label='Train Loss')
        plt.plot(test_losses, label='Test Loss')
        plt.legend()
        plt.savefig('bert_losses.png')
        model_path = PerReviewDataLoader.DATA_ROOT / 'bert_classifier_per_review'
    else:
        train_losses, test_losses, confounding_train_losses, confounding_test_losses = losses
        plt.plot(train_losses, label='Train Loss')
        plt.plot(test_losses, label='Test Loss')
        plt.plot(confounding_train_losses, label='Confounding Train Loss')
        plt.plot(confounding_test_losses, label='Confounding Test Loss')
        plt.legend()
        # plt.yscale('log')
        plt.savefig('bert_losses.png')
        model_path = PerReviewDataLoader.DATA_ROOT / ('bert_classifier_per_review' + causal_layer)
    model_path.mkdir(parents=True, exist_ok=True)
    # torch.save(model, model_path / 'model.pt')


from rest_framework.authentication import TokenAuthentication

from rest_auth_multitoken.rest_auth_multitoken.models import Token

def multitoken_create(token_model, user, serializer):
    token = token_model(user=user)
    token.save()
    user.auth_multitoken = token
    return token

class MultiTokenAuthentication(TokenAuthentication):
    model = Token

    def authenticate_credentials(self, key):
        user, token = super().authenticate_credentials(key)
        try:
            user.auth_multitoken = Token.objects.get(key=key)
        except:
            user.auth_multitoken = None
        return user, token


import os
import codecs
import numpy as np
import matplotlib.pyplot as plt
from Meth_blue_06_09 import get_cv_data
from Meth_blue_06_09 import calculate_graph_data
from Meth_blue_06_09 import get_data_paths

def main():
    directory = "/Users/st659/Google Drive/Meth Blue 12-09"

    paths = get_data_paths(directory)

    plt.style.use(['seaborn-white', 'seaborn-notebook'])

    print(paths)

    # sub_bw_anti_3, bw_anti_3, = calculate_graph_data(paths[0:3])
    # sub_bw_noanti_3, bw_noanti_3 = calculate_graph_data(paths[3:6])
    # sub_pk_anti_3, pk_anti_3 = calculate_graph_data(paths[6:9])
    # sub_pk_noanti_3, pk_noanti_3 = calculate_graph_data(paths[9:12])
    #
    # figure1 = plt.figure()
    # ax1 = figure1.add_subplot(111)
    # ax1.errorbar(sub_bw_anti_3[0], sub_bw_anti_3[1], sub_bw_anti_3[2])
    # ax1.errorbar(sub_bw_noanti_3[0], sub_bw_noanti_3[1], sub_bw_noanti_3[2])
    # ax1.errorbar(sub_pk_anti_3[0], sub_pk_anti_3[1], sub_pk_anti_3[2])
    # ax1.errorbar(sub_pk_noanti_3[0], sub_pk_noanti_3[1], sub_pk_noanti_3[2])
    #
    # plt.xlabel('Voltage vs reference (V)')
    # plt.ylabel('Current (mA)')
    # plt.xlim([-0.3, 0])
    # plt.ylim([-0.02, 0.01])
    # plt.legend(('BW25113 0$\mu g/ml$ Ampicillin', 'BW25133 50$\mu g/ml$ Ampicillin', 'BW25113 with PKD3 0$\mu g/ml$ Ampicillin', 'BW25113 with PKD3 50$\mu g/ml$ Ampicillin'),loc='lower right', fontsize = '10')
    # plt.title(r'Antibiotic Susceptibility of $10^{3}$ Bacteria after 5 hours incubation')

    # sub_bw_anti_4, bw_anti_4, l= calculate_graph_data(paths[12:15])
    # sub_bw_noanti_4, bw_noanti_4, l = calculate_graph_data(paths[15:18])
    # sub_pk_anti_4, pk_anti_4, l = calculate_graph_data(paths[18:21])
    # sub_pk_noanti_4, pk_noanti_4, l = calculate_graph_data(paths[21:24])
    #
    # figure2 = plt.figure()
    # ax2 = figure2.add_subplot(111)
    # ax2.errorbar(sub_bw_anti_4[0], sub_bw_anti_4[1], sub_bw_anti_4[2])
    # ax2.errorbar(sub_bw_noanti_4[0], sub_bw_noanti_4[1], sub_bw_noanti_4[2])
    # ax2.errorbar(sub_pk_anti_4[0], sub_pk_anti_4[1], sub_pk_anti_4[2])
    # ax2.errorbar(sub_pk_noanti_4[0], sub_pk_noanti_4[1], sub_pk_noanti_4[2])
    #
    # plt.xlabel('Voltage vs reference (V)')
    # plt.ylabel('Current (mA)')
    # plt.xlim([-0.3, 0])
    # plt.ylim([-0.02, 0.01])
    # plt.legend(('BW25113 50$\mu g/ml$ Ampicillin', 'BW25133 0$\mu g/ml$ Ampicillin', 'BW25113 with PKD3 50$\mu g/ml$ Ampicillin', 'BW25113 with PKD3 0$\mu g/ml$ Ampicillin'),loc='lower right', fontsize = '10')
    # plt.title(r'Antibiotic Susceptibility of $10^{4}$ CFU/ml after 5 hours incubation')
    # #plt.savefig(os.path.join(directory,'susceptibility 10_4.png'), dpi=1200)
    #
    # diff = np.subtract(sub_bw_anti_4[1], sub_bw_noanti_4[1])
    # diff_std = np.sqrt(np.add(np.square(sub_bw_anti_4[2]), np.square(sub_bw_noanti_4[2])))
    # diff2 = np.subtract(sub_pk_anti_4[1], sub_pk_noanti_4[1])
    # diff2_std = np.sqrt(np.add(np.square(sub_pk_anti_4[2]), np.square(sub_pk_noanti_4[2])))
    #
    # figure2 = plt.figure()
    # ax3 = figure2.add_subplot(211)
    # ax3.errorbar(sub_bw_anti_4[0], diff, diff_std, label='BW', color='b')
    # ax3.set_ylabel('$\Delta$ Current (mA)')
    #
    # plt.legend(loc = 'upper left')
    # ax4 = figure2.add_subplot(212)
    # ax4.errorbar(sub_bw_anti_4[0], diff2, diff2_std, label='PK', color='g')
    # ax4.set_ylabel('$\Delta$ Current (mA)')
    # ax4.set_xlabel('Voltage (V)')
    # plt.legend(loc = 'upper left')
    #
    #
    #
    #
    # sub_bw_anti_5, bw_anti_5, l = calculate_graph_data(paths[24:27])
    # sub_bw_noanti_5, bw_noanti_5, l = calculate_graph_data(paths[27:30])
    # sub_pk_anti_5, pk_anti_5, l = calculate_graph_data(paths[30:33])
    # sub_pk_noanti_5, pk_noanti_5, l = calculate_graph_data(paths[33:36])
    #
    # mean_matrix = np.array([bw_anti_5[1], bw_noanti_5[1], pk_noanti_5[1], pk_anti_5[1]])
    # mean_var = np.var(mean_matrix, axis=0)
    # print(len(mean_var))
    #
    # figure_var = plt.figure()
    # ax_var = figure_var.add_subplot(111)
    # ax_var.plot(bw_noanti_5[0], mean_var)
    # plt.show()

    # diff = np.subtract(sub_bw_anti_5[1], sub_bw_noanti_5[1])
    # diff_std = np.sqrt(np.add(np.square(sub_bw_anti_5[2]), np.square(sub_bw_noanti_5[2])))
    # diff2 = np.subtract(sub_pk_anti_5[1], sub_pk_noanti_5[1])
    # diff2_std = np.sqrt(np.add(np.square(sub_pk_anti_5[2]), np.square(sub_pk_noanti_5[2])))
    #
    # figure3 = plt.figure()
    # ax5 = figure3.add_subplot(211)
    # ax5.errorbar(sub_bw_anti_4[0], diff, diff_std, label='BW', color='b')
    # ax5.set_ylabel('$\Delta$ Current (mA)')
    #
    # plt.legend(loc = 'upper left')
    # ax6 = figure3.add_subplot(212)
    # ax6.errorbar(sub_bw_anti_4[0], diff2, diff2_std, label='PK', color='g')
    # ax6.set_ylabel('$\Delta$ Current (mA)')
    # ax6.set_xlabel('Voltage (V)')
    # plt.legend(loc = 'upper left')
    #
    # figure3 = plt.figure()
    # ax3 = figure3.add_subplot(111)
    # ax3.errorbar(sub_bw_anti_5[0], sub_bw_anti_5[1], sub_bw_anti_5[2])
    # ax3.errorbar(sub_bw_noanti_5[0], sub_bw_noanti_5[1], sub_bw_noanti_5[2])
    # ax3.errorbar(sub_pk_anti_5[0], sub_pk_anti_5[1], sub_pk_anti_5[2])
    # ax3.errorbar(sub_pk_noanti_5[0], sub_pk_noanti_5[1], sub_pk_noanti_5[2])
    #
    # plt.xlabel('Voltage vs reference (V)')
    # plt.ylabel('Current (mA)')
    # plt.xlim([-0.3, 0])
    # plt.ylim([-0.02, 0.01])
    # plt.legend(('BW25113 50$\mu g/ml$ Ampicillin', 'BW25133 00$\mu g/ml$ Ampicillin', 'BW25113 with PKD3 50$\mu g/ml$ Ampicillin', 'BW25113 with PKD3 0$\mu g/ml$ Ampicillin'),loc='lower right', fontsize = '10')
    # plt.title(r'Antibiotic Susceptibility of $10^{5}$ CFU/ml after 5 hours incubation')
    # #plt.savefig(os.path.join(directory,'susceptibility 10_5.png'), dpi=1200)
    #
    sub_bw_anti_6, bw_anti_6, leg= calculate_graph_data(paths[36:39])
    sub_bw_noanti_6, bw_noanti_6, leg = calculate_graph_data(paths[39:42])
    sub_pk_anti_6, pk_anti_6, leg = calculate_graph_data(paths[42:45])
    sub_pk_noanti_6, pk_noanti_6, leg = calculate_graph_data(paths[45:48])

    figure4 = plt.figure()
    ax4 = figure4.add_subplot(111)
    ax4.errorbar(sub_bw_anti_6[0], sub_bw_anti_6[1], sub_bw_anti_6[2])
    ax4.errorbar(sub_bw_noanti_6[0], sub_bw_noanti_6[1], sub_bw_noanti_6[2])
    ax4.errorbar(sub_pk_anti_6[0], sub_pk_anti_6[1], sub_pk_anti_6[2])
    ax4.errorbar(sub_pk_noanti_6[0], sub_pk_noanti_6[1], sub_pk_noanti_6[2])

    plt.xlabel('Voltage vs reference (V)')
    plt.ylabel('Current (mA)')
    plt.xlim([-0.3, 0])
    plt.ylim([-0.02, 0.01])
    plt.legend(('BW25113 50$\mu g/ml$ Ampicillin', 'BW25133 0$\mu g/ml$ Ampicillin', 'BW25113 with PKD3 50$\mu g/ml$ Ampicillin', 'BW25113 with PKD3 0$\mu g/ml$ Ampicillin'),loc='lower right', fontsize = '10')
    plt.title(r'Antibiotic Susceptibility of $10^{6}$ CFU/ml after 5 hours incubation')
    plt.savefig(os.path.join(directory,'susceptibility 10_6 300dpi.png'), dpi=300)

if __name__ == '__main__':
    main()