# -*- coding: utf-8 -*-
"""
Created on Fri Mar  6 14:03:49 2020

@author: Richie Bao-Chicago.IIT(driverless city project)
data:IIT(driverless city project)
reference:https://github.com/d2l-ai/d2l-zh  《动手学深度学习》(PyTorch版)
"""
import numpy as np
import matplotlib.pyplot as plt
from scipy.io import loadmat
import scipy.io as sio
import math
from decimal import *
from shapely.geometry import Point,MultiPoint
import networkx as nx
import operator
import torch
from sklearn import preprocessing
from tqdm import tqdm
import pickle

getcontext().prec = 28
np.set_printoptions(precision=28)

'''LandmarkMap 读取MatLab的.fig数据'''
LandmarkMap_fn=r"F:\data_02_Chicago\data_driverless City\IIT_data\LandmarkMap.fig"
def readMatLabFig_LandmarkMap(LandmarkMap_fn):
    LandmarkMap=loadmat(LandmarkMap_fn, squeeze_me=True, struct_as_record=False)
    y=loadmat(LandmarkMap_fn)
    print(sorted(LandmarkMap.keys()))

    LandmarkMap_dic={} #提取.fig值
    for object_idx in range(LandmarkMap['hgS_070000'].children.children.shape[0]):
        # print(object_idx)
        try:
            X=LandmarkMap['hgS_070000'].children.children[object_idx].properties.XData #good
            Y=LandmarkMap['hgS_070000'].children.children[object_idx].properties.YData
            LandmarkMap_dic[object_idx]=(X,Y)
        except:
            pass

    # print(LandmarkMap_dic)
    fig= plt.figure(figsize=(20,130))
    colors=['#7f7f7f','#d62728','#1f77b4','','','']
    markers=['.','+','o','','','']
    dotSizes=[200,3000,3000,0,0,0]
    linewidths=[2,10,10,0,0,0]
    i=0
    for key in LandmarkMap_dic.keys():
        plt.scatter(LandmarkMap_dic[key][0], LandmarkMap_dic[key][1],s=dotSizes[i],marker=markers[i], color=colors[i],linewidth=linewidths[i])
        i+=1
    plt.tick_params(axis='both',labelsize=80)
    plt.show()
    return LandmarkMap_dic

''''PHmi读取激光雷达导航评估值'''
PHMI_fn=r"F:\data_02_Chicago\data_driverless City\IIT_data\PHMI.fig"
def readMatLabFig_PHMI(PHMI_fn,LandmarkMap_dic):
    PHMI=loadmat(PHMI_fn, squeeze_me=True, struct_as_record=False)
    x=loadmat(PHMI_fn)
    print(sorted(PHMI.keys()))

    PHMI_dic={} #提取MatLab的.fig值
    for object_idx in range(PHMI['hgS_070000'].children.children.shape[0]):
        # print(object_idx)
        try:
            X=PHMI['hgS_070000'].children.children[object_idx].properties.XData #good
            Y=PHMI['hgS_070000'].children.children[object_idx].properties.YData
            Z=PHMI['hgS_070000'].children.children[object_idx].properties.ZData
            PHMI_dic[object_idx]=(X,Y,Z)
        except:
            pass

    # print(PHMI2_dic)
    fig= plt.figure(figsize=(20,130)) #figsize=(20,130)
    colors=['#7f7f7f','#d62728','#1f77b4','','','']
    markers=['.','+','o','','','']
    dotSizes=[200,3000,3000,0,0,0]
    linewidths=[2,10,10,0,0,0]

    ScalePhmi=math.pow(10,1)

    plt.plot(PHMI_dic[0][0], PHMI_dic[0][1],marker=markers[0], color=colors[0],linewidth=linewidths[0])

    ref=math.pow(10,-5)

    #for display clearly
    PHmiValue=PHMI_dic[1][2]
    # print(PHmiValue)
    # replaceValue=np.extract(PHmiValue<ref,PHmiValue)*-math.pow(10,5)
    # print(replaceValue)
    # PHmiValue[PHmiValue<ref]=replaceValue
    plt.plot(PHmiValue*ScalePhmi, PHMI_dic[0][1],marker=markers[0], color=colors[1],linewidth=1)

    # plt.plot(PHMI_dic[1][2]*ScalePhmi, PHMI_dic[0][1],marker=markers[0], color=colors[1],linewidth=1)
    plt.axvline(x=ref*ScalePhmi)

    plt.scatter(LandmarkMap_dic[1][0], LandmarkMap_dic[1][1],marker=markers[1], s=dotSizes[1],color=colors[2],linewidth=10)

    plt.tick_params(axis='both',labelsize=80)
    plt.show()

    return PHMI_dic

'''车载激光雷达导航的扫描区域影响landmarks提取 scanning circle buffer, extract the landmarks in the buffer at earch location'''
def scanCircleBuffer(locations,landmarks,radius):
    landmarks_pts=[Point(coordi[0],coordi[1]) for coordi in np.stack((landmarks[0], landmarks[1]), axis=-1)]
    # print(len(landmarks_pts))
    # print(landmarks_pts)
    locations_pts=[Point(coordi[0],coordi[1]) for coordi in np.stack((locations[0], locations[1]), axis=-1)]
    # print(len(locations_pts))
    scanCircleBuffer=[pt.buffer(radius) for pt in locations_pts]
    # print(scanCircleBuffer)
    targetPts={} #landmarks特征点
    targetPts_idx={} #建立location位置与其扫描范围内landmarks位置坐标的对应索引值
    for locBuffer_idx in range(len(scanCircleBuffer)):
        temp=[]
        temp_idx=[]
        for LM_idx in range(len(landmarks_pts)):
           if scanCircleBuffer[locBuffer_idx].contains(landmarks_pts[LM_idx]):
               temp.append(landmarks_pts[LM_idx])
               temp_idx.append(LM_idx)
        targetPts[locBuffer_idx]=temp
        targetPts_idx[locBuffer_idx]=temp_idx
    print("locations points:%d, target points:%d"%(len(locations_pts),len(targetPts.keys())))
    return targetPts,locations_pts,targetPts_idx

#展平函数
flatten_lst=lambda lst: [m for n_lst in lst for m in flatten_lst(n_lst)] if type(lst) is list else [lst]

'''#建立无人车位置点及对应扫描区范围特征点的网络结构，使用Networkx库 build the location network'''
def location_landmarks_network(targetPts_idx,locations,landmarks):
    LMs=np.stack((landmarks[0], landmarks[1]), axis=-1)
    LCs=np.stack((locations[0], locations[1]), axis=-1)
    ptsMerge=np.vstack((LMs,LCs))
    print("LMs shape:%s, LCs shaoe:%s, ptsMerge shape:%s"%(LMs.shape, LCs.shape,ptsMerge.shape))
    targetPts_idx_adj={}
    for key in targetPts_idx.keys():
        targetPts_idx_adj[key+LMs.shape[0]]=targetPts_idx[key]
    edges=[[(key,i) for i in targetPts_idx_adj[key]] for key in targetPts_idx_adj.keys()]
    edges=flatten_lst(edges)

    G=nx.Graph()
    G.position={}
    # G.targetPtsNum={}
    i=0
    for pts in ptsMerge:
        G.add_node(i)
        G.position[i]=(pts[0],pts[1])
        # G.targetPtsNum[LM]=(len(targetPts[key]))
        i+=1

    G.add_edges_from(edges)

    plt.figure(figsize=(20,130))
    nx.draw(G,G.position,linewidths=1,edge_color='gray')
    plt.show()

    return G

#应用numpy.histogram2d转换位置点及对应扫描区特征点为图片栅格数据
def colorMesh_phmi(landmarks,locations,targetPts_idx,Phmi):
    patternValuesDic={}
    for key in targetPts_idx.keys():
        patternValuesDic[key]=[0.5]*len(targetPts_idx[key])
        patternValuesDic[key].append(0.9)

    patternDic={}
    for key in patternValuesDic.keys():
        patternDic[key]={"x":landmarks[0][targetPts_idx[key]],
                         "y":landmarks[1][targetPts_idx[key]],
                         "z":patternValuesDic[key]
                         }
        patternDic[key]["x"]=np.append(patternDic[key]["x"],locations[0][key])
        patternDic[key]["y"]=np.append(patternDic[key]["y"],locations[1][key])

    histogram2dDic={}
    binNumber=(25,25) #32
    for key in patternDic.keys():
        zi, yi, xi = np.histogram2d(patternDic[key]["y"], patternDic[key]["x"], bins=binNumber, weights=patternDic[key]["z"], normed=False)
        counts, _, _ = np.histogram2d(patternDic[key]["y"], patternDic[key]["x"],bins=binNumber)
        # print(counts)
        histogram2dDic[key]={"xi":xi,"yi":yi,"zi":zi,"count":counts,"ziCount":zi / counts,"ziMasked":np.ma.masked_invalid(zi)}

    for key in histogram2dDic.keys():
        xi=histogram2dDic[key]["xi"]
        yi=histogram2dDic[key]["yi"]
        zi=histogram2dDic[key]["ziMasked"]

        x=patternDic[key]["x"]
        y=patternDic[key]["y"]
        z=patternDic[key]["z"]
        # print(x,y,z)
        #plot it
        fig, ax = plt.subplots(figsize=(25,20))
        ax.pcolormesh(xi, yi, zi, edgecolors='black')
        scat = ax.scatter(x, y, c=z, s=30)
        fig.colorbar(scat)
        ax.margins(0.05)

        plt.title("PHmi_%d:%f"%(key,Phmi[key]))
        plt.show()

        if key==20:
            break

    return histogram2dDic,patternDic

#显示评估值Phmi的数值分布
def singleBoxplot(array):
    import plotly.express as px
    import pandas as pd
    df=pd.DataFrame(array,columns=["value"])
    fig = px.box(df, y="value",points="all")
    fig.show()

# pyorth 深度学习 迭代数据集建立。迁移https://github.com/d2l-ai/d2l-zh  《动手学深度学习》(PyTorch版)
def data_iter(batch_size, features, labels):
    import random
    num_examples = len(features)
    indices = list(range(num_examples))
    random.shuffle(indices)  # 样本的读取顺序是随机的
    for i in range(0, num_examples, batch_size):
        j = torch.LongTensor(indices[i: min(i + batch_size, num_examples)]) # 最后一次可能不足一个batch
        yield  features.index_select(0, j), labels.index_select(0, j)

'''输出类别配置'''
# pytorch output值确定，用Percentile百分位数分类连续数值用作输出类别
def labelsPercentile(data):
    percentileNumber=[25,50,75]
    percentileValue=[np.percentile(data,i) for i in percentileNumber]
    for i in range(len(percentileValue)):
        if pow(10,-5)<percentileValue[i]:
            percentileValue.insert(i,pow(10,-5))
            break

    data_label=["e","d","c","b","a"]
    labels=[]
    for i in data:
        if i<=percentileValue[0]:
            labels.append(0)
        elif percentileValue[0]<i<=percentileValue[1]:
            labels.append(1)
        elif percentileValue[1]<i<=percentileValue[2]:
            labels.append(2)
        elif percentileValue[2]<i<=percentileValue[3]:
            labels.append(3)
        elif i>percentileValue[3]:
            labels.append(4)

    return labels

#pytorch output值确定，用Percentile百分位数分类连续数值用作输出类别——升级版
def labelsPercentile_upgrade(data):
    percentileNumber=[0,1,10,20, 30, 40,50,60, 70, 80, 90,100] #可以设置任意百分位切分值
    percentileValue=[np.percentile(data,i) for i in percentileNumber]
    print(percentileValue)

    bunchRange=list(zip(percentileValue, percentileValue[1:] + percentileValue[:1]))
    bunchIdx=list(range(len(bunchRange)))
    print(bunchRange,bunchIdx)
    valRange=[]

    i=0
    for val in data:
        val_Idx=[bunchIdx[k] for k in range(len(bunchRange)) if val>=bunchRange[k][0] and val<=bunchRange[k][1]]
        valRange.append(val_Idx[0])
        # print(val,val_Idx)
        # if i==0:
        #     break

    # print(valRange)
    print("bunchIdx amount:",len(bunchIdx),"\n",bunchIdx)
    return valRange

#输出类别设置，math.pow(10,-5)为评估标准值
def labels2Values(data):
    data[data>math.pow(10,-5)]=1
    data[data<=math.pow(10,-5)]=0
    print(data)

    return data

#pytorch output值确定，均分方式分类连续数值用作输出类别
def labelsPercentile_extent(data):
    # print(min(data))
    diff=max(data)-min(data)
    bunch=50.
    bunchList=[]
    i=0
    while 1:
        val=min(data)+diff/bunch*i
        if val<=max(data)+diff/bunch:
            bunchList.append(val)
            i+=1
        else:
            break
    bunchRange=list(zip(bunchList, bunchList[1:] + bunchList[:1]))
    bunchIdx=list(range(len(bunchRange)))
    # print(bunchRange,bunchIdx)
    valRange=[]

    i=0
    for val in data:

        val_Idx=[bunchIdx[k] for k in range(len(bunchRange)) if val>=bunchRange[k][0] and val<=bunchRange[k][1]]
        valRange.append(val_Idx[0])
        # print(val,val_Idx)
        # if i==0:
        #     break

    # print(valRange)
    print("bunchIdx amount:",len(bunchIdx),"\n",bunchIdx)
    return valRange

#提取返回值，以字典形式
def relationPts(landmarks,locations,targetPts_idx,Phmi):
    ptsVectors={}
    for key in targetPts_idx.keys():
        ptsVectors[key]={"vector":[(landmarks[0][i]-locations[0][key],landmarks[1][i]-locations[1][key]) for i in targetPts_idx[key]],
                         "distance":[math.sqrt((landmarks[0][i]-locations[0][key])**2+(landmarks[1][i]-locations[1][key])**2) for i in targetPts_idx[key]],
                         "quantity":len(targetPts_idx[key]),
                         "Phmi":Phmi[key]
                                   }

    return ptsVectors

#数据特征栅格——A。location无人车位置点与激光雷达扫描范围landmarks数据特征，将位置点作为栅格中心，图片栅格格式
def centricRaster(ptsCenter,radius,cellsize):
    pass
    Xi=list(np.arange(ptsCenter[0]-radius-cellsize/2,ptsCenter[0]+radius+cellsize,cellsize))
    Yi=list(np.arange(ptsCenter[1]-radius-cellsize/2,ptsCenter[1]+radius+cellsize,cellsize))
    XYi=np.array([(x,y) for x in Xi for y in Yi]).reshape(len(Xi),len(Yi),2)

    XRangei=list(zip(Xi, Xi[1:] + Xi[:1]))
    XRangei.pop(-1)

    YRangei=list(zip(Yi, Yi[1:] + Yi[:1]))
    YRangei.pop(-1)
    XYRangei=np.array([(x,y) for x in XRangei for y in  YRangei]).reshape(len( XRangei),len(YRangei),2,2)
    idxRange=np.array([1 for x in XRangei for y in  YRangei]).reshape(len( XRangei),len(YRangei),-1)

    z=idxRange
    featureLabel=np.array(range(z.shape[0]*z.shape[1])).reshape(z.shape[:-1])

    fig, ax = plt.subplots(figsize=(20,20))
    scat = ax.scatter(XYi[:,:,0], XYi[:,:,1], c="r", s=30)
    plt.show()

    return XYi,XYRangei,idxRange,featureLabel

#数据特征栅格——A。接函数 centricRaster()。
def featureRaster(landmarks,locations,targetPts_idx,Phmi,radius,cellsize):
    featureDic={}
    for key in tqdm(targetPts_idx.keys()):
        XYi,XYRangei,idxRange,featureLabel=centricRaster([locations[0][key],locations[1][key]],radius,cellsize)
        # fearureMinusValue=np.full(featureLabel.shape,-1)
        # print(XYi.shape,XYRangei.shape)

        relativeCellCoords=np.concatenate([v.reshape(-1,1) for v in np.where(idxRange)],axis=1) #返回数组值索引（2 维度），作为坐标位置
        rCellCoordsMatrix=relativeCellCoords.reshape(idxRange.shape[:-1]+(3,))
        idxList=[]
        labelList=[]
        for i in targetPts_idx[key]:
            valX=landmarks[0][i]
            valY=landmarks[1][i]
            a_bool=np.logical_and(XYRangei[:,:,0,0]<=valX,valX<=XYRangei[:,:,0,1])
            b_bool=np.logical_and(XYRangei[:,:,1,0]<=valY,valY<=XYRangei[:,:,1,1])
            c_bool=np.logical_and(a_bool,b_bool)
            idxList.append(rCellCoordsMatrix[c_bool].tolist()[0])
            labelList.append(featureLabel[c_bool].tolist()[0])

        # fearureMinusValueExtract=fearureMinusValue[]
        featureDic[key]={"idx":idxList,
                         "label":labelList,
                         "Phmi":Phmi[key],
                         # "featureValue":
                         }

        if key==1:
            break

    return featureDic

'''栅格图像显示'''
from IPython import display
from matplotlib import pyplot as plt
def use_svg_display():
    """Use svg format to display plot in jupyter"""
    display.set_matplotlib_formats('svg')
def colorMeshShow(histogram2dDic_part,patternDic_part,Phmi_part,condi):
    use_svg_display()
    xiArray=np.array([histogram2dDic_part[key]["xi"].tolist() for key in histogram2dDic_part.keys()])
    yiArray=np.array([histogram2dDic_part[key]["yi"].tolist() for key in histogram2dDic_part.keys()])
    ziArray=np.array([histogram2dDic_part[key]["ziMasked"].tolist() for key in histogram2dDic_part.keys()])

    xArray=np.array([patternDic_part[key]["x"].tolist() for key in patternDic_part.keys()])
    yArray=np.array([patternDic_part[key]["y"].tolist() for key in patternDic_part.keys()])
    zArray=np.array([patternDic_part[key]["z"] for key in patternDic_part.keys()])

    # print(xiArray)

    width=int(round(math.sqrt(len(xArray)),2))
    # print("+"*50)
    # print(width)
    fig, axs = plt.subplots(width, width, figsize=(10, 10),constrained_layout=True)
    for ax, xi, yi, zi, x, y,z,titleV,key in zip(axs.flat, xiArray,yiArray,ziArray,xArray, yArray,zArray,Phmi_part,condi):

        ax.pcolormesh(xi, yi, zi, edgecolors='black')
        scat = ax.scatter(x, y, c=z, s=30)
        fig.colorbar(scat)
        ax.margins(0.05)

        ax.set_title("PHmi_%d:%f"%(key,titleV))
        ax.axes.get_xaxis().set_visible(False)
        ax.axes.get_yaxis().set_visible(False)

    plt.show()

if __name__ == "__main__":
    LandmarkMap_dic=readMatLabFig_LandmarkMap(LandmarkMap_fn)
    PHMI_dic=readMatLabFig_PHMI(PHMI_fn,LandmarkMap_dic)

    locations=PHMI_dic[0]
    landmarks=LandmarkMap_dic[1]
    radius=25
    targetPts,locations_pts,targetPts_idx=scanCircleBuffer(locations,landmarks,radius)
    Phmi=PHMI_dic[1][2]
    singleBoxplot(Phmi)
    # G=location_landmarks_network(targetPts_idx,locations,landmarks)

    #b = (f >= 1.0 ? 255 : (f <= 0.0 ? 0 : (int)floor(f * 256.0)))
    # data=colorMesh_phmi(landmarks,locations,targetPts_idx,Phmi)

    histogram2dDic,patternDic=colorMesh_phmi(landmarks,locations,targetPts_idx,Phmi)
    condi={i for i in range(20)}
    histogram2dDic_part={key: histogram2dDic[key] for key in histogram2dDic.keys() & condi}
    patternDic_part={key:patternDic[key] for key in patternDic.keys() & condi}
    Phmi_part=[Phmi[i] for i in condi]
    colorMeshShow(histogram2dDic_part,patternDic_part,Phmi_part,condi)

    # features=torch.tensor(np.asarray([data[val]["zi"] for val in data.keys()]).reshape(len(data.keys()),-1))
    # LE=preprocessing.LabelEncoder()
    # lables=torch.tensor(LE.fit_transform(labelsPercentile(Phmi)))

    # features=torch.tensor(np.asarray([data[val]["zi"] for val in data.keys()]))
    # labels=torch.tensor(labelsPercentile(Phmi))
    # valRange=labelsPercentile_extent(Phmi)
    # labelsExtent=torch.tensor(labelsPercentile_extent(Phmi))

    # labelsUpgrade=labelsPercentile_upgrade(Phmi)
    # label2V=labels2Values(Phmi)
    # (unique, counts) = np.unique(label2V, return_counts=True)
    # print((unique, counts))
    # batch_size=100
    # data_iter(batch_size, features, labels)
    # for X, y in data_iter(batch_size, features, labels):
    #     print(X.shape, y.shape)
    #     break

    # ptsVectors=relationPts(landmarks,locations,targetPts_idx,Phmi)
    # ptsVectorsFn=r"C:\Users\richi\omen-richiebao\omen-code\Chicago_code\data\ptsVectors.pkl"
    # with open(ptsVectorsFn, 'wb') as handle:
    #     pickle.dump(ptsVectors, handle, protocol=pickle.HIGHEST_PROTOCOL)

    # ptsCenter=[0,0]
    # cellsize=1
    # XYi,XYRangei,idxRange,featureLabel=centricRaster(ptsCenter,radius,cellsize)
    # featureDic=featureRaster(landmarks,locations,targetPts_idx,Phmi,radius,cellsize)

    # featureDicFn=r"C:\Users\richi\omen-richiebao\omen-code\Chicago_code\data\phmiFeature.pkl"
    # with open(featureDicFn, 'wb') as handle:
    #     pickle.dump(featureDic, handle, protocol=pickle.HIGHEST_PROTOCOL)

    # with open(featureDicFn, 'rb') as handle:
    #     b = pickle.load(handle)

    # phmi_labelFn=r"C:\Users\richi\omen-richiebao\omen-code\Chicago_code\data\phmi_label.pkl"
    # with open(phmi_labelFn, 'wb') as handle:
    #     pickle.dump(featureLabel, handle, protocol=pickle.HIGHEST_PROTOCOL)


import cx_Freeze

executables = [cx_Freeze.Executable('teste.pygame.py')]

cx_Freeze.setup(
    name="Breakout GAME",
    options={'build_exe': {'packages':['pygame'],
                           'include_files':['Som']}},

    executables = executables

)

# No cmd instalar o cx-Freeze
# 'cx-Freeze'
# Abrir diretorio da pasta no cmd
# 'cd + diretorio da pasta'
# Executar para criar o executável do jogo
# 'python setup.py build'


import argparse
import serial
import serial.tools.list_ports
import matplotlib.pyplot as plt
import os

def display(ser):
    buffer_len = 50
    buffer = [0] * buffer_len
    fig, axs = plt.subplots(1, 1)
    while True:
        raw_line = ser.readline()
        line = raw_line.decode("utf-8").strip()
        print(line)
        if line.startswith("anom_score"):
            line_info = line.split(":")
            anomaly_score = float(line_info[1])
            # Add anomaly score to buffer
            buffer.append(anomaly_score)
            buffer = buffer[1:]
            # Plot buffer
            # Set axis limits
            axs.set_ylim(0, 1)
            axs.set_xlim(0, buffer_len)
            # Set title
            axs.set_title("Anomaly Score")
            axs.plot(buffer, color="red", linewidth=1)
            # Draw plot to screen
            fig.tight_layout()
            fig.canvas.draw()
            plt.pause(0.001)
            axs.clear()

if __name__ == "__main__":
    ports = list(serial.tools.list_ports.comports())
    for i, p in enumerate(ports):
        print(f"{i}: {p}")
        print(p.name)
    port_id = input("Select port to connect to: ")
    port_id = int(port_id)

    ser = serial.Serial(port=ports[port_id].device, timeout=0.1)

    display(ser)


# This script checks that exchanges strategies work

# ##############################################################################################
# Importations
from datetime import datetime

from lib.DefaultNatures.DefaultNatures import *

from src.common.Agent import Agent
from src.common.Aggregator import Aggregator
from src.common.World import World
from src.common.Strategy import *

from src.tools.GraphAndTex import GraphOptions
from src.tools.SubclassesDictionary import get_subclasses

# ##############################################################################################
# Minimum
# the following objects are necessary for the simulation to be performed
# you need exactly one object of each type
# ##############################################################################################

# ##############################################################################################
# Importation of subclasses
# all the subclasses are imported in the following dictionary
subclasses_dictionary = get_subclasses()

# ##############################################################################################
# Creation of the world
# a world contains all the other elements of the model
# a world needs just a name
name_world = "validation"
world = World(name_world)  # creation

# ##############################################################################################
# Definition of the path to the files
world.set_directory("cases/ValidationCases/Results/ExchangeStrategy")  # here, you have to put the path to your results directory

# ##############################################################################################
# Definition of the random seed
# The default seed is the current time (the value returned by datetime.now())
world.set_random_seed("seed")

# ##############################################################################################
# Time parameters
# it needs a start date, the value of an iteration in hours and the total number of iterations
start_date = datetime(year=1, month=1, day=1, hour=0, minute=0, second=0, microsecond=0)  # a start date in the datetime format
world.set_time(start_date,  # time management: start date
               1,  # value of a time step (in hours)
               24)  # number of time steps simulated

# ##############################################################################################
# Model
# ##############################################################################################

# ##############################################################################################
# Creation of nature
# low voltage electricity
LVE = load_low_voltage_electricity()

# ##############################################################################################
# Creation of daemons
price_manager_elec = subclasses_dictionary["Daemon"]["PriceManagerDaemon"]("prices", {"nature": LVE.name, "buying_price": 0, "selling_price": 0})  # sets prices for flat rate

subclasses_dictionary["Daemon"]["LimitPricesDaemon"]({"nature": LVE.name, "limit_buying_price": 1, "limit_selling_price": -1})  # sets prices for the system operator

# ##############################################################################################
# Creation of strategies
# the different distribution strategies
strategy_light_autarky = subclasses_dictionary["Strategy"]["LightAutarkyFullButFew"](get_emergency)
strategy_autarky = subclasses_dictionary["Strategy"]["AutarkyFullButFew"](get_emergency)
strategy_always_satisfied = subclasses_dictionary["Strategy"]["AlwaysSatisfied"]()
strategy_max_exchanges = subclasses_dictionary["Strategy"]["ExchangesFullButFew"](get_emergency)

# strategy grid, which always proposes an infinite quantity to sell and to buy
grid_strategy = subclasses_dictionary["Strategy"]["Grid"]()

# ##############################################################################################
# Manual creation of agents
light_autarky_owner = Agent("light_autarky_owner")

autarky_owner = Agent("autarky_owner")

always_satisfied_owner = Agent("always_satisfied_owner")

exchanges_owner = Agent("exchanges_owner")

aggregators_manager = Agent("aggregators_manager")

# ##############################################################################################
# Manual creation of contracts
BAU_contract = subclasses_dictionary["Contract"]["EgoistContract"]("BAU_contract", LVE, price_manager_elec)

curtailment_contract = subclasses_dictionary["Contract"]["CurtailmentContract"]("curtailment_contract", LVE, price_manager_elec)

cooperative_contract = subclasses_dictionary["Contract"]["CooperativeContract"]("cooperative_contract_elec", LVE, price_manager_elec)

# ##############################################################################################
# Creation of aggregators
aggregator_grid = Aggregator("national_grid", LVE, grid_strategy, aggregators_manager)

aggregator_light_autarky = Aggregator("local_grid_emergency", LVE, strategy_light_autarky, aggregators_manager, aggregator_grid, BAU_contract)
aggregator_autarky = Aggregator("local_grid_price", LVE, strategy_autarky, aggregators_manager, aggregator_grid, BAU_contract)
aggregator_always_satisfied = Aggregator("local_grid_quantity", LVE, strategy_always_satisfied, aggregators_manager, aggregator_grid, BAU_contract)
aggregator_max_exchanges = Aggregator("local_grid_exchanges", LVE, strategy_max_exchanges, aggregators_manager, aggregator_grid, BAU_contract, capacity={"buying": 12, "selling": inf})

# ##############################################################################################
# Manual creation of devices

# Each device is created 4 times
# light autarky strategy
device_BAU_light_autarky = subclasses_dictionary["Device"]["Background"]("device_BAU_light_autarky", BAU_contract, light_autarky_owner, aggregator_light_autarky, {"user": "dummy_user", "device": "dummy_usage"}, filename="cases/ValidationCases/AdditionalData/DevicesProfiles/Background.json")
device_curtailment_light_autarky = subclasses_dictionary["Device"]["Background"]("device_curtailment_light_autarky", curtailment_contract, light_autarky_owner, aggregator_light_autarky, {"user": "dummy_user", "device": "dummy_usage"}, filename="cases/ValidationCases/AdditionalData/DevicesProfiles/Background.json")
production_light_autarky = subclasses_dictionary["Device"]["DummyProducer"]("production_light_autarky", cooperative_contract, light_autarky_owner, aggregator_light_autarky, {"device": "dummy_usage"}, {"max_power": 12}, filename="cases/ValidationCases/AdditionalData/DevicesProfiles/DummyProducer.json")

# autarky strategy
device_BAU_autarky = subclasses_dictionary["Device"]["Background"]("device_BAU_autarky", BAU_contract, autarky_owner, aggregator_autarky, {"user": "dummy_user", "device": "dummy_usage"}, filename="cases/ValidationCases/AdditionalData/DevicesProfiles/Background.json")
device_curtailment_autarky = subclasses_dictionary["Device"]["Background"]("device_curtailment_autarky", curtailment_contract, autarky_owner, aggregator_autarky, {"user": "dummy_user", "device": "dummy_usage"}, filename="cases/ValidationCases/AdditionalData/DevicesProfiles/Background.json")
production_autarky = subclasses_dictionary["Device"]["DummyProducer"]("production_autarky", cooperative_contract, autarky_owner, aggregator_autarky, {"device": "dummy_usage"}, {"max_power": 12}, filename="cases/ValidationCases/AdditionalData/DevicesProfiles/DummyProducer.json")

# always satisfied strategy
device_BAU_always_satisfied = subclasses_dictionary["Device"]["Background"]("device_BAU_always_satisfied", BAU_contract, always_satisfied_owner, aggregator_always_satisfied, {"user": "dummy_user", "device": "dummy_usage"}, filename="cases/ValidationCases/AdditionalData/DevicesProfiles/Background.json")
device_curtailment_always_satisfied = subclasses_dictionary["Device"]["Background"]("device_curtailment_always_satisfied", curtailment_contract, always_satisfied_owner, aggregator_always_satisfied, {"user": "dummy_user", "device": "dummy_usage"}, filename="cases/ValidationCases/AdditionalData/DevicesProfiles/Background.json")
production_always_satisfied = subclasses_dictionary["Device"]["DummyProducer"]("production_always_satisfied", cooperative_contract, always_satisfied_owner, aggregator_always_satisfied, {"device": "dummy_usage"}, {"max_power": 12}, filename="cases/ValidationCases/AdditionalData/DevicesProfiles/DummyProducer.json")

# exchanges strategy
device_BAU_exchanges = subclasses_dictionary["Device"]["Background"]("device_BAU_exchanges", BAU_contract, exchanges_owner, aggregator_max_exchanges, {"user": "dummy_user", "device": "dummy_usage"}, filename="cases/ValidationCases/AdditionalData/DevicesProfiles/Background.json")
device_curtailment_exchanges = subclasses_dictionary["Device"]["Background"]("device_curtailment_exchanges", curtailment_contract, exchanges_owner, aggregator_max_exchanges, {"user": "dummy_user", "device": "dummy_usage"}, filename="cases/ValidationCases/AdditionalData/DevicesProfiles/Background.json")
production_exchanges = subclasses_dictionary["Device"]["DummyProducer"]("production_exchanges", cooperative_contract, exchanges_owner, aggregator_max_exchanges, {"device": "dummy_usage"}, {"max_power": 12}, filename="cases/ValidationCases/AdditionalData/DevicesProfiles/DummyProducer.json")

# ##############################################################################################
# Creation of the validation daemon
description = "This script checks that exchange strategies work"

filename = "exchange_strategy_validation"

reference_values = {"light_autarky_owner.LVE.energy_bought": [0, 2, 4, 6, 8, 10, 12, 12, 12, 12, 12, 12, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23],
                    "light_autarky_owner.LVE.energy_sold": [0, 2, 4, 6, 8, 10, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12],

                    "autarky_owner.LVE.energy_bought": [0, 2, 4, 6, 8, 10, 12, 12, 12, 12, 12, 12, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                    "autarky_owner.LVE.energy_sold": [0, 2, 4, 6, 8, 10, 12, 12, 12, 12, 12, 12, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],

                    "always_satisfied_owner.LVE.energy_bought": [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46],
                    "always_satisfied_owner.LVE.energy_sold": [12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12],

                    "exchanges_owner.LVE.energy_bought": [0, 2, 4, 6, 8, 10, 12, 12, 12, 12, 12, 12, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23],
                    "exchanges_owner.LVE.energy_sold": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
                    }

name = "LVE_light_autarky"
export_plot1 = {
    "name": name,
    "filename": "export_"+name,
    "options": GraphOptions(f"{name}_graph_options", ["csv", "LaTeX"], "multiple_series"),
    "X": {"catalog_name_entry": "iteration", "label": r"$t \, [\si{\hour}]$"},
    "Y": {"label": r"$\mathcal{P}_{\textrm{bought}} \, [$\euro{}$]$",
          "graphs": [ {"catalog_name_entry": "light_autarky_owner.LVE.energy_bought_reference", "style": "points", "legend": r"ref."},
                      {"catalog_name_entry": "light_autarky_owner.LVE.energy_bought_simulation", "style": "lines", "legend": r"num."}
                    ]
          },
    "Y2": {"label": r"$\mathcal{P}_{\textrm{sold}} \, [$\euro{}$]$",
          "graphs": [ {"catalog_name_entry": "light_autarky_owner.LVE.energy_sold_reference", "style": "points", "legend": r"ref."},
                      {"catalog_name_entry": "light_autarky_owner.LVE.energy_sold_simulation", "style": "lines", "legend": r"num."}
                    ]
          },
}

name = "LVE_autarky"
export_plot2 = {
    "name": name,
    "filename": "export_"+name,
    "options": GraphOptions(f"{name}_graph_options", ["csv", "LaTeX"], "multiple_series"),
    "X": {"catalog_name_entry": "iteration", "label": r"$t \, [\si{\hour}]$"},
    "Y": {"label": r"$\mathcal{P}_{\textrm{bought}} \, [$\euro{}$]$",
          "graphs": [ {"catalog_name_entry": "autarky_owner.LVE.energy_bought_reference", "style": "points", "legend": r"ref."},
                      {"catalog_name_entry": "autarky_owner.LVE.energy_bought_simulation", "style": "lines", "legend": r"num."}
                    ]
          },
    "Y2": {"label": r"$\mathcal{P}_{\textrm{sold}} \, [$\euro{}$]$",
          "graphs": [ {"catalog_name_entry": "autarky_owner.LVE.energy_sold_reference", "style": "points", "legend": r"ref."},
                      {"catalog_name_entry": "autarky_owner.LVE.energy_sold_simulation", "style": "lines", "legend": r"num."}
                    ]
          },
}

name = "LVE_always_satisfied"
export_plot3 = {
    "name": name,
    "filename": "export_"+name,
    "options": GraphOptions(f"{name}_graph_options", ["csv", "LaTeX"], "multiple_series"),
    "X": {"catalog_name_entry": "iteration", "label": r"$t \, [\si{\hour}]$"},
    "Y": {"label": r"$\mathcal{P}_{\textrm{bought}} \, [$\euro{}$]$",
          "graphs": [ {"catalog_name_entry": "always_satisfied_owner.LVE.energy_bought_reference", "style": "points", "legend": r"ref."},
                      {"catalog_name_entry": "always_satisfied_owner.LVE.energy_bought_simulation", "style": "lines", "legend": r"num."}
                    ]
          },
    "Y2": {"label": r"$\mathcal{P}_{\textrm{sold}} \, [$\euro{}$]$",
          "graphs": [ {"catalog_name_entry": "always_satisfied_owner.LVE.energy_sold_reference", "style": "points", "legend": r"ref."},
                      {"catalog_name_entry": "always_satisfied_owner.LVE.energy_sold_simulation", "style": "lines", "legend": r"num."}
                    ]
          },
}

name = "Bought_Energy_Alltogether"
export_plot4 = {
    "name": name,
    "filename": "export_"+name,
    "options": GraphOptions(f"{name}_graph_options", ["csv", "LaTeX"], "multiple_series"),
    "X": {"catalog_name_entry": "iteration", "label": r"$t \, [\si{\hour}]$"},
    "Y": {"label": r"$\mathcal{P}_{ref.} \, [\si{\watt}]$",
          "graphs": [ {"catalog_name_entry": "light_autarky_owner.LVE.energy_bought_reference", "style": "points", "legend": r"light autarky ref."},
                      {"catalog_name_entry": "autarky_owner.LVE.energy_bought_reference", "style": "points", "legend": r"autarky ref."},
                      {"catalog_name_entry": "always_satisfied_owner.LVE.energy_bought_reference", "style": "points", "legend": r"always satisfied ref."}
                      ]
          },
    "Y2": {"label": r"$\mathcal{P}_{num.} \, [\si{\watt}]$",
          "graphs": [ {"catalog_name_entry": "light_autarky_owner.LVE.energy_bought_simulation", "style": "lines", "legend": r"light autarky num."},
                      {"catalog_name_entry": "autarky_owner.LVE.energy_bought_simulation", "style": "lines", "legend": r"autarky num."},
                      {"catalog_name_entry": "always_satisfied_owner.LVE.energy_bought_simulation", "style": "lines", "legend": r"always satisfied num."}
                      ]
          }
}

name = "Sold_Energy_Alltogether"
export_plot5 = {
    "name": name,
    "filename": "export_"+name,
    "options": GraphOptions(f"{name}_graph_options", ["csv", "LaTeX"], "multiple_series"),
    "X": {"catalog_name_entry": "iteration", "label": r"$t \, [\si{\hour}]$"},
    "Y": {"label": r"$\mathcal{P}_{ref.} \, [\si{\watt}]$",
          "graphs": [ {"catalog_name_entry": "light_autarky_owner.LVE.energy_sold_reference", "style": "points", "legend": r"lgt autark."},
                      {"catalog_name_entry": "autarky_owner.LVE.energy_sold_reference", "style": "points", "legend": r"autark."},
                      {"catalog_name_entry": "always_satisfied_owner.LVE.energy_sold_reference", "style": "points", "legend": r"alws satis."}
                      ]
          },
    "Y2": {"label": r"$\mathcal{P}_{num.} \, [\si{\watt}]$",
          "graphs": [ {"catalog_name_entry": "light_autarky_owner.LVE.energy_sold_simulation", "style": "lines", "legend": r""},
                      {"catalog_name_entry": "autarky_owner.LVE.energy_sold_simulation", "style": "lines", "legend": r""},
                      {"catalog_name_entry": "always_satisfied_owner.LVE.energy_sold_simulation", "style": "lines", "legend": r""}
                      ]
          }
}

parameters = {"description": description, "filename": filename, "reference_values": reference_values, "tolerance": 1E-6, "export_plots": [export_plot1, export_plot2, export_plot3, export_plot4, export_plot5]}

validation_daemon = subclasses_dictionary["Daemon"]["ValidationDaemon"]("exchange_strategy_test", parameters)

# ##############################################################################################
# Simulation start
world.start()


import logging

from LibTqtk.Connectors import PriceFileConverter
import sys
from dateutil import parser as Prs

class Tick(PriceFileConverter.PriceFileConverter):

    def __init__(self, config_folder, config_file, parse):
        super().__init__(config_folder, config_file, parse)

    def parse_file_name(self, file_name):
        """
        Parse a Dukascopy Renko File name into
        internal format

        :return:
        """
        _comps = file_name.replace(".csv","").split("_")
        logging.info(f"Parsed file name elements {_comps}")

        assert _comps[1].lower() == "ticks", "Apparently wrong file type"
        _fn_info = dict()
        _fn_info["represent"] = "TICK"
        _fn_info["instrument"] = _comps[0]
        _fn_info["bar_size"] = "0"
        _fn_info["price_type"] = "BOTH"
        _fn_info["date_from"] = Prs.parse(_comps[2])
        _fn_info["date_to"] = Prs.parse(_comps[3])

        self.file_name_info = _fn_info

        return _fn_info

if __name__ == "__main__":
    try:
        _config_folder = sys.argv[1]  # config folder
        _config_file = sys.argv[2]  # config file name
        try:
            _split =  bool(sys.argv[6])  # 1: split, 0 no split
        except IndexError:
            _split = False

        _chunksize = None
        if _split:
            try:
                _chunksize = int(sys.argv[7])
            except IndexError:
                _chunksize = 100000

    except Exception as e:
        raise Exception(str(e))

    print("Processing for", _config_file)

    _rk = Tick(_config_folder, _config_file,  "")
    _rk.run()


from ninfo import PluginBase

class shodan_plug(PluginBase):
    """This plugin returns any information from Shodan"""
    name    =    'shodan'
    title   =    'Shodan'
    description   =  'Computer Search Engine'
    cache_timeout   =  60*60*2
    types   =    ['ip']
    remote = False

    def setup(self):
        from shodan import WebAPI
        self.api = WebAPI(self.plugin_config["api_key"])

    def get_info(self, arg):
        info = self.api.host(arg)
        return info

plugin_class = shodan_plug


from torch import nn
from typing import Optional, Union
import logging
import numpy as np
import torch
import torch.nn.functional as F

from .modeling_utils import (
    find_pruneable_heads_and_indices,
    prune_linear_layer
)

from typing import Callable, Optional, Union

class PerformerAttentionConfig:
    r"""
    This is the configuration class to store the configuration of a :class:`~transformers.PerformerAttention` module. It is used
    to define the behavior of a Performer/FAVOR+ attention module when it is initialized.

    Args:
        attention_dropout (:obj:`float`, `optional`, defaults to 0.1):
            The dropout ratio for the attention probabilities.
        causal (:obj:`bool`, `optional`, defaults to False):
            Whether to apply causal attention, where positions are prevented from attending to positions to ahead
            of themselves in the sequence, using the prefix-sum method.
        kernel_type (:obj:`str`,  `optional`, defaults to :obj:`'exp'`):
            The type of kernel function to use for comparing the queries and keys. Possible options are :obj:`'exp'`,
            :obj:`'cosh'`, and :obj:`'relu'`. The :obj:`'cosh'` option approximates softmax attention with a smaller
            variance than :obj:`'exp'`, but at the cost of using twice as many random features. :obj:`'relu'` may result
            in better performance than :obj:`'exp'` and :obj:`'cosh'` in certain circumstances, but it is not an unbiased
            estimator of softmax attention and thus should not be used with pretrained models that were pretrained with
            softmax attention.
        short_sequence_behavior (:obj:`str`, or `Callable` `optional` defaults to :obj:`'never_use_softmax'`):
            This parameter determines if and when the module should fall back to regular softmax attention. Softmax attention
            is generally faster than FAVOR+ when the sequence length is not significantly larger than the number of random
            features used— which is equal to round(d*log(d)), where d is the number of dimensions per attention head. The
            default behavior is to use FAVOR+ regardless of sequence length while training, but to use softmax attention at
            test time when the sequence length is less than twice the number of random features. Note that this means that for
            BERT-based models, where :obj:`'d_model'` = 768 and :obj:`'num_heads'` = 12, PerformerAttention will use softmax
            attention at test time when sequences do not exceed the usual maximum sequence length of 512 tokens.
            Possible values for this parameter are :obj:`'use_softmax_eval_only'`, :obj:`'use_softmax_eval_and_train'`,
            :obj:`'never_use_softmax'`. The option :obj:`'use_softmax_eval_and_train'` should probably only be used if the
            training set has a significant number of long sequences; otherwise, the model may not learn to deal with the
            random noise inherent in the FAVOR+ algorithm.
        kernel_epsilon (:obj:`float`, `optional`, defaults to 1e-4):
            Stabilizer term added to the output of the kernel function to avoid dividing by very small numbers.
        normalize_output (:obj:`bool`, `optional`, defaults to True):
            Whether to ensure that the output vectors are convex combinations of the input vectors; that is, that the rows of
            the implicit attention map sum to 1.
        normalization_stabilizer (:obj:`float`, `optional`, defaults to 1e-6):
            Stabilizer term used when normalizing the output to avoid dividing by very small numbers.
        num_random_features (:obj:`int`, `optional`, defaults to None):
            The dimensionality of the random feature vectors to use. When None, the dimensionality is set to
            D * log(D), where D is the dimensionality of each attention head.
        use_thick_features (:obj:`bool`, `optional`, defaults to False):
            Whether to generate a random feature tensor that has a batch dimension.
        use_orthogonal_features (:obj:`bool`, `optional`, defaults to True):
            Whether to use strictly orthogonal random features, as opposed to features drawn from a standard Gaussian
            distribution. Orthogonal features result in outputs that more closely approximate softmax attention, but at
            the cost of doing QR decomposition on the CPU every time the features are redrawn. Best combined with a
            reasonably large value of :obj:`feature_redraw_interval` (1-5k).
        use_qkv_linear_layers (:obj:`bool`, `optional`, defaults to True):
            Whether to transform the Q, K, and V inputs with a Linear layer before applying attention. Setting this
            to False may be useful if you want to use PerformerAttention as one component of a more complex
            attention mechanism.
        regularize_feature_norms (:obj:`bool`, `optional`, defaults to False):
            Whether to ensure that the random feature vectors have a norm of sqrt(`d`), where `d` is the dimensionality of
            each attention head.
        feature_redraw_interval (:obj:`int`, `optional`, defaults to 1000):
            The number of forward passes after which the random feature matrix should be redrawn. If None, then the feature
            matrix is never redrawn. It is recommended to set this property to some value on the order of 1-5k while
            training in order to get the best model performance. When combined with :obj:`redraw_stochastically`, this parameter
            determines the expected value of the redraw interval, rather than the interval itself.
        redraw_stochastically (:obj:`bool`, `optional`, defaults to False):
            If true, PerformerAttention will redraw its random features each forward pass with a probability equal to
            (1 / :obj:`feature_redraw_interval`), instead of deterministically redrawing once every N passes. This could be
            desirable in large models to ensure that the attention layers don't all redraw their features at the same time.
        redraw_verbose (:obj:`bool`, `optional`, defaults to False):
            Whether to log a message when random features are redrawn during training.
        dim (:obj:`int`, `optional`):
            Dimensionality of the queries, keys, and values.
        num_heads (:obj:`int`, `optional`):
            Number of attention heads.
    """

    attention_dropout: float = 0.1
    causal: bool = False
    kernel_type: str = 'cosh'

    # Default determined in PerformerAttention.__init__()
    # FAVOR+ takes more time compared with softmax over short sequence?
    # check: https://github.com/huggingface/transformers/issues/7675#issuecomment-734534302
    # __init__() set the value based on length of seq, roughly:
    # short sequence(L < 2 * M): fall back to transformer;
    # long sequence(L > 2 * M):  follow perfomer;
    short_sequence_behavior: Optional[Union[str, Callable]] = 'never_use_softmax'

    kernel_epsilon: float = 1e-4
    normalize_output: bool = True
    normalization_stabilizer: float = 1e-6

    num_random_features: Optional[int] = None
    use_thick_features: bool = False
    use_orthogonal_features: bool = True
    use_qkv_linear_layers: bool = True
    regularize_feature_norms: bool = True

    feature_redraw_interval: int = 1000
    redraw_stochastically: bool = False
    redraw_verbose: bool = False

    # Optional here so the user doesn't have to set redundant parameters, but must be set by model before config is
    # passed to PerformerAttention.__init__()
    d_model: Optional[int] = None
    num_heads: Optional[int] = None

KERNEL_CALLABLES = {
    'cosh': lambda x, h: torch.cat((torch.exp(h + x), torch.exp(h - x)), dim=-1),
    'exp': lambda x, h: torch.exp(h + x), # Default
    'elu': lambda x: F.elu(x) + 1,
    'relu': F.relu
}

SHORT_SEQUENCE_BEHAVIOR_CALLABLES = {
    'use_softmax_eval_only': lambda L, M, training: False if training else L < 2.0 * M,
    'use_softmax_eval_and_train': lambda L, M, training: L < 2.0 * M,
    'never_use_softmax': lambda L, M, training: False
}

class PerformerAttention(nn.Module):
    def __init__(self, config: Optional[PerformerAttentionConfig] = None, **kwargs):
        super().__init__()

        config = config or PerformerAttentionConfig()

        # kwargs take precedence over the default values that might be stored in the config object
        for k, v in kwargs.items():
            if not hasattr(config, k):
                raise AttributeError(f"PerformerAttention: '{k}' is an invalid config parameter")

            setattr(config, k, v)

        self.__dict__.update(config.__dict__)

        if self.num_heads is None or self.d_model is None:
            raise ValueError("PerformerAttention: num_heads and d_model must be non-None")

        self.dropout = nn.Dropout(p=self.attention_dropout)
        self.calls_since_last_redraw = 0
        self.random_features = None

        behavior = self.short_sequence_behavior
        if not behavior:
            behavior = 'never_use_softmax' if self.kernel_type == 'relu' else 'use_softmax_eval_only'
            self.should_fallback_to_softmax = SHORT_SEQUENCE_BEHAVIOR_CALLABLES[behavior]

        # if kernel is `relu`, then softmax attention cannot be used
        elif self.kernel_type == 'relu' and behavior != 'never_use_softmax':
            raise ValueError(f"PerformerAttention: short_sequence_behavior = {behavior} cannot be combined with the relu "
                             "kernel type")

        elif isinstance(behavior, str):
            self.should_fallback_to_softmax = SHORT_SEQUENCE_BEHAVIOR_CALLABLES[behavior]
        elif callable(behavior):
            self.should_fallback_to_softmax = behavior
        else:
            raise ValueError("PerformerAttention: short_sequence_behavior must be either str or Callable")

        self.kernel_fn = KERNEL_CALLABLES[self.kernel_type]

        assert self.d_model % self.num_heads == 0

        if self.use_qkv_linear_layers:
            self.q_lin = nn.Linear(in_features=self.d_model, out_features=self.d_model)
            self.k_lin = nn.Linear(in_features=self.d_model, out_features=self.d_model)
            self.v_lin = nn.Linear(in_features=self.d_model, out_features=self.d_model)

        self.out_lin = nn.Linear(in_features=self.d_model, out_features=self.d_model)

        self.pruned_heads = set()

    def prune_heads(self, heads):
        attention_head_size = self.d_model // self.num_heads
        if len(heads) == 0:
            return
        heads, index = find_pruneable_heads_and_indices(heads, self.num_heads, attention_head_size, self.pruned_heads)
        # Prune linear layers
        if self.use_qkv_linear_layers:
            self.q_lin = prune_linear_layer(self.q_lin, index)
            self.k_lin = prune_linear_layer(self.k_lin, index)
            self.v_lin = prune_linear_layer(self.v_lin, index)

        self.out_lin = prune_linear_layer(self.out_lin, index, dim=1)
        # Update hyper params
        self.num_heads = self.num_heads - len(heads)
        self.d_model = attention_head_size * self.num_heads
        self.pruned_heads = self.pruned_heads.union(heads)

    def redraw_features_now(self):
        device = self.random_features.device
        batch = self.random_features.shape[0]
        self._generate_feature_matrix(batch, device)

        if self.training and self.redraw_verbose:
            logging.info("PerformerAttention: Just redrew random features.")

        self.calls_since_last_redraw = 0

    def forward(self, query, key, value, mask=None, head_mask=None, output_attentions=False):
        """
        Parameters:
            query: torch.tensor(bs, seq_length, dim)
            key: torch.tensor(bs, seq_length, dim)
            value: torch.tensor(bs, seq_length, dim)
            mask: torch.tensor(bs, seq_length)

        Returns:
            weights: torch.tensor(bs, num_heads, seq_length, seq_length) Attention weights context: torch.tensor(bs,
            seq_length, dim) Contextualized layer. Optional: only if `output_attentions=True`
        """
        bs, q_length, dim = query.size()
        k_length = key.size(1)
        # assert dim == self.d_model, 'Dimensions do not match: %s input vs %s configured' % (dim, self.d_model)
        # assert key.size() == value.size()

        dim_per_head = self.d_model // self.num_heads
        mask_reshp = (bs, 1, 1, k_length)

        def shape(x):
            """ separate heads """
            return x.view(bs, -1, self.num_heads, dim_per_head).transpose(1, 2)

        if self.use_qkv_linear_layers:
            q = self.q_lin(query)
            k = self.k_lin(key)
            v = self.v_lin(value)
        else:
            q, k, v = query, key, value

        # (bs, num_heads, q_length, dim_per_head)
        q, k, v = (shape(x) for x in (q, k, v))

        # If the sequence length is short enough that FAVOR+ would use considerably more time and/or memory(?) than just using softmax attention,
        # putative reason: https://github.com/huggingface/transformers/issues/7675#issuecomment-734534302
        # For short enough sequence (L < 2 * d*log(d), remain to be tested) use softmax. This works because FAVOR+ is an unbiased estimator of softmax attention.
        m = self.num_random_features or round(dim_per_head * np.log(dim_per_head)) # m is the number of random features
        if self.should_fallback_to_softmax(q_length, m, self.training):
            scores = q @ k.transpose(-2, -1) / (dim ** 0.5)

            if mask is not None:
                mask = (mask == 0).view(mask_reshp).expand_as(scores)  # (bs, num_heads, q_length, k_length)
                scores.masked_fill_(mask, -float("inf"))  # (bs, num_heads, q_length, k_length)

            attn_map = nn.Softmax(dim=-1)(scores)
            attn_map = self.dropout(attn_map)  # (bs, num_heads, q_length, k_length)
            return self._finalize_attention_output(attn_map @ v, head_mask, attn_map)

        # When we're using FAVOR+ we can't output the attention matrix.
        # Why cannot, q'*k'^T ?
        #if output_attentions:
        #    raise ValueError("PerformerAttention: Can't output attention maps when using FAVOR+ linear attention.")

        self._redraw_features_if_needed(bs, q.device)

        # Get the transformed values of Q and K
        q_prime, k_prime = self.get_projected_queries_and_keys(q, k)
        return self.compute_attention_with_projected_queries_and_keys(q_prime, k_prime, v, mask, head_mask)

    # Turns Q into Q', K into K'
    def get_projected_queries_and_keys(self, q, k):
        # Instead of dividing the product QK^T by sqrt(d), we divide Q and K by the 4th root of d.
        q = q / (self.d_model ** 0.25)
        k = k / (self.d_model ** 0.25)

        projected_q = q @ self.random_features
        projected_k = k @ self.random_features

        # Special logic for kernels that attempt to approximate softmax
        if self.kernel_type in ('cosh', 'exp'):
            # The h(x) function is defined in Lemma 1 in Choromanski et al. pg. 4 as exp(-||x||**2 / 2). For numerical
            # stability we leverage the fact that exp(x)*exp(y) = exp(x + y) here and delay computing the exp().
            h_of_q = -torch.sum(q ** 2, dim=-1, keepdim=True) / 2
            h_of_k = -torch.sum(k ** 2, dim=-1, keepdim=True) / 2

            ## Need to check again ##:
            # In this implementation, q,k_stabilizer is max of h(*), but in original google implementation, they use max of q/k @ W_t

            # Compute the numerical stabilizer that we subtract from the input to exp(). For some reason the original
            # Jax implementation uses different types of stabilizers for queries vs. keys, and we follow that here.
            ## This is a workaround for very slow performance of torch.max(dim=N) on PyTorch 1.4 and earlier;
            ## see this GitHub discussion: https://github.com/pytorch/pytorch/issues/36900
            q_indices = h_of_q.argmax(-1).unsqueeze(-1)
            q_stabilizer = h_of_q.gather(-1, q_indices) # Note this is a (d_model, 1) matrix that gets broadcasted

            # This is just a scalar
            k_stabilizer = torch.max(h_of_k)

            q_kernel_output = self.kernel_fn(projected_q - q_stabilizer, h_of_q)
            k_kernel_output = self.kernel_fn(projected_k - k_stabilizer, h_of_k)

            # By multiplying by 1/sqrt(m), we ensure the final matrix product will contain a factor of 1/m. This means
            # each row of Q'K'^T can be interpreted as an average over the exp(omega^T * q) * exp(omega^T * k) terms.
            normalizing_constant = (q_kernel_output.shape[-1] ** -0.5)

            q_prime = normalizing_constant * (q_kernel_output + self.kernel_epsilon)
            k_prime = normalizing_constant * (k_kernel_output + self.kernel_epsilon)
            return q_prime, k_prime

        # Generalized attention (ReLU, ELU...)
        else:
            return (self.kernel_fn(x) + self.kernel_epsilon for x in (projected_q, projected_k))

    def compute_attention_with_projected_queries_and_keys(self, q_prime, k_prime, v, mask = None, head_mask = None):
        # Apply the padding mask to K'. Also applying it to Q' would be redundant.
        if mask is not None:
            k_prime *= mask.unsqueeze(1).unsqueeze(-1).expand_as(k_prime)

        if self.causal:
            output = _headwise_causal_numerator(q_prime, k_prime, v)
        else:
            k_prime_t = k_prime.transpose(-2, -1)
            output = q_prime @ (k_prime_t @ v)

        # Ensure that the output vectors are convex combinations of input vectors; that is,
        # the implied attention scores sum to 1
        if self.normalize_output:
            if self.causal:
                sums = k_prime.cumsum(dim=-2)
                d = torch.einsum("bhlm,bhlm->bhl", q_prime, sums).unsqueeze(-1)
            else:
                # Equivalent to multiplying K'^T by a ones vector
                d = q_prime @ k_prime.sum(dim=-2).unsqueeze(-1)

            # Avoid dividing by very small numbers
            d += 2 * self.normalization_stabilizer * (torch.abs(d) <= self.normalization_stabilizer)
            output /= d

        # TODO: output attention map
        return self._finalize_attention_output(output, head_mask)

    def _finalize_attention_output(self, context, head_mask=None, att_map_to_output=None):
        def unshape(x):
            """ group heads """
            return x.transpose(1, 2).contiguous().view(x.shape[0], -1, x.shape[1] * x.shape[-1])

        # Mask heads if we want to
        if head_mask is not None:
            context = context * head_mask

        context = unshape(context)  # (bs, q_length, dim)
        context = self.out_lin(context)  # (bs, q_length, dim)

        if att_map_to_output:
            return context, att_map_to_output
        else:
            return context,

    def _generate_feature_matrix(self, batch_size, device):
        dim_per_head = self.d_model // self.num_heads
        num_rows = self.num_random_features or round(dim_per_head * math.log(dim_per_head))
        batch = batch_size if self.use_thick_features else 1

        if not self.use_orthogonal_features:
            return torch.randn(batch, num_rows, dim_per_head, device=device)

        # The Givens rotation algorithm currently only supports matrices with an even number of rows/columns.
        # This should be fine for all widely used Transformer models, but we can use QR decomposition as a backup.
        num_full_blocks = num_rows // dim_per_head
        orthog_func = _get_square_orthogonal_block_givens if dim_per_head % 2 == 0 else _get_square_orthogonal_block_qr
        block_list = [orthog_func(batch, dim_per_head, device) for _ in range(num_full_blocks)]

        remaining_rows = num_rows - num_full_blocks * dim_per_head
        if remaining_rows > 0:
            q = orthog_func(batch, dim_per_head, device)
            block_list.append(q[:, :remaining_rows])

        final_tensor = torch.cat(block_list, dim=1)

        # This option yields SMREG
        if self.regularize_feature_norms:
            final_tensor *= dim_per_head ** 0.5
        else:
            # Hack to make the matrix columns have the norm we would expect them to have if they were sampled straight
            # from a Gaussian, instead of being all norm 1 since they went through QR decomposition
            multiplier = torch.randn(batch, num_rows, dim_per_head, device=device).norm(dim=2)
            final_tensor = torch.diag(multiplier) @ final_tensor

        # Add an attention head dimension
        final_tensor.unsqueeze_(1)
        new_shape = list(final_tensor.shape)
        new_shape[0] = batch_size  # This is redundant if use_thick_features == True, but not otherwise
        new_shape[1] = self.num_heads
        self.random_features = final_tensor.expand(*new_shape).transpose(-2, -1)

    def _redraw_features_if_needed(self, batch, device):
        # We haven't created the projection matrix yet, let's create it
        if self.random_features is None:
            self._generate_feature_matrix(batch, device)

        elif self.feature_redraw_interval is not None:
            if self.redraw_stochastically:
                # random.random() returns a float between 0.0 and 1.0, so this expression
                # evaluates to True with probability 1. / self.feature_redraw_interval
                if random.random() < 1. / self.feature_redraw_interval:
                    self.redraw_features_now()

            # It's time to redraw the projection matrix
            elif self.calls_since_last_redraw >= self.feature_redraw_interval:
                self.redraw_features_now()

            # Keep track of how many forward passes we do before we redraw again
            else:
                self.calls_since_last_redraw += 1

# Not currently used unless dim_per_head is odd
def _get_square_orthogonal_block_qr(batch, size, device=None):
    unstructured_block = torch.randn(batch, size, size, device=device)
    q, r = torch.qr(unstructured_block, some=True)
    return q.transpose(-2, -1)

# Not ideal but all we can do until https://github.com/pytorch/pytorch/issues/42502 gets implemented
def _batch_randperm(batch, n, dtype=torch.int64, device=None):
    out_tensor = torch.empty(batch, n, dtype=dtype, device=device)
    for i in range(batch):
        torch.randperm(n, out=out_tensor[i], dtype=dtype, device=device)

    return out_tensor

# Fast parallelizable way of making random orthogonal matrices- O(n log(n)) vs. O(n^3) for QR
def _get_square_orthogonal_block_givens(batch, num_rows, device=None):
    r"""Constructs a 2D-tensor which is a product of Givens random rotations.
    Constructs a 2D-tensor of the form G_1 * ... * G_k, where G_i is a Givens
    random rotation. The resulting tensor mimics a matrix taken uniformly at
    random form the orthogonal group.
    Args:
      num_rows: number of rows/columns of the resulting 2D-tensor.
    Returns:
      The product of Givens random rotations.
    """
    q = torch.eye(num_rows, device=device)  # Start with identity matrix
    q = q.expand(batch, num_rows, num_rows)

    # Compute the cosines and sines of the rotations up front
    num_iterations = 2 * int(math.ceil(math.log(num_rows)))
    angles = math.pi * torch.rand(batch, num_iterations, num_rows // 2, 1, device=device)
    cosines, sines = torch.cos(angles), torch.sin(angles)
    q.unsqueeze_(2)

    # Group the matrix into random, non-overlapping pairs of rows. Because these pairs are non-overlapping, we can
    # perform each set of rotations in parallel.
    for n in range(num_iterations):
        shuffled_rows = _batch_randperm(batch, num_rows, device=device).view(batch, -1, 1, 1)
        random_row_pairs = q.gather(1, shuffled_rows.expand_as(q)).view(batch, -1, 2, num_rows)

        rows1, rows2 = random_row_pairs[:, :, 0], random_row_pairs[:, :, 1]
        new_rows1 = cosines[:, n] * rows1 + sines[:, n] * rows2
        new_rows2 = -sines[:, n] * rows1 + cosines[:, n] * rows2

        random_row_pairs[:, :, 0], random_row_pairs[:, :, 1] = new_rows1, new_rows2

        q = random_row_pairs.view(batch, -1, 1, num_rows)  # Ungroup all the rows again

    return q.squeeze(-2)

def _headwise_causal_numerator(q_prime, k_prime, v):
    results = []

    # Iterate over the attention heads to avoid allocating a very large tensor
    for head in range(q_prime.shape[1]):
        # Outer products- a sorta biggish tensor
        outer_prods = torch.einsum('blm,bld->blmd', k_prime[:, head], v[:, head])
        prefix_sums = outer_prods.cumsum(dim=1)

        query_prods = torch.einsum('blmd,blm->bld', prefix_sums, q_prime[:, head])
        results.append(query_prods.unsqueeze(1))

    return torch.cat(results, dim=1)

# Not currently used
def _lengthwise_causal_numerator(q_prime, k_prime, v):
    batch_size, num_heads, seq_len, num_features = k_prime.shape
    d_model = v.shape[-1]

    # Merge the batch and attention head dimensions so we can use baddbmm_()
    prefix_sums = torch.zeros(batch_size * num_heads, num_features, d_model, device=v.device)
    q_prime = q_prime.view(-1, seq_len, num_features)
    k_prime = k_prime.view(-1, seq_len, num_features, 1)  # Add singleton dimension for outer product
    v = v.reshape(-1, seq_len, 1, d_model)  # Add singleton dimension for outer product

    result = []
    for pos in range(seq_len):
        prefix_sums.baddbmm_(k_prime[:, pos], v[:, pos])  # Fused in-place addition of outer products
        result.append(torch.einsum("bmd,bm->bd", prefix_sums, q_prime[:, pos]).unsqueeze(-2))

    # Unmerge the attention head and batch size dimensions
    return torch.cat(result, dim=-2).view(batch_size, num_heads, seq_len, d_model)


import cv2
import numpy as np
from PyQt5.QtCore import pyqtSignal, Qt, QThread

class VideoThread(QThread):
  """Provides OpenCV video feed from IP camera based on CameraController object provided"""
  change_pixmap_signal = pyqtSignal(np.ndarray)

  def __init__(self, camera):
    super().__init__()
    self._run_flag = True
    self.camera = camera

  def run(self):
    # capture from web cam
    if not self.camera.isConnected():
      no_cam_img = np.zeros((600,600,3), np.uint8)
      cv2.putText(no_cam_img, 'No Camera Available', (150,300), cv2.FONT_HERSHEY_DUPLEX, 1, (255,255,255))
      self.change_pixmap_signal.emit(no_cam_img)
    else:
      cap = cv2.VideoCapture(self.camera.getVideo())
      while self._run_flag:
        ret, cv_img = cap.read()
        if ret:
          self.change_pixmap_signal.emit(cv_img)
      cap.release()

  def stop(self):
    """Sets run flag to False and waits for thread to finish"""
    self._run_flag = False
    self.wait()


# -*- coding: utf-8 -*-
"""
Created on Fri Jun 12 08:48:06 2020

@author: dalong
"""

import jieba
import pickle
import numpy as np
from keras.models import load_model

INPUT_LENGTH=13
num_encoder_symbols=7885
class ChatbotRespnse():
    words = pickle.load(open('../pickle_and_h5/words.pkl','rb'))
    word_index = pickle.load(open('../pickle_and_h5/word_index.pkl','rb'))
    index_word = pickle.load(open('../pickle_and_h5/index_word.pkl','rb'))
    encoder_model=load_model('../pickle_and_h5/encoder_model.h5', compile=False)
    decoder_model=load_model('../pickle_and_h5/decoder_model.h5', compile=False)

    def response(self,source,encoder_inference, decoder_inference):
        states_value = encoder_inference.predict(source)

        # 生成一个size=1的空序列
        target_seq = np.zeros((1, 1, num_encoder_symbols))
        # 将这个空序列的内容设置为开始字符
        target_seq[0, 0, self.word_index['START']] = 1.

        # 进行字符恢复
        # 简单起见，假设batch_size = 1
        stop_condition = False
        decoded_sentence = ''

        while not stop_condition:
            output_tokens, h, c = decoder_inference.predict([target_seq] + states_value)
    #        print(output_tokens)

            # sample a token
            sampled_token_index = np.argmax(output_tokens[0, -1, :])
            sampled_char = self.index_word[sampled_token_index]
            decoded_sentence += sampled_char

            # 退出条件：生成 EOS 或者 超过最大序列长度
            if sampled_char == 'EOS' or len(decoded_sentence) >INPUT_LENGTH  :
                stop_condition = True

            # 更新target_seq
            target_seq = np.zeros((1, 1, num_encoder_symbols))
            target_seq[0, 0, sampled_token_index] = 1.

            # 更新中间状态
            states_value = [h, c]

        return decoded_sentence[:-3]

    def chatbot_response(self,ss):

        input_seq=np.zeros((1,INPUT_LENGTH,num_encoder_symbols))
        ss_seg=jieba.lcut(ss)
        for t,char in enumerate(ss_seg):
            try:
                input_seq[0,t,self.word_index[char]]=1.0
            except:
                return "你说啥呢？我的脑洞没那么大~~~百度去吧"
        decoded_sentence = self.response(input_seq,self.encoder_model,self.decoder_model)

        return decoded_sentence


import imp
from django.contrib.auth import get_user_model
from django.contrib.auth.models import User
from django.contrib.auth.mixins import LoginRequiredMixin
from django.views.generic import ListView, TemplateView

from accounts.forms import CustomUserCreationForm, LoginForm
from resume.models import Resume

class IndexView(TemplateView):
    template_name = 'index.html'

    def get_context_data(self, **kwargs):
        context = super().get_context_data()
        context['form'] = CustomUserCreationForm
        context['form_log'] = LoginForm
        context['vacancies'] = ''
        context['yeap'] = False
        context['resumes'] = ''
        if self.request.user.is_authenticated:
            user = self.request.user
            if user.is_worker:
                context['vacancies'] = 'Help me!'
            else:
                context['resumes'] = Resume.objects.all()
        return context

    def get(self, request, *args, **kwargs):
        cont = super().get(request, *args, **kwargs)
        return cont


import keras
from keras.models import Sequential
from keras.layers.core import Dense
from keras.optimizers import Adam
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import StandardScaler
import numpy as np
import random
from keras.constraints import max_norm
from keras.models import model_from_yaml

MUTATION_RANGE = 0.5
MUTATION_CHANCE = 0.1
MUTATION_BIAS_CHANCE = 0.1

CROSSOVER_RANGE = 0.9
CROSSOVER_CHANCE = 0.5
CROSSOVER_BIAS_CHANCE = 0.5

class Mlp():

	def __init__(self):
		self.model = Sequential([
			Dense(2, input_shape=(2,), activation="relu", kernel_constraint=max_norm(1.)),
			Dense(6, activation="relu", kernel_constraint=max_norm(1.)),
			Dense(2, activation="softmax", kernel_constraint=max_norm(1.))
			])
		#print(self.model.summary())
		self.model.compile(Adam(lr=0.0001), loss='sparse_categorical_crossentropy', metrics=['accuracy'])
		#w = self.model.layers[0].get_weights()
		#print('------------w-----------------')
		#w = self.model.get_weights()
		#print(w)
		#print(w)

	def set_train(self, train, labels):
		print(train)
		scaler = MinMaxScaler(feature_range=(0,1))
		scaled_train = scaler.fit_transform((train))
		self.scaled_train = scaled_train
		self.train_labels = labels

	def train(self):
		self.model.fit(self.scaled_train, self.train_labels, batch_size=1, epochs=1, shuffle=True, verbose=2)
		w = self.model.layers[0].get_weights()
		print(w)
		new_w = []
		new_w.append(np.array([[1, 2], [3,4]], float))
		new_w.append(w[1])
		self.model.layers[0].set_weights(new_w)
		w = self.model.layers[0].get_weights()
		print(w)

	def predict(self, test):
		#predictions = self.model.predict(self.scaled_train, batch_size=10, verbose=0)
		scaler = MinMaxScaler(feature_range=(0,1))
		test = np.array([test[0][0], test[0][1]], dtype=np.float)
		test = test.reshape(1,-1)
		test = np.vstack((test, [-500, -300]))
		test = np.vstack((test, [500, 300]))

		#print(test)
		scaled_test = scaler.fit_transform(test)
		#print('parameterssss scaled', scaled_test[0].reshape(1,-1))
		rounded_predictions = self.model.predict_classes(scaled_test[0].reshape(1,-1), batch_size=1, verbose=0)
		#rounded_predictions = self.model.predict(scaled_test, batch_size=10, verbose=0)
		#print('rounded', rounded_predictions)
		#if(rounded_predictions[0][0] > rounded_predictions[0][1]):
		#	rounded_predictions = [0]
		#else:
		#		rounded_predictions = [1]
		return rounded_predictions

	def weight_mutation(self):
		number_of_layers = int(len(self.model.get_weights())/2)
		layer_len = [2]
		layer_len += [len(self.model.layers[i].get_weights()[0][0]) for i in range(number_of_layers)]
		#print(layer_len)
		#print(self.model.layers[0].get_weights())
		#print(self.model.layers[0].get_weights()[0][0][0])
		for layer in range(1, number_of_layers+1):
			for i in range(int(layer_len[layer]*layer_len[layer-1]*MUTATION_RANGE)):
				var = random.randint(1,100)
				if (var <= 100*MUTATION_CHANCE):
					new_weigth = self.model.layers[layer-1].get_weights()
					index_i = random.randint(0, layer_len[layer-1]-1)
					index_j = random.randint(0, layer_len[layer]-1)
					#print('layer', layer, layer_len[layer]-1, layer_len[layer-1]-1)
					value = random.uniform(-1,1)
					#print(new_weigth)
					new_weigth[0][index_i][index_j] = value
					self.model.layers[layer-1].set_weights(new_weigth)
			self.bias_mutation(layer-1, layer_len[1:])

	def bias_mutation(self, layer, layer_len):
		for i in range(int(layer_len[layer]*MUTATION_RANGE)):
			var = random.randint(1,100)
			if (var <= 100*MUTATION_BIAS_CHANCE):
				new_weigth = self.model.layers[layer].get_weights()
				index = random.randint(0, layer_len[layer]-1)
				value = random.uniform(-1,1)
				new_weigth[1][index] = value
				self.model.layers[layer].set_weights(new_weigth)
		#print('layer 2', self.model.layers[3].get_weights())

	def crossover(self, mlp):
		number_of_layers = int(len(self.model.get_weights())/2)
		layer_len = [2]
		layer_len += [len(self.model.layers[i].get_weights()[0][0]) for i in range(number_of_layers)]

		for layer in range(1, number_of_layers+1):
			for i in range(int(layer_len[layer]*layer_len[layer-1]*CROSSOVER_RANGE)):
				var = random.randint(1,100)
				if (var <= 100*CROSSOVER_CHANCE):
					new_weigth = self.model.layers[layer-1].get_weights()
					parent_weight = mlp.model.layers[layer-1].get_weights()
					index_i = random.randint(0, layer_len[layer-1]-1)
					index_j = random.randint(0, layer_len[layer]-1)
					new_weigth[0][index_i][index_j] = (new_weigth[0][index_i][index_j]+
														parent_weight[0][index_i][index_j])/2
					self.model.layers[layer-1].set_weights(new_weigth)
			layerb = layer-1
			layer_lenb = layer_len[1:]
			for i in range(int(layer_lenb[layerb]*CROSSOVER_RANGE)):
				var = random.randint(1,100)
				if (var <= 100*CROSSOVER_BIAS_CHANCE):
					new_weigth = self.model.layers[layerb].get_weights()
					parent_weight = mlp.model.layers[layerb].get_weights()
					index = random.randint(0, layer_lenb[layerb]-1)

					new_weigth[1][index] = (new_weigth[1][index]+parent_weight[1][index])/2
					self.model.layers[layerb].set_weights(new_weigth)

	def save_mlp(self):
		# serialize model to YAML
		model_yaml = self.model.to_yaml()
		with open("model.yaml", "w") as yaml_file:
			yaml_file.write(model_yaml)
		# serialize weights to HDF5
		self.model.save_weights("model.h5")

	def load_mlp(self):
		# load YAML and create model
		yaml_file = open('model.yaml', 'r')
		loaded_model_yaml = yaml_file.read()
		yaml_file.close()
		self.model = model_from_yaml(loaded_model_yaml)
		# load weights into new model
		self.model.load_weights("model.h5")


#!/usr/bin/env python
# -*- encoding: utf-8 -*-
'''
@File    :   __init__.py
@Time    :   2019/04/12 16:35:09
@Author  :   jiegemena
@Version :   1.0
@Contact :   jiegemena@outlook.com
@License :   https://github.com/jiegemena
@Desc    :   None
'''

# here put the import lib
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
'''
@File    :   main.py
@Time    :   2019/04/12 16:33:48
@Author  :   jiegemena
@Version :   1.0
@Contact :   jiegemena@outlook.com
@License :   https://github.com/jiegemena
@Desc    :   None
'''

# here put the import lib
from flask import Flask, request
import os
from datetime import timedelta
from jgpycshare.LogTools import LogTools
import area
import json
import tools

basedir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))

def create_app():

    app = Flask(__name__)
    app.config.from_object('config')
    if app.config['DEBUG']:
        app.config['SECRET_KEY'] = os.urandom(24)  # 设置为24位的字符,每次运行服务器都是不同的，所以服务器启动一次上次的session就清除。
    app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(days=7)  # 设置session的保存时间。

    app.secret_key = app.config['SECRET_KEY']

    flask_logging(app)
    flask_init(app)
    flask_blueprints(app)
    # register_commands(app)
    register_errors(app)
    # register_shell_context(app)
    flask_template_context(app)

    sql_intercept = app.config['REQUEST_INTERCEPT']
    sql_intercept = sql_intercept.split('|')
    flask_request_handlers(app, sql_intercept)
    return app

def flask_logging(app):
    log = tools.qxlog()
    app.config['log'] = log
    log.info("info")
    log.debug("debug")
    log.error("error")
    log.info("info")

# 初始化应用
def flask_init(app):
    if os.path.exists(app.config['SQL_INIT'] + '.lock') is False:
        file = open(app.config['SQL_INIT'] + '.lock', 'w')
        file.write('del it create new database!')
        file.close()

        db = tools.db_sqlite3(sql_conn_str=app.config['SQL_CONN'])
        with open(app.config['SQL_INIT'], 'r') as f:
            db.get_db().executescript(f.read())
            print("db init successfully")
    ### register consul
    tools.unregister()
    if app.config['CONSUL_STATE']:
        tools.register()

def flask_blueprints(app):
    # app.register_blueprint(area.api_bp, url_prefix='/api')
    # app.register_blueprint(area.admin_bp, url_prefix='/admin')
    app.register_blueprint(area.guest_bp, url_prefix='/')
    # app.register_blueprint(web_bp)

# def register_shell_context(app):
#     @app.shell_context_processor
#     def make_shell_context():
#         return dict(db='db')

# 可以在 模板处 : {{test}}
def flask_template_context(app):
    @app.context_processor
    def make_template_context():
        return dict(test='test')

def register_errors(app):
    @app.errorhandler(400)
    def bad_request(e):
        return '400 error'

    @app.errorhandler(404)
    def page_not_found(e):
        return '500 error'

    @app.errorhandler(500)
    def internal_server_error(e):
        return '500 error'

def flask_request_handlers(app, sql_intercept):
    @app.after_request
    def after_request(response):
        # token = 'asdfaasdf'
        # response.set_cookie('csrf_token', token)
        # print('request end')
        return response

    @app.before_request
    def print_request_info():
        log = app.config['log']
        log.info("请求地址:" + str(request.path))
        log.info("GET参数：" + str(request.args))
        log.info("POST参数：" + str(request.form))
        for key in request.args:
            for k in sql_intercept:
                if (request.args[key]).find(k) >= 0:
                    return 'have get sql_intercept par:' + k

        for key in request.form:
            for k in sql_intercept:
                if (request.form[key]).find(k) >= 0:
                    return 'have post sql_intercept par:' + k


#!/usr/bin/python3
"""
This module defines the State class,
a sub class of the Base class
"""
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, Integer, String
Base = declarative_base()

class State(Base):
    """This class is a sub class of the Base class"""
    __tablename__ = 'states'

    id = Column(Integer, primary_key=True, autoincrement=True, nullable=False)
    name = Column(String(128), nullable=False)


import pygame
from pygame.sprite import Sprite

class Bullet(Sprite):
    """A class to manage bullets fired from the ship"""

    def __init__(self, configs, screen, ship):
        super().__init__()
        self.screen = screen

        self.rect = pygame.Rect(0, 0, configs.bullet_width,
          configs.bullet_height)

        self.rect.centerx = ship.rect.centerx
        self.rect.top = ship.rect.top

        self.y = float(self.rect.y)

        self.color = configs.bullet_color
        self.speed = configs.bullet_speed

    def update(self):

        self.y -= self.speed
        self.rect.y = self.y

    def draw_bullet(self):
        pygame.draw.rect(self.screen, self.color, self.rect)


import pandas as pd
import surprise
import torch
from surprise import Dataset
from typing import Tuple
import numpy as np
import torch.utils.data

def get_movielens_1m() -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
    movielens = Dataset.load_builtin("ml-1m", prompt=False)

    movies_file = f"{surprise.get_dataset_dir()}/ml-1m/ml-1m/movies.dat"
    users_file = f"{surprise.get_dataset_dir()}/ml-1m/ml-1m/users.dat"
    ratings_file = f"{surprise.get_dataset_dir()}/ml-1m/ml-1m/ratings.dat"

    movies_df = pd.read_csv(
        movies_file, sep="::", names=["MovieID", "Title", "Genres"], engine="python"
    )

    users_df = pd.read_csv(
        users_file,
        sep="::",
        names=["UserID", "Gender", "Age", "Occupation", "Zip-code"],
        engine="python",
    )
    ratings_df = pd.read_csv(
        ratings_file,
        sep="::",
        names=["UserID", "MovieID", "Rating", "Timestamp"],
        engine="python",
    )

    return movies_df, users_df, ratings_df

class MovieLens1MDataset(torch.utils.data.Dataset):

    def __init__(self):
        _, _, ratings_df = get_movielens_1m()

        self.items = ratings_df.iloc[:, :2].astype(np.longlong) - 1  # -1 because ID begins from 1
        self.ratings = ratings_df.iloc[:, 2]

        self.dims = np.max(self.items, axis=0) + 1

        self.user_field_idx = np.array((0,), dtype=np.longlong)
        self.item_field_idx = np.array((1,), dtype=np.longlong)

    def __len__(self):
        return self.ratings.shape[0]

    def __getitem__(self, index):
        return np.array(self.items.iloc[index]), np.array(self.ratings.iloc[index])


#!/usr/bin/env python3
from sklearn import tree
import sys

data=sys.argv
wgt=int(data[1])
txtre=int(data[2])

#		   wgt txtre
features=[[110,0],[140,0],[120,0],[80,1],[90,1],[100,1]]

output=["Apple","Apple","Apple","Orange","Orange","Orange"]

algo=tree.DecisionTreeClassifier()
#train the algo for data set
trained=algo.fit(features,output)
#test the algo
output=trained.predict([[wgt,txtre]])

print(output)


import torch
from fvcore.common.registry import Registry

META_ARCH_REGISTRY = Registry("META_ARCH")
META_ARCH_REGISTRY.__doc__ = """
Registry for meta-architectures, i.e. the whole model

The registered object will be called with `obj(cfg)`
and expected to return a `nn.Module` object.
"""
def build_model(cfg):
    """
        build the whole model architecture, defined by configs.MODEL.META_ARCHITECTURE
    """
    name = cfg.training.model.meta_arch
    model = META_ARCH_REGISTRY.get(name)(cfg)
    model.to(torch.device(cfg.training.model.device))

    return model


#The part where the face data of the person is taken and recorded in the system
import cv2
# Camera Settings
cameraWidth = 1020
cameraHeight = 720

# Face Detect Class
classifierFolder = 'haarcascade_frontalface_default.xml'
face_detector = cv2.CascadeClassifier(classifierFolder)

# Face Data Folder
faceDataFolder = "Datas/FaceDatas"

# Create Camera
#camera = cv2.VideoCapture(1)
camera = cv2.VideoCapture(0,cv2.CAP_DSHOW).release()
camera=cv2.VideoCapture(0,cv2.CAP_DSHOW)

# Camera Settings Late
camera.set(3, cameraWidth)
camera.set(4, cameraHeight)

# region Face Scan Function
def FaceCapture():
    # Captured face count
    count = 0
    # Face capture process
    while True:
        ret, img = camera.read()
        img = cv2.flip(img, 1)
        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        faces = face_detector.detectMultiScale(gray, 1.4, 5)
        for (x, y, w, h) in faces:
            cv2.rectangle(img, (x, y), (x + w, y + h), (255, 0, 0), 2)
            count += 1
            # Save captured face to FaceData folder
            cv2.imwrite(f"{faceDataFolder}/user.{str(userToken)}.{str(count)}.jpg", gray[y:y + h, x:x + w])
            cv2.imshow('image', img)
        k = cv2.waitKey(100) & 0xff  # 'ESC' to stop capturing
        if k == 27:
            break
        elif count >= 50:  # Take 50 faces to stop capture
            break

# endregion

userID = input("User ID : ")

userToken = userID

print("Initializing face capture. Look the camera and wait..")
FaceCapture()
print("Face is captured.")

# Clean and close camera window
camera.release()
cv2.destroyAllWindows()


import re
from wordcloud import WordCloud, STOPWORDS
import pandas as pd
import matplotlib.pyplot as plt

# Import pulled Tweets
data_pull = pd.read_csv('C:/Users/huffm/tweets.csv', header=None)
raw_tweets = data_pull.iloc[:,1]

# eliminate special characters, links, short words, and stopwords
raw_string = ''.join(raw_tweets)
no_links = re.sub(r'http\S+', '', raw_string)
no_unicode = re.sub(r"\\[a-z][a-z]?[0-9]+", '', no_links)
no_special_characters = re.sub('[^A-Za-z ]+', '', no_unicode)

words = no_special_characters.split(" ")
words = [w for w in words if len(w) > 2]
words = [w.lower() for w in words]
words = [w for w in words if w not in STOPWORDS]

#print(words)
wc = WordCloud(background_color='white',
        width=1600,
        height=800,
        random_state=21,
        colormap='jet',
        max_words=2000,
        max_font_size=200)

clean_string = ','.join(words)
wc.generate(clean_string)

plt.figure(figsize=(12, 10))
plt.axis('off')
plt.imshow(wc, interpolation="bilinear")
plt.title("Colorado Twitter Word Cloud")
plt.show()


# upload to S3 bucket through S3 client

import boto3

bucket_name = 'demobucket250-e67018'
file_name = 'abc.txt'

# s3_resource = boto3.resource('s3')
# s3_client = s3_resource.meta.client
# s3_client.upload_file(Bucket=bucket_name, Filename=file_name, Key=file_name)

#OR

s3_client = boto3.client('s3')
s3_client.upload_file(Bucket=bucket_name, Filename=file_name, Key=file_name)

print(f'{file_name} has been uploaded to {bucket_name} bucket')


from __future__ import division

import torch
from torch.utils.data import Dataset
from torchvision.transforms import Normalize
import numpy as np
import cv2
from os.path import join

import config
import constants
from utils.imutils import crop, flip_img, flip_pose, flip_kp, transform, rot_aa
import glob
import pytorch3d
from pytorch3d import ops, io, transforms

class AtlasDataset(Dataset):
    """
    Base Dataset Class - Handles data loading and augmentation.
    Able to handle heterogeneous datasets (different annotations available for different datasets).
    You need to update the path to each dataset in utils/config.py.
    """

    def __init__(self, options, dataset, ignore_3d=False, use_augmentation=True, is_train=True):
        super(AtlasDataset, self).__init__()
        self.dataset = dataset
        self.is_train = is_train
        self.options = options
        self.img_dir = config.DATASET_FOLDERS[dataset]
        self.normalize_img = Normalize(mean=constants.IMG_NORM_MEAN, std=constants.IMG_NORM_STD)
        self.data = np.load(config.DATASET_FILES[is_train][dataset])
        self.imgname = self.data['imgname']
        self.mapname = self.data['mapname']
        self.simname = self.data['simname']

        # Get paths to gt masks, if available
        try:
            self.maskname = self.data['maskname']
        except KeyError:
            pass
        try:
            self.partname = self.data['partname']
        except KeyError:
            pass

        # Bounding boxes are assumed to be in the center and scale format
        self.scale = self.data['scale']
        self.center = self.data['center']

        # If False, do not do augmentation
        self.use_augmentation = use_augmentation

        # Get gt SMPL parameters, if available
        self.pose = self.data['pose'].astype(np.float)
        self.betas = self.data['shape'].astype(np.float)
        self.trans = self.data['trans'].astype(np.float)
        if 'has_smpl' in self.data:
            self.has_smpl = self.data['has_smpl']
        else:
            self.has_smpl = np.ones(len(self.imgname))
        if ignore_3d:
            self.has_smpl = np.zeros(len(self.imgname))

        # Get gt 3D pose, if available
        try:
            self.pose_3d = self.data['S']
            self.has_pose_3d = 1
        except KeyError:
            self.has_pose_3d = 0
        if ignore_3d:
            self.has_pose_3d = 0

        # Get 2D keypoints
        try:
            keypoints_gt = self.data['part']
        except KeyError:
            keypoints_gt = np.zeros((len(self.imgname), 24, 3))
        try:
            keypoints_openpose = self.data['openpose']
        except KeyError:
            keypoints_openpose = np.zeros((len(self.imgname), 25, 3))
        self.keypoints = np.concatenate([keypoints_openpose, keypoints_gt], axis=1)

        # Get gender data, if available
        try:
            gender = self.data['gender']
            self.gender = np.array([0 if str(g) == 'm' else 1 for g in gender]).astype(np.int32)
        except KeyError:
            self.gender = -1*np.ones(len(self.imgname)).astype(np.int32)

        self.length = self.scale.shape[0]

    def augm_params(self):
        """Get augmentation parameters."""
        flip = 0            # flipping
        pn = np.ones(3)  # per channel pixel-noise
        rot = 0            # rotation
        sc = 1            # scaling
        if self.is_train:
            # We flip with probability 1/2
            if np.random.uniform() <= 0.5:
                flip = 1

            # Each channel is multiplied with a number
            # in the area [1-opt.noiseFactor,1+opt.noiseFactor]
            pn = np.random.uniform(1-self.options.noise_factor, 1+self.options.noise_factor, 3)

            # The rotation is a number in the area [-2*rotFactor, 2*rotFactor]
            rot = min(2*self.options.rot_factor,
                    max(-2*self.options.rot_factor, np.random.randn()*self.options.rot_factor))

            # The scale is multiplied with a number
            # in the area [1-scaleFactor,1+scaleFactor]
            sc = min(1+self.options.scale_factor,
                    max(1-self.options.scale_factor, np.random.randn()*self.options.scale_factor+1))
            # but it is zero with probability 3/5
            if np.random.uniform() <= 0.6:
                rot = 0

        return flip, pn, 0, sc

    def load_points(self, path):
        lists = glob.glob(path)
        mesh = io.load_objs_as_meshes(lists)
        pts, nms = ops.sample_points_from_meshes(mesh, self.options.num_points // len(lists), return_normals=True)
        pts = pts.reshape([-1,3])
        return pts, nms.reshape([-1,3]) * 0.1

    def map_processing(self, map_img, center, scale, rot, flip, pn):
        """data aug for pytorch3d"""
        tr = pytorch3d.transforms.RotateAxisAngle(rot, axis='Z')
        pt = tr.transform_points(map_img)
        if flip:
            pt[:,0] = -pt[:,0]
        return pt

    def map_processing_legacy(self, map_img, center, scale, rot, flip, pn):
        """Process rgb image and do augmentation."""
        map_img = np.roll(map_img, 64, axis=1)
        rot_mat = np.eye(3)
        if not rot == 0:
            rot_rad = rot * np.pi / 180
            sn,cs = np.sin(rot_rad), np.cos(rot_rad)
            rot_mat[0,:2] = [cs, -sn]
            rot_mat[1,:2] = [sn, cs]
        map_img = np.reshape(map_img, [256*256,4])
        map_img[:,:3] = np.einsum('ij,kj->ki', rot_mat, map_img[:,:3])
        map_img = np.reshape(map_img, [256, 256, 4])
        # flip the image
        if flip:
            map_img = map_img[:,::-1,:]
            map_img[:,:,0] = -map_img[:,:,0]
        # (3,224,224),float,[0,1]
        map_img = np.transpose(map_img.astype('float32'),(2,0,1))
        return map_img

    def pose_processing(self, pose, r, f):
        """Process SMPL theta parameters  and apply all augmentation transforms."""
        # rotation or the pose parameters
        pose[:3] = rot_aa(pose[:3], r)
        # flip the pose parameters
        if f:
            pose = flip_pose(pose)
        # (72),float
        pose = pose.astype('float32')
        return pose

    def __getitem__(self, index):
        # index = 0
        item = {}
        scale = self.scale[index].copy()
        center = self.center[index].copy()

        # Get augmentation parameters
        flip, pn, rot, sc = self.augm_params()
        #print(rot)
        # Get SMPL parameters, if available
        if self.has_smpl[index]:
            pose = self.pose[index].copy()
            betas = self.betas[index].copy()
            trans = torch.from_numpy(self.trans[index].copy()).float()
        else:
            pose = np.zeros(72)
            betas = np.zeros(10)

        # Store image before normalization to use it in visualization
        clothmap, normmap = self.load_points(join(self.img_dir, self.simname[index]))
        clothmap = self.map_processing(clothmap - trans, center, sc*scale, rot, flip, pn)

        normmap = self.map_processing(normmap, center, sc*scale, rot, flip, pn)
        item['map'] = clothmap.float()
        item['normmap'] = normmap.float()
        item['pose'] = torch.from_numpy(self.pose_processing(pose, rot, flip)).float()
        item['betas'] = torch.from_numpy(betas).float()

        item['rot_angle'] = np.float32(rot)
        item['gender'] = self.gender[index]
        item['sample_index'] = index
        item['dataset_name'] = self.dataset
        item['dataname'] = self.imgname[index]

        return item

    def __len__(self):
        return len(self.imgname)


import requests
from bs4 import BeautifulSoup
#模拟浏览器
header = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36'
          }
#请求url
print('笔趣库网站www.biquku.la')
print('例子：小说《飞剑问道》 格式：http://www.biquku.la/0/65/')
urls=input('请输入笔趣库需要下载的小说的url：')
url=urls
r = requests.get(url,headers = header)
r.encoding = 'utf-8'
soup = BeautifulSoup(r.text, "lxml")
content = soup.find_all('div', id="list")
xsmz=soup.find_all('div', id="info")
for xs in xsmz:
    xs= xs.find_all('h1')
    for xs in xs:
     print('准备下载：'+xs.text)
for list in content:
    urls = list.find_all('a')
    for td in urls:
        zj_url=td["href"]
        section_name=td.text
        xsurl=url+zj_url
        res2 = requests.get(xsurl, headers=header)
        res2.encoding = 'utf-8'
        soup = BeautifulSoup(res2.text, "lxml")
        content = soup.find_all('div', id="content")
        mc = soup.find_all('script', language="javascript")
        for nr,mc in zip(content,mc):
         section_text = nr.text
         mc=mc.text
         mc=mc.split(';')[5]
         section_mc = mc.split('"')[1]
         fo = open(section_mc+'.txt', "ab+")  # 打开小说文件
         # 以二进制写入章节题目 需要转换为utf-8编码，否则会出现乱码
         fo.write(('\r' + section_name + '\r\n').encode('UTF-8'))
         # 以二进制写入章节内容
         fo.write((section_text).encode('UTF-8'))
         fo.close()
         print(section_name+'    '+'写入文本成功')  # 关闭小说文件


from modules.common.module import BotModule
import urllib.request

class MatrixModule(BotModule):
    async def matrix_message(self, bot, room, event):
        args = event.body.split()
        args.pop(0)
        day = 0
        hour = 12
        if len(args) >= 1:
            day = int(args[0]) - 1
        if len(args) == 2:
            hour = int(args[1])

        imgurl = 'http://ennuste.ilmailuliitto.fi/' + str(day) + '/wstar_bsratio.curr.' + str(hour) + '00lst.d2.png'
        await bot.upload_and_send_image(room, imgurl, f"RASP Day {day+1} at {hour}:00", no_cache=True)

    def help(self):
        return 'RASP Gliding Weather forecast, Finland only'