from datetime import datetime
from src.extension import db
from src.pbl5_ma import CheckInSchema
from src.model import CheckIn
from flask import jsonify, request, jsonify

check_in_schema = CheckInSchema()
check_ins_schema = CheckInSchema(many=True)

def get_all_check_ins_service():
    check_ins = CheckIn.query.all()
    return check_ins_schema.dump(check_ins)

def create_check_in_service():
    data = request.json

    if (data and ('plate_number' in data) and ('student_id' in data) and ('img_check_in' in data)):
        plate_number = data['plate_number']
        student_id = data['student_id']
        img_check_in = data['img_check_in']
        try:
            time_check_in = datetime.now()
            check_in = CheckIn(plate_number, student_id, time_check_in, img_check_in)
            db.session.add(check_in)
            db.session.commit()
            return jsonify({
                    "check_in": {
                        "id": check_in.id,
                        "student_id": student_id,
                        "plate_number": plate_number,
                        "time_check_in": time_check_in,
                        "img_check_in": img_check_in
                    },
                    "message": "Check in success",
                    "status": 1
                }), 201
        except Exception:
            db.rollback()
            return jsonify({
                    "message": "Check in failed",
                    "status": 0
                }), 400
    else:
        return jsonify({
                    "message": "Validation request error",
                    "status": 0
                }), 400

def find_by_student_id_and_plate_number_service(student_id, plate_number_input):
    check_in_find = db.session.query(CheckIn).filter(CheckIn.student_id == student_id,
                                                    CheckIn.plate_number == plate_number_input)\
                                                    .order_by(CheckIn.time_check_in.desc())\
                                                    .first()
    return check_in_find


# !/usr/bin/env python3
# -*- coding: utf-8 -*-

from Predictor import Predictor
from sklearn.neighbors import KNeighborsRegressor
#Used to get command line arguments
import sys
import os
#Used to check validity of date
from datetime import datetime, timedelta

from TFMLP import MLPR

import Utils
from Utils import fetch_data_from_api, plot_data

#Display usage information
def print_usage():
    print('Usage:\n')
    print('\tpython main.py <source type> <market> <currency pair> <period> <start time> <end time>')

    print('\t')
    print('\tsource type: K_NEIGHBORS_V1')
    print('\tsource type: MLPR_V1')
    print('\t')
    print('\tmarket: BITSTAMP')
    print('\t')
    print('\tcurrency pair: BTC_USD')
    print('\t')
    print('\tperiod: ONE_HOUR')
    print('\tperiod: ONE_DAY')
    print('\tperiod: WEEK')
    print('\t')
    print('\tstart time: NOW | 2017-02-09T08:00:00')
    print('\t')
    print('\tend time (optional): 2017-03-14T16:00:00')

#Main program
def main(args):
    if(len(args) != 5 and len(args) != 6):
        print("Invalid parameters count")
        print_usage()
        return

    print("-------------------------------------------------------------------")
    print ("Start time = %s" % datetime.now())
    print("\n");

    #API Client ID
    try:
        api_client_id = os.environ["API_CLIENT_ID"]
    except KeyError:
        print("Env. variable API_CLIENT_ID is not defined or is invalid!")
        print_usage()
        return

    #API Client secret
    try:
        api_client_secret = os.environ["API_CLIENT_SECRET"]
    except KeyError:
        print("Env. variable API_CLIENT_SECRET is not defined or is invalid!")
        print_usage()
        return

    #Source type
    pred_source = args[0].upper()

    #Market type
    market = args[1].upper()

    #Currency pair
    currency_pair = args[2].upper()

    #Period
    period = args[3].upper()

    # Start and end time
    if(args[4] == "NOW"):
        if(period == "ONE_HOUR"):
            startTime = datetime.today()
            # startTime = startTime - timedelta(hours=24 + 3)
            startTime = startTime - timedelta(hours=0)
            startTime = startTime.replace(minute=0, second=0)
            endTime = startTime + timedelta(hours=24*3) # 3 dni
            endTime = endTime.replace(minute=0, second=0)
        elif(period == "ONE_DAY"):
            startTime = datetime.today()
            # startTime = startTime - timedelta(days=14) # 16 trochu rozdielne
            startTime = startTime - timedelta(days=0)
            startTime = startTime.replace(hour=0, minute=0, second=0)
            endTime = startTime + timedelta(days=30*5) # 5mesiace
            endTime = endTime.replace(hour=0, minute=0, second=0)
        elif(period == "WEEK"):
            startTime = datetime.today()
            # startTime = startTime - timedelta(weeks=14)
            startTime = startTime - timedelta(weeks=0)
            while startTime.weekday() != 0: #0 for monday
                startTime -= timedelta(days=1)

            startTime = startTime.replace(hour=0, minute=0, second=0)
            endTime = startTime + timedelta(weeks=150)
            while endTime.weekday() != 0: #0 for monday
                endTime -= timedelta(days=1)
            endTime = endTime.replace(hour=0, minute=0, second=0)

            startTime += timedelta(days=1)
            endTime += timedelta(days=1)

        start = startTime.strftime("%Y-%m-%dT%H:%M:%S")
        end = endTime.strftime("%Y-%m-%dT%H:%M:%S")
    else:
        #Test validity of start date string
        try:
            datetime.strptime(args[4], '%Y-%m-%dT%H:%M:%S').timestamp()
        except Exception as e:
            print('Error parsing date: ' + args[1])
            print_usage()
            return
        #Test validity of end date string
        try:
            datetime.strptime(args[5], '%Y-%m-%dT%H:%M:%S').timestamp()
        except Exception as e:
            print('Error parsing date: ' + args[2])
            PrintUsage()
            return
        start = args[4]
        end = args[5]

    print("api_client_id: {}".format(api_client_id))
    print("api_client_secret: {}".format(api_client_secret))
    print("start: {}".format(start))
    print("start: {}".format(start))
    print("end: {}".format(end))
    print("pred_source: {}".format(pred_source))
    print("market: {}".format(market))
    print("currency_pair: {}".format(currency_pair))
    print("period: {}".format(period))
    print("\n")

    #Everything looks okay; proceed with program
    #Grab the data frame
    D = fetch_data_from_api(api_client_id, api_client_secret, market, currency_pair, period)

    # print("D: {}".format(D))

    #The number of previous days of data used
    #when making a prediction
    num_past_days = 16

    plot_data(D)

    #Number of neurons in the input layer
    i = num_past_days * 7 + 1
    #Number of neurons in the output layer
    o = D.shape[1] - 1
    #Number of neurons in the hidden layers
    h = int((i + o) / 2)
    #The list of layer sizes
    layers = [i, h, h, h, h, h, h, o]

    if(pred_source.startswith('K_NEIGHBORS')):
        R = KNeighborsRegressor(n_neighbors = 5)
    elif(predSource.startswith('MLPR')):
        R = MLPR(layers, maxItr = 1000, tol = 0.60, reg = 0.001, verbose = False)
    else:
        print("Source not implemented yet!")
        print_usage()
        return

    sp = Predictor(R, nPastDays = num_past_days)
    #Learn the dataset and then display performance statistics
    sp.Learn(D)
    sp.TestPerformance()
    #Perform prediction for a specified date range

    P = sp.PredictDate(start, end, period)
    print("P.shape[0]: {}".format(P.shape[0]))
    # print("P: {}".format(P))

    #Keep track of number of predicted results for plot
    n = P.shape[0]
    #Append the predicted results to the actual results
    D = P.append(D)

    #Predicted results are the first n rows
    plot_data(D, range(n + 1))

    print("\n");
    print("End time = %s" % datetime.now())
    print("-------------------------------------------------------------------")

    return (P, n)

#Main entry point for the program
if __name__ == "__main__":
    p, n = main(sys.argv[1:])


"""
Compuatational Physics II
Project 10
Lalit Chauhdary
l.chaudhary@jacobs-university.de
"""
import numpy as np
import matplotlib.pyplot as plt

#Discrete Fourier Transform
def DFT(ft):
    N = len(ft)
    fw= []                     #to store result of DFT
    for m in range(N):
        sum = 0.0
        for n in range(N):
            sum += ft[n] * np.exp(-1j * 2*np.pi * m * n / N)
        fw.append((sum / np.sqrt(N)))
    return fw

#Inverse Discrete Fourier Transform
def InverseDFT(fw):
    N = len(fw)
    ft = []
    for n in range(N):
        sum = 0.0
        for m in range(N):
            sum += fw[m] * np.exp(1j * 2* np.pi * m * n / N)
        ft.append(sum/np.sqrt(N))
    return ft

#PART A
#Input Gaussian
Gaussian_data = []
mu  = 0                       #Mean
sigma = 0.5                   #Standard deviation
N = 50                        #Number of data points

x = np.linspace(-2, 2, N)

def gaussian (x):
    return 1/(sigma * np.sqrt(2*np.pi)) * np.exp (-(x - mu)**2 /(2*sigma**2))

#Input data array
for xx in x:
    Gaussian_data.append(gaussian(xx))

plt.plot(x, Gaussian_data, label = 'Original data')

DFT_gaussian = DFT(Gaussian_data)

#Separate real, imaginary and absolute value of result
DFT_gaussian_real = [d.real for d in DFT_gaussian]
DFT_gaussian_imag = [d.imag for d in DFT_gaussian]
DFT_gaussian_abs = [np.abs(d) for d in DFT_gaussian]

plt.plot(x,DFT_gaussian_abs, label = 'magnitude')
plt.plot(x,DFT_gaussian_real, label = 'real part')
plt.plot(x,DFT_gaussian_imag, label = 'imaginary part')

plt.legend()
plt.show()

#PART B
def f(x):
    return np.sin(x)**2 * np.exp(-(x-np.pi/2)**2)

x = np.linspace(0, 3, 50)

data = [f(t) for t in x]

DFT_data = DFT(data)
Inverse_data = InverseDFT(DFT_data)

plt.figure(2)
plt.plot(x, data, 'k.', label = 'original data')
plt.plot(x, [i.real for i in DFT_data], label = 'After DFT')
plt.plot(x, [i.real for i in Inverse_data], label = 'Retrieved data after IDFT')
plt.legend()
plt.show()

#Compare the original data and the back transformed data
diff = 0
for i in range (len(data)):
    diff += np.abs (data[i] - Inverse_data[i].real)

if diff >= 1*10**(-7):
    print('There is some deviation between the original and the retrieved data')
else:
    print('The original and the retrieved data are similar')


import hashlib
import sys
# your gen-py dir
sys.path.append('./gen-py')
from ril.ttypes import *
import json

class FileData():

    def __init__(self, instanceName=None):
        self.inputfilename=f"/var/lib/data-test/{instanceName}_inputs.json"
        self.outputfilename=f"/var/lib/data-test/{instanceName}_outputs.json"
        self.outputs={}
        try:
            with open(self.inputfilename) as f:
                self.inputs = json.load(f)
        except:
            self.inputs ={}
        self.step_idx = 0

    def doStep(self):
        self.step_idx+=1

    def read(self, var_ref):
        if (str(var_ref) in self.inputs):
            var_inputs = self.inputs.get(str(var_ref))
            if (len(var_inputs) > self.step_idx):
                return var_inputs[self.step_idx]
            else:
                return None
        else:
            return None

    def write(self, var_ref, value):
        if (not str(var_ref) in self.outputs):
            self.outputs[str(var_ref)] = []
        var_outputs = self.outputs.get(str(var_ref))
        if (len(var_outputs) <= self.step_idx ):
            var_outputs.append(value)
        else:
            var_outputs[self.step_idx] = value

    def dump(self):
        with open(self.outputfilename, 'w') as outfile:
            json.dump( self.outputs, outfile)


import torch
from transformers import (
    DistilBertForSequenceClassification,
    DistilBertForTokenClassification,
    DistilBertTokenizer,
)

MODEL_SA = "./app/model/model_sa_v7"
MODEL_MLC = "./app/model/model_mlc_v7"
MODEL_NER = "./app/model/model_ner_v7"

MODEL_SA_OUT = "./out/sa/"
MODEL_MLC_OUT = "./out/mlc/"
MODEL_NER_OUT = "./out/ner/"

########### SA ###########
tokenizer = DistilBertTokenizer.from_pretrained(
    "distilbert-base-uncased", do_lower_case=True
)
model = DistilBertForSequenceClassification.from_pretrained(
    "distilbert-base-uncased", num_labels=3
)
model.load_state_dict(torch.load(MODEL_SA, map_location=torch.device("cpu")))
model.save_pretrained(MODEL_SA_OUT)
tokenizer.save_pretrained(MODEL_SA_OUT)
# tf_model = TFDistilBertForSequenceClassification.from_pretrained(
#     MODEL_SA_OUT, from_pt=True
# )
# tf_model.save_pretrained(MODEL_SA_OUT)

########### MLC ###########
tokenizer = DistilBertTokenizer.from_pretrained(
    "distilbert-base-uncased", do_lower_case=True
)
model = DistilBertForSequenceClassification.from_pretrained(
    "distilbert-base-uncased", num_labels=4
)
model.load_state_dict(torch.load(MODEL_MLC, map_location=torch.device("cpu")))
model.save_pretrained(MODEL_MLC_OUT)
tokenizer.save_pretrained(MODEL_MLC_OUT)

########### NER ###########
tokenizer = DistilBertTokenizer.from_pretrained(
    "distilbert-base-cased", do_lower_case=False
)
model = DistilBertForTokenClassification.from_pretrained(
    "distilbert-base-cased", num_labels=10
)
model.load_state_dict(torch.load(MODEL_NER, map_location=torch.device("cpu")))
model.save_pretrained(MODEL_NER_OUT)
tokenizer.save_pretrained(MODEL_NER_OUT)


from matplotlib import pyplot as plt
from collections import deque
from threading import Lock, Thread
import myo
import time
import numpy as np
import pandas as pd
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from socket import socket, AF_INET, SOCK_DGRAM
from statistics import mode
from scipy.spatial import distance

emg_train=np.array([[] for i in range(8)])

def queue_init(dim,q_length):   #dim x lengthのキューを作成
    tmp=deque(maxlen=q_length)
    a=0
    if dim==2:
        a=[0,0]
    for i in range(q_length):
        tmp.append(a)
    return tmp

finger_2d = queue_init(2,10)    #直近10回のデータサンプルの2次元座標を保存
finger_predict = queue_init(1,10)   #上の座標に対応する直近10回のクラス判別結果を保存

testdata_2d_rec=[]  #評価タスク時のデータを格納

finger_label=[]
emg_features=[]
lda_finger=LinearDiscriminantAnalysis(n_components=2)   #線形判別分析の学習器作成

count=0
ovr_l = 20  #クラス判別に使用するデータサンプルのオーバーラップ分は20sample
win_l = 40  #解析窓の長さは40sample分

fps=0.001   #fpsというより、視覚フィードバックの図の更新頻度(とりあえず0.001s感覚でフィードバックに関するwhile部分が動作します)

queue_len=6
do_record=True

class_f = 9  # 学習データの動作クラス数(使用データに合わせて変更してください)

ax1=0    #pltのリアルタイムプロット用のグローバル変数(0は仮で代入)(変更不要)
scat_finger=0

def update_plot(scat1,ax1): #
    tmp = np.array([x for x in list(finger_2d)])
    scat1.set_offsets(tmp)
    label_ = ["グー", "人差し指","中薬指","小指","パー","ピース","内屈","外屈","無動作"]
    ax1.set_title("finger motion : " + label_[finger_predict[-1]],fontname="MS Gothic")
    plt.pause(fps)

class EmgCollector(myo.DeviceListener): #Myo armbandを用いた近電位記録に関数するクラス

  def __init__(self, n):
    self.idle=False  #Trueの間計測を行う
    self.lock = Lock()
    self.time_pre=0
    self.emg_data_queue = deque(maxlen=win_l)  #クラス判別に使用するデータが格納されるキュー
    self.start_time=time.time() #計測時間の記録に使用
    #以下2つはなぜ入れたか覚えてないです
    self.sampleamount = 0
    self.n = n

  def get_emg_queue(self):  #呼び出し時に各センサの筋電位データを要素数8のリストで返す
    with self.lock:
      return list(self.emg_data_queue)  #リスト化が必要

  def on_connected(self, event):
    event.device.stream_emg(True)

  def start(self):  #筋電位のリアルタイム判別開始に呼び出す
      for i in range(ovr_l):
        self.emg_data_queue.append((0, [0,0,0,0,0,0,0,0]))  #一番最初のデータサンプルだけ解析窓に含まれるデータ半分が0になる
      self.idle=True

  def predict(self):    #クラスの予測
      tmp=self.get_emg_queue()  #その段階での解析窓を切り出し
      tmp=np.array([x[1] for x in tmp]).T
      test_features=[feature_calc(tmp, win_l)]  #特徴量を計算してリストを得る
      df = pd.DataFrame(test_features)
      global finger_predict,wrist_predict,testdata_2d_rec   #いらない変数があるかもしれない
      lda_fingerresult=int(lda_finger.predict(df.values))   #特徴量のデータからクラス判別を実施
      finger_predict.append(lda_fingerresult)   #判別結果のラベルを保存
      a= lda_finger.transform(df.values)    #先ほどの特徴量を2次元座標に変換
      global finger_2d
      finger_2d.append(a[0])    #座標データの保存
      testdata_2d_rec.append([lda_fingerresult, a[0][0], a[0][1]])  #記録データ用に判別クラス、座標を保存

  def end(self):
      self.idle=False

  def on_emg(self, event):
    with self.lock:
        if self.idle:
            tmp=event.emg
            self.sampleamount+=1
            self.emg_data_queue.append((event.timestamp, event.emg))

            if self.sampleamount==ovr_l:
                self.sampleamount=0
                thread = Thread(target=self.predict)
                thread.start()
                thread.join(0.0001)

def Record():   #Unityとの同期確認
    print("wait msg")
    ADDR = ''
    PORT = 50004 # 受信ポート
    M_SIZE = 1024
    msg=0
    rcv = socket(AF_INET, SOCK_DGRAM)
    VR_test_start = 0
    rcv.bind((ADDR, PORT))
    while True:
        msg, address = rcv.recvfrom(8192)
        msg_=int(msg[0] - 48)

        if VR_test_start==0 and msg_==1:
            VR_test_start=1
            global testdata_2d_rec
            testdata_2d_rec=[]
            print("msg received record start")

        if VR_test_start==1 and msg_==9:
            VR_test_start=0
            print("msg received record finished")
            df = pd.DataFrame(testdata_2d_rec)
            df.to_csv(msg.decode()[1:]+".csv")  # Unity側で設定した名前で記録したcsvデータを保存

class Train(object):    #視覚フィードバックに関するクラス
    def __init__(self,listener):
        self.n = listener.n
        self.listener = listener

    def Show_emg_fb(self):
        ADDR = '127.0.0.1'
        PORT_TO = 50007  # 送信ポート
        M_SIZE = 1024
        global ax1,finger_2d,scat_finger,finger_predict
        snd = socket(AF_INET, SOCK_DGRAM)
        self.listener.start()
        while True:
            update_plot(scat_finger,ax1)
            msg = str(finger_predict[-1]) + "0"
            snd.sendto(msg.encode(),(ADDR,PORT_TO))

    def Show_emg_nfb(self,df,ff,fm):
        ADDR = '127.0.0.1'
        PORT_TO = 50007  # 送信ポート
        M_SIZE = 1024
        global finger_2d,finger_predict
        fig = plt.figure(figsize=(4, 12))
        ax1 = plt.subplot(111)
        snd = socket(AF_INET, SOCK_DGRAM)
        base_df = df
        features_finger = ff
        base2d_finger = fm

        basedata_finger = []  # 0番目から各クラスの重心

        for k in range(class_f):
            tmp = []
            tmp.append([base2d_finger[j][0] for j in range(len(base2d_finger)) if features_finger[j] == (k)])
            tmp.append([base2d_finger[j][1] for j in range(len(base2d_finger)) if features_finger[j] == (k)])
            basedata_finger.append(tmp)

        basedata_f_centers = []

        for i in range(class_f):
            tmp = []
            tmp.append(np.mean(basedata_finger[i][0]))
            tmp.append(np.mean(basedata_finger[i][1]))
            basedata_f_centers.append(tmp)

        # --ここまでが初回計測データからの重心計算と、使用ファイル読み込み
        # ここからが計測データの処理
        basedata_f_cov = []

        for i in range(class_f):
            tmp = np.cov(basedata_finger[i][0], basedata_finger[i][1])
            basedata_f_cov.append(np.linalg.pinv(tmp))

        label_ =["グー", "人差し指","中薬指","小指","パー","ピース","内屈","外屈","無動作"]
        self.listener.start()
        while True:
            msg=str(finger_predict[-1])+"0"
            snd.sendto(msg.encode(),(ADDR,PORT_TO))
            ax1.cla()
            ax1.set_title("finger motion : " + label_[finger_predict[-1]], fontname="MS Gothic")
            f_maharanobis=distance.mahalanobis(list(finger_2d[-1]),basedata_f_centers[finger_predict[-1]],basedata_f_cov[finger_predict[-1]])
            ax1.bar([1], f_maharanobis)
            ax1.set_ylim(0,8)
            plt.pause(0.00001)

def feature_calc(emg,win_l):    #特徴量計算を行う関数
    FEATURES = []
    for i in range(8):
        tmp = emg[i]
        FEATURES.append(np.mean(np.abs(tmp)))  # MAV

        FEATURES.append(np.var(tmp))  # VAR
        zero = 0
        for j in range(0, win_l - 1):
            if tmp[j] * tmp[j + 1] < 0:
                zero += 1

        FEATURES.append(zero)   #ZC

        diff = np.diff(tmp, n=1)    # WL
        FEATURES.append(np.sum(np.abs(diff)))
        # freq = np.abs(np.fft.fft(tmp))  # 周波数領域
        # FEATURES.append(np.max(freq))  # PKF
        # FEATURES.append(np.mean(freq))  # MKF

    return FEATURES

def main():
    finger_2d = queue_init(2,queue_len)
    finger_predict = queue_init(1,queue_len)
#-----------------------------------------
    myo.init()
    hub = myo.Hub()
    listener = EmgCollector(512)

    features_finger=[]
    features_wrist=[]
    with hub.run_in_background(listener.on_event):
        global lda_finger, lda_wrist
        print("training")
        df = pd.read_csv('nkn_r_base0_.csv', header=0, index_col=0)
        features_finger = df.values[:,-1]
        df=df.values[:,0:-1]
        finger_motion = lda_finger.fit(df, features_finger).transform(df)

        fb = input("start Train with FeedBack? y/n:")
        thread = Thread(target=Record)
        thread.start()
        if fb == "y":
            fig = plt.figure(figsize=(18, 12))
            global ax1, ax2, scat_finger, scat_wrist
            ax1 = plt.subplot(111)
            label_ = ["グー", "人差し指","中薬指","小指","パー","ピース","内屈","外屈","無動作"]
            for k in range(class_f):  # 手描画
                tmp = []
                tmp.append([finger_motion[j][0] for j in range(len(finger_motion)) if features_finger[j] == (k)])
                tmp.append([finger_motion[j][1] for j in range(len(finger_motion)) if features_finger[j] == (k)])
                tmp = np.array(tmp)
                scat_finger = ax1.scatter(tmp[0], tmp[1], label=label_[k], cmap='viridis', edgecolor='blacK')
            ax1.set_title("finger_pattern")
            ax1.legend(labels=label_, fontsize=12,prop={"family":"MS Gothic"})
            scat_finger = ax1.scatter(0, 0, label="current", c="crimson", s=100,
                                      marker="X")  # 空撃ちすることでリアルタイム分類時のset_datasに備える

            plt.pause(0.05)
            Train(listener).Show_emg_fb()
        if fb =="n":
            Train(listener).Show_emg_nfb(df,features_finger,finger_motion)

if __name__ == '__main__':  #一応ここから開始
  main()


"""
We start with mnist, then do something else
"""
import numpy as np
import torch.optim

import torch
import torchvision.datasets as datasets
import pandas as pd
import sklearn.decomposition
import sklearn.metrics

from pathlib import Path
import torch
import torchvision

import torchvision.datasets as datasets

class MNISTRandomLabels(datasets.MNIST):
  def __init__(self, corrupt_prob=0.0, num_classes=10, shfld_pxls=False , rnd_pxls=False, gaussian=False, **kwargs):
    super(MNISTRandomLabels, self).__init__(**kwargs)
    self.n_classes = num_classes

    if corrupt_prob > 0:
      print("Introducing Noise: Corrupting labels ...")
      self.corrupt_labels(corrupt_prob)
    elif shfld_pxls:
      print("Introducing Noise: Shuffling pixels")
      self.shuffled_pixels()
    elif rnd_pxls:
      print("Introducing Noise: Randomizing pixels")
      self.random_pixels()
    elif gaussian:
      print("Introducing Noise: Generating pixels from a Gaussian distribution")
      self.gaussian()

  def corrupt_labels(self, corrupt_prob):
    """
    Labels are corrupted with a corrput_probability
    :param corrupt_prob:
    :return:
    """
    np.random.seed(12345)
    labels = np.array(self.targets)
    mask = np.random.rand(len(labels)) <= corrupt_prob/100.
    rnd_labels = np.random.choice(self.n_classes, mask.sum())
    labels[mask] = rnd_labels
    # we need to explicitly cast the labels from npy.int64 to
    # builtin int type, otherwise pytorch will fail...
    labels = [int(x) for x in labels]
    self.targets = labels

  def shuffled_pixels(self):
    """
    All images are permuted using the same permutation
    :return:
    """
    np.random.seed(12345)
    images = np.array(self.data)
    data = np.swapaxes(np.reshape(images, (images.shape[0], -1, images.shape[-1])), 0, 1)
    np.random.shuffle(data)
    self.data = np.reshape(np.swapaxes(data, 0, 1), images.shape)

  def random_pixels(self):
    """
    Each image is permuted using a different permutation matrix
    :return:
    """
    np.random.seed(12345) # sets the seed for the ensemble of permutations
    images = np.array(self.data)
    for i, img in enumerate(images):
      rndpxls = np.reshape(img, (-1, img.shape[2])).copy()
      np.random.shuffle(rndpxls)
      images[i] = np.reshape(rndpxls, img.shape)
    self.data = images

  def gaussian(self):
    """
    Each pixel in the data is ind. sampled from a Gaussian dist. with mean and variance matching the original dataset's
    :return:
    """
    np.random.seed(12345) # sets the seed for the ensemble of generated pixels
    images = self.data
    print(images.shape)
    mean = np.mean(images, axis=(0,1,2))
    std = np.std(images, axis=(0,1,2))
    data = np.clip(np.random.multivariate_normal(mean=mean, cov=np.diag(std), size=images.shape[:-1]), 0, 255).astype(np.uint8)     # we need to explicitly cast the data as int otherwise pytorch will fail
    self.data = data

def transform_truncated(pca, X, n_components):
    X = pca._validate_data(X, dtype=[np.float64, np.float32], reset=False)
    if pca.mean_ is not None:
        X = X - pca.mean_
    X_transformed = np.dot(X, pca.components_[:n_components, :].T)
    #start = np.random.choice(range(0,len(pca.components_)-n_components))
    #X_transformed = np.dot(X, pca.components_[start:start+n_components, :].T)
    if pca.whiten:
        X_transformed /= np.sqrt(pca.explained_variance_)
    return X_transformed

def inv_transform(pca, X, n_components):
    return np.dot(X, pca.components_[:n_components, :]) + pca.mean_

def inv_forward_transform(pca, X, n_components):
    return inv_transform(
        pca, transform_truncated(pca, X, n_components), n_components
    )

def get_pca_transformed_fmnist(n_components, train=True):

  transform = torchvision.transforms.Compose([torchvision.transforms.ToTensor()])
  data = torchvision.datasets.FashionMNIST(Path() / "data", train=train, download=True, transform=transform)
  dataloader = torch.utils.data.DataLoader(dataset=data, batch_size=len(data))
  images_all, _ = next(iter(dataloader))
  # convert to 1D
  images_flat = images_all.reshape(images_all.shape[0], -1)

  pca = sklearn.decomposition.PCA(n_components=784)
  images_flat_hat = pca.inverse_transform(pca.fit_transform(images_flat))
  images_hat = inv_forward_transform(pca, X=images_flat, n_components=n_components).reshape(-1,28,28)

  return images_hat

def normalize_img(x):
    """
    Over all three channels; there should be numpy way
    :param x:
    :return:
    """
    if len(x.shape) == 2:
        try:
            x=x.numpy()
        except:
            pass
        x = (x - np.amin(x)) / (np.amax(x) - np.amin(x))
    elif len(x.shape) ==3:
        for i in range(3):
            x[...,i] = (x[...,i] - np.amin(x[...,i])) / (np.amax(x[...,i]) - np.amin(x[...,i]))
    return (255*x).astype(np.uint8)

def normalize_dataset(data):
    data_norm = np.array([normalize_img(data[i]) for i in range(len(data))])
    return data_norm

class MNISTDatasetNoise(datasets.MNIST):
    """CIFAR10 dataset, with support for random labels and pixels

    Params
    ------
    corrupt_prob: float
      Default 0.0. The probability of a label being replaced with
      random label.
    num_classes: int
      Default 10. The number of classes in the dataset.
    """

    def __init__(self, n_components=0, num_classes=10, **kwargs):
        super(MNISTDatasetNoise, self).__init__(**kwargs)
        self.n_classes = num_classes
        self.n_components = n_components
        self.train = kwargs['train']
        if n_components > 0:
            print("Adding dataset noise ...")
            self.add_fmnist(n_components)

    def add_fmnist(self, n_components):
        fmnist = get_pca_transformed_fmnist(self.n_components, self.train)
        norm_fmnist = normalize_dataset(fmnist)
        d = np.mean([norm_fmnist, normalize_dataset(self.data.numpy())], axis=0).astype(np.uint8)
        self.data = torch.from_numpy(d)


#!/usr/bin/env python

import numpy as np
import matplotlib.pyplot as plt
import matplotlib

import MITgcmutils as mit

plt.ion()

matplotlib.rcParams['ps.useafm'] = True
matplotlib.rcParams['pdf.use14corefonts'] = True
matplotlib.rcParams['text.usetex'] = True

dir0 = '../run/'

file1 = 'diagU*'
file2 = 'diagV*'
file3 = 'diagKEU*'
file4 = 'diagKEV*'
file5 = 'diagKEs*'

flag_uv = 5 # 1: u , 2: v, 3:KEu, 4:KEv, 5:KEE
flag_grid = 1

#%==================== LOAD FIELDS ===================================

# load grid
if flag_grid:
  XC    = mit.rdmds(dir0+'XC*')
  YC    = mit.rdmds(dir0+'YC*')
  XG    = mit.rdmds(dir0+'XG*')
  YG    = mit.rdmds(dir0+'YG*')
  DXC   = mit.rdmds(dir0+'DXC*')
  DYC   = mit.rdmds(dir0+'DYC*')
  hFacC = mit.rdmds(dir0+'hFacC*')
  hFacS = mit.rdmds(dir0+'hFacS*')
  hFacW = mit.rdmds(dir0+'hFacW*')
  RAS   = mit.rdmds(dir0+'RAS*')
  RAW   = mit.rdmds(dir0+'RAW*')
  RAC   = mit.rdmds(dir0+'RAC*')
  RAZ   = mit.rdmds(dir0+'RAZ*')
  RC    = mit.rdmds(dir0+'RC*')
  RF    = mit.rdmds(dir0+'RF*')
  DRC   = mit.rdmds(dir0+'DRC*')
  DRF   = mit.rdmds(dir0+'DRF*')
  Depth = mit.rdmds(dir0+'Depth*')

if flag_uv == 1:
  filer = file1
elif flag_uv == 2:
  filer = file2
elif flag_uv == 3:
  filer = file3
elif flag_uv == 4:
  filer = file4
elif flag_uv == 5:
  filer = file5

i = 4
iters1 = mit.mds.scanforfiles(dir0 + filer)

utot   = mit.rdmds(dir0 + filer,iters1[i],rec=0)
uadv   = mit.rdmds(dir0 + filer,iters1[i],rec=1)
upress = mit.rdmds(dir0 + filer,iters1[i],rec=2)
u_eta  = mit.rdmds(dir0 + filer,iters1[i],rec=3)
udissh = mit.rdmds(dir0 + filer,iters1[i],rec=4)
udissv = mit.rdmds(dir0 + filer,iters1[i],rec=5)
uext   = mit.rdmds(dir0 + filer,iters1[i],rec=6)
u_ab   = mit.rdmds(dir0 + filer,iters1[i],rec=7)

utot = utot/86400

si_z,si_y,si_x = utot.shape
ix = np.int(si_x/2)

def yzplot(psi,*args, **kwargs):

  vmax = np.max(np.abs((psi)))
  vmax = kwargs.get('vmax', vmax)
  vmin = -vmax
  psi = np.where(psi<vmin,vmin,psi)
  psi = np.where(psi>vmax,vmax,psi)

  title = kwargs.get('title',None)

  fgrid = kwargs.get('fgrid', 0)

  if fgrid:
    xx = YC[:,ix]*1e-3
    yy = RC[:,0,0]
  else:
    si_y,si_x = psi.shape
    xx = np.arange(si_x)
    yy = np.arange(si_y)

  plt.figure()
  plt.contourf(xx,yy,psi,100,cmap=plt.cm.seismic,vmin=vmin,vmax=vmax,extend='both')
  plt.colorbar(format='%.0e')
  plt.title(title)
  if fgrid:
    plt.xlabel('x (km)')
    plt.ylabel('z (m)')

psi = utot[:,:,ix]
yzplot(psi,title=r"tottend (m\,s$^{-2}$)",fgrid=flag_grid,vmax=1e-6)

psi = uadv + upress + udissv + udissh + u_eta + u_ab + uext
psi = psi[:,:,ix]
yzplot(psi,title=r"sum (m\,s$^{-2}$)",fgrid=flag_grid,vmax=1e-6)

error = np.abs(uadv) + np.abs(upress) + np.abs(udissv) + np.abs(udissh) + np.abs(u_eta) + np.abs(u_ab) + np.abs(uext)
error2 = error[:,:,ix]/np.abs(utot[:,:,ix])

psi3 = (psi - utot[:,:,ix])/utot[:,:,ix]/error2
yzplot(psi3,title=r"relative error (m\,s$^{-2}$)",fgrid=flag_grid,vmax=1e-7)


"""
Create a CHATGPT chatbot that can perform Q&A over Joplin notes.
Created on Jun 2023

@author: Dina Berenbaum
"""

import os
import glob

import weaviate

from langchain.document_loaders import UnstructuredMarkdownLoader
from langchain.embeddings import OpenAIEmbeddings
from langchain.indexes import VectorstoreIndexCreator
from langchain.indexes.vectorstore import VectorStoreIndexWrapper
from langchain.llms import AzureOpenAI
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.vectorstores import Weaviate

class JoplinChatbot:
    def __init__(self, sandbox_url, index_name='Joplin_test',
                 embedder=OpenAIEmbeddings,
                 splitter=RecursiveCharacterTextSplitter,
                 vectorstore=Weaviate, **kwargs):
        """
        Initializer for the JoplinChatbot class. Vectorestore and Embedder are needed both for adding new files to the
        datastore and also for the queries. The splitter is only needed for adding new files to the datastore.

        :param sandbox_url: URL for the sandbox.
        :param index_name: Index name for the chatbot.
        :param embedder: Class for the embedder (defaults to OpenAIEmbeddings).
        :param splitter: Class for the splitter (defaults to RecursiveCharacterTextSplitter).
        :param vectorstore: Class for the vectorstore (defaults to Weaviate).
        :param splitter_args: Additional arguments for the splitter class (optional).
        :param embedder_args: Additional arguments for the embedder class (optional).
        :param chain_type: Type of chain to use (defaults to "stuff").
        """
        self._index_name = index_name
        self._client = weaviate.Client(sandbox_url)

        # Chosen classes
        self._embedder_class = embedder
        self._splitter_class = splitter
        self._vectorstore_class = vectorstore

        # Initialize the modules
        self._embedding = self._get_embedding(kwargs.get("embedder_params", {}))
        self._vectorstore = self._get_vectorstore(kwargs.get("vectorstore_params", {}))
        self.db = VectorStoreIndexWrapper(vectorstore=self._vectorstore)

        self.chain_type = kwargs.get("chain_type", "stuff")  # 4 types of chains: stuff, map_reduce, refine, map_rerank

    def _read_markdown_files(self, path_to_dir):
        """
        Reads markdown files from a directory.

        :param path_to_dir: The directory to read markdown files from.
        :return: List of loaded markdown files.
        """
        markdown_files = glob.glob(os.path.join(path_to_dir, "*.md"))
        return [UnstructuredMarkdownLoader(f).load()[0] for f in markdown_files]

    def _get_splitter(self, splitter_args):
        """
        Returns a splitter instance.

        :param splitter_args: Arguments for the splitter class.
        :return: An instance of a splitter.
        """
        return self._splitter_class(**splitter_args)

    def _get_embedding(self, embedder_args):
        """
        Returns an embedder instance.

        :param embedder_args: Arguments for the embedder class.
        :return: An instance of an embedder.
        """
        return self._embedder_class(**embedder_args)

    def _get_vectorstore(self, vectorstore_args):
        """
        Returns a vectorstore instance.

        :return: An instance of a vectorstore.
        """
        return self._vectorstore_class(self._client, index_name=self._index_name, text_key='text',
                                       embedding=self._embedding,
                                       by_text=False, **vectorstore_args)

    def index_new_files(self, path_to_dir, splitter_args=None):
        """
        Indexes new files.

        :param path_to_dir: Directory containing the new files.
        :param splitter_args: Additional arguments for the splitter (optional).
        """
        documents = self._read_markdown_files(path_to_dir)
        splitter = self._get_splitter(splitter_args if splitter_args else {})

        # currently for the from_text method of Weaviate you need to pass the
        # arguments outside of the initialized vectorestore, so it means you pass them twice :/
        vectorstore_kwargs = {"client": self._client, "index_name": self._index_name}

        self.db = VectorstoreIndexCreator(embedding=self._embedding, vectorstore_cls=self._vectorstore_class,
                                          text_splitter=splitter, vectorstore_kwargs=vectorstore_kwargs).from_documents(
            documents)

    def query(self, text):
        """
        Queries the database.

        :param text: The query text.
        :return: The result of the query.
        """
        deployment_name = os.environ.get('OPENAI_DEPLOYMENT')  # todo: make sure this deployment is really running
        llm = AzureOpenAI(deployment_name=deployment_name)
        return self.db.query(text, chain_type=self.chain_type, llm=llm)


# -*- coding: utf-8 -*-
import scrapy
import requests
import os

class TukuspiderSpider(scrapy.Spider):
    name = 'tukuspider'
    allowed_domains = ['www.tuku.cc']
    custom_settings = {
        "USER_AGENT": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",
    }
    input = input('comic number:')
    try:
        start_urls = ['http://www.tuku.cc/comic/'+str(input)]
    except IOError :
        print('wrong comic number')

    def parse(self, response):
        list = response.css('div[id="chapterlistload"]').css('a::attr(href)').extract()
        urls = ['http://www.tuku.cc'+l for l in list]
        #print(urls)
        for url in urls:
            #print(url)
            yield scrapy.Request(url, meta={'chapter': url.split('/')[-2], 'page': 0}, callback=self.parse_chapter_contents)

    def parse_chapter_contents(self, response):
        print(response.url)

        img_url = response.css('img[id="cp_image"]::attr(src)').extract()[0]
        page = response.meta['page']
        chapter = 'qilongzhu/'+response.meta['chapter']
        #print(next)
        #print(img_url)
        if not os.path.exists(chapter):
            os.mkdir(chapter)

        if requests.get(img_url).status_code == 200:
            with open(os.path.join(chapter, str(page)+'.jpg'), 'wb') as imgf:
                imgf.write(requests.get(img_url).content)
            next = 'http://www.tuku.cc' + response.css('a[href*="/p"]::attr(href)').extract()[-1]
            yield scrapy.Request(next, meta={'chapter': response.meta['chapter'], 'page': page + 1},
                                   callback=self.parse_chapter_contents)

        else:
            print('chapter download finished: ' + response.meta['chapter'])
        #print('download finished ' + response.meta['title'])


import pygame
from Block import Block
from settings import *
from typing import Tuple

class Sudoku:

    board = [
        [7, 8, 0, 4, 0, 0, 1, 2, 0],
        [6, 0, 0, 0, 7, 5, 0, 0, 9],
        [0, 0, 0, 6, 0, 1, 0, 7, 8],
        [0, 0, 7, 0, 4, 0, 2, 6, 0],
        [0, 0, 1, 0, 5, 0, 9, 3, 0],
        [9, 0, 4, 0, 6, 0, 0, 0, 5],
        [0, 7, 0, 3, 0, 0, 0, 1, 2],
        [1, 2, 0, 0, 0, 7, 4, 0, 0],
        [0, 4, 9, 2, 0, 6, 0, 0, 7]
    ]

    def __init__(self):
        self.width = WIDTH
        self.height = HEIGHT
        self.sudoku_width = SUDOKU_WIDTH
        self.sudoku_height = SUDOKU_HEIGHT
        self.x_delta = X_DELTA
        self.y_delta = Y_DELTA
        self.size = SIZE
        self.grid = None
        self.win = None
        self.sudokuWin = None
        self.clock = None
        self.fps = FPS
        self.selected = None
        self.font = None

    def display_init(self):

        pygame.init()
        pygame.font.init()

        self.win = pygame.display.set_mode((self.width, self.height))
        pygame.display.set_caption("Sudoku Solver")
        self.win.fill(BACKGROUND_COLOR)

        self.sudokuWin = self.win.subsurface((self.x_delta, self.y_delta, self.sudoku_width, self.sudoku_width))

        self.clock = pygame.time.Clock()

        self.font = pygame.font.SysFont("comicsansms", 40)
        self.title = self.font.render('Sudoku solver', 1, MID_BLACK)
        w, h = self.title.get_size()

        self.win.blit(self.title, ((self.width - w) // 2, (self.y_delta - h) // 2))
        pygame.display.update()

    def grid_init(self):
        self.grid = list()
        for row in range(9):
            self.grid.append(list())
            for col in range(9):
                self.grid[row].append(Block(row, col))

        for i, row in enumerate(self.board):
            for j, element in enumerate(row):
                self.grid[i][j].set_number(element)
                if element != 0:
                    self.grid[i][j].make_readonly()

    def get_row_col(self, pos: Tuple) -> Tuple:
        x, y = pos
        row = (y - self.y_delta) // self.size
        col = (x - self.x_delta) // self.size

        return row, col

    def is_valid_dims(self, row: int, col: int) -> bool:
        return row in range(9) and col in range(9)

    def draw_grid(self, win: pygame.Surface) -> None:
        width, height = win.get_size()
        color = MID_BLACK

        # pygame.draw.line(win, color, (0, 0), (width, 0), 2)
        # pygame.draw.line(win, color, (0, 0), (0, height), 2)

        for i in range(9):
            linewidth = 2 if i % 3 == 0 else 1

            pygame.draw.line(win, color, (0, i * self.size), (width, i * self.size), linewidth)
            pygame.draw.line(win, color, (i * self.size, 0), (i * self.size, height), linewidth)

        pygame.draw.line(win, color, (width - 2, 0), (width - 2, height), 2)
        pygame.draw.line(win, color, (0, height - 2), (width, height - 2), 2)

    def draw_board(self, win: pygame.Surface) -> None:
        if self.grid is None:
            return

        for row in self.grid:
            for block in row:
                block.draw(win)

    def verify_temp(self, row: int, col: int, num: int) -> bool:
        if self.selected:
            # row, col = self.selected.get_dims()
            # num = self.selected.get_number()

            for i in range(9):
                if self.grid[row][i] == num and i != col:
                    return False

                if self.grid[i][col] == num and i != row:
                    return False

            boxRow = row // 3
            boxCol = col // 3

            for i in range(3 * boxRow, 3 * boxRow + 3):
                for j in range(3 * boxCol, 3 * boxCol + 3):
                    if self.grid[i][j] == num and i != row and j != col:
                        return False

            return True

        return False

    def solve_gui(self):
        pos = self.find_empty()

        if pos is None:
            return True

        row, col = pos

        for i in range(1, 10):
            if self.verify_temp(row, col, i):
                self.grid[row][col].set_number(i)
                self.grid[row][col].make_readonly()
                self.draw()
                pygame.time.delay(100)

                if self.solve_gui():
                    return True

                self.grid[row][col].set_number(0)
                self.grid[row][col].remove_readonly()
                self.grid[row][col].remove_temp()
                self.draw()
                pygame.time.delay(100)

        return False

    def find_empty(self) -> Tuple:
        for row in self.grid:
            for block in row:
                if block.is_empty():
                    row, col = block.get_dims()
                    return row, col

        return None

    def draw(self):
        self.sudokuWin.fill(CREAM)

        self.draw_grid(self.sudokuWin)
        self.draw_board(self.sudokuWin)

        pygame.display.update()

    def quit(self):
        pygame.font.quit()
        pygame.quit()

    def run(self):

        self.display_init()
        self.grid_init()

        run = True
        while run:
            self.clock.tick(self.fps)
            for event in pygame.event.get():

                if event.type == pygame.QUIT:
                    run = False

                if event.type == pygame.KEYDOWN:

                    if event.key in (pygame.K_1, pygame.K_KP1) and self.selected:
                        self.selected.set_temp(1)

                    if event.key in (pygame.K_2, pygame.K_KP2) and self.selected:
                        self.selected.set_temp(2)

                    if event.key in (pygame.K_3, pygame.K_KP3) and self.selected:
                        self.selected.set_temp(3)

                    if event.key in (pygame.K_4, pygame.K_KP4) and self.selected:
                        self.selected.set_temp(4)

                    if event.key in (pygame.K_5, pygame.K_KP5) and self.selected:
                        self.selected.set_temp(5)

                    if event.key in (pygame.K_6, pygame.K_KP6) and self.selected:
                        self.selected.set_temp(6)

                    if event.key in (pygame.K_7, pygame.K_KP7) and self.selected:
                        self.selected.set_temp(7)

                    if event.key in (pygame.K_8, pygame.K_KP8) and self.selected:
                        self.selected.set_temp(8)

                    if event.key in (pygame.K_9, pygame.K_KP9) and self.selected:
                        self.selected.set_temp(9)

                    if event.key == pygame.K_ESCAPE:
                        if self.selected:
                            self.selected.remove_temp()

                    if event.key in (pygame.K_RETURN, pygame.K_KP_ENTER):
                        if self.selected:
                            row, col = self.selected.get_dims()
                            num = self.selected.get_number()
                            if self.verify_temp(row, col, num):
                                self.selected.set_valid()

                    if event.key == pygame.K_SPACE:
                        self.solve_gui()

                if event.type == pygame.MOUSEBUTTONDOWN:
                    pos = pygame.mouse.get_pos()
                    row, col = self.get_row_col(pos)

                    if self.is_valid_dims(row, col):
                        if self.selected:
                            self.selected.deselect()
                            self.selected = None

                        if not self.grid[row][col].is_readonly():
                            self.selected = self.grid[row][col]
                            self.selected.select()

                    print(row, col, sep='\t')

            self.draw()

        self.quit()

if __name__ == "__main__":
    X = Sudoku()
    X.run()


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from util import get_data100k,read_process,get_edgelist,pa_index
import networkx as nx
from networkx.algorithms import bipartite

import numpy as np
import pandas as pd
import time
from collections import deque

import tensorflow as tf
from six import next
from sklearn import preprocessing
import sys
from scipy.sparse import lil_matrix
from scipy.sparse import coo_matrix

BATCH_SIZE = 1000
USER_NUM = 943
ITEM_NUM = 1682
df_train, df_test = get_data100k()
#df_train, df_test = get_data1m()
##preferential attachment
paUsers,paItems=pa_index(df_train,df_test)
################laplacian
la_movie,la_user,norm_la_movie,norm_la_user=laplacian_graph(df_train,df_test)

img=pd.read_csv('./data/cnn_100k.csv')
del img['Unnamed: 0']
img['path']-=1
img=img.set_index(np.arange(len(img)))
img_ma = np.zeros((ITEM_NUM,ITEM_NUM), dtype=np.float32) #np.asarray([[0 for x in range(USER_NUM)] for y in range(ITEM_NUM)],dtype=np.float16)
for index, row in img.iterrows():
    itemid=int(row['path'])
    img_ma[itemid]=row[1:]

AdjacencyUsers = np.zeros((USER_NUM,ITEM_NUM), dtype=np.float32) #np.asarray([[0 for x in range(ITEM_NUM)] for y in range(USER_NUM)],dtype=np.float16)
DegreeUsers_norm = np.zeros((USER_NUM,USER_NUM), dtype=np.float32)# np.asarray([[0 for x in range(1)] for y in range(USER_NUM)],dtype=np.float16)
AdjacencyItems = np.zeros((ITEM_NUM,USER_NUM), dtype=np.float32) #np.asarray([[0 for x in range(USER_NUM)] for y in range(ITEM_NUM)],dtype=np.float16)
DegreeItems_norm =  np.zeros((ITEM_NUM,ITEM_NUM), dtype=np.float32) #np.asarray([[0 for x in range(1)] for y in range(ITEM_NUM)],dtype=np.float16)
for index, row in df_train.iterrows():
    userid=int(row['user'])
    itemid=int(row['item'])
    AdjacencyUsers[userid][itemid]=row['rate']/5
    AdjacencyItems[itemid][userid]=row['rate']/5
    DegreeUsers_norm[userid][userid]+=1
    DegreeItems_norm[itemid][itemid]+=1

for i in range(len(DegreeUsers_norm)):
    for j in range(len(DegreeUsers_norm[i])):
        if DegreeUsers_norm[i][j] !=0:
            DegreeUsers_norm[i][j]=DegreeUsers_norm[i][j]**-0.5
            break

for i in range(len(DegreeItems_norm)):
    for j in range(len(DegreeItems_norm[i])):
        if DegreeItems_norm[i][j] !=0:
            DegreeItems_norm[i][j]=DegreeItems_norm[i][j]**-0.5
            break

class ShuffleIterator(object):

    def __init__(self, inputs, batch_size=10):
        self.inputs = inputs
        self.batch_size = batch_size
        self.num_cols = len(self.inputs)
        self.len = len(self.inputs[0])
        self.inputs = np.transpose(np.vstack([np.array(self.inputs[i]) for i in range(self.num_cols)]))

    def __len__(self):
        return self.len

    def __iter__(self):
        return self

    def __next__(self):
        return self.next()

    def next(self):
        ids = np.random.randint(0, self.len, (self.batch_size,))
        out = self.inputs[ids, :]
        return [out[:, i] for i in range(self.num_cols)]

class OneEpochIterator(ShuffleIterator):
    def __init__(self, inputs, batch_size=10):
        super(OneEpochIterator, self).__init__(inputs, batch_size=batch_size)
        if batch_size > 0:
            self.idx_group = np.array_split(np.arange(self.len), np.ceil(self.len / batch_size))
        else:
            self.idx_group = [np.arange(self.len)]
        self.group_id = 0

    def next(self):
        if self.group_id >= len(self.idx_group):
            self.group_id = 0
            raise StopIteration
        out = self.inputs[self.idx_group[self.group_id], :]
        self.group_id += 1
        return [out[:, i] for i in range(self.num_cols)]

def inferenceDense(phase,user_batch, item_batch,idx_user,idx_item, user_num, item_num,UReg=0.05,IReg=0.1):

    user_batch = tf.nn.embedding_lookup(idx_user, user_batch, name="embedding_user")
    item_batch = tf.nn.embedding_lookup(idx_item, item_batch, name="embedding_item")

    ul1mf=tf.layers.dense(inputs=user_batch, units=MFSIZE,activation=tf.nn.relu, kernel_initializer=tf.random_normal_initializer(stddev=0.01))
    il1mf=tf.layers.dense(inputs=item_batch, units=MFSIZE,activation=tf.nn.relu, kernel_initializer=tf.random_normal_initializer(stddev=0.01))
    InferInputMF=tf.multiply(ul1mf, il1mf)

    infer=tf.reduce_sum(InferInputMF, 1, name="inference")

    regularizer = tf.add(UW*tf.nn.l2_loss(ul1mf), IW*tf.nn.l2_loss(il1mf), name="regularizer")

    return infer, regularizer

def optimization(infer, regularizer, rate_batch, learning_rate=0.0005, reg=0.1):

    global_step = tf.train.get_global_step()
    assert global_step is not None
    cost_l2 = tf.nn.l2_loss(tf.subtract(infer, rate_batch))
    cost = tf.add(cost_l2, regularizer)
    train_op = tf.train.AdamOptimizer(learning_rate).minimize(cost, global_step=global_step)
    return cost, train_op

def clip(x):
    return np.clip(x, 1.0, 5.0)

def GraphRec_image(train, test,ver, Dataset='100k'):
    AdjacencyUsers = np.zeros((USER_NUM,ITEM_NUM), dtype=np.float32) #np.asarray([[0 for x in range(ITEM_NUM)] for y in range(USER_NUM)],dtype=np.float16)
    DegreeUsers = np.zeros((USER_NUM,1), dtype=np.float32)# np.asarray([[0 for x in range(1)] for y in range(USER_NUM)],dtype=np.float16)

    AdjacencyItems = np.zeros((ITEM_NUM,USER_NUM), dtype=np.float32) #np.asarray([[0 for x in range(USER_NUM)] for y in range(ITEM_NUM)],dtype=np.float16)
    DegreeItems =  np.zeros((ITEM_NUM,1), dtype=np.float32) #np.asarray([[0 for x in range(1)] for y in range(ITEM_NUM)],dtype=np.float16)
    for index, row in train.iterrows():
      userid=int(row['user'])
      itemid=int(row['item'])
      AdjacencyUsers[userid][itemid]=row['rate']/5.0
      AdjacencyItems[itemid][userid]=row['rate']/5.0
      DegreeUsers[userid][0]+=1
      DegreeItems[itemid][0]+=1

    DUserMax=np.amax(DegreeUsers)
    DItemMax=np.amax(DegreeItems)
    DegreeUsers=np.true_divide(DegreeUsers, DUserMax)
    DegreeItems=np.true_divide(DegreeItems, DItemMax)

    AdjacencyUsers=np.asarray(AdjacencyUsers,dtype=np.float32)
    AdjacencyItems=np.asarray(AdjacencyItems,dtype=np.float32)

    if ver=='ver1':
        UserFeatures= np.concatenate((np.identity(USER_NUM,dtype=np.bool_), AdjacencyUsers), axis=1)
        ItemFeatures= np.concatenate((np.identity(ITEM_NUM,dtype=np.bool_), AdjacencyItems,img_ma), axis=1)
    if ver=='ver2':
        UserFeatures= np.concatenate((np.identity(USER_NUM,dtype=np.bool_), AdjacencyUsers,DegreeUsers), axis=1)
        ItemFeatures= np.concatenate((np.identity(ITEM_NUM,dtype=np.bool_), AdjacencyItems,DegreeItems,img_ma), axis=1)
    if ver=='ver3':
        UserFeatures= np.concatenate((np.identity(USER_NUM,dtype=np.bool_), AdjacencyUsers,DegreeUsers,paUsers), axis=1)
        ItemFeatures= np.concatenate((np.identity(ITEM_NUM,dtype=np.bool_), AdjacencyItems,DegreeItems,paItems_img_ma), axis=1)
    if ver=='ver4':
        UserFeatures= np.concatenate((np.identity(USER_NUM,dtype=np.bool_), AdjacencyUsers,DegreeUsers_norm,paUsers), axis=1)
        ItemFeatures= np.concatenate((np.identity(ITEM_NUM,dtype=np.bool_), AdjacencyItems,DegreeItems_norm,paItems_img_ma), axis=1)
    if ver=='ver5':
        UserFeatures= np.concatenate((np.identity(USER_NUM,dtype=np.bool_), AdjacencyUsers,norm_la_user,paUsers), axis=1)
        ItemFeatures= np.concatenate((np.identity(ITEM_NUM,dtype=np.bool_), AdjacencyItems,norm_la_movie,paItems,img_ma), axis=1)
    if ver=='ver6':
        UserFeatures= np.concatenate((np.identity(USER_NUM,dtype=np.bool_), AdjacencyUsers,la_user,paUsers), axis=1)
        ItemFeatures= np.concatenate((np.identity(ITEM_NUM,dtype=np.bool_), AdjacencyItems,la_movie,paItems,img_ma), axis=1)

    UserFeaturesLength=UserFeatures.shape[1]
    ItemFeaturesLength=ItemFeatures.shape[1]

    samples_per_batch = len(train) // BATCH_SIZE
    iter_train = ShuffleIterator([train["user"],train["item"],train["rate"]],batch_size=BATCH_SIZE)
    iter_test = OneEpochIterator([test["user"],test["item"],test["rate"]],batch_size=10000)

    user_batch = tf.placeholder(tf.int32, shape=[None], name="id_user")
    item_batch = tf.placeholder(tf.int32, shape=[None], name="id_item")
    rate_batch = tf.placeholder(tf.float64, shape=[None])
    phase = tf.placeholder(tf.bool, name='phase')

    w_user = tf.constant(UserFeatures,name="userids", shape=[USER_NUM, UserFeatures.shape[1]],dtype=tf.float64)
    w_item = tf.constant(ItemFeatures,name="itemids", shape=[ITEM_NUM, ItemFeatures.shape[1]],dtype=tf.float64)

    infer, regularizer = inferenceDense(phase,user_batch, item_batch,w_user,w_item, user_num=USER_NUM, item_num=ITEM_NUM)
    global_step = tf.contrib.framework.get_or_create_global_step()
    _, train_op = optimization(infer, regularizer, rate_batch, learning_rate=LR, reg=0.09)

    init_op = tf.global_variables_initializer()
    config = tf.ConfigProto()
    config.gpu_options.per_process_gpu_memory_fraction = 0.5
    finalerror=-1
    with tf.Session(config=config) as sess:
        sess.run(init_op)
        print("{} {} {} {}".format("epoch", "train_error", "val_error", "elapsed_time"))
        errors = deque(maxlen=samples_per_batch)
        start = time.time()
        for i in range(EPOCH_MAX * samples_per_batch):
            #users, items, rates,y,m,d,dw,dy,w = next(iter_train)
            users, items, rates = next(iter_train)
            _, pred_batch = sess.run([train_op, infer], feed_dict={user_batch: users,
                                                                   item_batch: items,
                                                                   rate_batch: rates,
                                                                   phase:True})
            pred_batch = clip(pred_batch)
            errors.append(np.power(pred_batch - rates, 2))
            if i % samples_per_batch == 0:
                train_err = np.sqrt(np.mean(errors))
                test_err2 = np.array([])
                degreelist=list()
                predlist=list()
                for users, items, rates in iter_test:
                    pred_batch = sess.run(infer, feed_dict={user_batch: users,
                                                            item_batch: items,
                                                            phase:False})

                    pred_batch = clip(pred_batch)
                    test_err2 = np.append(test_err2, np.power(pred_batch - rates, 2))
                end = time.time()
                test_err = np.sqrt(np.mean(test_err2))
                finalerror=test_err
                print("{:3d},{:f},{:f},{:f}(s)".format(i // samples_per_batch, train_err, test_err, end - start))
                start = end

MFSIZE=50
UW=0.05
IW=0.02
LR=0.00003
EPOCH_MAX = 300
tf.reset_default_graph()
GraphRec_image(df_train, df_test,ver='ver1',Dataset='100k')


import hashlib
import json
from http import HTTPStatus

from fastapi import Request
from fastapi.param_functions import Query
from lnurl.types import LnurlPayMetadata
from fastapi.exceptions import HTTPException
from fastapi.responses import HTMLResponse

from lnbits.core.services import create_invoice

from . import donations_ext
from .crud import get_donation

@donations_ext.get(
    "/lnurl/{donation_id}", response_class=HTMLResponse, name="donation.lnurl_response"
)
async def lnurl_response(req: Request, donation_id: str):
    donation = await get_donation(donation_id)
    if not donation:
        raise HTTPException(
            status_code=HTTPStatus.NOT_FOUND, detail="Donation not found"
        )

    payResponse = {
        "tag": "payRequest",
        "callback": req.url_for("donation.lnurl_callback", donation_id=donation_id),
        "metadata": LnurlPayMetadata(json.dumps([["text/plain", str(donation.title)]])),
        "maxSendable": 500000000,
        "minSendable": 10000,
    }
    return json.dumps(payResponse)

@donations_ext.get(
    "/lnurl/cb/{donation_id}", response_class=HTMLResponse, name="donation.lnurl_callback"
)
async def lnurl_callback(
    donation_id: str, amount: str = Query(None)
):
    donation = await get_donation(donation_id)
    if not donation:
        raise HTTPException(
            status_code=HTTPStatus.NOT_FOUND, detail="Donation not found"
        )
    amount_received = int(amount)

    if amount_received < 10000:
        raise HTTPException(
            status_code=HTTPStatus.FORBIDDEN,
            detail="Amount {round(amount_received / 1000)} is smaller than minimum 10 sats.",
        )
    elif amount_received / 1000 > 500000000:
        raise HTTPException(
            status_code=HTTPStatus.FORBIDDEN,
            detail="Amount {round(amount_received / 1000)} is greater than maximum 500000.",
        )
    _, payment_request = await create_invoice(
        wallet_id=donation.wallet,
        amount=int(amount_received / 1000),
        memo=donation.title,
        unhashed_description=(
            LnurlPayMetadata(json.dumps([["text/plain", str(donation.title)]]))
        ).encode(),
        extra={"tag": "donations", "donationsId": donation_id},
    )
    payResponse = {"pr": payment_request, "routes": []}
    return json.dumps(payResponse)


# CoC Discord Bot.
# Mainly created to improve API/Discord Bot knowledge. Improving python alongside.

# imports for coc scripts
import coc
import os

# imports for discord bot
from discord.ext import commands

# imports for data handling and gathering
import pandas as pd
from datetime import datetime, timedelta
from dotenv import load_dotenv

# Login to CoC and init bot
load_dotenv()
client = coc.login(
    os.getenv("EMAIL"),
    os.getenv("PASS"),
    client=coc.EventsClient
)
bot = commands.Bot(command_prefix="?")

# Load Variables and Data Structures
save_file = os.path.dirname(os.path.realpath(__file__)) + "\PlayerData.xlsx"
# Dataframe Structure: Columns = PlayerTag. Index 0-23 = 24hr time. 24 = last update time
data = pd.read_excel(save_file)
clanTag = '#J9R8RR20'
memberTag = []
channel_id = 865440614643269644

# This function is called once when starting the program and updates the dataframe with members that
# have left or been added. It backs up the saved data before performing operations.
async def updateData():
    # backup
    backupData()
    # get member tags in clan
    for member in await client.get_members(clanTag):
        memberTag.append(member.tag)
    # remove left members
    for tag in data.columns:
        if tag not in memberTag:
            print(f'Need to Remove {tag}')
            data.drop([tag], axis=1, inplace=True)
    # add new members
    for tag in memberTag:
        if tag not in data.columns:
            data[tag] = [None] * 26
            print(f'Added {tag}')
    # prep clan and player updates
    client.add_clan_updates(clanTag)
    mem_list = await client.get_members(clanTag)
    for m in mem_list:
        client.add_player_updates(m.tag)
    # signal done
    print("Loaded PlayerTag updates")

# When pinged with a tag, it will check data to see when last updated. If not within 15min it will reupdate the tag
def ping(name):
    global data
    if isRecentlyUpdated(name):
        return
    time = datetime.now()
    # if not init
    if data.isnull().at[time.hour, name]:
        data.at[time.hour, name] = 0
    data.at[time.hour, name] += 1
    await saveData()

# When given name checks if update happened in last 15 min (true) or not (false)
def isRecentlyUpdated(name):
    time = datetime.now()
    # if null then never updated, update time and return true
    if data.isnull().at[24, name]:
        data.at[24, name] = time
        return False
    # parse time
    last_update = datetime.strptime(str(data.at[24, name]), '%Y-%m-%d %H:%M:%S.%f')
    # check if in last 15 min
    if last_update + timedelta(minutes=15) > time:
        return True
    # if not then set new update return false
    data.at[24, name] = time
    return False

# Saves Dataframe to file
async def saveData():
    data.to_excel(save_file, sheet_name='Player Ping Online')
    await bot.get_channel(channel_id).send("Saved Data")

# Backs data up
def backupData():
    data.to_excel(os.path.dirname(os.path.realpath(__file__)) + "\PlayerDataBackup.xlsx", sheet_name='yep')
    print("Backed up")

# This event handler detects when a player donates and pings tag
@client.event
@coc.PlayerEvents.donations()
async def playerDonated(oldMem, newMem):
    await bot.get_channel(channel_id).send(f'{newMem.tag}({newMem}) donated at {datetime.now()}')
    ping(newMem.tag)

# This event handler detects when player attacks and pings tag
@client.event
@coc.PlayerEvents.versus_trophies()
async def playerVersusAttack(oldMem, newMem):
    await bot.get_channel(channel_id).send(f'{newMem.tag}({newMem}) versus battle at {datetime.now()}')
    ping(newMem.tag)

# Removes tag and data when player leaves
@client.event
@coc.ClanEvents.member_leave()
async def playerLeave(member, clan):
    data.drop([member.tag], axis=1, inplace=True)
    client.remove_player_updates(member)
    await bot.get_channel(channel_id).send(f'Removed {member.tag}({member})')

# When player joins while running
@client.event
@coc.ClanEvents.member_join()
async def playerJoin(member, clan):
    data[member.tag] = []
    client.add_player_updates(member)
    await bot.get_channel(channel_id).send(f'Added {member.tag}({member})')

# Bot commands

@bot.command(name="listp", help='Lists updating player tags')
async def list_tags(ctx):
    await ctx.send(memberTag)

@bot.command(name='data', help="Returns data of given tag")
async def print_data(ctx, tag):
    await ctx.send(data[tag])

@bot.command(name='save', help='Saves current data')
async def savecmd():
    await saveData()

@bot.command(name='pinfo', help='Returns player info from tag')
async def pinfo(ctx, tag):
    await bot.get_channel(channel_id).send(await client.get_player(tag))

@bot.command(name='tagof', help='Gets tag of name from clan')
async def ptag(ctx, input_name):
    for member in await client.get_members(clanTag):
        if member.name == input_name:
            await bot.get_channel(channel_id).send(member.tag)

# Run scripts and bot
# client.loop.run_forever()
client.loop.run_until_complete(updateData())
bot.run(os.getenv("DISCORD_TOKEN"))


import pandas as pd
from nltk.tokenize import NLTKWordTokenizer
from nltk.stem import WordNetLemmatizer
import nltk
from nltk.corpus import wordnet
from nltk.corpus import stopwords
import json
import string
from math import floor

# naming convention: preprocessing_<W|C>[L][S][P][H<cutoff>]
#   W: with words
#   C: with chunks
#   L: with lemmatization
#   S: with stopwords
#   P: with punctuation
#   H: with highpass

def preprocessing_W(path):
    """Return preprocessed dataframe

    Parameters
    ----------
    path : str
        path of data

    Returns
    -------
    pd.DataFrame
        Preprocessed Dataframe. Data columns are as follows:

        ======  ==============================================================
        id      id of document, e.g. "id18154" (as `str`)
        token   a token in the dataset, e.g. "the" (as `list(str)`)
        author  author of the document that this token belongs to (as `str`)
        ======  ==============================================================
    """
    df = _load(path)
    df.token = _lower(df.token)
    df.token = _tokenize(df.token)
    df.token = _remove_punctuation(df.token)
    df.token = _remove_stopwords(df.token)
    return df

def preprocessing_L(path):
    df = _load(path)
    df.token = _lower(df.token)
    df.token = _tokenize(df.token)
    df.token = _lemmatize(df.token)
    df.token = _remove_punctuation(df.token)
    df.token = _remove_stopwords(df.token)
    return df

def preprocessing_SP(path):
    df = _load(path)
    df.token = _lower(df.token)
    df.token = _tokenize(df.token)
    return df

def preprocessing_SPC(path):
    df = _load(path)
    df.token = _lower(df.token)
    df.token = _tokenize(df.token)
    df.token = _chunks(df.token)
    return df

def preprocessing_C(path):
    df = _load(path)
    df.token = _lower(df.token)
    df.token = _tokenize(df.token)
    df.token = _remove_punctuation(df.token)
    df.token = _remove_stopwords(df.token)
    df.token = _chunks(df.token)
    return df

def preprocessing_SPH(path, cutoff):
    df = _load(path)
    df.token = _lower(df.token)
    df.token = _tokenize(df.token)
    df.token = _high_pass(df.token, cutoff)
    return df

def preprocessing_SPHC(path, cutoff):
    df = _load(path)
    df.token = _lower(df.token)
    df.token = _tokenize(df.token)
    df.token = _chunks(df.token)
    df.token = _high_pass(df.token, cutoff)
    return df

# utilitiy functions to facilitate preprocessing

def _load(path):
    file =  open(path, "r")
    data = json.load(file)
    return pd.DataFrame(data, columns=['id', 'token', 'author'])

def _lower(docs: pd.Series):
    return docs.apply(lambda s: s.lower())

def _tokenize(docs: pd.Series):
    return docs.apply(lambda s: [s[start:end] for start, end in NLTKWordTokenizer().span_tokenize(s)])

def _lemmatize(docs: pd.Series):
    def pos_tagger(tag):
        if tag.startswith("J"):
            return wordnet.ADJ
        elif tag.startswith("V"):
            return wordnet.VERB
        elif tag.startswith("N"):
            return wordnet.NOUN
        elif tag.startswith("R"):
            return wordnet.ADV
        else:
            return None

    lemm = WordNetLemmatizer()
    return docs.apply(lambda doc: nltk.pos_tag(doc)) \
        .apply(lambda doc: list(map(lambda x: (x[0], pos_tagger(x[1])), doc))) \
        .apply(lambda doc: list(map(lambda x: x[0] if x[1] == None else lemm.lemmatize(x[0], x[1]), doc)))

def _remove_stopwords(docs: pd.Series):
    stop_words = set(stopwords.words('english'))
    return docs.apply(lambda doc: [w for w in doc if not w in stop_words])

def _remove_punctuation(docs: pd.Series):
    return docs.apply(lambda doc: [w for w in doc if not w in string.punctuation])

def _high_pass(docs: pd.Series, cutoff: float):
    tokens = docs.explode()
    dist = nltk.FreqDist(tokens)
    n = len(tokens)
    return docs.apply(lambda doc: [w for w in doc if dist[w] > n * cutoff])

def _chunks(doc: pd.Series):
    joined = doc.apply(lambda doc: "".join(doc))
    return joined.apply(lambda doc: ["".join([doc[3*i], doc[3*i+1], doc[3*i+2]]) for i in range(floor(len(doc)/3))])

if __name__ == "__main__":
    for path in ["new_train.json", "new_test.json"]:
        preprocessing_W(path).to_json(f'preprocessing_output/preprocessed_{"train" if "train" in path else "test"}_W.json', orient='records')
        preprocessing_L(path).to_json(f'preprocessing_output/preprocessed_{"train" if "train" in path else "test"}_WL.json', orient='records')
        preprocessing_SP(path).to_json(f'preprocessing_output/preprocessed_{"train" if "train" in path else "test"}_WSP.json', orient='records')
        preprocessing_SPC(path).to_json(f'preprocessing_output/preprocessed_{"train" if "train" in path else "test"}_CSP.json', orient='records')
        preprocessing_C(path).to_json(f'preprocessing_output/preprocessed_{"train" if "train" in path else "test"}_C.json', orient='records')
        preprocessing_SPH(path, 5e-4).to_json(f'preprocessing_output/preprocessed_{"train" if "train" in path else "test"}_WSPH5e-4.json', orient='records')
        preprocessing_SPHC(path, 5e-4).to_json(f'preprocessing_output/preprocessed_{"train" if "train" in path else "test"}_CSPH5e-4.json', orient='records')
        preprocessing_SPH(path, 1e-4).to_json(f'preprocessing_output/preprocessed_{"train" if "train" in path else "test"}_WSPH1e-4.json', orient='records')
        preprocessing_SPHC(path, 1e-4).to_json(f'preprocessing_output/preprocessed_{"train" if "train" in path else "test"}_CSPH1e-4.json', orient='records')
        preprocessing_SPH(path, 5e-5).to_json(f'preprocessing_output/preprocessed_{"train" if "train" in path else "test"}_WSPH5e-5.json', orient='records')
        preprocessing_SPHC(path, 5e-5).to_json(f'preprocessing_output/preprocessed_{"train" if "train" in path else "test"}_CSPH5e-5.json', orient='records')
        preprocessing_SPH(path, 1e-5).to_json(f'preprocessing_output/preprocessed_{"train" if "train" in path else "test"}_WSPH1e-5.json', orient='records')
        preprocessing_SPHC(path, 1e-5).to_json(f'preprocessing_output/preprocessed_{"train" if "train" in path else "test"}_CSPH1e-5.json', orient='records')
        preprocessing_SPH(path, 1e-6).to_json(f'preprocessing_output/preprocessed_{"train" if "train" in path else "test"}_WSPH1e-6.json', orient='records')
        preprocessing_SPHC(path, 1e-6).to_json(f'preprocessing_output/preprocessed_{"train" if "train" in path else "test"}_CSPH1e-6.json', orient='records')


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('triumph_app', '0003_auto_20171115_0403'),
    ]

    operations = [
        migrations.AlterField(
            model_name='challenge',
            name='sequence',
            field=models.CharField(max_length=999, verbose_name=b'\xd0\x92\xd1\x8b\xd1\x80\xd0\xb0\xd0\xb6\xd0\xb5\xd0\xbd\xd0\xb8\xd0\xb5'),
            preserve_default=True,
        ),
    ]


import requests
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.select import Select

def collect_proxies():
    driver = webdriver.Chrome(executable_path='chromedriver')
    url = 'https://free-proxy-list.net/'
    driver.get(url)
    proxies = []
    url = 'https://free-proxy-list.net/'
    timeout = 30
    try:
        WebDriverWait(driver, timeout).until(EC.visibility_of_element_located((By.ID, "proxylisttable")))
    except TimeoutException:
        driver.quit()

    selectXpath = '//*[@id="proxylisttable"]/tfoot/tr/th[7]/select'
    select_https = Select(driver.find_element(By.XPATH, selectXpath))

    #Change the filter to only show https proxies
    select_https.select_by_visible_text('yes')

    nextXpath = '//*[@id="proxylisttable_next"]/a'

    proxy_table_body = driver.find_element(By.XPATH, '//*[@id="proxylisttable"]/tbody')
    for pageNum in range(4):
        rows = proxy_table_body.find_elements(By.CSS_SELECTOR, 'tr')
        for tr in rows:
            td = tr.find_elements(By.CSS_SELECTOR, 'td')
            proxy = f'{td[0].text}:{td[1].text}'
            proxies.append(proxy)
        next_Page = driver.find_element(By.XPATH, nextXpath)
        next_Page.click()

    #webdriver.ActionChains(driver).move_to_element(select_https).click(element).perform()
    return proxies


import os
from typing import Dict, Optional, Union, Tuple

import torch
import torch.nn as nn
import torch.optim as optim

def get_device(
    device_type: str, device_index: Optional[Union[int, Tuple[int, ...]]] = None
) -> torch.device:
    if device_type == "cpu":
        return torch.device(device_type)
    elif device_type == "cuda":
        if isinstance(device_index, int):
            return torch.device(device_type + f":{device_index}")
        elif isinstance(device_index, list):
            raise NotImplementedError()
        else:
            raise ValueError("ConfigError: Invalid device index")
    else:
        raise ValueError("ConfigError: Invalid device type")

def get_optimizer(cfg, model):
    optimizer_params = cfg["train_config"]["parameters"]["optimizer"]
    optimizer_kind = optimizer_params.pop("kind")
    if optimizer_kind == "adam":
        return optim.Adam(model.parameters(), **optimizer_params)
    elif optimizer_kind == "sgd":
        return optim.SGD(model.parameters(), **optimizer_params)
    elif optimizer_kind == "adamw":
        return optim.AdamW(model.parameters(), **optimizer_params)
    else:
        raise NotImplementedError()

def get_loss(loss):
    if loss == "logit":
        return nn.BCEWithLogitsLoss()
    elif loss=="cross_entropy":
        return nn.CrossEntropyLoss()


from django.shortcuts import render, redirect
from .models import *
from authapp.models import Master_Table, Service_Provider, User
from userapp.models import Request_Details, User_Query
from .util import *

# Page rendering
def Index(request):
   return My_Request(request)

# Adding garage details
def AddGarageDetails(request):
    gname= request.POST['gname']
    gaddress= request.POST['gaddress']
    gcity= request.POST['gcity']
    gmobileno= request.POST['gmobileno']
    gimage= request.FILES['gimage']
    sp_id= Master_Table.objects.get(Email= request.session['Email'])

    if "general_service" in request.POST:
        serv1= "General Service"
        serv1_price= request.POST['general_service_price']
    else:
        serv1= ""
        serv1_price= 0
    if "electrical_issue" in request.POST:
        serv2= "Electrical Issue"
        serv2_price= request.POST['electrical_issue_price']
    else:
        serv2= ""
        serv2_price= 0
    if "engine/silencer_noise" in request.POST:
        serv3= "Engine/Silencer Noise"
        serv3_price= request.POST['engine/silencer_noise_price']
    else:
        serv3= ""
        serv3_price= 0
    if "repainting/scratch_removal" in request.POST:
        serv4= "Repainting/Scratch Removal"
        serv4_price= request.POST['repainting/scratch_removal_price']
    else:
        serv4= ""
        serv4_price= 0
    if "oil_changing" in request.POST:
        serv5= "Oil Changing"
        serv5_price= request.POST['oil_changing_price']
    else:
        serv5= ""
        serv5_price= 0
    if "tyre_puncture/replacement" in request.POST:
        serv6= "Tyre puncture/Replacement"
        serv6_price= request.POST['tyre_puncture/replacement_price']
    else:
        serv6= ""
        serv6_price= 0

    if "chain_and_spocket_issue" in request.POST:
        serv7= "Chain and Spocket Issue"
        serv7_price= request.POST['chain_and_spocket_issue_price']
    else:
        serv7= ""
        serv7_price= 0

    pro_name= Service_Provider.objects.filter(SP_Id= sp_id).first()

    new_garage= GarageDetails.objects.create(
        Gname= gname,
        Gaddress= gaddress,
        City= gcity,
        Mobile_No= gmobileno,
        Gimage=  gimage,
        Provider_Name= pro_name.UserName,
        SP_ID= sp_id,
        Ser1= serv1,
        Ser2= serv2,
        Ser3= serv3,
        Ser4= serv4,
        Ser5= serv5,
        Ser6= serv6,
        Ser7= serv7,
        Ser1_Price= serv1_price,
        Ser2_Price= serv2_price,
        Ser3_Price= serv3_price,
        Ser4_Price= serv4_price,
        Ser5_Price= serv5_price,
        Ser6_Price= serv6_price,
        Ser7_Price= serv7_price,
        )

    return render(request,'providerapp/index_pro.html')

# Request Handling

def My_Request(request): # retrieving provider specific request
    username= Service_Provider.objects.get(SP_Id= request.session['Email'])
    my_request= Request_Details.objects.filter(Provider_ID= request.session['Email'])

    return render(request, 'providerapp/index_pro.html', {'requests': my_request, 'name': username.UserName })

def Request_Handle(request,req_id): # accepting or deleting request
    user_req= Request_Details.objects.get(id= req_id)
    user_det= User.objects.get(UserName= user_req.User_Name)
    provider_det= Service_Provider.objects.get(SP_Id= user_req.Provider_ID)
    username= user_det.UserName
    useremail= str(user_det.User_Id)
    providername= provider_det.UserName
    #email= user_det.User_Id
    if "accept" in request.POST:
        sendmail('Request Accepted','request_accept',useremail, {'u_name': username, 'pro_name': providername} )
    elif "reject" in request.POST:
        req= Request_Details.objects.get(id= req_id)
        req.delete()
        sendmail('Request Rejected', 'request_reject', useremail, {'pro_name': providername, 'u_name': username})

    return My_Request(request)
    #sendmail('Urgent service needed','service_need',provider_id, {'name':  request.session['UserName'],'pro_name': providername.UserName})

def LogOut(request):
   del request.session['Email']
   del request.session['UserName']
   del request.session['Role']

   return redirect('/authapp/')

def UserQuery(request):
    name= request.POST['name']
    email= request.POST['email']
    subject= request.POST['subject']
    message= request.POST['message']

    user_query= User_Query.objects.create(User_Name= name, Email= email, Subject= subject, Message= message)

    return render(request, 'userapp/contact.html')


#!/usr/bin/env python
# coding: utf-8

# In[ ]:

#Libraries importing:
import findspark
findspark.init()
findspark.find()
import pyspark
findspark.find()
from operator import add
from pyspark import SparkContext, SparkConf
from pyspark.sql import SparkSession
import pandas as pd

#Opening PySpark Session:
conf = pyspark.SparkConf().setAppName('appName').setMaster('local')
sc = pyspark.SparkContext(conf=conf)
spark = SparkSession(sc)

#Defining computeContribs function:
def computeContribs(urls, rank):
    """Calculates URL contributions to the rank of other URLs."""
    num_urls = len(urls)
    for url in urls:
        yield (url, rank / num_urls)

#Defining parseNeighbors function:
def parseNeighbors(urls):
    """Parses a urls pair string into urls pair."""
    parts = urls.split("	")
    return parts[0], parts[1]

#Defining Diff function between 2 lists:
def Diff(li1, li2):
    return (list(list(set(li1)-set(li2)) + list(set(li2)-set(li1))))

#Defining intersection function between 2 lists:
def intersection(lst1, lst2):
    return list(set(lst1) & set(lst2))

#Reading all the lines of Google web graph:
lines = sc.textFile('web-Google.txt')
print("Reading and skipping")
lines = lines.zipWithIndex().filter(lambda tup: tup[1] > 3).map(lambda tup: tup[0])
links = lines.map(lambda urls: parseNeighbors(urls)).distinct().groupByKey().cache()

# Find node count
N = links.count()
print(N)
ranks = links.map(lambda url_tuple: (url_tuple[0], 1.0))

old_ranks = ranks
delta = 1

temp_struct=ranks.map(lambda tupla : tupla[0])
temp_struct_collect=temp_struct.collect()
itPR = 0

#PageRank algorithm.
print("Starting PageRank... ")
while(delta > 1.0e-6):
    print("qui")
    itPR = itPR + 1
    contribs = links.join(old_ranks).flatMap(lambda tupla : computeContribs(tupla[1][0],tupla[1][1]))
    dict_contr=contribs.collectAsMap()
    temp_struct_current=list(dict_contr.keys())
    ris=Diff(temp_struct_collect, temp_struct_current)
    val=intersection(ris, temp_struct_collect)
    result = map(lambda e: (e,0),val)
    result=list(result)
    result=sc.parallelize(result)
    contribs=contribs.union(result)
    n_ranks = contribs.reduceByKey(add).mapValues(lambda rank: (rank * 0.90) + 0.10)
    if(itPR!=1):
        n_ranks_df = pd.DataFrame(n_ranks.sortByKey().collect(), columns =['Node', 'Score'])
        old_ranks_df = pd.DataFrame(old_ranks.sortByKey().collect(), columns =['Node', 'Score'])
        df1 = abs(n_ranks_df['Score'].sub(old_ranks_df['Score'],axis=0))
        delta=df1.sum()
    old_ranks = n_ranks
    n_ranks=None
    n_ranks_df=None
    old_ranks_df=None
    del contribs
    print("Delta: " , delta)
print("Finish PageRank... ")
print("Number of iterations: ", itPR)

# In[ ]:

#transformation in df to save in CSV
old_ranks_df = pd.DataFrame(old_ranks.sortByKey().collect(), columns =['Node', 'Score'])
old_ranks_df.sort_values(by=['Score'], inplace=True, ascending=False)

# Write CSV
import tkinter as tk
from tkinter import filedialog
from pandas import DataFrame

old_ranks_df.to_csv(path_or_buf="csv_PR_0.10_v3")

# In[ ]:

#Saving PR dataframe
import pickle
with open('dataframePR_0.10_v3.pkl', 'wb') as f:  # Python 3: open(..., 'wb')
    pickle.dump(old_ranks_df,  f)

# In[ ]:

from pyspark.sql.types import *
from functools import reduce
from pyspark.sql.functions import col, lit, when
from graphframes import *

# Auxiliar functions
def equivalent_type(f):
    if f == 'datetime64[ns]': return TimestampType()
    elif f == 'int64': return LongType()
    elif f == 'int32': return IntegerType()
    elif f == 'float64': return FloatType()
    else: return StringType()

def define_structure(string, format_type):
    try: typo = equivalent_type(format_type)
    except: typo = StringType()
    return StructField(string, typo)

# Given pandas dataframe, it will return a spark's dataframe.
def pandas_to_spark(pandas_df):
    columns = list(pandas_df.columns)
    types = list(pandas_df.dtypes)
    struct_list = []
    for column, typo in zip(columns, types):
      struct_list.append(define_structure(column, typo))
    p_schema = StructType(struct_list)
    return sqlContext.createDataFrame(pandas_df, p_schema)

links = lines.map(lambda urls: parseNeighbors(urls))

from pyspark.sql import SQLContext
sqlContext = SQLContext(sc)

# In[ ]:

#use this row only if you use the saved resut
old_ranks_df=old_ranks_df.rename(columns={"Node":"id"})

# In[ ]:

vertices= pandas_to_spark(old_ranks_df)
edges = sqlContext.createDataFrame(links, ["src", "dst"])

#graph graphframes
from graphframes import *
g = GraphFrame(vertices, edges)

# In[ ]:

#analysis connected components
sc.setCheckpointDir("/tmp/graphframes-example-connected-components")
result = g.connectedComponents()

#grouping and counting all the connected components
import pyspark.sql.functions as f
sorted_connected=result.groupBy('component').count().select('component', f.col('count').alias('n')).orderBy('n', ascending=False)

# In[ ]:

#community detection
result2 = g.labelPropagation(maxIter=3)
result2_df = pd.DataFrame(result2.collect(), columns =['id', 'label', 'score'])

#saving result community
with open('result2_v3.pkl', 'wb') as f:
     pickle.dump(result2_df,  f)

#Write CSV
import tkinter as tk
from tkinter import filedialog
from pandas import DataFrame

result2_df.to_csv(path_or_buf="result2_df_3")

# In[ ]:

#For each label (community) counting how many nodes
import pyspark.sql.functions as f
sorted_l=result2.groupBy('label').count().select('label', f.col('count').alias('n')).orderBy('n', ascending=False)

# In[ ]:

#I check which node belongs to the node to choose the community
result2.filter(result2.id ==  41909).show()#661424963782

# In[ ]:

#selection of community
component_selected=result2.filter(result2.label == 661424963782 )
component_selected=component_selected.select("id")

#community's vertices
result_app=result2.select("id","label")
vertices_sub_graph = vertices.join(result_app, vertices.id ==result_app.id,how="left").drop(result_app.id)
vertices_sub_graph = vertices_sub_graph.filter(vertices_sub_graph.label  ==661424963782)

#community's edges
edges_sub_graph = edges.join(result_app, edges.src ==result_app.id,how="left").drop(result_app.id)
edges_sub_graph = edges_sub_graph.withColumnRenamed("label", "labelsrc")
edges_sub_graph = edges_sub_graph.join(result_app, edges_sub_graph.dst == result_app.id,how="left").drop(result_app.id)
edges_sub_graph=edges_sub_graph.filter( (edges_sub_graph.label  == 661424963782) | (edges_sub_graph.label  == 661424963782) )
edges_sub_graph=edges_sub_graph.drop(edges_sub_graph.labelsrc)
edges_sub_graph=edges_sub_graph.drop(edges_sub_graph.label)

def listOfTuples(l1, l2):
    return list(map(lambda x, y:(x,y), l1, l2))

edges_sub_graph_src=list(edges_sub_graph.select('src').toPandas()['src'])
edges_sub_graph_dst=list(edges_sub_graph.select('dst').toPandas()['dst'])
edges_sub_graph_list_tuple=listOfTuples(edges_sub_graph_src, edges_sub_graph_dst)

# In[ ]:

#list creartioon of vertices for networkX
list_app=[]
for vertex in vertices_sub_graph.collect():
    my_dict = {
      "Score": vertex.Score ,
    }
    my_tupla=(vertex.id,my_dict)
    list_app.append(my_tupla)

# In[ ]:

#list ofvertices
list_vertices=list_app

# In[ ]:

#creation graph networkX for Cytoscape
import networkx as nx
G = nx.Graph()
G.add_nodes_from(list_vertices)
G.add_edges_from(edges_sub_graph_list_tuple)

#centrality measures
betweenness_centrality = nx.betweenness_centrality(G)
nx.set_node_attributes(G, betweenness_centrality, "betweenness")
closeness_centrality = nx.closeness_centrality(G)
nx.set_node_attributes(G, closeness_centrality, "closeness")
degreee_centrality = nx.degree_centrality(G)
nx.set_node_attributes(G, degreee_centrality, "degree")

# In[ ]:

G=nx.read_gml("test.gml")
h,a=nx.hits(G, max_iter=100, tol=1e-06, nstart=None, normalized=True)#hubs and authorities

# In[ ]:

nx.set_node_attributes(G, h, name="Hubs")

# In[ ]:

nx.set_node_attributes(G, a, name="Authorities")

# In[ ]:

nx.write_gml(G, "test.gml")

# In[ ]:

#file for Cytoscape
nx.write_gml(G, "test.gml")


# -*- coding: utf-8 -*-

# nn_benchmark
# author - Quentin Ducasse
# https://github.com/QDucasse
# quentin.ducasse@ensta-bretagne.org

import sys
import torch
from nn_benchmark.core import Exporter
from nn_benchmark.networks import QuantTFC

if __name__ == "__main__":
    acq_list = [2, 3, 4, 5, 6, 7, 8, 16, 32]
    weq_list = [2, 3, 4, 5, 6, 7, 8, 16, 32]
    inq_list = [8, 8, 8, 8, 8, 8, 8, 32, 32]

    exporter = Exporter()

    # TFC
    for acq in acq_list:
        for weq in weq_list:
            if ((acq <= 8) and (weq <= 8)):
                inq = 8
            else:
                inq = 32
            # Load correct model
            tfc = QuantTFC(in_channels=1, weight_bit_width=weq, act_bit_width=acq, in_bit_width=inq)
            tfc_model = "/workspace/finn/trained_models/QuantTFC_A{0}W{1}I{2}/checkpoints/best.tar".format(acq, weq, inq)
            package = torch.load(tfc_model, map_location='cpu')
            model_state_dict = package['state_dict']
            tfc.load_state_dict(model_state_dict)
            # Generate ONNX counterpart
            output_path = "/workspace/finn/trained_onnx/QuantTFC_A{0}W{1}I{2}".format(acq, weq, inq)
            print("Exporting QuantTFC_A{0}W{1}I{2}.onnx".format(acq,weq,inq))
            exporter.export_onnx(model = tfc, output_dir_path = output_path, in_channels = 1,
                                 act_bit_width = acq, weight_bit_width = weq, input_bit_width = inq,
                                 epoch = 40)


from typing import List
from topsdk.client import BaseRequest
from topsdk.util import convert_struct_list,convert_basic_list,convert_struct,convert_basic
from datetime import datetime

class TaobaoWlbWmsInventoryQueryRequest(BaseRequest):

    def __init__(
        self,
        page_size: int = None,
        page_no: int = None,
        channel_code: str = None,
        due_date: datetime = None,
        produce_date: datetime = None,
        batch_code: str = None,
        type: int = None,
        inventory_type: int = None,
        store_code: str = None,
        item_id: str = None
    ):
        """
            每页多少条，最大50条
        """
        self._page_size = page_size
        """
            分页的第几页
        """
        self._page_no = page_no
        """
            渠道编码，type=3时字段传值有效。（TB 淘系， OTHERS 其他）
        """
        self._channel_code = channel_code
        """
            失效日期，type=2时字段传值有效。
        """
        self._due_date = due_date
        """
            生产日期，type=2时字段传值有效。
        """
        self._produce_date = produce_date
        """
            库存批次号，type=2时字段传值有效。 商品设置为批次管理时，商品产生批次库存。当商品为批次管理时，此字段不传值，返回所有批次库存信息。
        """
        self._batch_code = batch_code
        """
            库存查询类型 1-	汇总库存，不区分批次和渠道 2-	批次库存，库存按商品批次维度划分 3-	渠道库存，库存按渠道维度划分 （当前业务不支持批次库存和渠道库存共存，批次库存无渠道属性，渠道库存无批次属性）
        """
        self._type = type
        """
            库存类型。 (1 正品 101 残次 102 机损 103 箱损 201 冻结库存 301 在途库存 )
        """
        self._inventory_type = inventory_type
        """
            仓库编码
        """
        self._store_code = store_code
        """
            菜鸟商品ID
        """
        self._item_id = item_id

    @property
    def page_size(self):
        return self._page_size

    @page_size.setter
    def page_size(self, page_size):
        if isinstance(page_size, int):
            self._page_size = page_size
        else:
            raise TypeError("page_size must be int")

    @property
    def page_no(self):
        return self._page_no

    @page_no.setter
    def page_no(self, page_no):
        if isinstance(page_no, int):
            self._page_no = page_no
        else:
            raise TypeError("page_no must be int")

    @property
    def channel_code(self):
        return self._channel_code

    @channel_code.setter
    def channel_code(self, channel_code):
        if isinstance(channel_code, str):
            self._channel_code = channel_code
        else:
            raise TypeError("channel_code must be str")

    @property
    def due_date(self):
        return self._due_date

    @due_date.setter
    def due_date(self, due_date):
        if isinstance(due_date, datetime):
            self._due_date = due_date
        else:
            raise TypeError("due_date must be datetime")

    @property
    def produce_date(self):
        return self._produce_date

    @produce_date.setter
    def produce_date(self, produce_date):
        if isinstance(produce_date, datetime):
            self._produce_date = produce_date
        else:
            raise TypeError("produce_date must be datetime")

    @property
    def batch_code(self):
        return self._batch_code

    @batch_code.setter
    def batch_code(self, batch_code):
        if isinstance(batch_code, str):
            self._batch_code = batch_code
        else:
            raise TypeError("batch_code must be str")

    @property
    def type(self):
        return self._type

    @type.setter
    def type(self, type):
        if isinstance(type, int):
            self._type = type
        else:
            raise TypeError("type must be int")

    @property
    def inventory_type(self):
        return self._inventory_type

    @inventory_type.setter
    def inventory_type(self, inventory_type):
        if isinstance(inventory_type, int):
            self._inventory_type = inventory_type
        else:
            raise TypeError("inventory_type must be int")

    @property
    def store_code(self):
        return self._store_code

    @store_code.setter
    def store_code(self, store_code):
        if isinstance(store_code, str):
            self._store_code = store_code
        else:
            raise TypeError("store_code must be str")

    @property
    def item_id(self):
        return self._item_id

    @item_id.setter
    def item_id(self, item_id):
        if isinstance(item_id, str):
            self._item_id = item_id
        else:
            raise TypeError("item_id must be str")

    def get_api_name(self):
        return "taobao.wlb.wms.inventory.query"

    def to_dict(self):
        request_dict = {}
        if self._page_size is not None:
            request_dict["page_size"] = convert_basic(self._page_size)

        if self._page_no is not None:
            request_dict["page_no"] = convert_basic(self._page_no)

        if self._channel_code is not None:
            request_dict["channel_code"] = convert_basic(self._channel_code)

        if self._due_date is not None:
            request_dict["due_date"] = convert_basic(self._due_date)

        if self._produce_date is not None:
            request_dict["produce_date"] = convert_basic(self._produce_date)

        if self._batch_code is not None:
            request_dict["batch_code"] = convert_basic(self._batch_code)

        if self._type is not None:
            request_dict["type"] = convert_basic(self._type)

        if self._inventory_type is not None:
            request_dict["inventory_type"] = convert_basic(self._inventory_type)

        if self._store_code is not None:
            request_dict["store_code"] = convert_basic(self._store_code)

        if self._item_id is not None:
            request_dict["item_id"] = convert_basic(self._item_id)

        return request_dict

    def get_file_param_dict(self):
        file_param_dict = {}
        return file_param_dict