import torch
import json
import numpy as np

from sklearn.cluster import KMeans
from scipy.stats import norm
import copy as cp
from sklearn.svm import SVC

from torch.quasirandom import SobolEngine
from mpl_toolkits.mplot3d import axes3d, Axes3D
from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels import ConstantKernel, Matern

import matplotlib.pyplot as plt
from matplotlib import cm

# from turbo_1.turbo_1 import Turbo1

# the input will be samples!
class Classifier():
    def __init__(self, samples, dims, kernel_type, gamma_type = "auto"):
        self.training_counter = 0
        assert dims >= 1
        assert type(samples)  ==  type([])
        self.dims    =   dims

        #create a gaussian process regressor
        noise        =   0.1
        m52          =   ConstantKernel(1.0) * Matern(length_scale=1.0, nu=2.5)
        self.gpr     =   GaussianProcessRegressor(kernel=m52, alpha=noise**2) #default to CPU
        self.kmean   =   KMeans(n_clusters=2)
        #learned boundary
        self.svm     =   SVC(kernel = kernel_type, gamma=gamma_type)
        #data structures to store
        self.samples = []
        self.X       = np.array([])
        self.fX      = np.array([])

        #good region is labeled as zero
        #bad  region is labeled as one
        self.good_label_mean  = -1
        self.bad_label_mean   = -1

        self.update_samples(samples)

    def is_splittable_svm(self):
        plabel = self.learn_clusters()
        self.learn_boundary(plabel)
        svm_label = self.svm.predict( self.X )
        if len( np.unique(svm_label) ) == 1:
            return False
        else:
            return True

    def get_max(self):
        return np.max(self.fX)

    def plot_samples_and_boundary(self, func, name):
        assert func.dims == 2

        plabels   = self.svm.predict( self.X )
        good_counts = len( self.X[np.where( plabels == 0 )] )
        bad_counts  = len( self.X[np.where( plabels == 1 )] )
        good_mean = np.mean( self.fX[ np.where( plabels == 0 ) ] )
        bad_mean  = np.mean( self.fX[ np.where( plabels == 1 ) ] )

        if np.isnan(good_mean) == False and np.isnan(bad_mean) == False:
            assert good_mean > bad_mean

        lb = func.lb
        ub = func.ub
        x = np.linspace(lb[0], ub[0], 100)
        y = np.linspace(lb[1], ub[1], 100)
        xv, yv = np.meshgrid(x, y)
        true_y = []
        for row in range(0, xv.shape[0]):
            for col in range(0, xv.shape[1]):
                x = xv[row][col]
                y = yv[row][col]
                true_y.append( func( np.array( [x, y] ) ) )
        true_y = np.array( true_y )
        pred_labels = self.svm.predict( np.c_[xv.ravel(), yv.ravel()] )
        pred_labels = pred_labels.reshape( xv.shape )

        fig, ax = plt.subplots()
        ax.contour(xv, yv, true_y.reshape(xv.shape), cmap=cm.coolwarm)
        ax.contourf(xv, yv, pred_labels, alpha=0.4)

        ax.scatter(self.X[ np.where(plabels == 0) , 0 ], self.X[ np.where(plabels == 0) , 1 ], marker='x', label="good-"+str(np.round(good_mean, 2))+"-"+str(good_counts) )
        ax.scatter(self.X[ np.where(plabels == 1) , 0 ], self.X[ np.where(plabels == 1) , 1 ], marker='x', label="bad-"+str(np.round(bad_mean, 2))+"-"+str(bad_counts)    )
        ax.legend(loc="best")
        ax.set_xlabel('x1')
        ax.set_ylabel('x2')
        ax.set_xlim([-10, 10])
        ax.set_ylim([-10, 10])
        plt.savefig(name)
        plt.close()

    def get_mean(self):
        return np.mean(self.fX)

    def update_samples(self, latest_samples):
        assert type(latest_samples) == type([])
        X  = []
        fX  = []
        for sample in latest_samples:
            X.append(  sample[0] )
            fX.append( sample[1] )

        self.X          = np.asarray(X, dtype=np.float32).reshape(-1, self.dims)
        self.fX         = np.asarray(fX,  dtype=np.float32).reshape(-1)
        self.samples    = latest_samples

    def train_gpr(self, samples):
        X  = []
        fX  = []
        for sample in samples:
            X.append(  sample[0] )
            fX.append( sample[1] )
        X  = np.asarray(X).reshape(-1, self.dims)
        fX = np.asarray(fX).reshape(-1)

        # print("training GPR with ", len(X), " data X")
        self.gpr.fit(X, fX)

    ###########################
    # BO sampling with EI
    ###########################

    def expected_improvement(self, X, xi=0.0001, use_ei = True):
        ''' Computes the EI at points X based on existing samples X_sample and Y_sample using a Gaussian process surrogate model.
        Args: X: Points at which EI shall be computed (m x d). X_sample: Sample locations (n x d).
        Y_sample: Sample values (n x 1). gpr: A GaussianProcessRegressor fitted to samples.
        xi: Exploitation-exploration trade-off parameter.
        Returns: Expected improvements at points X. '''
        X_sample = self.X
        Y_sample = self.fX.reshape((-1, 1))

        gpr = self.gpr

        mu, sigma = gpr.predict(X, return_std=True)

        if not use_ei:
            return mu
        else:
            #calculate EI
            mu_sample = gpr.predict(X_sample)
            sigma = sigma.reshape(-1, 1)
            mu_sample_opt = np.max(mu_sample)
            with np.errstate(divide='warn'):
                imp = mu - mu_sample_opt - xi
                imp = imp.reshape((-1, 1))
                Z = imp / sigma
                ei = imp * norm.cdf(Z) + sigma * norm.pdf(Z)
                ei[sigma == 0.0] = 0.0
            return ei

    def plot_boundary(self, X):
        if X.shape[1] > 2:
            return
        fig, ax = plt.subplots()
        ax.scatter( X[ :, 0 ], X[ :, 1 ] , marker='.')
        ax.scatter(self.X[ : , 0 ], self.X[ : , 1 ], marker='x')
        ax.set_xlim([-10, 10])
        ax.set_ylim([-10, 10])
        plt.savefig("boundary.pdf")
        plt.close()

    def get_sample_ratio_in_region( self, cands, path ):
        total = len(cands)
        for node in path:
            boundary = node[0].classifier.svm
            if len(cands) == 0:
                return 0, np.array([])
            assert len(cands) > 0
            cands = cands[ boundary.predict( cands ) == node[1] ]
            # node[1] store the direction to go
        ratio = len(cands) / total
        assert len(cands) <= total
        return ratio, cands

    def propose_rand_samples_probe(self, nums_samples, path, lb, ub):

        seed   = np.random.randint(int(1e6))
        sobol  = SobolEngine(dimension = self.dims, scramble=True, seed=seed)

        center = np.mean(self.X, axis = 0)
        #check if the center located in the region
        ratio, tmp = self.get_sample_ratio_in_region( np.reshape(center, (1, len(center) ) ), path )
        if ratio == 0:
            print("==>center not in the region, using random samples")
            return self.propose_rand_samples(nums_samples, lb, ub)
        # it is possible that the selected region has no points,
        # so we need check here

        axes    = len( center )

        final_L = []
        for axis in range(0, axes):
            L       = np.zeros( center.shape )
            L[axis] = 0.01
            ratio   = 1

            while ratio >= 0.9:
                L[axis] = L[axis]*2
                if L[axis] >= (ub[axis] - lb[axis]):
                    break
                lb_     = np.clip( center - L/2, lb, ub )
                ub_     = np.clip( center + L/2, lb, ub )
                cands_  = sobol.draw(10000).to(dtype=torch.float64).cpu().detach().numpy()
                cands_  = (ub_ - lb_)*cands_ + lb_
                ratio, tmp = self.get_sample_ratio_in_region(cands_, path )
            final_L.append( L[axis] )

        final_L   = np.array( final_L )
        lb_       = np.clip( center - final_L/2, lb, ub )
        ub_       = np.clip( center + final_L/2, lb, ub )
        print("center:", center)
        print("final lb:", lb_)
        print("final ub:", ub_)

        count         = 0
        cands         = np.array([])
        while len(cands) < 10000:
            count    += 10000
            cands     = sobol.draw(count).to(dtype=torch.float64).cpu().detach().numpy()

            cands     = (ub_ - lb_)*cands + lb_
            ratio, cands = self.get_sample_ratio_in_region(cands, path)
            samples_count = len( cands )

        #extract candidates

        return cands

    def propose_rand_samples_sobol(self, nums_samples, path, lb, ub):

        #rejected sampling
        selected_cands = np.zeros((1, self.dims))
        seed   = np.random.randint(int(1e6))
        sobol  = SobolEngine(dimension = self.dims, scramble=True, seed=seed)

        # scale the samples to the entire search space
        # ----------------------------------- #
        # while len(selected_cands) <= nums_samples:
        #     cands  = sobol.draw(100000).to(dtype=torch.float64).cpu().detach().numpy()
        #     cands  = (ub - lb)*cands + lb
        #     for node in path:
        #         boundary = node[0].classifier.svm
        #         if len(cands) == 0:
        #             return []
        #         cands = cands[ boundary.predict(cands) == node[1] ] # node[1] store the direction to go
        #     selected_cands = np.append( selected_cands, cands, axis= 0)
        #     print("total sampled:", len(selected_cands) )
        # return cands
        # ----------------------------------- #
        #shrink the cands region

        ratio_check, centers = self.get_sample_ratio_in_region(self.X, path)
        # no current samples located in the region
        # should not happen
        # print("ratio check:", ratio_check, len(self.X) )
        # assert ratio_check > 0
        if ratio_check == 0 or len(centers) == 0:
            return self.propose_rand_samples( nums_samples, lb, ub )

        lb_    = None
        ub_    = None

        final_cands = []
        for center in centers:
            center = self.X[ np.random.randint( len(self.X) ) ]
            cands  = sobol.draw(2000).to(dtype=torch.float64).cpu().detach().numpy()
            ratio  = 1
            L      = 0.0001
            Blimit = np.max(ub - lb)

            while ratio == 1 and L < Blimit:
                lb_    = np.clip( center - L/2, lb, ub )
                ub_    = np.clip( center + L/2, lb, ub )
                cands_ = cp.deepcopy( cands )
                cands_ = (ub_ - lb_)*cands_ + lb_
                ratio, cands_ = self.get_sample_ratio_in_region(cands_, path)
                if ratio < 1:
                    final_cands.extend( cands_.tolist() )
                L = L*2
        final_cands      = np.array( final_cands )
        if len(final_cands) > nums_samples:
            final_cands_idx  = np.random.choice( len(final_cands), nums_samples )
            return final_cands[final_cands_idx]
        else:
            if len(final_cands) == 0:
                return self.propose_rand_samples( nums_samples, lb, ub )
            else:
                return final_cands

    def propose_samples_bo( self, nums_samples = 10, path = None, lb = None, ub = None, samples = None):
        ''' Proposes the next sampling point by optimizing the acquisition function.
        Args: acquisition: Acquisition function. X_sample: Sample locations (n x d).
        Y_sample: Sample values (n x 1). gpr: A GaussianProcessRegressor fitted to samples.
        Returns: Location of the acquisition function maximum. '''
        assert path is not None and len(path) >= 0
        assert lb is not None and ub is not None
        assert samples is not None and len(samples) > 0

        self.train_gpr( samples ) # learn in unit cube

        dim  = self.dims
        nums_rand_samples = 10000
        if len(path) == 0:
            return self.propose_rand_samples(nums_samples, lb, ub)

        X    = self.propose_rand_samples_sobol(nums_rand_samples, path, lb, ub)
        # print("samples in the region:", len(X) )
        # self.plot_boundary(X)
        if len(X) == 0:
            return self.propose_rand_samples(nums_samples, lb, ub)

        X_ei = self.expected_improvement(X, xi=0.001, use_ei = True)
        row, col = X.shape

        X_ei = X_ei.reshape(len(X))
        n = nums_samples
        if X_ei.shape[0] < n:
            n = X_ei.shape[0]
        indices = np.argsort(X_ei)[-n:]
        proposed_X = X[indices]
        return proposed_X

    ###########################
    # sampling with turbo
    ###########################
    # version 1: select a partition, perform one-time turbo search

    # def propose_samples_turbo(self, num_samples, path, func):
    #     #throw a uniform sampling in the selected partition
    #     X_init = self.propose_rand_samples_sobol(30, path, func.lb, func.ub)
    #     #get samples around the selected partition
    #     print("sampled ", len(X_init), " for the initialization")
    #     turbo1 = Turbo1(
    #         f  = func,              # Handle to objective function
    #         lb = func.lb,           # Numpy array specifying lower bounds
    #         ub = func.ub,           # Numpy array specifying upper bounds
    #         n_init = 30,            # Number of initial bounds from an Latin hypercube design
    #         max_evals  = num_samples, # Maximum number of evaluations
    #         batch_size = 1,         # How large batch size TuRBO uses
    #         verbose=True,           # Print information from each batch
    #         use_ard=True,           # Set to true if you want to use ARD for the GP kernel
    #         max_cholesky_size=2000, # When we switch from Cholesky to Lanczos
    #         n_training_steps=50,    # Number of steps of ADAM to learn the hypers
    #         min_cuda=1024,          #  Run on the CPU for small datasets
    #         device="cpu",           # "cpu" or "cuda"
    #         dtype="float32",        # float64 or float32
    #         X_init = X_init,
    #     )
    #
    #     proposed_X, fX = turbo1.optimize( )
    #     fX = fX*-1
    #
    #     return proposed_X, fX

    ###########################
    # random sampling
    ###########################

    def propose_rand_samples(self, nums_samples, lb, ub):
        x = np.random.uniform(lb, ub, size = (nums_samples, self.dims) )
        return x

    def propose_samples_rand( self, nums_samples = 10):
        return self.propose_rand_samples(nums_samples, self.lb, self.ub)

    ###########################
    # learning boundary
    ###########################

    def get_cluster_mean(self, plabel):
        assert plabel.shape[0] == self.fX.shape[0]

        zero_label_fX = []
        one_label_fX  = []

        for idx in range(0, len(plabel)):
            if plabel[idx] == 0:
                zero_label_fX.append( self.fX[idx]  )
            elif plabel[idx] == 1:
                one_label_fX.append( self.fX[idx] )
            else:
                print("kmean should only predict two clusters, Classifiers.py:line73")
                os._exit(1)

        good_label_mean = np.mean( np.array(zero_label_fX) )
        bad_label_mean  = np.mean( np.array(one_label_fX) )
        return good_label_mean, bad_label_mean

    def learn_boundary(self, plabel):
        assert len(plabel) == len(self.X)
        self.svm.fit(self.X, plabel)

    def learn_clusters(self):
        assert len(self.samples) >= 2, "samples must > 0"
        assert self.X.shape[0], "points must > 0"
        assert self.fX.shape[0], "fX must > 0"
        assert self.X.shape[0] == self.fX.shape[0]

        tmp = np.concatenate( (self.X, self.fX.reshape([-1, 1]) ), axis = 1 )
        assert tmp.shape[0] == self.fX.shape[0]

        self.kmean  = self.kmean.fit(tmp)
        plabel      = self.kmean.predict( tmp )

        # the 0-1 labels in kmean can be different from the actual
        # flip the label is not consistent
        # 0: good cluster, 1: bad cluster

        self.good_label_mean , self.bad_label_mean = self.get_cluster_mean(plabel)

        if self.bad_label_mean > self.good_label_mean:
            for idx in range(0, len(plabel)):
                if plabel[idx] == 0:
                    plabel[idx] = 1
                else:
                    plabel[idx] = 0

        self.good_label_mean , self.bad_label_mean = self.get_cluster_mean(plabel)

        return plabel

    def split_data(self):
        good_samples = []
        bad_samples  = []
        train_good_samples = []
        train_bad_samples  = []
        if len( self.samples ) == 0:
            return good_samples, bad_samples

        plabel = self.learn_clusters( )
        self.learn_boundary( plabel )

        for idx in range(0, len(plabel)):
            if plabel[idx] == 0:
                #ensure the consistency
                assert self.samples[idx][-1] - self.fX[idx] <= 1
                good_samples.append( self.samples[idx] )
                train_good_samples.append( self.X[idx] )
            else:
                bad_samples.append( self.samples[idx] )
                train_bad_samples.append( self.X[idx] )

        train_good_samples = np.array(train_good_samples)
        train_bad_samples  = np.array(train_bad_samples)

        assert len(good_samples) + len(bad_samples) == len(self.samples)

        return  good_samples, bad_samples


from flask import Flask, render_template, jsonify
import sqlalchemy
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import Session
from sqlalchemy import create_engine, func, desc
import sqlalchemy.dialects.sqlite
import datetime as dt
import pandas as pd
import os

app = Flask(__name__,
            static_folder='static',
            template_folder='templates')

@app.route('/')
def home():
    return render_template('viz1.html')

@app.route('/covid_data', methods=['GET'])
def covid_data():
    #create engine to connect to SQL database
    engine = create_engine("postgresql+psycopg2://llnvzqitayvosz:0c1511137a621fdcc484c0a1afdccf84d1b10e28b5370f0af994ca3113fad4d8@ec2-3-213-85-90.compute-1.amazonaws.com:5432/d3livmv6adnqas")
    #connect to SQL database
    connection = engine.connect()

    # creat dataframe of wins per country from database
    covid_data = pd.read_sql('SELECT * FROM infected;', connection)
    print(covid_data)
    # convert dataframe to list of lists with header
    covid_data_list=[covid_data.columns.values.tolist()] \
        + covid_data.values.tolist()

    response = jsonify(covid_data_list)
    response.headers.add("Access-Control-Allow-Origin", "*")
    return response

@app.route('/state_data', methods=['GET'])
def state_data():
    #create engine to connect to SQL database
    engine = create_engine("postgresql+psycopg2://llnvzqitayvosz:0c1511137a621fdcc484c0a1afdccf84d1b10e28b5370f0af994ca3113fad4d8@ec2-3-213-85-90.compute-1.amazonaws.com:5432/d3livmv6adnqas")
    #connect to SQL database
    connection = engine.connect()

    # creat dataframe of wins per country from database
    state_data = pd.read_sql('SELECT * FROM percent6;', connection)

    #print(state_data)

    state_data_list=[state_data.columns.values.tolist()] \
        + state_data.values.tolist()
    response = jsonify(state_data_list)
    response.headers.add("Access-Control-Allow-Origin", "*")
    return response

@app.route('/filtered_data', methods=['GET'])
def filtered_data():
    #create engine to connect to SQL database
    engine = create_engine("postgresql+psycopg2://llnvzqitayvosz:0c1511137a621fdcc484c0a1afdccf84d1b10e28b5370f0af994ca3113fad4d8@ec2-3-213-85-90.compute-1.amazonaws.com:5432/d3livmv6adnqas")
    #connect to SQL database
    connection = engine.connect()

    # creat dataframe of wins per country from database
    filtered_data = pd.read_sql('SELECT * FROM filtered_data4;', connection)

    #print(filtered_data)

    filtered_data_list=[filtered_data.columns.values.tolist()] \
        + filtered_data.values.tolist()
    response = jsonify(filtered_data_list)
    response.headers.add("Access-Control-Allow-Origin", "*")
    return response

if __name__ == '__main__':
    port = int(os.environ.get("PORT", 5000))
    app.run(host='0.0.0.0', port=port)


import pygame,sys
from pygame.locals import *

pygame.init()

#setting up the window
DISPLAYSURF = pygame.display.set_mode((500,400), 0, 32)
pygame.display.set_caption("Drawing")

#setting up the colors
BLACK = (  0,   0,   0)
WHITE = (255, 255, 255)
RED   = (255,   0,   0)
GREEN = (  0, 255,   0)
BLUE  = (  0,   0, 255)

#drawing on the surface
DISPLAYSURF.fill(WHITE)
pygame.draw.polygon(DISPLAYSURF, GREEN, ((146,0), (291, 106), (236, 277), (56, 277), (0, 106)))
pygame.draw.line(DISPLAYSURF, BLUE, (60,60), (120,60), 4)
pygame.draw.line(DISPLAYSURF, BLUE, (120,60), (60,120))
pygame.draw.aaline(DISPLAYSURF, BLACK, (240,60), (60,120))
pygame.draw.line(DISPLAYSURF, BLUE, (60, 120), (120,120), 4)
pygame.draw.circle(DISPLAYSURF, BLUE, (300,50), 20, 0)
pygame.draw.ellipse(DISPLAYSURF, RED, (300,250,40,80), 1)
pygame.draw.rect(DISPLAYSURF, RED, (200,150,100,50))

pixObj = pygame.PixelArray(DISPLAYSURF)
pixObj[480][380] = BLACK
pixObj[482][382] = BLACK
pixObj[484][384] = BLACK
pixObj[486][386] = BLACK
pixObj[488][388] = BLACK
del pixObj #Deleting the PixelArrayobjectwill ―unlock‖ the Surfaceobject so that you can once again draw images on it

#run the game loop
while True:
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            pygame.quit()
            sys.exit()
    pygame.display.update()


#!/usr/bin/python
# -*- coding: utf-8 -*-

from flask import Flask, jsonify, request
from tinydb import Query, TinyDB
from crypt import AESCipher, bcrypt_checkpw, bcrypt_encrypt
import time
from totp import Totp
from expiringdict import ExpiringDict

app = Flask(__name__)
app.secret_key = '19#^f8kV8D55h!TXruYum%^VQh$zsCeQ'

DB = TinyDB('./db.json')
query = Query()

passphrase = "6V5#645L5FJn#Zc9i^i$b^3*38dShnk$"
aes = AESCipher(passphrase)

CACHE = ExpiringDict(max_len=10000, max_age_seconds=60)

def response(status, message, result=None):
    return jsonify({
        "status": status,
        "message": message,
        "result": result
    })

def response_success(result=None):
    return response(2000, "success", result)

@app.route('/api/create_seed', methods=['POST'])
def create_seed():
    """生成一个seed和对应的url"""
    username = request.form.get('username')
    tp = Totp()
    tp.create_seed()
    return response_success(result={"url": tp.create_url(username), "seed": tp.seed})

@app.route('/api/save_seed', methods=['POST'])
def save_seed():
    """提供用户名、seed和最近两次token，保存数据"""
    username = request.form.get('username')
    seed = request.form.get('seed')
    pin = request.form.get('pin')
    first_token = request.form.get("first_token")
    second_token = request.form.get("second_token")
    # first_token和second_token不能重复
    if first_token == second_token:
        return response(status=4000, message="verify seed fail")
    # 已存在的用户不允许再添加seed
    if DB.search(query.username == username):
        return response(status=4004, message="username exists")

    tp = Totp(seed=seed)
    # 计算服务器和本地的时间差
    jet_lag_unit = tp.calc_jet_lag_unit(first_token, second_token)
    if jet_lag_unit is None:
        # 如果超过了允许的时间差，则验证不通过
        return response(status=4000, message="verify seed fail")
    else:
        # seed aes256加密
        seed_aes = aes.encrypt(seed)
        # 不允许出现两个一样的seed，提高安全性
        if DB.search(query.seed == seed_aes):
            return response(status=4001, message="seed repeat")
        # 保存到db
        DB.insert({'username': username, 'seed': seed_aes, 'created': int(time.time()), "jet_lag_unit": jet_lag_unit, "pin": bcrypt_encrypt(pin)})
        return response_success()

@app.route('/api/resync', methods=['POST'])
def resync():
    """时间同步"""
    username = request.form.get('username')
    first_token = request.form.get("first_token")
    second_token = request.form.get("second_token")
    # first_token和second_token不能重复
    if first_token == second_token:
        return response(status=4000, message="verify seed fail")
    # 获取当前用户
    users = DB.search(query.username == username)
    if not users:
        return response(status=4002, message="username not exists")

    user = users[0]
    seed = aes.decrypt(user.get("seed"))
    tp = Totp(seed=seed)
    # 计算服务器和本地的时间差
    jet_lag_unit = tp.calc_jet_lag_unit(first_token, second_token)
    if jet_lag_unit is None:
        # 如果超过了允许的时间差，则验证不通过
        return response(status=4000, message="verify seed fail")
    else:
        # 更新时间差
        DB.update({'jet_lag_unit': jet_lag_unit}, query.username == username)
        return response_success()

@app.route('/api/verify_token', methods=['POST'])
def verify_token():
    """验证用户的token是否正确"""
    username = request.form.get('username')
    ptoken = request.form.get("ptoken")

    pin = ptoken[:-6]
    token = ptoken[-6:]

    # 如果用户不存在，抛出错误
    users = DB.search(query.username == username)
    if not users:
        return response(status=4002, message="username not exists")

    user = users[0]
    seed = aes.decrypt(user.get("seed"))
    jet_lag_unit = user.get("jet_lag_unit", 0)
    pin_hashed = user.get("pin")
    tp = Totp(seed=seed)
    if tp.token_verify(token, jet_lag_unit=jet_lag_unit) and bcrypt_checkpw(pin, pin_hashed):
        # 不能直接使用token缓存，防止多个人的token在一个时间内一致导致token不可用的情况。
        cache_key = username + token
        # 确保token只能被成功使用一次
        if CACHE.get(cache_key):
            return response(status=4003, message="token verify fail")
        else:
            CACHE[cache_key] = 1
        return response_success()
    else:
        return response(status=4003, message="token verify fail")

if __name__ == '__main__':
    app.run(port=8080)


from bs4 import BeautifulSoup
import requests
try:
    from crawlers.utils import clean
    from crawlers.utils import save_preview
except:
    from utils import clean
    from utils import save_preview

def url_scheme():
    return "https://chapmanganato.com/manga-"

def type():
    return "bs4"

def get_preview(RAW_URL, title):
    temp = RAW_URL.split("/")
    URL = "/".join(temp[:-1]) +"/"

    r = requests.get(URL)
    if r.status_code == 200:
        soup = BeautifulSoup(r.text, "html.parser")
        temp = soup.find("span", class_="info-image")
        url = clean(temp.contents)[0].attrs['src']

        return save_preview(title, url)

def get_chapter_list(RAW_URL):
    temp = RAW_URL.split("/")
    URL = "/".join(temp[:-1]) +"/"

    r = requests.get(URL)
    if r.status_code == 200:
        soup = BeautifulSoup(r.text, "html.parser")
        chapterList = soup.find_all("li", class_="a-h")
        chapList = []
        for li in chapterList:
            a = clean(li.contents)[0]
            url = a.attrs['href']
            chapter_name = a.contents[0]
            chapList.append((chapter_name, url))
        return chapList[::-1]

def get_title(RAW_URL):
    temp = RAW_URL.split("/")
    URL = "/".join(temp[:-1]) +"/"

    r = requests.get(URL)
    if r.status_code == 200:
        soup = BeautifulSoup(r.text, "html.parser")
        name = clean(soup.find('div', class_ = "panel-breadcrumb").contents)
        title = name[2].attrs['title']
        return title


import json

from django.test import TestCase, Client
from django.urls import reverse

from rest_framework import status

from ..models import FieldType
from ..serializer import FieldTypeSerializer

client = Client()

def get_valid_object():
    valid_object = {
        "name": "Test Name",
        "description": "Test Description"
    }
    return valid_object

def get_invalid_object():
    invalid_object = {
        "name": "",
        "description": ""
    }
    return invalid_object

class GetAllFieldTypes(TestCase):
    """Test to GET every field type from API"""

    def setUp(self):
        FieldType.objects.create(**get_valid_object())
        FieldType.objects.create(**get_valid_object())

    def test_get_every_field_type(self):

        response = client.get(reverse("fieldtype-list"), format="json")

        # get data from database
        field_types = FieldType.objects.all()
        serializer = FieldTypeSerializer(field_types, many=True)

        self.assertEqual(response.data, serializer.data)
        self.assertEqual(response.status_code, status.HTTP_200_OK)

class GetSingleFieldTypes(TestCase):
    """Test to GET single field type from API"""

    def setUp(self):
        self.field_type1 = FieldType.objects.create(**get_valid_object())
        self.field_type2 = FieldType.objects.create(**get_valid_object())

    def test_get_valid_single_field_type(self):

        response = client.get(reverse(
            "fieldtype-detail",
            kwargs={"pk": self.field_type1.pk}))

        # get data from database
        field_type = FieldType.objects.get(pk=self.field_type1.pk)
        serializer = FieldTypeSerializer(field_type)

        self.assertEqual(response.data, serializer.data)
        self.assertEqual(response.status_code, status.HTTP_200_OK)

    def test_get_invalid_single_field_type(self):
        invalid_id = 450
        response = client.get(reverse(
            "fieldtype-detail",
            kwargs={"pk": invalid_id}))

        self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)

class CreateNewFieldTypes(TestCase):
    """Test for creaating a new field type"""

    def setUp(self):
        self.valid_field_type = get_valid_object()
        self.invalid_field_type = get_invalid_object()

    def test_create_valid_field_type(self):

        response = client.post(reverse("fieldtype-list"),
                               data=json.dumps(self.valid_field_type),
                               content_type="application/json")

        self.assertEqual(response.status_code, status.HTTP_201_CREATED)

    def test_create_invalid_field_type(self):

        response = client.post(reverse("fieldtype-list"),
                               data=json.dumps(self.invalid_field_type),
                               content_type="application/json")

        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)

class UpdateSingleFieldTypes(TestCase):
    """Test to Update a single field type"""

    def setUp(self):
        self.field_type = FieldType.objects.create(**get_valid_object())

        self.valid_field_type = get_valid_object()
        self.invalid_field_type = get_invalid_object()

    def test_update_valid_field_type(self):

        response = client.put(reverse(
            "fieldtype-detail",
            kwargs={"pk": self.field_type.pk}),
            data=json.dumps(self.valid_field_type),
            content_type="application/json")

        self.assertEqual(response.status_code, status.HTTP_200_OK)

    def test_update_invalid_single_field_type(self):
        response = client.put(reverse(
            "fieldtype-detail",
            kwargs={"pk": self.field_type.pk}),
            data=json.dumps(self.invalid_field_type),
            content_type="application/json")

        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)

class DeleteSingleFieldTypes(TestCase):
    """Test to DELETE single field type from API"""

    def setUp(self):
        self.field_type = FieldType.objects.create(**get_valid_object())

    def test_delete_valid_single_field_type(self):

        response = client.delete(reverse(
            "fieldtype-detail",
            kwargs={"pk": self.field_type.pk}))

        self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)

    def test_delete_invalid_single_field_type(self):
        invalid_id = 450
        response = client.get(reverse(
            "fieldtype-detail",
            kwargs={"pk": invalid_id}))

        self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)


import pandas
import requests
import json
import streamlit as st
url = "http://apis.data.go.kr/B552584/ArpltnInforInqireSvc/getMsrstnAcctoRltmMesureDnsty?stationName=주엽동&dataTerm=DAILY&pageNo=1&numOfRows=100&returnType=json&serviceKey=KQRR%2BJLPRITcRv6CvRB1QUxmDQ%2BKmcKWMjK1A19g%2BiHLEbXTpqjWmut5pwHfKkH6O7KfqLSXxEmrLt6Ctooliw%3D%3D"

geturl = requests.get(url)
rawData = geturl.text

json_ob = json.loads(rawData)

jsonData = json_ob['response']['body']['items']
data = pandas.DataFrame(jsonData)

for i in range(len(data['khaiValue'])):
    if data['khaiValue'][i] == '-':
        data['khaiValue'][i] = '0'
for i in range(len(data['pm10Value'])):
    if data['pm10Value'][i] == '-':
        data['pm10Value'][i] = '0'
for i in range(len(data['coValue'])):
    if data['coValue'][i] == '-':
        data['coValue'][i] = '0'
for i in range(len(data['so2Value'])):
    if data['so2Value'][i] == '-':
        data['so2Value'][i] = '0'
for i in range(len(data['o3Value'])):
    if data['o3Value'][i] == '-':
        data['o3Value'][i] = '0'
for i in range(len(data['no2Value'])):
    if data['no2Value'][i] == '-':
        data['no2Value'][i] = '0'

data['환경 수치'] = pandas.to_numeric(data['khaiValue'])
data['미세먼지'] = pandas.to_numeric(data['pm10Value'])
data['일산화탄소'] = pandas.to_numeric(data['coValue'])
data['아황산가스'] = pandas.to_numeric(data['so2Value'])
data['오존'] = pandas.to_numeric(data['o3Value'])
data['이산화질소'] = pandas.to_numeric(data['no2Value'])

time = data['dataTime']
figure = data['환경 수치']
fineDust = data['미세먼지']
co = data['일산화탄소']
so2 = data['아황산가스']
o3 = data['오존']
no2 = data['이산화질소']

st.write('# 일산 주엽동 대기질')
st.write('***')
st.write('#### 선 그래프')
st.line_chart(figure)
st.line_chart(fineDust)
st.line_chart(co)
st.line_chart(so2)
st.line_chart(o3)
st.line_chart(no2)
st.write('***')
st.write('#### 막대 그래프')
st.bar_chart(figure)
st.bar_chart(fineDust)
st.bar_chart(co)
st.bar_chart(so2)
st.bar_chart(o3)
st.bar_chart(no2)


from collections import defaultdict
import logging

from ghidra.program.model.mem import MemoryAccessException

from .instr import instruction_finder
from .arch import LE, BE
from .mem_markers import UniqueMarker

logger = logging.getLogger(__name__)

class InvalidAddrException(Exception):
    pass

class NotCodeException(Exception):
    pass

class InvalidRegException(Exception):
    pass

class CodeWord(list):
    """
    This class represents a memory location containing code.  It will
    store a set of PCODE instruction classes.

    Note that this implementation leads to a serious deficiency - code and
    data are not interchangable here!  For instance - if code is
    self-modifying then the modified result will not be interpreted as code
    by the emulator, and the emulator will die when it tries to execute
    that (see Ram's get_code for that...).  Of course - in order to interpret
    self-modifying code, we also need a way to turn that code into PCODE,
    which is beyond the scope of this software, therefore, this serious
    deficiency is reasonable.
    """
    pass

class MemChunk(defaultdict):
    """
    This is intended to be an implementation of both reg and ram
    """
    def __init__(self, api_base, arch):
        super(MemChunk, self).__init__(int)

        self.arch = arch
        self.api_base = api_base

    def _validate_code(self, addr):
        addr = hex(addr)
        if addr.endswith("L"):
            addr = addr[:-1]
        return self.api_base.getInstructionAt(
                self.api_base.toAddr(addr)) is not None

    def _validate_writeable(self, addr):
        # TODO: check for actual writeableness, maybe
        return not self._validate_code(addr)

    def store(self, address, length, value):
        """Store a value at the given address, of "length" bytes, with
        endianness matching the architecture

        :param address: The index at which to store value
        :type addres: int
        :param length: The number of bytes over which to store value
        :type length: int
        :param value: The value to store
        :type value: int

        :raises InvalidAddrException: When the address was not writeable
        """
        if not self._validate_writeable(address):
            raise InvalidAddrException("Tried write at non-writeable spot")

        address = long(address)
        value = long(value)

        # Note that python handles bitwise operation on negative numbers as
        # 2s complement and like there are an infinite number of 1's in
        # front of the most significant bit.

        # This means that the below operations are already sign extended,
        # and this is what we'd expect a processor to do.

        # Thus - negative numbers just work.
        cur_val = value
        for ind in range(length):
            if self.arch.endian is LE:
                st_loc = address + ind
            else:
                st_loc = address + ((length - 1) - ind)
            self[st_loc] = cur_val & (2**self.arch.bits_per_byte - 1)
            cur_val >>= self.arch.bits_per_byte

    def load_byte(self, address):
        """Load just one byte from address
        """
        return self[address]

    def load(self, address, length):
        """Load a value from the given address, of "length" bytes, with
        endianness matching the architecture.

        :param address: The index from which to load
        :type addres: int
        :param length: The number of bytes to load
        :type length: int

        :return: The value loaded
        :rtype: int
        """
        address = long(address)

        cur_val = 0
        for ind in range(length):
            if self.arch.endian is LE:
                st_loc = address + ind
            else:
                st_loc = address + ((length - 1) - ind)
            one_byte = self.load_byte(st_loc) % 256
            cur_val += one_byte << (ind * self.arch.bits_per_byte)
        return long(cur_val)

    def __str__(self):
        sorted_keys = sorted(self.keys())
        return ", ".join("0x{:x}: {}".format(key, hex(self[key])) for key in sorted_keys)

class Registers(MemChunk):
    def _validate_writeable(self, addr):
        # This assumes that Ghidra will only try to write to writeable
        # registers.
        return True

    def __str__(self):
        def fmt_key(key):
            reg = None
            try:
                # Get the register object if possible
                reg = self.arch.lookup_reg_by_offset(key)
            except IndexError as e:
                logging.debug("Register not found {} {}".format(key, e))

            reg_size = 1
            reg_name = key
            if reg is not None:
                reg_size = reg.getMinimumByteSize()
                reg_name = "{}({:x})".format(reg.name, key)
            out_txt = "{}: 0x{:x}".format(reg_name, self.load(key, reg_size))
            used_keys = set(range(key, key + reg_size))
            return out_txt, used_keys
        sorted_keys = sorted(self.keys())
        vals = list()
        all_used_keys = set()
        for key in sorted_keys:
            if key in all_used_keys:
                continue
            out_txt, used_keys = fmt_key(key)
            all_used_keys = all_used_keys.union(used_keys)
            vals.append(out_txt)
        return ", ".join(vals)

class Uniques(MemChunk):
    def _validate_writeable(self, addr):
        # This assumes that Ghidra will only try to write to writeable
        # uniques.
        return True

    def store(self, address, length, value):
        """Store a value just like for the parent class, however, if
        the value is a UniqueMarker instance, store it as a special case
        at only the address.
        """
        if isinstance(value, UniqueMarker):
            self[address] = value
        else:
            super(Uniques, self).store(address, length, value)
    def load(self, address, length):
        """Load a value just like for the parent class, however, if
        the value is a UniqueMarker instance return only it.
        """
        if isinstance(self[address], UniqueMarker):
            return self[address]
        else:
            return super(Uniques, self).load(address, length)

class Ram(MemChunk):
    def load_byte(self, address):
        if address in self:
            return self[address]
        else:
            try:
                # It handles 64 bit values better when they're hex strings
                # without an L at the end
                addr = self.api_base.toAddr(hex(long(address))[:-1])
                return self.api_base.getByte(addr)
            except MemoryAccessException as e:
                logger.debug("mem access except")
                return 0

    def get_code(self, address):
        if not self._validate_code(address):
            raise InvalidAddrException("No code at address")

        inst = self.api_base.getInstructionAt(self.api_base.toAddr(address))
        inst_size = inst.length
        pcodes = inst.getPcode()

        instrs = [instruction_finder(pcode, self.arch) for pcode in pcodes]

        return instrs, inst_size


import numpy as np
import matplotlib.pyplot as plt
from scipy import signal
from bandit import Bandits

import glob
import os
import imageio

class UCB_Single_Freq():
    """
    The upper confidence bound algorithm
    (one in RL course slides)
    N      - Number of experiment
    l_na   - number of times a played up to t
    theta  - empirical reward of a up to t
    bandit - bandit class that store info about this
    """
    def __init__(self,bandit,sigma,N,name,tag):
        self.name      = name
        self.bandit    = bandit
        self.N         = N
        self.l_na      = np.zeros((self.bandit.K,1))
        self.sigma     = sigma
        self.regret    = [0]
        self.emp_rewards = np.random.random((self.bandit.K,1)) ### empirical rewards
        self.MSE         = []
        self.MSE1        = []
        self.tag = tag
        self.current_arm = 0
#         self.logregret   = []
    def compute_regret(self,arms):

        current_regret   = self.regret[-1] + self.bandit.maxnorm - self.bandit.norm[arms]

        self.regret.append(current_regret)

    def training(self,SHOWED = True,savefig=False,frame = 100):
        #### clear the previous values // re-initialize the values

        self.regret      = [0]
        self.l_na        = np.zeros((self.bandit.K,1))
        self.regret      = [0]
        self.MSE         = []
        self.MSE1        = []

        mse              = 0
        #self.theta       = np.zeros((bandit.K,2))
        self.emp_rewards = np.random.random((self.bandit.K,1))
        self.SUM_PX      = np.zeros((self.bandit.K,2))
        if savefig:
            if not os.path.exists(os.path.join('./Resultfig',"Result_GIF")):
                os.makedirs(os.path.join('./Resultfig',"Result_GIF"))
            filename = os.path.join('./Resultfig',self.name + "_"+self.tag+".gif")
            writer = imageio.get_writer(filename, format='GIF-PIL', mode='I', fps= 10)

        for t in range(self.N):
            ### SELECT ARMS
            if t < self.bandit.K:
                self.current_arm = t
            else:
                self.current_arm = np.argmax(self.emp_rewards +  self.sigma*np.sqrt(2*np.log(t+2)/self.l_na))

            self.l_na[self.current_arm] +=1

            ### perform the experiment and obtain X
            mean = [self.bandit.hnorm[self.current_arm].real,self.bandit.hnorm[self.current_arm].imag]
            cov  = np.identity(2) * self.sigma**2 / 2
            x_t  = np.random.multivariate_normal(mean,cov,1) ### rewards

            self.SUM_PX[self.current_arm][:] += x_t[0]

            ### update the emp_rewards
            self.emp_rewards[self.current_arm]    = (self.SUM_PX[self.current_arm][0]**2 + self.SUM_PX[self.current_arm][1]**2) / self.l_na[self.current_arm]**2

            ### update regret and rewards
            mse  = (np.nanmax(self.emp_rewards) - self.bandit.real_norm)**2
            self.MSE.append(mse)
            mse1  = (self.emp_rewards[np.nanargmax(self.l_na)] - self.bandit.real_norm)**2
            self.MSE1.append(mse1)
            self.compute_regret(self.current_arm)
            if savefig and t < frame:
                files = self.__create_png(t)
                image = imageio.imread(files)
                writer.append_data(image)
                os.remove(files)

    def __create_png(self,i):
        if not os.path.exists(os.path.join("./Resultfig","Result_png")):
            os.makedirs(os.path.join("./Resultfig","Result_png"))
        filename = os.path.join("./Resultfig","Result_png","empirical_"+str(i)+".png")
        plt.figure()
        plt.plot(self.bandit.arms,self.bandit.norm, label = '|G(e^{j$\omega_k$})|')
        dy  = self.sigma*np.sqrt(2*np.log(i+2)/(self.l_na+1e-10))
        y1  = [float(self.emp_rewards[inn] - dy[inn]) for inn in range(self.bandit.K)]
        y2  = [float(self.emp_rewards[inn] + dy[inn]) for inn in range(self.bandit.K)]
        plt.fill_between(self.bandit.arms, y1, y2,color='gray', alpha=0.2)
        plt.scatter(self.bandit.arms,self.emp_rewards,color = 'orange',label = r'|$\^G$(e^{j$\omega$})|')
        plt.scatter(self.bandit.arms[self.current_arm],self.emp_rewards[self.current_arm],color = 'red',label = r'|$\^G$(e^{j$\omega^*$})|')
        plt.legend()
        plt.xlabel('Frequency $\omega$ [rad/s]')
        plt.ylabel(r'|$\hat{G(j\omega)}$)|')
        plt.title("The empirical reward in "+self.name)
        plt.ylim([-1,2])
        plt.savefig(filename)
        plt.close()
        return filename

class UCB_Power_Allocation :
    def __init__(self,bandit,sigma,N,prior,name,beta,tag):
        self.name        = name
        self.bandit      = bandit
        self.N           = N
        self.pho         = prior
        self.prior       = prior
        self.emp_rewards = np.random.random((self.bandit.K,1)) ### empirical rewards
        self.l_na        = np.zeros((self.bandit.K,1))
        self.SUM_PX      = np.zeros((self.bandit.K,2))
        self.sigma       = sigma
        self.regret      = [0]
        self.MSE         = []
        self.MSE1        = []
        self.run_list    = [i for i in range(self.bandit.K)]
        self.beta        = beta
        self.tag         = tag

    def Calculate_Bounds(self,t):

        self.bounds = [self.emp_rewards[arms] + np.sqrt(2*self.sigma**2)*np.sqrt((np.log(t+1)/self.l_na[arms])) for arms in range(self.bandit.K)]
        self.bounds = np.array(self.bounds)

    def compute_regret(self,t):
        delta_t         = [self.pho[i] * (self.bandit.maxnorm - self.bandit.norm[i]) for i in range(self.bandit.K)]
        current_regret  = self.regret[-1] + np.sum(delta_t)
        self.regret.append(np.copy(current_regret))
    def reset_prior(self):
        self.prior            = [1/self.bandit.K] * self.bandit.K
        self.prior           /= np.sum(self.prior)

    def training(self, SHOWED = True,savefig=False,frame=0):
        #### clear the previous values // re-initialize the values
        self.reset_prior
        self.regret      = [0]
        self.l_na        = np.zeros((self.bandit.K,1))
        self.SUM_PX      = np.zeros((self.bandit.K,2))
        self.emp_rewards = np.zeros((self.bandit.K,1))
        self.pho         = self.prior
        self.bounds      = np.zeros((self.bandit.K,1))
        self.MSE         = []
        self.run_list    = [i for i in range(self.bandit.K)]
        if savefig:
            if not os.path.exists(os.path.join('./Resultfig',"Result_GIF")):
                os.makedirs(os.path.join('./Resultfig',"Result_GIF"))
            filename = os.path.join('./Resultfig',"Result_GIF",self.name + "_Power_Allocation_"+self.tag+".gif")
            writer = imageio.get_writer(filename, format='GIF-PIL', mode='I', fps= 10)

        for t in range(self.N):
            ### SELECT ARMS
            for arms in self.run_list :

                ### perform the experiment and obtain X
                if self.pho[arms] != 0:
                    mean = [self.bandit.hnorm[arms].real,self.bandit.hnorm[arms].imag]
                    cov  = np.identity(2) * self.sigma**2 / (2*self.pho[arms])
                    x_t  = np.random.multivariate_normal(mean,cov,1) ### rewards

                    ### calculate rewards
                    self.SUM_PX[arms][0] += self.pho[arms] * x_t[0][0]
                    self.SUM_PX[arms][1] += self.pho[arms] * x_t[0][1]
                    self.l_na[arms]      += self.pho[arms]

                    ### update the emp_rewards
                    self.emp_rewards[arms]    = (self.SUM_PX[arms][0]**2 + self.SUM_PX[arms][1]**2) / self.l_na[arms]**2

            self.Calculate_Bounds(t)
            if savefig and t < frame:
                files = self.__create_png(t)
                image = imageio.imread(files)
                writer.append_data(image)
                os.remove(files)
            self.Update_Pho(t,SHOWED)
            mse  = (np.nanmax(self.emp_rewards) - self.bandit.real_norm)**2
            self.MSE.append(mse)
            mse  = (self.emp_rewards[np.nanargmax(self.l_na)] - self.bandit.real_norm)**2
            self.MSE1.append(mse)
            ### update regret and reward

            self.compute_regret(t)

    def Update_Pho(self,t,SHOWED = False):
        r             = self.beta*np.sqrt(2*self.sigma**2)*(np.log(t+2))/(2+t)
        z             = 0
        T             = 0
        bar           = list(self.bounds.T[0])
        a             = np.nanmax(bar)
        s             = [i  for i in bar if i != a]
        N             = self.bandit.K - len(s)
        b             = 0
        self.run_list = []
        while (T < r) and (N < self.bandit.K) :
            b         = np.max(s)
            T         = T + N * (a - b)
            if T > r:
                break
            a          = b
            s          = [i  for i in s if i!= b]
            N          = self.bandit.K - len(s)
            # print('b = ',b,'T = ',T,'a = ',a,'N = ',N,'s = ',s)
        if t == 0:
            self.pho = [1/self.bandit.K] *self.bandit.K
        else:
            if N >= self.bandit.K:
                self.pho       = [self.bounds[i] for i in range(self.bandit.K)]
                self.run_list  = np.nonzero(self.pho)[0]
                self.pho      /= np.sum(self.pho)
            else:
                z              = b + (T-r)/N
                self.pho       = [a - z if a > z else 0 for a in self.bounds.T[0] ]
                self.run_list  = np.nonzero(self.pho)[0]
                self.pho      /= np.sum(self.pho)

        if SHOWED == True:
            print("t = ",t," pho = ",self.pho)
            plt.figure()
            plt.scatter(range(self.bandit.K),self.bounds,color = 'blue')
            plt.plot(range(self.bandit.K),self.bounds,color = 'blue')
            plt.bar(range(self.bandit.K),self.pho,color = 'r')
            plt.show()
            input("Press enter to continue...")

    def __create_png(self,i):
        if not os.path.exists(os.path.join("./Resultfig","Result_png")):
            os.makedirs(os.path.join("./Resultfig","Result_png"))
        filename = os.path.join("./Resultfig","Result_png","empirical_"+str(i)+".png")
        plt.figure()
        plt.plot(self.bandit.arms,self.bandit.norm, label = '|G(e^{j$\omega_k$})|')
        dy  = self.sigma*np.sqrt(2*np.log(i+2)/(self.l_na+1e-10))
        y1  = [float(self.emp_rewards[inn] - dy[inn]) for inn in range(self.bandit.K)]
        y2  = [float(self.emp_rewards[inn] + dy[inn]) for inn in range(self.bandit.K)]
        plt.fill_between(self.bandit.arms, y1, y2,color='gray', alpha=0.2)
        for current_arm,i_pho in enumerate(self.pho):

            if float(i_pho) > 0.0:
                plt.scatter(self.bandit.arms[current_arm],self.emp_rewards[current_arm],color = 'red',label = r'|$\^G$(e^{j$\omega^*$})|')
            else:
                plt.scatter(self.bandit.arms,self.emp_rewards,color = 'orange',label = r'|$\^G$(e^{j$\omega$})|')

        plt.xlabel('Frequency $\omega$ [rad/s]')
        plt.ylabel(r'|$\hat{G(j\omega)}$)|')
        plt.title("The empirical reward in "+self.name)
        plt.ylim([-1,2])
        plt.savefig(filename)
        plt.close()
        return filename


import logging
import os
import sys
import warnings
from datetime import datetime
from os import listdir
from os.path import isfile, join

import numpy as np
import pandas as pd

from util_functions.csv_export.csv_export import csv_export
from util_functions.fixes.fix_col_dates import check_na_col_dates
from util_functions.fixes.fix_enum import fix_enum
from util_functions.fixes.fix_lot_productCode import fix_lot_product_code
from util_functions.fixes.fix_urls import check_urls
from util_functions.fixes.masterid_modification import masterid_modification
from util_functions.importing.clean_text_columns import clean_text_columns
from util_functions.importing.import_file import import_data_file, reading_mapper
from util_functions.mini_jsons.indicators import indicators_mini_json
from util_functions.mini_jsons.sanctions import sanctions
from util_functions.set_up_logging import set_up_logging
from util_functions.util.move_util_files import move_util_files

def generate_indicators(filename_csv, country, chunk_it=None, chunk_size=10):
    """
    Takes in a data frame and generates indicators
    :param chunk_size: The number of chunks to split the data to using IDs
    :param chunk_it: chunk the data
    :param filename_csv: The name of the starting CSV file
    :param country: ISO-2 country code
    :return: Main data frame with indicators
    """
    print(os.getcwd())

    df, column_names = import_data_file(filename_csv=filename_csv, country=country)
    print(f"processing {country} with {len(df)} rows")
    # rename variables
    logging.info(f"Renaming columns...")
    # rename_mapper = reading_mapper(filename=f'../country_codes/{country}/name_mapper.txt')
    rename_mapper = reading_mapper(country=country, sheet_name="columns")
    df = df.rename(columns=rename_mapper)
    # PT FIX for
    df.loc[df.tender_id == "64c85640-8297-4ab7-b0c3-866880f0bd9a", "tender_addressofimplementation_country"] = "AT"
    # FIX CO master IDs
    if country == "CO":
        df = df.loc[df.buyer_masterid.notnull()]
        df = df.loc[df.bidder_masterid.notnull()]
        print(f"processing {country} after dropping missing master IDs with {len(df)} rows")

    col_text = ['buyer_name',
                'buyer_id',
                'buyer_masterid',
                'bidder_name',
                'bidder_id',
                'bidder_masterid',
                'lot_title']

    df = clean_text_columns(df, col_text)
    df = masterid_modification(df=df, country=country)
    # lot_productCode -- > lot_productcode
    df = fix_lot_product_code(df=df, country=country)
    # col_dates
    col_dates = ['tender_awarddecisiondate',
                 'tender_contractsignaturedate',
                 'tender_publications_firstdcontractawarddate']
    df = check_na_col_dates(df=df, col_dates=col_dates, country=country)

    if country in ['UK', 'DE', 'ES']:
        logging.info(f"special fix for {country}")
        logging.info(f"current number of rows {len(df)}")
        df = df.dropna(subset=['bidder_name']).reset_index()
        logging.info(f"dropping missing bidder_name...")
        logging.info(f"current number of rows {len(df)}")

    # logging.info(f'setting [""] in {col_name} no NA')
    # df.loc[df['buyer_mainactivities'] == '[""]', 'buyer_mainactivities'] = np.nan
    df = fix_enum(df=df, country=country)
    url_values = []
    url_list = ["tender_publications_lastcontractawardurl", "notice_url", "source"]
    df = check_urls(df=df, url_list=url_list)
    prices = ['bid_priceusd']
    logging.info(f"Finished renaming and cleaning columns")
    logging.info(f"Generating indicators' JSONs starting...")
    for entity_country in ['buyer_country', 'bidder_country', 'tender_country']:
        print(f"Fixing country codes in {entity_country} that are GB to UK")
        logging.info(f"Fixing country codes in {entity_country} that are GB to UK")
        try:
            df.loc[df[entity_country] == "GB", entity_country] = "UK"
            df.loc[df[entity_country] == 'British lndian Ocean Territory', entity_country] = "IO"

        except KeyError:
            print(f"{entity_country} is not available")
            pass

    # generate CSV export
    print("generate CSV export..")
    df_csv = csv_export(df=df, country=country)
    df_csv.to_csv(f"csv_export/{country}_export.csv.gz",
                  index=False, encoding='utf-8')
    del df_csv
    print(f"generate CSV export done. File in csv_export/{country}_export.csv")
    # generate indicators data
    if chunk_it:
        print(f"Chunking data for {country} into {chunk_size} chunks")
        logging.info(f"Chunking data for {country} into {chunk_size} chunks")
        df_groups = df.groupby('tender_id').ngroups
        range_groups_df = range(df_groups + 1)
        range_groups_df = np.array_split(np.array(range_groups_df), chunk_size)
        df['id_group'] = df.groupby('tender_id').ngroup()
        df_full = pd.DataFrame()
        for index_df, range_out_df in enumerate(range_groups_df):
            df_chunk = df[df['id_group'].isin(range_out_df)]
            print(f"processing range {index_df+1}, with chunk size of {len(df_chunk)}")
            logging.info(f"processing range {index_df}, with chunk size of {len(df_chunk)}")
            df_chunk = indicators_mini_json(df=df_chunk)
            logging.info(f"Selecting columns....")
            final_cols = list(rename_mapper.values()) + ["indicators"]
            for col_to_drop in ['bidder_previousSanction',
                                'bidder_hasSanction', 'sanct_startdate',
                                'sanct_enddate', 'sanct_name',
                                'lot_bidscount']:
                try:
                    final_cols.remove(col_to_drop)
                except ValueError:
                    continue
            df_chunk = df_chunk[final_cols]
            df_full = pd.concat([df_full, df_chunk])
            df_full = df_full.replace({'[""]': np.nan})  # temp
        logging.info(f"Processing sanctions...")
        df_full = sanctions(df=df_full, country=country)
        return df_full

    else:
        df = indicators_mini_json(df=df)
        logging.info(f"Selecting columns....")
        final_cols = list(rename_mapper.values()) + ["indicators"]
        for col_to_drop in ['bidder_previousSanction',
                            'bidder_hasSanction', 'sanct_startdate',
                            'sanct_enddate', 'sanct_name',
                            'lot_bidscount']:
            try:
                final_cols.remove(col_to_drop)
            except ValueError:
                continue
        df = df[final_cols]
        df = df.replace({'[""]': np.nan})  # temp
        logging.info(f"Processing sanctions...")
        df = sanctions(df=df, country=country)
        return df

    # processing sanctions output from matching in R

if __name__ == '__main__':
    warnings.simplefilter(action='ignore', category=FutureWarning)
    time_in = datetime.now()
    logging.info("making directories")
    os.makedirs("mod_ind_df", exist_ok=True)
    os.makedirs("csv_export", exist_ok=True)
    os.makedirs("validation_node/datasets", exist_ok=True)
    os.makedirs("validation_node/logs", exist_ok=True)

    country = str(sys.argv[1].split('/')[-1]).upper()
    try:
        chunk = str(sys.argv[2].split('/')[-1])
    except IndexError:
        chunk = False
        print("processing data as whole")
        pass
    try:
        chunk_it = str(sys.argv[3].split('/')[-1])
        chunk_it = True
        chunk_size = int(sys.argv[4].split('/')[-1])
    except IndexError:
        chunk_it = False
        chunk_size = 0
        print("processing data as whole with no chunks")
        pass
    set_up_logging(sys.argv[1].split('/')[-1], time_in)
    fileName_CSV = f"../utility_data/country/{country}/{country}_mod.csv"
    logging.info(f"Starting process...")

    current_path = f"../utility_data/country/{country}/"
    country_files = [f for f in listdir(current_path) if isfile(join(current_path, f)) and f[-3:] == 'zip']

    df_final = generate_indicators(filename_csv=fileName_CSV, country=country,
                                   chunk_it=chunk_it, chunk_size=chunk_size)

    if chunk == "chunk":
        df_final_groups = df_final.groupby('tender_id').ngroups
        range_groups = range(df_final_groups + 1)
        range_groups = np.array_split(np.array(range_groups), chunk_size)
        df_final['id_group'] = df_final.groupby('tender_id').ngroup()
        os.makedirs(f"mod_ind_df/{country}", exist_ok=True)
        for index, range_out in enumerate(range_groups):
            # df_final_chunk = df_final.loc[range_out, :]
            df_final_chunk = df_final[df_final.id_group.isin(range_out)]
            os.makedirs(f"mod_ind_df/{country}/{country}_{index + 1}/{country}", exist_ok=True)
            outfile = f"mod_ind_df/{country}/{country}_{index + 1}/{country}/{country}_mod_ind.csv"
            move_util_files(country=country, des_file=f"mod_ind_df/{country}/{country}_{index + 1}/{country}")
            print(f"copying tool to {outfile} dir")
            source_file = "reverse-flatten-tool"
            des_file = f"mod_ind_df/{country}/{country}_{index + 1}/reverse-flatten-tool-{country.lower()}-{index + 1}"
            os.system(f"\cp -fr {source_file} -T {des_file}")
            # shutil.copytree(source_file, des_file,)
            os.system(f"bash run_country_chunks.sh -c {country} -x {index+1}")
            logging.info(f"Exporting {outfile}...")
            df_final_chunk.to_csv(outfile, sep=';', index=False, encoding='utf-8')
            print(f"File saved in {outfile}")
        # move_util_files(country=country, des_file=f"mod_ind_df/{country}")
    else:
        outfile = f"../country_codes/{country}/{country}_mod_ind.csv"
        logging.info(f"Exporting {outfile}...")

        df_final.to_csv(outfile, sep=';', index=False, encoding='utf-8'
                        # , quoting=csv.QUOTE_NONNUMERIC
                        )
        print(f"File saved in {outfile}")

        move_util_files(country=country)
    time_out_temp = datetime.now()
    logging.info(f"Finished")
    logging.info(f"Run Time of Processing {time_out_temp - time_in}")
    print(f"Run Time of Processing for {country} {time_out_temp - time_in}")


from airflow import DAG
from airflow.operators.bash_operator import BashOperator

from datetime import datetime, timedelta

default_args = {
    'owner': 'airflow',
    'depends_on_past': False,
    'start_date': datetime(2015, 6, 1),
    'email': ['abhi.darkness@gmail.com'],
    'email_on_failure': True,
    'email_on_retry': False,
    'retries': 1,
    'retry_delay': timedelta(minutes=5),
    # 'queue': 'bash_queue',
    # 'pool': 'backfill',
    # 'priority_weight': 10,
    # 'end_date': datetime(2016, 1, 1),
}

dag = DAG('hello_world3', default_args=default_args, schedule_interval=timedelta(1))

t1 = BashOperator(
    task_id='print_date',
    bash_command='date',
    dag=dag)

t2 = BashOperator(
    task_id='sleep',
    bash_command='sleep 5',
    retries=3,
    dag=dag)

templated_command = """
    {% for i in range(5) %}
        echo "{{ ds }}"
        echo "{{ macros.ds_add(ds, 7)}}"
        echo "{{ params.my_param }}"
    {% endfor %}
"""

t3 = BashOperator(
    task_id='templated',
    bash_command=templated_command,
    params={'my_param': 'Parameter I passed in'},
    dag=dag)

t2.set_upstream(t1)

# This means that t2 will depend on t1
# running successfully to run
# It is equivalent to
# t1.set_downstream(t2)

t3.set_upstream(t1)

# all of this is equivalent to
# dag.set_dependency('print_date', 'sleep')
# dag.set_dependency('print_date', 'templated')


'''
 CS677 - Data Science with Python
 lecture - K-means
 By: Farshid Alizadeh-Shabdiz

'''

### Simple example
import pandas as pd
from sklearn.datasets import make_blobs
features , _ = make_blobs(n_samples=50, n_features=2, centers=3,  random_state=0)

#cluster_std=0.6,
dataframe = pd.DataFrame(features, columns=["feature_1", "feature_2"])
MyCluster = KMeans(n_clusters=3, random_state=0)

MyCluster.fit(features)

dataframe['group'] = MyCluster.predict(features)

dataframe.head(5)

print(MyCluster.labels_)

centers = MyCluster.cluster_centers_
print(centers)

plt.scatter(features[:, 0], features[:, 1], c=MyCluster.labels_, s=32, cmap="viridis")
plt.scatter(centers[:, 0], centers[:, 1], c='red', s=128, alpha=0.4);

MyCluster.predict([[0.5, 2.6], [2, 2], [3, 2], [1.23, 2.34]])

X = np.array([[1, 2], [1, 4], [1, 0],
              [2, 1], [3, 1],
              [4, 2], [4, 4], [4, 0]])

MyCluster.fit_predict(X) 		# Compute cluster centers and predict cluster index for each sample.
								# Convenience method; equivalent to calling fit(X) followed by predict(X).
MyCluster.fit_transform(X)		# Compute clustering and transform X to cluster-distance space.
								# In the new space, each dimension is the distance to the cluster centers.
								# Note that even if X is sparse, the array returned by transform will typically be dense.

##############
import pandas as pd
import numpy as np
from sklearn.cluster import KMeans

data = pd.DataFrame(
		{'id': [ 1,2,3,4,5,6,7,8],
		'Label': ['green','green','green','green', 'red','red','red','red'],
		'Height': [5, 5.5, 5.33, 5.75,
			6.00 , 5.92 , 5.58 , 5.92] ,
		'Weight': [100, 150, 130, 150, 180, 190, 170, 165],
		'Foot': [6, 8, 7, 9, 13, 11, 12, 10]},
		columns = ['id', 'Height', 'Weight',
			'Foot', 'Label'] )

##############################
### k-Means

init_centers = np.array([[5.0,7.0],[5.5, 9.0]])
colmap = {0: 'blue', 1: 'grey'}
n_clusters = 2
x = data[['Height', 'Foot']].values
kmeans_classifier = KMeans(n_clusters=n_clusters ,
		init=init_centers)
y_kmeans=kmeans_classifier.fit_predict(x)

kmeans_classifier.cluster_centers_
y_kmeans

### Plotting
import matplotlib.pyplot as plt

fig = plt.figure()
centroids = kmeans_classifier.cluster_centers_

for i in range(n_clusters):
	new_df = data[y_kmeans==i]
	plt.scatter(new_df['Height'],new_df['Foot'], color=colmap[i],
		s=100, label='points in cluster ' + str(i+1))
for i in range(n_clusters):
	plt.scatter(centroids[i][0], centroids[i][1], color=colmap[i],
		marker='x', s=300,label='centroid ' + str(i+1))
for i in range(len(data)):
	x_text = data['Height'].iloc[i] + 0.05
	y_text = data['Foot'].iloc[i] + 0.2
	id_text = data['id'].iloc[i]
	plt.text(x_text, y_text, str(id_text), fontsize=14)

plt.legend(loc='upper left')
plt.xlim(4, 7)
plt.ylim(5, 15)
plt.xlabel('Height')
plt.ylabel('Foot')
plt.show()

##############################
### Random initial centroids
colmap = {0: 'blue', 1: 'grey'}
x = data[['Height', 'Foot']].values
kmeans_classifier = KMeans(n_clusters=n_clusters)
y_kmeans=kmeans_classifier.fit_predict(x)
centroids = kmeans_classifier.cluster_centers_

centroids
y_kmeans
##############################
### IRIS data clustering
from sklearn import datasets

iris = datasets.load_iris()
x = iris.data

kmeans_classifier = KMeans(n_clusters=3)
y_kmeans = kmeans_classifier.fit_predict(x)
centroids = kmeans_classifier.cluster_centers_

centroids
y_kmeans

### Plotting
import matplotlib.pyplot as plt
fig, ax = plt.subplots(1,figsize=(7,5))
plt.scatter(x[y_kmeans == 0, 0], x[y_kmeans == 0, 1],
			s = 75, c = 'red', label = 'Iris-setosa')
plt.scatter(x[y_kmeans == 1, 0], x[y_kmeans == 1, 1],
			s = 75, c = 'blue', label = 'Iris-versicolour')
plt.scatter(x[y_kmeans == 2, 0], x[y_kmeans == 2, 1],
			s = 75, c = 'green', label = 'Iris-virginica')
plt.scatter(centroids[:, 0], centroids[:,1],
			s = 200, c = 'black', label = 'Centroids')

x_label = iris.feature_names[0]
y_label = iris.feature_names[1]

plt.legend()
plt.xlabel(x_label)
plt.ylabel(y_label)
plt.tight_layout()
plt.show()

### Code to compute K
from sklearn import datasets
import matplotlib.pyplot as plt
from sklearn.cluster import KMeans

iris = datasets.load_iris()
x = iris.data
inertia_list = []
for k in range (1 ,26):
	kmeans_classifier = KMeans(n_clusters=k)
	y_kmeans = kmeans_classifier.fit_predict(x)
	inertia = kmeans_classifier.inertia_
	inertia_list.append(inertia)

fig, ax = plt.subplots(1,figsize=(7,5))
plt.plot(range(1, 26), inertia_list , marker='o', color='green')

plt.legend()
plt.xlabel('number of clusters: k')
plt.ylabel('inertia')
plt.tight_layout()
plt.show()


import cv2
import numpy as np
import os

def nothing(x):
    pass

# Get current dir
base_path = os.getcwd()

# Gives width and height of frame
width = 640
height = 480

# Window name
window_name = 'img'

# Create a window
cv2.namedWindow(window_name, cv2.WINDOW_NORMAL)

# cv2.resizeWindow(window_name, width, height)

# Create trackbars for color change
# Hue is from 0-179 for Opencv
cv2.createTrackbar('HMin', window_name, 0, 179, nothing)
cv2.createTrackbar('SMin', window_name, 0, 255, nothing)
cv2.createTrackbar('VMin', window_name, 0, 255, nothing)
cv2.createTrackbar('HMax', window_name, 0, 179, nothing)
cv2.createTrackbar('SMax', window_name, 0, 255, nothing)
cv2.createTrackbar('VMax', window_name, 0, 255, nothing)

# Set default value for Max HSV trackbars
cv2.setTrackbarPos('HMax', window_name, 179)
cv2.setTrackbarPos('SMax', window_name, 255)
cv2.setTrackbarPos('VMax', window_name, 255)

# Initialize HSV min/max values
hMin = sMin = vMin = hMax = sMax = vMax = 0
phMin = psMin = pvMin = phMax = psMax = pvMax = 0

for my_path, my_subdirs, my_files in os.walk(base_path):
    print(my_path)
    for my_file in my_files:
        if(my_file.endswith(".jpg")):

            # Concatenate strings so we can access images in subdirs
            image = cv2.imread(my_path + "/" + my_file)

            while(1):
                # Get current positions of all trackbars
                hMin = cv2.getTrackbarPos('HMin', window_name)
                sMin = cv2.getTrackbarPos('SMin', window_name)
                vMin = cv2.getTrackbarPos('VMin', window_name)
                hMax = cv2.getTrackbarPos('HMax', window_name)
                sMax = cv2.getTrackbarPos('SMax', window_name)
                vMax = cv2.getTrackbarPos('VMax', window_name)

                # Set minimum and maximum HSV values to display
                lower = np.array([hMin, sMin, vMin])
                upper = np.array([hMax, sMax, vMax])

                # Convert to HSV format and color threshold
                hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
                mask = cv2.inRange(hsv, lower, upper)
                result = cv2.bitwise_and(image, image, mask=mask)

                # Print if there is a change in HSV value
                if((phMin != hMin) | (psMin != sMin) | (pvMin != vMin) | (phMax != hMax) | (psMax != sMax) | (pvMax != vMax) ):
                    print("(hMin = %d , sMin = %d, vMin = %d), (hMax = %d , sMax = %d, vMax = %d)" % (hMin , sMin , vMin, hMax, sMax , vMax))
                    phMin = hMin
                    psMin = sMin
                    pvMin = vMin
                    phMax = hMax
                    psMax = sMax
                    pvMax = vMax

                # Display result image
                cv2.imshow(window_name, result)
                if cv2.waitKey(10) & 0xFF == ord('q'):
                    break
                elif cv2.waitKey(10) & 0xFF == ord('e'): # Right now you've got to hold this down to make it work
                    exit()

cv2.destroyAllWindows()


from PyQt5 import QtCore, QtGui, QtWidgets
from PyQt5.QtWidgets import QInputDialog, QDialog

class Ui_Dialog(QDialog):
    def setupUi(self, Dialog):
        Dialog.setObjectName("Dialog")
        Dialog.resize(502, 250)
        self.list = QtWidgets.QListWidget(Dialog)
        self.list.setGeometry(QtCore.QRect(40, 10, 256, 192))
        self.list.setObjectName("list")
        self.Add = QtWidgets.QPushButton(Dialog)
        self.Add.setGeometry(QtCore.QRect(400, 10, 89, 25))
        self.Add.setObjectName("Add")
        self.pushButton_2 = QtWidgets.QPushButton(Dialog)
        self.pushButton_2.setGeometry(QtCore.QRect(400, 50, 89, 25))
        self.pushButton_2.setObjectName("pushButton_2")

        self.retranslateUi(Dialog)
        QtCore.QMetaObject.connectSlotsByName(Dialog)

        self.Add.clicked.connect(self.AddData)

    def AddData(self):
        row = self.list.currentRow()
        text, ok = QInputDialog.getText(self, "Employee Dialog", "Enter EMployee")
        if ok and text is not None:
            self.list.insertItem(row, text)

    def retranslateUi(self, Dialog):
        _translate = QtCore.QCoreApplication.translate
        Dialog.setWindowTitle(_translate("Dialog", "Dialog"))
        self.Add.setText(_translate("Dialog", "Add"))
        self.pushButton_2.setText(_translate("Dialog", "Delete"))

if __name__ == "__main__":
    import sys
    app = QtWidgets.QApplication(sys.argv)
    Dialog = QtWidgets.QDialog()
    ui = Ui_Dialog()
    ui.setupUi(Dialog)
    Dialog.show()
    sys.exit(app.exec_())


import sys
sys.path.append('..')
from CryptoPrediction  import CryptoPrediction
import pandas as pd
import numpy as np
import statsmodels as sm
import matplotlib.pyplot as plt
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
#.stats.diagnostic.acorr_ljungbox

cp = CryptoPrediction()
btc_df = pd.read_csv("../data/gemini_BTCUSD_2019_1min.csv")
data = btc_df["Close"][::-1].to_numpy()
lags = 14
X, y = cp.create_examples(data, lags)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, shuffle=False)

y_pred_naive = []
for i in range(len(X)):
    y_pred_naive.append(cp.naive_forecast(X[i]))

y_pred_avg_3 = []
for i in range(len(X)):
    y_pred_avg_3.append(cp.avg_forecast(X[i], 3))

lin_reg = LinearRegression().fit(X_train, y_train)
y_pred_lin_reg = lin_reg.predict(X_test)

residuals = np.subtract(y_test, np.array(y_pred_lin_reg))
res_df = pd.DataFrame(residuals)
print(res_df.describe())
print("NAIVE MSE: ", cp.mse(np.array(y_pred_naive), y))
print("AVERAGE MSE (3)", cp.mse(np.array(y_pred_avg_3), y))
print("Linear Regression MSE: ", cp.mse(y_pred_lin_reg, y_test))
# res_df.plot()
# res_df.hist()
# res_df.plot(kind='kde')
# sm.graphics.gofplots.qqplot(res_df)

sm.graphics.tsaplots.plot_acf(res_df, lags=lags)
#lbvalue, pvalue = sm.stats.diagnostic.acorr_ljungbox(res_df)
#plt.plot(pvalue[1:])

#pd.plotting.autocorrelation_plot(res_df)
plt.show()


"""A plugin that adds the ``config-wizard`` command to RepoBee. It runs through
a short configuration wizard that lets the user set RepoBee's defaults.

.. module:: configwizard
    :synopsis: Plugin that adds a configuration wizard to RepoBee.

.. moduleauthor:: Simon Larsén
"""
import argparse
import configparser
import collections
import os

from typing import Mapping, List

import bullet  # type: ignore

import repobee_plug as plug

from _repobee import constants

class Wizard(plug.Plugin, plug.cli.Command):
    __settings__ = plug.cli.command_settings(
        category=plug.cli.CoreCommand.config,
        help="Interactive configuration wizard to set up the config file.",
        description=(
            "A configuration wizard that sets up the configuration file."
            "Warns if there already is a configuration file, as it will be "
            "overwritten."
        ),
    )

    def command(self) -> None:
        return callback(self.args)

def callback(args: argparse.Namespace) -> None:
    """Run through a configuration wizard."""
    parser = configparser.ConfigParser()

    if constants.DEFAULT_CONFIG_FILE.exists():
        plug.echo(
            "Editing config file at {}".format(
                str(constants.DEFAULT_CONFIG_FILE)
            )
        )
        parser.read(str(constants.DEFAULT_CONFIG_FILE))

    os.makedirs(
        str(constants.DEFAULT_CONFIG_FILE.parent), mode=0o700, exist_ok=True
    )
    if constants.CORE_SECTION_HDR not in parser:
        parser.add_section(constants.CORE_SECTION_HDR)

    configurable_args = [
        plug.ConfigurableArguments(
            config_section_name=constants.CORE_SECTION_HDR,
            argnames=list(constants.ORDERED_CONFIGURABLE_ARGS),
        )
    ] + plug.manager.hook.get_configurable_args()

    configurable_args_dict: Mapping[str, List[str]] = collections.defaultdict(
        list
    )
    for ca in configurable_args:
        configurable_args_dict[ca.config_section_name] += ca.argnames

    section = bullet.Bullet(
        prompt="Select a section to configure:",
        choices=list(configurable_args_dict.keys()),
    ).launch()

    plug.echo(
        f"""
Configuring section: {section}
Type config values for the options when prompted.
Press ENTER without inputing a value to pick existing default.

Current defaults are shown in brackets [].
"""
    )
    for option in configurable_args_dict[section]:
        prompt = "Enter default for '{}': [{}] ".format(
            option, parser.get(section, option, fallback="")
        )
        default = input(prompt)
        if default:
            if section not in parser:
                parser.add_section(section)
            parser[section][option] = default

    with open(str(constants.DEFAULT_CONFIG_FILE), "w", encoding="utf8") as f:
        parser.write(f)

    plug.echo(
        "Configuration file written to {}".format(
            constants.DEFAULT_CONFIG_FILE
        )
    )


import pdb
import copy
import threading
from itertools import product
import datetime
import logging

from . import setup_environment
from . import utils
from .features import class_map
from .features import officers_collate

log = logging.getLogger(__name__)

def populate_features_table(config, schema):
    """Calculate values for all features which are set to True (in the config file)
    for the appropriate run type (officer/dispatch)
    """
    engine = setup_environment.get_database()
    if config['unit'] == 'officer':
        populate_officer_features_table(config, schema, engine)

def populate_dispatch_features_table(config, table_name, engine):
    """Calculate all the feature values and store them in the features table in the database"""

    # Get a list of all the features that are set to true.
    feature_list = [feat for feat, is_set_true in config['dispatch_features'].items() if is_set_true]
    num_features = len(feature_list)

    # make sure we have at least 1 feature
    assert num_features > 0, 'List of features to build is empty'

    feature_threads = []

    # run the build_and_insert of a set of features
    def run_thread(feature_sublist, engine):

        db_conn = engine.connect()

        for feature_name in feature_sublist:
            log.debug('... building feature {}'.format(feature_name))

            feature_obj = class_map.lookup(feature_name,
                                           unit='dispatch',
                                           from_date=config['raw_data_from_date'],
                                           to_date=config['raw_data_to_date'],
                                           fake_today=datetime.datetime.today(),
                                           table_name=table_name)
            feature_obj.build_and_insert(db_conn)

        db_conn.close()

    def chunks(l, n):
        """Yield successive n-sized chunks from l."""
        for i in range(0, len(l), n):
            yield l[i:i + n]

    # build each feature and store it in its own table in features_prejoin
    # start a new thread for each set of 5 features
    for feature_sublist in chunks(feature_list, 5):
        t = threading.Thread(target=run_thread, args=(feature_sublist, engine,))
        feature_threads.append(t)
        t.start()

    # join each thread and wait for it to be done to make sure we're done building them all
    # before we move on to joining them
    for i, thread in enumerate(feature_threads):
        log.debug('Waiting for feature thread: {}/{})'.format(i, (num_features / 5)))
        thread.join()

    # join each single-feature to the main table one at a time
    for i, feature_name in enumerate(feature_list):
        log.debug("Adding feature {}/{} ({})".format(i, len(feature_list), feature_name))

        update_query = ("UPDATE features.{table_name} AS feature_table "
                        "SET {feature} = prejoin_table.{feature} "
                        "FROM features_prejoin.{feature} AS prejoin_table "
                        "WHERE feature_table.dispatch_id = prejoin_table.dispatch_id "
                        .format(table_name=table_name,
                                feature=feature_name))

        engine.execute(update_query)

def join_feature_table(engine, list_prefixes, schema, features_table_name):
    """
    This function joins the block tables into the features_table_name
    using the prefix of the aggregated tables specified in each class

    :param engine: engine to connect to db
    :param list list_prefixes: list of prefixes specified in each block class
    :param str schema: name of schema where collate table are stored
    :param str features_table_name: name of the table to create in the features schema
    """
    table_names = ['{}_aggregation'.format(prefix) for prefix in list_prefixes]

    # seperate the tables by block that have a date column or not
    table_names_no_date = [x for x in table_names if 'ND' in x]
    table_names_with_date = [x for x in table_names if x not in set(table_names_no_date)]

    query = ""
    if len(table_names) > 0:
        if table_names_with_date:
            query = """ select * from {}."{}" """.format(schema, table_names_with_date[0])
            table_names_with_date = table_names_with_date[1:]
        for table_name in table_names_with_date:
            query += """ full outer join {}."{}"  using (officer_id, as_of_date)""".format(schema, table_name)

        # check if in the first loop above a table was added
        if len(query) == 0:
            query = """ select * from {}."{}" """.format(schema, table_names_no_date[0])
            table_names_no_date = table_names_no_date[1:]

        for table_name in table_names_no_date:
            query += """ full outer join {}."{}"  using (officer_id)""".format(schema, table_name)

        drop_table_query = """DROP TABLE IF EXISTS features."{}";""".format(features_table_name)
        engine.execute(drop_table_query)

        create_table_query = """CREATE TABLE features."{0}" as ({1});""".format(features_table_name,
                                                                                query)
        engine.execute(create_table_query)

        create_as_of_date_index = """CREATE INDEX on features."{0}" (as_of_date);  """.format(features_table_name)
        engine.execute(create_as_of_date_index)

        create_officer_date_index = """CREATE INDEX on features."{0}" (as_of_date, officer_id);  """.format(
            features_table_name)
        engine.execute(create_officer_date_index)

def add_feature_indexes(engine, list_prefixes, schema):
    """
    This function add indexes to each feature table created to speed up the matrix creation

    :param engine: engine to connect to db
    :param list list_prefixes: list of prefixes specified in each block class
    :param str schema: name of schema where collate table are stored
    """

    table_names = ['{}_aggregation'.format(prefix) for prefix in list_prefixes]

    # seperate the tables by block that have a date column or not
    table_names_no_date = [x for x in table_names if 'ND' in x]
    table_names_with_date = [x for x in table_names if x not in set(table_names_no_date)]

    for table_name in table_names_with_date:
        create_as_of_date_index = """ALTER TABLE "{0}"."{1}" ADD PRIMARY KEY (as_of_date,officer_id);  """.format(schema,
                                                                                table_name)
        engine.execute(create_as_of_date_index)

    for table_name in table_names_no_date:
        create_officer_index = """ALTER TABLE  "{0}"."{1}" ADD PRIMARY KEY (officer_id);  """.format(schema, table_name)
        engine.execute(create_officer_index)

def populate_officer_features_table(config, schema, engine):
    """
     Calculate all the feature values and store them in the features table in the database
     using collate method that creates a table of feature for each block and stores them in a
     given schema.

     Args:
        config: Python dict read in from YAML config file containing
                user-supplied details of the experiments to be run
        schema: schama name for storing collate tables
     """
    temporal_info = config['temporal_info'].copy()
    # get the list of fake todays specified by the config file
    as_of_dates = utils.generate_feature_dates(temporal_info)
    log.debug(as_of_dates)

    list_prefixes = []
    # get a list of all features that are set to true.
    for block_name in config["officer_features"]:
        log.debug('block_name: {}'.format(block_name))
        block = config['feature_blocks'][block_name]
        feature_list = [key for key in block if block[key] == True]

        ## Need to find a way of calling the class given the block_name
        block_class = class_map.lookup_block(block_name,
                                             module=officers_collate,
                                             lookback_durations=temporal_info['timegated_feature_lookback_duration'],
                                             n_cpus=config['n_cpus'])

        # Build collate tables and returns table name
        block_class.build_collate(engine, as_of_dates, feature_list, schema)
        block_class.build_post_features(engine, feature_list, schema)
        list_prefixes.extend(block_class.prefix)

    # Join all tables into one
    log.debug(list_prefixes)
    add_feature_indexes(engine, list_prefixes, schema)

#    join_feature_table(engine, list_prefixes, schema, table_name)


# Question: https://leetcode.com/problems/rotating-the-box/
# Medium
# Can be optimized
from typing import Optional, List

class Solution:
    # O(HEIGHT * (WIDTH^2)) Time and O(1) Space
    def rotateTheBox(self, box: List[List[str]]) -> List[List[str]]:
        HEIGHT, WIDTH = len(box), len(box[0])

        for row in range(HEIGHT):
            for col in reversed(range(WIDTH)):
                if box[row][col] == '#':
                    idx = col + 1

                    while idx < WIDTH and box[row][idx] == '.':
                        idx += 1

                    idx -= 1
                    box[row][col] = '.'
                    box[row][idx] = '#'

        return zip(*box[::-1])
'''

# Kunal Wadhwa

'''


#-------------------------------------------------------------------------
# AUTHOR: Isaiah Hessler
# FILENAME: Assignment 2 - Part 1
# SPECIFICATION: Complete the Python program (decision_tree_2.py) that will read the files
# contact_lens_training_1.csv, contact_lens_training_2.csv, and contact_lens_training_3.csv. Each of
# those training sets has a different number of instances. You will observe that now the trees are being
# created setting the parameter max_depth = 3, which it is used to define the maximum depth of the tree
# (pre-pruning strategy) in sklearn. Your goal is to train, test, and output the performance of the 3 models
# created by using each training set on the test set provided (contact_lens_test.csv). You must repeat
# this process 10 times (train and test by using a different training set), choosing the average accuracy
# as the final classification performance of each model.
# FOR: CS 4210- Assignment #2
# TIME SPENT: 4 Hours
#-----------------------------------------------------------*/

#IMPORTANT NOTE: DO NOT USE ANY ADVANCED PYTHON LIBRARY TO COMPLETE THIS CODE SUCH AS numpy OR pandas. You have to work here only with standard
# dictionaries, lists, and arrays

#importing some Python libraries
from sklearn import tree
import csv

dataSets = ['contact_lens_training_1.csv', 'contact_lens_training_2.csv', 'contact_lens_training_3.csv']
testData = 'contact_lens_test.csv'
counter = 1

for ds in dataSets:

    dbTraining = []
    X = []
    Y = []
    XTest = []
    YTest = []
    sum = 0

    #reading the training data in a csv file
    with open(ds, 'r') as csvfile:
         reader = csv.reader(csvfile)
         for i, row in enumerate(reader):
             if i > 0: #skipping the header
                dbTraining.append (row)

    #transform the original categorical training features to numbers and add to the 4D array X. For instance Young = 1, Prepresbyopic = 2, Presbyopic = 3
    # so X = [[1, 1, 1, 1], [2, 2, 2, 2], ...]]
    for row in dbTraining:
        temp = []
        for j, value in enumerate(row):
            if j == 0:
                if value == 'Young':
                    temp.append(1)
                elif value == 'Prepresbyopic':
                    temp.append(2)
                else:
                    temp.append(3)
            if j == 1:
                if value == 'Myope':
                    temp.append(1)
                else:
                    temp.append(2)
            if j == 2:
                if value == 'No':
                    temp.append(1)
                else:
                    temp.append(2)
            if j == 3:
                if value == 'Reduced':
                    temp.append(1)
                else:
                    temp.append(2)
        X.append(temp)

    #transform the original categorical training classes to numbers and add to the vector Y. For instance Yes = 1, No = 2, so Y = [1, 1, 2, 2, ...]
    for row in dbTraining:
        if row[-1] == 'Yes':
            Y.append(1)
        else:
            Y.append(2)

    #loop your training and test tasks 10 times here
    for i in range (10):

       #fitting the decision tree to the data setting max_depth=3
        clf = tree.DecisionTreeClassifier(criterion = 'entropy', max_depth=3)
        clf = clf.fit(X, Y)

       #read the test data and add this data to dbTest
        dbTest = []
        with open(testData, 'r') as csvfile:
            reader = csv.reader(csvfile)
            for i, row in enumerate(reader):
                if i > 0: #skipping the header
                    dbTest.append (row)

        #transform the features of the test instances to numbers following the same strategy done during training,
        #and then use the decision tree to make the class prediction. For instance: class_predicted = clf.predict([[3, 1, 2, 1]])[0]
        #where [0] is used to get an integer as the predicted class label so that you can compare it with the true label
        #--> add your Python code here
        for row in dbTest:
            temp = []
            for j, value in enumerate(row):
                if j == 0:
                    if value == 'Young':
                        temp.append(1)
                    elif value == 'Prepresbyopic':
                        temp.append(2)
                    else:
                        temp.append(3)
                if j == 1:
                    if value == 'Myope':
                        temp.append(1)
                    else:
                        temp.append(2)
                if j == 2:
                    if value == 'No':
                        temp.append(1)
                    else:
                        temp.append(2)
                if j == 3:
                    if value == 'Reduced':
                        temp.append(1)
                    else:
                        temp.append(2)
            XTest.append(temp)

        for row in dbTest:
            if row[-1] == 'Yes':
                YTest.append(1)
            else:
                YTest.append(2)

        class_predicted = clf.predict(XTest)
           #compare the prediction with the true label (located at data[4]) of the test instance to start calculating the accuracy.
           #--> add your Python code here

        TP = 0
        TN = 0
        FP = 0
        FN = 0
        accuracy = 0
        x = 0
        for class_row in YTest:
            if class_row == 1:
                if class_predicted[x] == 1:
                    TP += 1
                else:
                    FN += 1
            elif class_row == 2:
                if class_predicted[x] == 2:
                    TN += 1
                else:
                    FP += 1
            x += 1

        accuracy = (TP + TN) / (TP + TN + FP + FN)
        XTest.clear()
        YTest.clear()

        #find the average accuracy of this model during the 10 runs (training and test set)
        sum += accuracy

    #print the average accuracy of this model during the 10 runs (training and test set).
    #your output should be something like that: final accuracy when training on contact_lens_training_1.csv: 0.2
    sum = sum / 10
    print(f'Final accuracy when training on contact_lens_training_{counter}.csv: {sum}')
    counter += 1


import pygame as pg
from pygame import image
import src.animation as animation
from .utils import *
from copy import copy

class UI:
    def __init__(self, screen, fruit, fps: int, player):
        self.screen, self.w, self.h = screen, screen.get_width(), screen.get_height()
        self.fruit_class = fruit
        self.time_left: int = 0
        self.fps: int = fps
        self.player = player
        self.clock_animation: animation.Animation = animation.Animation(
            544, 6, [load_alpha(f"data/assets/clock/clock{i+1}.png") for i in range(8)], 3
        )

        img = load_alpha("data/assets/frut_background.png")
        self.ui_fruits: pg.Surface = pg.transform.scale(img, (img.get_width()*2, int(img.get_height()*1.2)))

        self.font = pg.font.Font("data/fonts/Minecraft.ttf", 25)
        self.ui_fr_rect: pg.Rect = self.ui_fruits.get_rect(centerx=self.w//2, bottom=self.h-5)
        self.images = self.fruit_class.current_fruit
        self.images = [resizex(img, 0.8) for img in self.images]
        self.rects = [img.get_rect(center=(self.ui_fr_rect.centerx-(len(self.images)//2)*id_, self.ui_fr_rect.centery)) for id_, img in enumerate(self.images)]

    def reset(self, time_for_level):
        self.time_left: int = time_for_level

    def tick(self):
        self.time_left -= (1/self.fps) if self.time_left > 0 else 0

    def get_time(self):
        if self.time_left > 0:
            return f"""{
                int(self.time_left // 60) if len(str(int(self.time_left // 60))) == 2 else f"0{int(self.time_left // 60)}"
            }:{
                int(self.time_left - self.time_left // 60 * 60) if len(str(int(self.time_left - self.time_left // 60 * 60))) == 2 else f"0{int(self.time_left - self.time_left // 60 * 60)}"
            }"""

    def update(self):
        self.screen.blit(self.ui_fruits, self.ui_fr_rect)

        self.screen.blit(self.ui_fruits, (self.ui_fr_rect.x, 0))
        score = self.font.render(f'{"0" * (5 - len(str(self.player.score))) + str(int(self.player.score))}', True, (0, 0, 0))
        self.screen.blit(score, (self.ui_fr_rect.centerx - score.get_width()//2, 5))

        prev = copy(self.images)
        self.images = self.fruit_class.current_fruit

        if prev != self.images:
            # update images if they change (avoid doing this everytime and adding a func to init fruits)
            self.images = [resize(img, (int(img.get_width()*0.8), int(img.get_height()*0.8))) for img in self.images]
        self.rects = [img.get_rect(center=(self.ui_fr_rect.centerx-(((len(self.images)//2)-id_)*32), self.ui_fr_rect.centery)) for id_, img in enumerate(self.images)]

        for id_, img in enumerate(self.images):
            self.screen.blit(img, self.rects[id_])

        st_rct = self.rects[self.fruit_class.state]
        pg.draw.rect(self.screen, (0, 0, 0), [st_rct.x-2, st_rct.y-2, st_rct.width+4, st_rct.height+4], width=2)