import os
os.environ['PYGAME_HIDE_SUPPORT_PROMPT'] = "hide"
import pygame
from solver import *
from tkinter import *
from tkinter import messagebox

board = None
def clearBoard():
  global board
  board = [
        [0, 0, 0, 0, 0, 0, 0, 0, 0],
        [0, 0, 0, 0, 0, 0, 0, 0, 0],
        [0, 0, 0, 0, 0, 0, 0, 0, 0],
        [0, 0, 0, 0, 0, 0, 0, 0, 0],
        [0, 0, 0, 0, 0, 0, 0, 0, 0],
        [0, 0, 0, 0, 0, 0, 0, 0, 0],
        [0, 0, 0, 0, 0, 0, 0, 0, 0],
        [0, 0, 0, 0, 0, 0, 0, 0, 0],
        [0, 0, 0, 0, 0, 0, 0, 0, 0]]
  screen.fill((255,255,255))
  drawGrid()
  drawButtons()
  pygame.display.update()

pygame.font.init()
myFont = pygame.font.SysFont(None, 65)
pygame.display.set_caption("Sudoku Solver")
(width, height) = (700, 503)
screen = pygame.display.set_mode((width, height))
background_colour = (255,255,255)
screen.fill(background_colour)
resetButton = None
solveButton = None
#sets up a window with a white background and a title

x = 0
y = 0
actualx = 0
actualy = 0

def drawGrid():
    pygame.draw.rect(screen, (0,0,0), (1, 1, 500 ,500), 4)
    for i in range(8):
        if (i+1) % 3 == 0 and i != 0:
            thick = 3
        else:
            thick = 1
        gridInterval = ((i+1)/9)*501
        pygame.draw.line(screen, (0,0,0), (0, gridInterval), (501, gridInterval), thick)
        pygame.draw.line(screen, (0, 0, 0), (gridInterval, 0), (gridInterval, 501), thick)
        #draws a square and 8 lines across and down, changing thickness to make the board

def getSelected(pos):
    global x
    global y
    global actualx
    global actualy
    global board
    if pos <= (501, 501):
        x = int(((pos[0]-1) // 55.5556))
        y = int(((pos[1]-1) // 55.5556))
        actualx = ((x/9)*500)+1
        actualy = ((y/9)*500)+1
    elif resetButton.collidepoint(pos):
        clearBoard()
        pygame.display.update()
    elif solveButton.collidepoint(pos):
        valid = validateBoard(board)
        if all(valid):
          solve(board)
          drawNumbers(board)
          pygame.display.update()
        else:
          if valid[0] == False:
            Tk().wm_withdraw()
            messagebox.showinfo('Error','The Sudoku entered does not have enough numbers (Minimum is 17).')
          if valid[1] == False:
            Tk().wm_withdraw()
            messagebox.showinfo('Error','The Sudoku entered violates Sudoku Rules.')

    #calculates which box or button the mouse has clicked on and where to draw the blue outline

def drawSelectedCell(pos):
    global x
    global y
    getSelected(pos)
    screen.fill((255,255,255))
    drawGrid()
    drawButtons()
    drawNumbers(board)
    pygame.draw.rect(screen, (0, 85, 255), (actualx, actualy, 55.5556, 55.5556), 3)
    pygame.display.update()
    #wipes the screen and draws the grid and new blue box

def drawNumbers(board):
    screen.fill((255,255,255))
    drawGrid()
    drawButtons()
    pygame.draw.rect(screen, (0, 85, 255), (actualx, actualy, 55.5556, 55.5556), 3)
    for i in range(9):
        for j in range(9):
            if board[i][j] != 0:
              text = myFont.render(str(board[i][j]), False, (0,0,0))
              screen.blit(text, ((j/9*500) + 17, (i/9*500) + 8))
    pygame.display.update()
    #draws numbers from left to right, top to bottom after clearing the image and drawing the grid and cursor

def drawButtons():
    global resetButton
    global solveButton
    buttonBackground = pygame.image.load(r'C:\Users\brett\OneDrive\Pictures\Sudoku\buttonBackground.png')
    buttonBackground = pygame.transform.scale(buttonBackground, (126, 48))
    resetButton = screen.blit(buttonBackground, (545, 145))
    solveButton = screen.blit(buttonBackground, (545, 310))
    text = myFont.render("Reset", False, (0,0,0))
    screen.blit(text, (550, 150))
    text = myFont.render("Solve", False, (0,0,0))
    screen.blit(text, (550, 315))
    #draws the buttons and button text

def validateBoard(board):
    count = 0
    for i in range(9):
        count = count + board[i].count(0)
    sudokuRules = True
    for i in range(9):
      for j in range(9):
        if board[i][j] != 0 and valid(board, board[i][j], (i,j)) == False:
            sudokuRules = False
    conditions = [count <= 64, sudokuRules]
    return conditions
    #checks the board can be solved

clearBoard()
drawGrid()
drawButtons()
pygame.display.update()
pygame.display.flip()

running = True
while running:
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
          running = False
        if event.type == pygame.KEYDOWN:
                if event.key == pygame.K_1:
                    board[y][x] = 1
                    drawNumbers(board)
                if event.key == pygame.K_2:
                    board[y][x] = 2
                    drawNumbers(board)
                if event.key == pygame.K_3:
                    board[y][x] = 3
                    drawNumbers(board)
                if event.key == pygame.K_4:
                    board[y][x] = 4
                    drawNumbers(board)
                if event.key == pygame.K_5:
                    board[y][x] = 5
                    drawNumbers(board)
                if event.key == pygame.K_6:
                    board[y][x] = 6
                    drawNumbers(board)
                if event.key == pygame.K_7:
                    board[y][x] = 7
                    drawNumbers(board)
                if event.key == pygame.K_8:
                    board[y][x] = 8
                    drawNumbers(board)
                if event.key == pygame.K_9:
                    board[y][x] = 9
                    drawNumbers(board)
                if event.key == pygame.K_DELETE or event.key == pygame.K_BACKSPACE:
                    board[y][x] = 0
                    drawNumbers(board)
                if pos:
                  if event.key == pygame.K_UP:
                    if y > 0:
                      y = y - 1
                      pos = (pos[0], (y * 55.5556)+5)
                      drawSelectedCell(pos)
                  if event.key == pygame.K_DOWN:
                    if y < 8:
                      y = y + 1
                      pos = (pos[0], (y * 55.5556)+5)
                      drawSelectedCell(pos)
                  if event.key == pygame.K_LEFT:
                    if x > 0:
                      x = x - 1
                      pos = ((x * 55.5556)+5, pos[1])
                      drawSelectedCell(pos)
                  if event.key == pygame.K_RIGHT:
                    if x < 8:
                      x = x + 1
                      pos = ((x * 55.5556)+5, pos[1])
                      drawSelectedCell(pos)

        if event.type == pygame.MOUSEBUTTONDOWN:
          pos = pygame.mouse.get_pos()
          drawSelectedCell(pos)
          #main loop: checks for any input

pygame.quit()


import random
import os
from shutil import copyfile
import torchvision
from PIL import Image
from torchvision import transforms
import os

def resizeAndNormalizeImage(inputPath, outputPath):
    ''' resize and normalize an image '''

    img = Image.open(inputPath)
    img = torchvision.transforms.ToTensor()(img)
    img = torchvision.transforms.Normalize(mean=(0.485,0.456,0.406), std=(0.229,0.224,0.225))(img)
    img = torchvision.transforms.ToPILImage()(img)
    img = torchvision.transforms.Resize((350,350))(img)
    img.save(outputPath)

def resizeAndNormalizeFolder(inputFolder, targetDirectory):

    ''' resize and normalize all the images in a folder '''

    folderType = inputFolder.split("/")[-1]
    for filename in os.listdir(inputFolder):
        if filename.endswith(".jpg"):
            inputPath = inputFolder + "/" + filename
            outputPath = targetDirectory + "/" + folderType + "/" + filename
            resizeAndNormalizeImage(inputPath, outputPath)

def initializeDirectory(root):
    lf = "/Left_Fracture"
    lnf = "/Left_Non-Fracture"
    rf = "/Right_Fracture"
    rnf = "/Right_Non-Fracture"

    if not os.path.exists(root):
        print("a")
        os.mkdir(root)
    if not os.path.exists(root + lf):
        print("b")
        os.mkdir(root + lf)
    if not os.path.exists(root + lnf):
        print("c")
        os.mkdir(root + lnf)
    if not os.path.exists(root + rf):
        print("d")
        os.mkdir(root + rf)
    if not os.path.exists(root + rnf):
        print("e")
        os.mkdir(root + rnf)

def getSubGroups(root):

    ''' generate the names of the data folders '''

    ''' the following are the various directories (already existing in the file system) '''

    lf = "/Left_Fracture"
    lnf = "/Left_Non-Fracture"
    rf = "/Right_Fracture"
    rnf = "/Right_Non-Fracture"

    lfFolder = root + lf
    lnfFolder = root + lnf
    rfFolder = root + rf
    rnfFolder = root + rnf

    return [lfFolder, lnfFolder, rfFolder, rnfFolder]

valDirectory = "/home/sethbw/Documents/FourierResNet__Depreciated/DeepLearningRadiographs/Validation"
testDirectory = "/home/sethbw/Documents/FourierResNet__Depreciated/DeepLearningRadiographs/Test"
trainDirectory = "/home/sethbw/Documents/FourierResNet__Depreciated/DeepLearningRadiographs/TrainAugmented"
trainNADirectory = "/home/sethbw/Documents/FourierResNet__Depreciated/DeepLearningRadiographs/Train"

inputFolders = getSubGroups(trainNADirectory)
targetDirectory = "/home/sethbw/Documents/HipFractureDiagnosis/DeepLearningRadiographs/TrainMediumSmall"

initializeDirectory(targetDirectory)
for folder in inputFolders:
    resizeAndNormalizeFolder(folder, targetDirectory)

inputFolders = getSubGroups(testDirectory)
targetDirectory = "/home/sethbw/Documents/HipFractureDiagnosis/DeepLearningRadiographs/TestMediumSmall"
initializeDirectory(targetDirectory)
for folder in inputFolders:
    resizeAndNormalizeFolder(folder, targetDirectory)

inputFolders = getSubGroups(valDirectory)
targetDirectory = "/home/sethbw/Documents/HipFractureDiagnosis/DeepLearningRadiographs/ValidationMediumSmall"
initializeDirectory(targetDirectory)
for folder in inputFolders:
    resizeAndNormalizeFolder(folder, targetDirectory)

inputFolders = getSubGroups(trainDirectory)
targetDirectory = "/home/sethbw/Documents/HipFractureDiagnosis/DeepLearningRadiographs/TrainAugmentedMediumSmall"
initializeDirectory(targetDirectory)
for folder in inputFolders:
    resizeAndNormalizeFolder(folder, targetDirectory)


"""changed column name

Revision ID: d117cba5172d
Revises: 07cc036446ac
Create Date: 2021-12-21 13:01:29.513999

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'd117cba5172d'
down_revision = '07cc036446ac'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('Broup', sa.Column('is_left', sa.Boolean(), nullable=True))
    op.drop_column('Broup', 'left')
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('Broup', sa.Column('left', sa.BOOLEAN(), autoincrement=False, nullable=True))
    op.drop_column('Broup', 'is_left')
    # ### end Alembic commands ###


import pickle
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim

import pandas as pd
import os
import heapq
import sys
import_path = os.path.split(os.path.realpath(__file__))[0] + '/..'
if import_path not in sys.path:
    sys.path.append(import_path)
import time
import random
from itertools import product
from sklearn.preprocessing import MinMaxScaler

from utils.train import Trainer, Plugin
from metrics.RootRelativeSquaredError import RRSE
from metrics.EmpiricalCorrelaionCoefficient import CORR
from metrics.MeanAbsoluteError import MAE
from metrics.MeanSquaredError import MSE
from itertools import product

def ensure_path(path):
    if not os.path.exists(path):
        os.makedirs(path)

def grid_search(*hyper_params):
    for hyper_param in product(*hyper_params):
        yield hyper_param

def get_name_from_params(params):
    params = [str(p) for p in params]
    f = '_'.join(params)
    return f

def get_with_pickle(fn):
    with open(fn, 'rb') as f:
        result = pickle.load(f)
    return result

def save_with_pickle(obj, fn):
    with open(fn, 'wb') as f:
        pickle.dump(obj, f)

class MTSDataset(torch.utils.data.Dataset):
    # data :numpy.array
    # data.shape = (N, feature)
    #mode in ['train', 'valid', 'test']
    def __init__(self, data, window, horizon, mm, mode='train'):
        N, _ = data.shape
        self.window = window
        self.horizon = horizon
        self.mode = mode

        self.nums = N - window - horizon + 1
        self.train_nums = int(self.nums*0.6)
        self.valid_nums = int(self.nums*0.2)

        self.data = torch.from_numpy(mm.fit_transform(data)).float()
        self.mm = mm

    def set_mode(self, mode):
        self.mode = mode

    def __getitem__(self, index):
        begin = None
        if self.mode == 'train':
            begin = 0
        elif self.mode == 'valid':
            begin = self.train_nums
        elif self.mode == 'test':
            begin = self.train_nums + self.valid_nums

        x = self.data[begin+index:begin+index+self.window, :]
        y = self.data[begin+index+self.window+self.horizon-1, :]
        return x, y

    def __len__(self):
        if self.mode == 'train':
            return self.train_nums
        elif self.mode == 'valid':
            return self.valid_nums
        elif self.mode == 'test':
            return self.nums - self.train_nums - self.valid_nums

def setup_seed(seed):
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    np.random.seed(seed)
    random.seed(seed)
    torch.backends.cudnn.deterministic = True

def get_electricity():
    current_file_name = os.path.realpath(__file__)
    current_file_path = os.path.split(current_file_name)[0]
    electricity_file_name = os.path.join(current_file_path, 'electricity/electricity.npy')
    return np.load(electricity_file_name)

def get_exchange_rate():
    current_file_name = os.path.realpath(__file__)
    current_file_path = os.path.split(current_file_name)[0]
    exchange_rate_file_name = os.path.join(current_file_path, 'exchange_rate/exchange_rate.npy')
    return np.load(exchange_rate_file_name)

def get_solar_energy():
    current_file_name = os.path.realpath(__file__)
    current_file_path = os.path.split(current_file_name)[0]
    solar_energy_file_name = os.path.join(current_file_path, 'solar-energy/solar_AL.npy')
    return np.load(solar_energy_file_name)

def get_traffic():
    current_file_name = os.path.realpath(__file__)
    current_file_path = os.path.split(current_file_name)[0]
    traffic_file_name1 = os.path.join(current_file_path, 'traffic/traffic1.npy')
    traffic_file_name2 = os.path.join(current_file_path, 'traffic/traffic2.npy')
    traffic1 =  np.load(traffic_file_name1)
    traffic2 = np.load(traffic_file_name2)
    traffic = np.concatenate([traffic1, traffic2], 0)
    return traffic

def get_dataset(dataset_name, window, horizon):
    dataset_dict = {
        'solar_energy':get_solar_energy,
        'traffic':get_traffic,
        'electricity':get_electricity,
        'exchange_rate':get_exchange_rate
    }
    get = dataset_dict[dataset_name]
    data = get()
    dataset = MTSDataset(data, window, horizon, MinMaxScaler())
    return dataset

class MTSTrainer(Trainer):
    def __init__(self, model, criterion, optimizer, dataset):
        super(MTSTrainer, self).__init__(model, criterion, optimizer, dataset)
        self.jump_out = False
        self.original_dataset = dataset
        self.dataset = None

    def run(self, epochs=1, batch_size=64, shuffle=True):
        for q in self.plugin_queues.values():
            heapq.heapify(q)
        self.dataset = torch.utils.data.DataLoader(self.original_dataset, batch_size=batch_size, shuffle=shuffle, pin_memory=True)
        for i in range(1, epochs + 1):
            self.train(batch_size, shuffle)
            self.call_plugins('epoch', i, i == epochs)
            if self.jump_out:
                break
        self.dataset = None

    def train(self, batch_size, shuffle=True):
        self.original_dataset.set_mode('train')
        super(MTSTrainer, self).train()

class MTSPlugin(Plugin):
    def __init__(self, file_path, limit=10):
        super(MTSPlugin, self).__init__([(1, 'epoch'), (1, 'iteration')])
        self.file_path = file_path
        self.iteration_nums = 0
        self.iteration_loss = 0
        self.limit = limit
        self.counter = 0
        self.validation_best_corr = -1
        self.validation_best_rse = 1
        self.mm = None
        #self.last_time = time.time()

    def register(self, trainer):
        self.trainer = trainer
        self.mm = trainer.original_dataset.mm
        self.trainer.stats['valid_RSE'] = []
        self.trainer.stats['valid_CORR'] = []
        self.trainer.stats['valid_MSE'] = []
        self.trainer.stats['valid_MAE'] = []

        self.trainer.stats['test_RSE'] = []
        self.trainer.stats['test_CORR'] = []
        self.trainer.stats['test_MSE'] = []
        self.trainer.stats['test_MAE'] = []

        self.trainer.stats['train_Loss'] = []

    def epoch(self, time, last_time):
        self.trainer.model.eval()
        stats = self.trainer.stats

        train_loss = self.iteration_loss/self.iteration_nums
        stats['train_Loss'].append(train_loss)
        self.iteration_nums = 0
        self.iteration_loss = 0

        valid_rrse = 0.0
        valid_corr = 0.0
        valid_mse = 0.0
        valid_mae = 0.0

        test_rrse = 0.0
        test_corr = 0.0
        test_mse = 0.0
        test_mae = 0.0

        #mse = nn.MSELoss()
        with torch.no_grad():
            self.trainer.original_dataset.set_mode('valid')
            dataloader = torch.utils.data.DataLoader(self.trainer.original_dataset, batch_size=64, shuffle=False, pin_memory=True)
            valid_pred = []
            valid_y = []
            for (X, Y) in dataloader:
                X = X.cuda()
                predY = self.trainer.model(X)
                valid_pred.append(predY)
                valid_y.append(Y)

            Y = torch.cat(valid_y, 0)
            predY = torch.cat(valid_pred, 0)

            predY = predY.detach().cpu().numpy()
            Y = Y.detach().cpu().numpy()
            predY = self.mm.inverse_transform(predY)
            Y = self.mm.inverse_transform(Y)

            rrse, corr = RRSE(Y, predY), CORR(Y, predY)
            valid_corr = float(corr)
            valid_rrse= float(rrse)
            valid_mse = float(MSE(Y, predY))
            valid_mae = float(MAE(Y, predY))

            self.trainer.original_dataset.set_mode('test')
            test_pred = []
            test_y = []
            for (X, Y) in dataloader:
                X = X.cuda()
                #Y = Y.cuda()
                predY = self.trainer.model(X)
                test_pred.append(predY)
                test_y.append(Y)

            Y = torch.cat(test_y, 0)
            predY = torch.cat(test_pred, 0)

            predY = predY.detach().cpu().numpy()
            Y = Y.detach().cpu().numpy()
            predY = self.mm.inverse_transform(predY)
            Y = self.mm.inverse_transform(Y)

            rrse, corr = RRSE(Y, predY), CORR(Y, predY)
            test_corr = float(corr)
            test_rrse = float(rrse)
            test_mse = float(MSE(Y, predY))
            test_mae = float(MAE(Y, predY))

        stats['valid_RSE'].append(valid_rrse)
        stats['valid_CORR'].append(valid_corr)
        stats['valid_MSE'].append(valid_mse)
        stats['valid_MAE'].append(valid_mae)
        stats['test_RSE'].append(test_rrse)
        stats['test_CORR'].append(test_corr)
        stats['test_MSE'].append(test_mse)
        stats['test_MAE'].append(test_mae)

        self.counter += 1
        if valid_corr - valid_rrse >= self.validation_best_corr - self.validation_best_rse:
            self.validation_best_corr = valid_corr
            self.validation_best_rse = valid_rrse
            self.counter = 0
            self.save_model()

        if self.counter >= self.limit:#提前结束需要保存数据
            self.trainer.jump_out = True
            self.save_stats()
        elif last_time:#如果是最后一次运行也需要保存数据
            self.save_stats()
        self.trainer.model.train()

        #print(f'epochs={time},train_mse={mse},valid_corr={valid_mean_corr},valid_rse={valid_mean_rrse},test_corr={test_mean_corr},test_rse={test_mean_rrse}')
        print(f'epochs={time}')
        print(f'train:loss={train_loss}')
        print(f'valid:mse={valid_mse},mae={valid_mae},rse={valid_rrse},corr={valid_corr}')
        print(f'test:mse={test_mse},mae={test_mae},rse={test_rrse},corr={test_corr}')

    def iteration(self, time, batch_input, batch_target, batch_output, loss):
        L = loss.item()
        #c = time.time()
        #print(f'{c-self.last_time}')
        #self.last_time = c
        self.iteration_loss += L
        self.iteration_nums += 1

    def save_model(self):
        model_file = self.file_path + '.model'
        #with open(model_file, 'wb') as f:
        #    pickle.dump(self.trainer.model.cpu(), f)
        #self.trainer.model.cuda()
        torch.save(self.trainer.model.state_dict(), model_file)

    # def load_model(self):
    #     model_file = self.file_path + '.model'
    #     return torch.load(model_file)

    def save_stats(self):
        stats_file = self.file_path + '.stats'
        with open(stats_file, 'wb') as f:
            pickle.dump(self.trainer.stats, f)

    # def load_stats(self):
    #     stats_file = self.file_path + '.stats'
    #     result = None
    #     with open(stats_file, 'rb') as f:
    #         result = pickle.load(f)
    #     return result

if __name__ == '__main__':
    electricity = get_electricity()
    print('electricity', electricity.shape)
    exchange_rate = get_exchange_rate()
    print('exchange_rate', exchange_rate.shape)
    solar_energy = get_solar_energy()
    print('solar_energy', solar_energy.shape)
    traffic = get_traffic()
    print('traffic', traffic.shape)


import datetime

import asterix4py

b = open('../sample/cat062.ast', 'rb').read()
decoder = asterix4py.AsterixParser(b)
print('decode: ', decoder.get_result())

midnight = datetime.datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)

# each plot has a sequence number
for nr, plot in decoder.get_result().items():
    cat = plot.get('cat')
    if cat != 62:
        print(f"cat{cat:03d}")
        continue

    print(
        f"cat{cat:03d}"
        f" tot:{midnight + datetime.timedelta(seconds=plot['070']['ToT']):%H:%M:%S.%f}"
        , end='')

    if plot.get('105'):
        print(
            f" lat:{plot['105']['Lat']:0<10.9}"
            f" lon:{plot['105']['Lon']:0<10.9}"
        , end='')

    if plot.get('060'):
        print(
            f" a:{plot['060']['Mode3A']}"
        , end='')

    if plot.get('380'):
        print(
            f" s:{plot['380'].get('ADR'):06X}"
            f" acid:{plot['380'].get('ACID')}"
            f" fss:{plot['380'].get('FSS')}"
            f" gsp:{plot['380'].get('GSP')}"
            f" ias:{plot['380'].get('IAS')}"
        , end='')

    if plot.get('390'):
        print(
            f" cs:{plot['390'].get('ADR')}"
            f" type:{plot['390'].get('TYPE')}"
            f" wtc:{plot['390'].get('WTC')}"
            f" adep:{plot['390'].get('DEP')}"
            f" ades:{plot['390'].get('DES')}"
            f" cfl:{plot['390'].get('CFL')}"
        , end='')
    print('')


import numpy as np
import os
import pandas as pd
import matplotlib.pyplot as plt
import tensorflow as tf
import random as python_random
import joblib

from Utils import *
from Metrics import*

from tensorflow import keras
from tensorflow.keras import optimizers, regularizers,Input, Model
from tensorflow.keras.backend import reshape, clear_session
from keras.layers import Dense, Bidirectional, Flatten, Dropout, LSTM, LeakyReLU,Activation
from keras.models import Sequential, load_model
from keras.utils import to_categorical
from focal_loss import BinaryFocalLoss
from plot_keras_history import plot_history

from sklearn import metrics
import seaborn as sn

# Import the model we are using
from sklearn.ensemble import RandomForestClassifier
from tcn import *

def run_exp_hist(x_tr,y_tr, x_te, y_te, repeats=5, gamma = 2,epochs=2, node = 600, dropout = 0.1, reg = 0, m_type = 1, LSTM = True, TCN = False, CV = True):
    """ Runs several experiments and averages the results
    Arguments: (x_tr, y_tr) training set
               (x_te, y_te) test set
               repeats = number of training on which we average
               gamma = parameter for the focal loss
               epochs= number of times, it passes through the training dataset.
               node = number of nodes of the neural network
               dropout = dropout value
               m_type = type of model to train/evaluate (on LSTM layers)
                        0: 1 LSTM layer model
                        1: 1 bidirectional LSTM layer model
                        2: 2 bidirectional LSTM layers model
               LSTM/TCN boolean defines which model is used.
               CV = true if cross validation (then test set is left untouched)
                    false if repeated runs (then test set is used against whole train set)
    Prints the average weighted, macro and proportional F1 score, mean F1 score,
    precision, recall per label, and loss evolution
                  """
    #wf1_
    f1_scores = list()
    acc_scores = list()
    loss_scores = list()
    train = pd.DataFrame()
    val = pd.DataFrame()
    tab = []

    if not CV:
      print("Training versus test")
      x_1, x_2, y_1, y_2 = x_tr, x_te, y_tr, y_te

    for r in range(repeats):
        #if cross validation folds (5 folds)
        if CV:
          print("Cross validation")
          x_1, x_2, y_1, y_2 = train_test_split(x_tr, y_tr, test_size = 0.2)

        if LSTM:
          hist, loss, accuracy, wf1, wf1_, mf1, F1_tab, Ptab, Rtab = evaluate_model(x_1, y_1, x_2, y_2, nodes_nb = node, drop = dropout, model_type = m_type, reg = reg)

        if TCN:

            hist, loss, accuracy, wf1,wf1_, mf1, F1_tab, Ptab, Rtab= evaluate_model_TCN(x_1, y_1, x_2, y_2, gamma,epochs)

        wf1 = wf1 * 100.0
        mf1 = mf1 * 100.0
        wf1_ = wf1_ * 100.0
        accuracy = accuracy * 100.0
        f1_scores.append([wf1, mf1, wf1_])
        acc_scores.append(accuracy)
        loss_scores.append(loss)
        tab.append(F1_tab)
        train[str(r)] = hist.history['loss']
        val[str(r)] = hist.history['val_loss']

    f1_scores = np.array(f1_scores)
    summarize_scores([f1_scores[:,0], f1_scores[:,1], f1_scores[:,2], acc_scores, loss_scores], ['weighted F1 score', 'Macro F1 score', 'Proportional F1 score', 'Accuracy', 'Loss'])
    print("Mean F1 score per label: ", np.mean(np.array(tab), axis = 0))
    plt.plot(train, color='blue', label='train')
    plt.plot(val, color='orange', label='test')
    plt.title('model train vs test loss')
    plt.ylabel('Focal loss')
    plt.xlabel('epoch')
    plt.show()
    return f1_scores, tab

def evaluate_model(x_tr, y_tr, x_te, y_te, model_type = 1, gamma=2, nodes_nb=600, drop = 0.1, epochs = 200, reg = 0, verbose = 0, plot = 0, single_run = 0):
    """Training function, to evaluate train set against test set or train set againts validation set
        Arguments: (x_tr, y_tr) training data
                   (x_te, y_te) testing/validation data
                   model_type: type of model to train/evaluate (on LSTM layers)
                              0: 1 LSTM layer model
                              1: 1 bidirectional LSTM layer model
                              2: 2 bidirectional LSTM layer model
                   gamma: focal loss parameter
                   nodes_nb: number of neurons in the LSTM layers
                   drop: dropout value
                   verbose: if true, print all the metrics
                   plot: if true, print built in plot
                   single_run: fix random seed to ensure reproducibility
          Returns: loss: Last binary focal loss value on test/validation set
                  accuracy: accuracy on test/validation set
                  wf1: weighted F1 score
                  wf1_: custom weighted F1 score (with proportional weights)
                  mf1: macro F1 score
                  F1_tab: F1 score per label
                  Ptab: precision per label
                  Rtab: recall per label"""

    batch_size = 32
    n_features, n_outputs = x_tr.shape[2], y_tr.shape[2]
    w = class_weights(y_tr)
    clear_session()

    if single_run:
      np.random.seed(2020)
      python_random.seed(2020)
      tf.random.set_seed(2020)

    #-------------------------------------model definition-------------------------------------#
    model = Sequential()

    #model types
    if model_type == 0:
      model.add(LSTM(nodes_nb,input_shape = (None, n_features), return_sequences = True))
      model.add(Dropout(drop))
      model.add(Dense(n_outputs, activation = 'sigmoid'))
    if model_type == 1:
      model.add(Bidirectional(LSTM(nodes_nb,input_shape = (None, n_features), return_sequences = True)))
      model.add(Dropout(drop))
      model.add(Dense(n_outputs, activation = 'sigmoid'))
    if model_type == 2:
      model.add(Bidirectional(LSTM(nodes_nb,input_shape = (None, n_features), return_sequences = True)))
      model.add(Dropout(drop))
      model.add(Bidirectional(LSTM(nodes_nb,input_shape = (None, n_features), return_sequences = True)))
      model.add(Dropout(drop))
      model.add(Dense(n_outputs, activation = 'sigmoid'))

    model.compile(loss=BinaryFocalLoss(gamma), optimizer = 'adam', metrics = [BinaryAccuracy(), Precision(), Recall(), FalseNegatives(), FalsePositives()])

    #------------------------------------fit network---------------------------------------------#

    hist = model.fit(x_tr, y_tr, epochs = epochs, batch_size = batch_size, verbose = 0, validation_data = (x_te, y_te))
    if verbose:
      model.summary()

    #---------------------------------evaluate model----------------------------------------------#

    #evaluate model on test set (over all classes)
    loss, accuracy, P, R, FN, FP= model.evaluate(x_te, y_te, batch_size = batch_size, verbose = verbose)

    #save model
    if single_run:
        model.save('Results/opt_LSTM_model')
        print("Model saved to Results")

    y_pred = model.predict(x_te, batch_size = batch_size, verbose = 0)
    y_pred = reshape(y_pred, (y_pred.shape[0]* y_pred.shape[1], 8))
    y_te = reshape(y_te, (y_te.shape[0]*y_te.shape[1],8))

    #evaluate F1 score for each label
    F1_tab, Ptab, Rtab, wf1_ = F1_score(y_te, y_pred, w)
    #evaluate accuracy per label
    acc_tab = Acc(y_te, y_pred)

    #test f1 score built in
    f = F1Score(8, threshold = 0.5, average = 'weighted')
    f.update_state(y_te, y_pred)
    wf1 = f.result().numpy()
    f.reset_states()
    f = F1Score(8, threshold = 0.5, average = 'macro')
    f.update_state(y_te, y_pred)
    mf1 = f.result().numpy()
    f.reset_states()

    #-----------------------------------------print---------------------------------------------#

    #print all
    if verbose :
      print(" -> Accuracy: ", accuracy, "; Mean of labelwise accuracy: ", np.mean(acc_tab))
      print("Per label accuracy: ", acc_tab)
      print("-> Proportional F1 score: ", wf1_, "; Weighted F1 score: ", wf1, "; Macro F1 score: ", mf1)
      print("-> F1 score per label: ", F1_tab)
      print("-> Precision: ", P, "; Recall: ", R)
      print("-> Precision per label: ", Ptab)
      print("-> Recall per label: ", Rtab)
      print("-> Loss: ", loss)

    if plot:
      plot_history(hist.history)

    return hist, loss, accuracy, wf1, wf1_, mf1, F1_tab, Ptab, Rtab

def evaluate_model_TCN(x_tr, y_tr, x_te, y_te, gamma=2,epochs= 200, verbose = 0,plot = 0, single_run = 0):
    """Training function, to evaluate train set against test set or train set againts validation set
        Arguments: (x_tr, y_tr) training data
                   (x_te, y_te) testing/validation data
                   gamma: focal loss parameter
                   epochs= number of times, it passes through the training dataset.
                   verbose: if true, print all the metrics
                   plot: if true, print built in plot
                   single_run: fix random seed to ensure reproducibility

          Returns: loss: Last binary focal loss value on test/validation set
                  accuracy: accuracy on test/validation set
                  wf1: weighted F1 score
                  wf1_: custom weighted F1 score (with proportional weights)
                  mf1: macro F1 score
                  F1_tab: F1 score per label
                  Ptab: precision per label
                  Rtab: recall per label"""

    batch_size=1;
    w = class_weights(y_tr)
    clear_session()

    if single_run:
      np.random.seed(123)
      python_random.seed(123)
      tf.random.set_seed(1234)

    #-------------------------------------model definition---------------------------#
    #Creation TCN object
    Tcn=TCN(nb_filters=64,kernel_size=2,nb_stacks=1,dilations=(8,16,32,64,128,256,512,1024),return_sequences=True,
activation=LeakyReLU(0.01),kernel_initializer='he_normal')

    i = Input(batch_shape=(1,None,  x_tr.shape[2]))
    o = Tcn(i)
    o = Dense(200, activation=LeakyReLU(0.01), kernel_regularizer=keras.regularizers.l1_l2(0.00001))(o)
    o = Dense(8, activation='sigmoid')(o)

    model = Model(inputs=[i], outputs=[o])
    model.compile(optimizer='adam', loss=BinaryFocalLoss(gamma), metrics=[BinaryAccuracy(), Precision(), Recall(), FalseNegatives(),FalsePositives()])

    model.summary()

    #---------------------fit network---------------------------------------------#
    hist = model.fit(x_tr, y_tr, epochs = epochs, batch_size = batch_size, verbose = 0, validation_data = (x_te, y_te))
    #---------------------------------evaluate model----------------------------------------------#

    #evaluate model on test set (over all classes)
    loss, accuracy, P, R, FN, FP= model.evaluate(x_te, y_te, batch_size = batch_size, verbose = verbose)

    #save model
    if single_run:
        model.save('Results/opt_TCN_model')
        print("Model saved to Results")

    y_pred = model.predict(x_te, batch_size = batch_size, verbose = 0)
    y_pred[y_pred<0.5]=0.
    y_pred[y_pred>0.5]=1.

    y_pred = reshape(y_pred, (y_pred.shape[0]* y_pred.shape[1], 8))
    y_te = reshape(y_te, (y_te.shape[0]*y_te.shape[1],8))

    #evaluate F1 score for each label
    F1_tab, Ptab, Rtab, wf1_ = F1_score(y_te, y_pred, w)

    #evaluate accuracy per label
    acc_tab = Acc(y_te, y_pred)
    print("-> F1 score per label: ", F1_tab)
    print("-> y_pred " ,y_pred[:,4])

    #test f1 score built in
    f = F1Score(8, threshold = 0.5, average = 'weighted')
    f.update_state(y_te, y_pred)
    wf1 = f.result().numpy()
    print("weighted F1 score built in: ", wf1 )
    f.reset_states()
    f = F1Score(8, threshold = 0.5, average = 'macro')
    f.update_state(y_te, y_pred)
    mf1 = f.result().numpy()
    print("macro F1 score built in: ", mf1 )
    f.reset_states()

    #-----------------------------------------print---------------------------------------------#

    #print all
    if verbose :
      print(" -> Accuracy: ", accuracy, "; Mean of labelwise accuracy: ", np.mean(acc_tab))
      print("Per label accuracy: ", acc_tab)
      print("-> Weighted F1 score: ", wf1_)
      print("-> F1 score per label: ", F1_tab)
      print("-> Precision: ", P, "; Recall: ", R)
      print("-> Precision per label: ", Ptab)
      print("-> Recall per label: ", Rtab)
      print("-> Loss: ", loss)

    if plot:
      plot_history(hist.history)

    return hist, loss, accuracy, wf1,wf1_, mf1, F1_tab, Ptab, Rtab

def build_model_RF(X,Y):

    #train and test split
    x_tr, x_te, y_tr, y_te = train_test_split(X, Y, test_size = 0.2, random_state = 200)

    #feature expansion
    x_tr_expanded=feature_expansion(x_tr)
    x_te_expanded=feature_expansion(x_te)

    #time window flattening
    x_tr_expanded,y_tr_t=time_window_sample(x_tr_expanded, y_tr, 120)
    x_te_expanded,y_te_t=time_window_sample(x_te_expanded, y_te, 120)

    #reshape into one video
    x_tr_reshaped=np.reshape(x_tr_expanded, (x_tr_expanded.shape[0]*x_tr_expanded.shape[1],x_tr_expanded.shape[2]))
    y_tr_reshaped=np.reshape(y_tr_t, (y_tr_t.shape[0]*y_tr_t.shape[1],y_tr_t.shape[2]))

    x_te_reshaped=np.reshape(x_te_expanded, (x_te_expanded.shape[0]*x_te_expanded.shape[1],x_te_expanded.shape[2]))
    y_te_reshaped=np.reshape(y_te_t, (y_te.shape[0]*y_te_t.shape[1],y_te_t.shape[2]))

    #train classifier
    clf = RandomForestClassifier(n_estimators=10, criterion = 'gini', max_depth=10, random_state = 42)
    clf.fit(x_tr_reshaped,y_tr_reshaped)

    #save model
    joblib.dump(clf, "Results/opt_random_forest_model.joblib")

def predict(X,Y, flag, batch_size = 32, epochs = 200):
  """ Predicts labels for X given and compares predictions to ground truth Y
      Arguments: (X,Y) data and corresponding labels,
                    X dim = (number of samples, number of timesteps, number of features = 75)
                    Y dim = (number of samples, number of timesteps, number of labels = 8)
                  flag: model to use
                  batch_size : should be set to 1 if only 1 sample
                  repeats: the averaged scores are computed over #number of repeats since stochastic approach
      Prints scores and downloads prediction
  """

  ##RESHAPE GROUND TRUTH Y
  y_te = reshape(Y, (Y.shape[0]*Y.shape[1],Y.shape[2]))

  if flag == "LSTM":
    ## LOAD MODEL
    loaded_model = load_model('Results/opt_LSTM_model')
    print("Loaded model from disk")
    loaded_model.compile(loss=BinaryFocalLoss(2), optimizer = 'adam', metrics = [BinaryAccuracy(), Precision(), Recall(), FalseNegatives(), FalsePositives()])
    #predict and reshape predictions
    y_pred = loaded_model.predict(X, batch_size = batch_size)
    y_pred = reshape(y_pred, (y_pred.shape[0]* y_pred.shape[1], 8))
    #build-in predictions
    loss, accuracy, P, R, FN, FP = loaded_model.evaluate(X,Y, batch_size = batch_size)

  if flag == "TCN":
    loaded_model = load_model('Results/opt_TCN_model')
    print("Loaded model from disk")
    loaded_model.compile(optimizer='adam', loss=BinaryFocalLoss(5), metrics=[BinaryAccuracy(), Precision(), Recall(), FalseNegatives(), FalsePositives()])

    #predict and reshape predictions
    y_pred = loaded_model.predict(X, batch_size = batch_size)
    y_pred = reshape(y_pred, (y_pred.shape[0]* y_pred.shape[1], 8))
    #build-in predictions
    loss, accuracy, P, R, FN, FP = loaded_model.evaluate(X,Y, batch_size = batch_size)

  if flag == "Random_Forest":

    ##Load MODEL
    loaded_rf = joblib.load("Results/opt_random_forest_model.joblib")

    #feature expansion
    X_expanded=feature_expansion(X)

    #time window flattening
    X_expanded,Y=time_window_sample(X_expanded, Y, 120)

    #reshape into one video
    X_reshaped=np.reshape(X_expanded, (X_expanded.shape[0]*X_expanded.shape[1],X_expanded.shape[2]))
    y_te =np.reshape(Y, (Y.shape[0]*Y.shape[1],Y.shape[2]))

    ##prediction
    y_pred=loaded_rf.predict(X_reshaped)

  #customed predictions
  wf1, mf1, pf1, F1_tab, Ptab, Rtab, acc_tab = custom_scoring(y_te, y_pred)
  Tab=perf_measure(y_te,y_pred)

  print("F1 score per label: ", F1_tab)
  print("Precision per label: ", Ptab)
  print("Recall per label: ", Rtab)
  print("Macro F1 score: ", mf1, " ; Weighted F1 score: ", wf1, " ; Proportional F1 score: ", pf1)
  for name, tab in zip(["True Positive ", "True Negative ", "False Positive ", "False Negatives "], Tab):
      for i, t in enumerate(tab):
        print(name, "number for label ", i, " :", t)

  y_pred = y_pred > 0.5
  tf.cast(y_pred, tf.int32)
  print("Prediction will be saved into Results/")
  name = 'Results/'+flag+'_Annotation.csv'
  pd.DataFrame(tf.get_static_value(y_pred), columns=["arch", "burrow + arch", "drag + arch", "groom","tap + arch","egg", "proboscis", "idle"]).to_csv(name)


import matplotlib
matplotlib.use("TkAgg")
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg
from matplotlib.figure import Figure
import tkinter as tk
from tkinter import ttk
import csv

class Grapher(tk.Tk):
    def __init__(self, *args, **kwargs):
        tk.Tk.__init__(self, *args, **kwargs)

        tk.Tk.iconbitmap(self)
        tk.Tk.wm_title(self, "ComMeN grapher")

        container = tk.Frame(self)
        container.pack(side="top", fill="both", expand=True)
        container.grid_rowconfigure(0, weight=1)
        container.grid_columnconfigure(0, weight=1)

        self.frames = {}

        for F in [StartPage]:
            frame = F(container)
            self.frames[F] = frame
            frame.grid(row=0, column=0, sticky="nsew")

        self.show_frame(StartPage)

    def show_frame(self, cont):
        frame = self.frames[cont]
        frame.tkraise()

class StartPage(tk.Frame):
    def __init__(self, parent):
        self.comp_data = {}
        self.time_data = []
        self.node_vars = {}

        tk.Frame.__init__(self, parent)

        tk.Label(self, text="Data file:").grid(row=0, column=0)

        filename = tk.Entry(self)
        filename.grid(row=0, column=1)
        filename.insert(0, 'Experiments/0/0.csv')

        data_button = ttk.Button(self, text="Load data", command=lambda: self.load_data(filename.get()))
        data_button.grid(row=0, column=2)

        self.data_message = tk.StringVar()
        tk.Label(self, textvariable=self.data_message).grid(row=1, column=1)
        self.data_message.set("")

        f = Figure(figsize=(5, 5), dpi=100)
        self.subplot = f.add_subplot(111)
        self.compartment_variable = tk.StringVar(self)

        self.canvas = FigureCanvasTkAgg(f, self)
        # self.canvas.get_tk_widget().pack(side=tk.BOTTOM, fill=tk.BOTH, expand=True)
        self.canvas.get_tk_widget().grid(row=2, rowspan=20, columnspan=3)
        self.canvas.show()

        self.compartment_menu = tk.OptionMenu(self, self.compartment_variable, "")
        self.compartment_menu.config(width=20)
        self.compartment_menu.grid(row=23, column=1)

        plot_button = ttk.Button(self, text="Plot", command=lambda: self.update_graph())
        plot_button.grid(row=24, column=1)

        self.pack()

    def load_data(self, filename):
        try:
            csv_file = open(filename, 'r')
            self.data_message.set("Data loaded")
        except IOError:
            self.data_message.set("File {0} not found".format(filename))
            return
        self.subplot.clear()
        self.canvas.show()
        self.comp_data = {}
        self.time_data = []
        csv_reader = csv.DictReader(csv_file)
        compartments = sorted(list(csv_reader.fieldnames))
        compartments.remove('node_id')
        compartments.remove('timestep')

        self.compartment_menu['menu'].delete(0, 'end')
        for choice in compartments:
            self.compartment_menu['menu'].add_command(label=choice, command=tk._setit(self.compartment_variable, choice))

        for c in compartments:
            self.comp_data[c] = {}

        current_time = -1.0
        for row in csv_reader:
            if float(row['timestep']) != current_time:
                self.time_data.append(float(row['timestep']))
                current_time = float(row['timestep'])
            for c in compartments:
                node_data = self.comp_data[c]
                if row['node_id'] not in node_data:
                    node_data[row['node_id']] = []
                node_data[row['node_id']].append(row[c])

        self.pack()

        nodes = self.comp_data.values()[0].keys()
        nodes = sorted(nodes)
        row = 2
        for n in nodes:
            self.node_vars[n] = tk.IntVar()
            tk.Checkbutton(self, text=n, variable=self.node_vars[n]).grid(row=row, column=3)
            row += 1

    def update_graph(self,):
        self.data_message.set("")
        self.subplot.clear()
        if not self.compartment_variable.get():
            return
        compartment_chosen_data = self.comp_data[self.compartment_variable.get()]
        nodes = [n for n in self.node_vars if self.node_vars[n].get() == 1]

        if not nodes:
            nodes = self.node_vars.keys()

        for key in nodes:
            self.subplot.plot(self.time_data, compartment_chosen_data[key], label=key)
        self.canvas.show()

if __name__ == '__main__':
    app = Grapher()
    app.mainloop()


from textbookdownloader.file_downloader import *
import json

def main():
    url = input('Enter URL (online reader): ')
    file_format = input('Download format (pdf[/png]): ').lower()
    with open('./config.json') as f:
        config = json.load(f)
    print('-'*75)
    if file_format == 'png':
        download_pictures(url, **config)
    else:
        download_pdf(url, **config)

if __name__ == '__main__':
    main()


import pickle
import numpy as np
from scipy.io import loadmat
import matplotlib.pyplot as plt

from configs.config import Config
from utils.plot_utils import PlotUtils
from forward_model.model import MethodOfMomentModel
from inverse_model.solve import InverseProblemSolver

def get_grid_permittivity(grid_positions):

    center_x = 0.4
    center_y = 0.4
    radius = 0.15
    epsilon_r = 1.1

    m = Config.doi["forward_grids"]
    scatterer = np.ones((m, m), dtype=float)

    # circle
    scatterer[(grid_positions[0] - center_x) ** 2 + (grid_positions[1] - center_y) ** 2 <= radius ** 2] = epsilon_r

    # square
    # mask = ((grid_positions[0] <= center_x + radius) & (grid_positions[0] >= center_x - radius) &
    #         (grid_positions[1] <= center_y + radius) & (grid_positions[1] >= center_y - radius))
    # scatterer[mask] = epsilon_r

    fig, (ax1, ax2) = plt.subplots(ncols=2)

    original = ax1.imshow(np.real(scatterer), cmap=plt.cm.hot, extent=PlotUtils.get_doi_extent())
    fig.colorbar(original, ax=ax1, fraction=0.046, pad=0.04)
    ax1.title.set_text("Scatterer: real")

    reconstructed_real = ax2.imshow(np.imag(scatterer), cmap=plt.cm.hot, extent=PlotUtils.get_doi_extent())
    fig.colorbar(reconstructed_real, ax=ax2, fraction=0.046, pad=0.04)
    ax2.title.set_text("Scatterer: imaginary")

    return scatterer

def solve_many(A, data, prior):
    values = np.linspace(0, 1e3, 20)
    fig1, axes1 = plt.subplots(nrows=4, ncols=5)
    fig2, axes2 = plt.subplots(nrows=4, ncols=5)

    real_rec = []
    imag_rec =  []
    for value in values:
        params = {"alpha": value}
        chi_real, chi_imag = InverseProblemSolver.solve(A, data, prior, params)
        real_rec.append(chi_real)
        imag_rec.append(chi_imag)

    fig1.subplots_adjust(hspace=0.2)
    fig1.suptitle('Reconstructions for real part')

    for ax, rec, in zip(axes1.flatten(), real_rec):
        ax.imshow(rec, cmap=plt.cm.hot, extent=PlotUtils.get_doi_extent())

    fig2.subplots_adjust(hspace=0.2)
    fig2.suptitle('Reconstructions for imaginary part')

    for ax, rec, in zip(axes2.flatten(), imag_rec):
        ax.imshow(rec, cmap=plt.cm.hot, extent=PlotUtils.get_doi_extent())

def plot_results(scatterer, chi_real, chi_imag):

    fig, (ax1, ax2, ax3) = plt.subplots(ncols=3)

    original = ax1.imshow(scatterer, cmap=plt.cm.hot)
    fig.colorbar(original, ax=ax1, fraction=0.046, pad=0.04)
    ax1.title.set_text("Original scatterer")

    reconstructed_real = ax2.imshow(chi_real, cmap=plt.cm.hot, extent=PlotUtils.get_doi_extent())
    fig.colorbar(reconstructed_real, ax=ax2, fraction=0.046, pad=0.04)
    ax2.title.set_text("Real Reconstruction")

    reconstructed_imag = ax3.imshow(chi_imag, cmap=plt.cm.hot, extent=PlotUtils.get_doi_extent())
    fig.colorbar(reconstructed_imag, ax=ax3, fraction=0.046, pad=0.04)
    ax3.title.set_text("Imaginary Reconstruction")

if __name__ == '__main__':

    """" Solve the Forward Problem """
    # Instantiate forward model
    forward = MethodOfMomentModel()

    """" Generate / Read the scatterer """
    scatterer = get_grid_permittivity(forward.grid_positions)

    # Generate forward data for scatterer
    direct_field, direct_power, total_field, total_power = forward.generate_forward_data(scatterer)

    """" Solve the Inverse Problem """

    model_type = "prytov_complex"
    prior = "qs2D_complex"
    params = {"alpha": 10}

    A = InverseProblemSolver.get_inverse_model(model_type)
    data = InverseProblemSolver.get_measurement_data(model_type, direct_power, total_power)

    chi_real, chi_imag = InverseProblemSolver.solve(A, data, prior, params)
    plot_results(scatterer, chi_real, chi_imag)

    # solve_many(A, data, prior)


import warnings

warnings.filterwarnings('ignore')

import datetime

import numpy as np
import pandas as pd

# global vars

first_n_funds=5
aggressive_rate=6

def log(text): print(str(datetime.datetime.now()).split('.')[0],text)

# read data
log('reading data...')
funds_code_df=pd.read_csv('data/funds_table.csv',usecols=['Symbol','Name'])
funds_code_df.columns=[x.lower() for x in funds_code_df.columns]
funds_history=pd.read_csv('data/funds_data.csv')
funds_history['Date']=pd.to_datetime(funds_history['Date'])

revenue_days=[1,5,10,22,66,120,250]
revenue_cols=['day','week','biweek','month','3_month','6_month','year']
revenue_zip=zip(revenue_days,revenue_cols)

col1=[]
col2=[]

funds_history['%_change']=(funds_history['Adj Close']-funds_history['Adj Close'].shift(1))/funds_history['Adj Close'].shift(1)*100

## add holidays
funds_history=funds_history[funds_history['%_change']!=0]

log('calculating revenue and max withdraw...')
for day in revenue_days:
    funds_history['lag_%s'%day]=funds_history.groupby('symbol')['Adj Close'].shift(day)

for day,col in revenue_zip:
    funds_history['%s_revenue'%col]=(funds_history['Adj Close']-funds_history['lag_%s'%day])/funds_history['lag_%s'%day]*100
    col2.append('%s_revenue'%col)
    if not col in ['day','year']:
        funds_history['%s_max_withdraw'%col]=funds_history.groupby('symbol')['Adj Close'].transform(lambda x:(x.rolling(day).max()-x)/x.rolling(day).max()*100)
        col1.append('%s_max_withdraw'%col)

cols=['symbol','Date', 'Adj Close']+col1+col2
funds_history=funds_history[cols]
funds_withdraw=funds_history[funds_history.Date>pd.to_datetime(datetime.datetime.now().date()-datetime.timedelta(days=500))].groupby('symbol',as_index=False)[col1].max()
aa=funds_history[funds_history.Date>pd.to_datetime(datetime.datetime.now().date()-datetime.timedelta(500))]
to_del=[str(x).zfill(6) for x in set(aa[aa.month_revenue.isnull()].symbol)]
funds_history=funds_history[~funds_history.symbol.isin(to_del)]

funds_select=funds_history.sort_values('Date').groupby('symbol',as_index=False).apply(lambda x:x.iloc[-1])[[x for x in funds_history.columns if x not in col1]]
funds_select=funds_select.merge(funds_withdraw,on='symbol')
funds_select=funds_select.merge(funds_code_df,on='symbol',how='left')
funds_select.reset_index(drop=True,inplace=True)

log('calculating sharpe...')
def downward_std(df):
    ls=df.iloc[-180:].values
    return np.std([x for x in ls if x <1][-180:])

funds_history['day_revenue']=funds_history['day_revenue'].astype(float)

df_dstd=funds_history.groupby('symbol',as_index=False)['day_revenue'].apply(downward_std)
df_dstd['week_revenue']=funds_history.groupby('symbol')['week_revenue'].apply(downward_std).values
df_dstd['month_revenue']=funds_history.groupby('symbol')['month_revenue'].apply(downward_std).values
df_dstd['3_month_revenue']=funds_history.groupby('symbol')['3_month_revenue'].apply(downward_std).values
df_dstd.columns=[x.replace('revenue','downward_std') for x in df_dstd.columns]
funds_select=funds_select.merge(df_dstd,on='symbol')

col3=['week_sharpe','month_sharpe','3_month_sharpe']
funds_select['week_sharpe']=funds_select['week_revenue']/funds_select['week_downward_std']
funds_select['month_sharpe']=funds_select['month_revenue']/funds_select['month_downward_std']
funds_select['3_month_sharpe']=funds_select['3_month_revenue']/funds_select['3_month_downward_std']

log('filtering data...')
funds_select=funds_select[funds_select['week_sharpe']!=np.inf]
funds_select=funds_select[funds_select['month_sharpe']!=np.inf]
funds_select=funds_select[funds_select['3_month_sharpe']!=np.inf]

funds_select=funds_select[funds_select.month_revenue.astype(float)>0]
funds_select=funds_select[funds_select['3_month_revenue'].astype(float)>0]
funds_select=funds_select[funds_select.year_revenue.astype(float)>0]

funds_select=funds_select[funds_select.year_revenue>np.quantile(funds_select.year_revenue,0.8)]

log('normalization...')
for col in col1+col2:
    funds_select[col]=funds_select[col].transform(lambda x:round(x,2))
funds_select.drop_duplicates(inplace=True)

for col in col1:
    funds_select['%s_norm_1'%col]=1/funds_select[col]
    funds_select=funds_select[funds_select['%s_norm_1'%col]!=np.inf]
    funds_select['%s_norm_1'%col]=funds_select['%s_norm_1'%col]-funds_select['%s_norm_1'%col].min()
    funds_select['%s_norm_1'%col]=funds_select['%s_norm_1'%col]/funds_select['%s_norm_1'%col].max()

for col in col2:
    funds_select['%s_norm_2'%col]=funds_select[col]-funds_select[col].min()
    funds_select['%s_norm_2'%col]=funds_select['%s_norm_2'%col]/funds_select['%s_norm_2'%col].max()

for col in col3:
    funds_select['%s_norm_3'%col]=funds_select[col]-funds_select[col].min()
    funds_select['%s_norm_3'%col]=funds_select['%s_norm_3'%col]/funds_select['%s_norm_3'%col].max()

eval_col1=[x for x in funds_select.columns if 'norm_1' in x]
eval_col2=[x for x in funds_select.columns if 'norm_2' in x]
eval_col3=[x for x in funds_select.columns if 'norm_3' in x]

log('socring...')
result=funds_select.copy()
result['withdraw_score']=np.sum(result[eval_col1],axis=1)
result['withdraw_score']=result['withdraw_score'].transform(lambda x:x-x.min())
result['withdraw_score']=result['withdraw_score'].transform(lambda x:x/x.max())
result['withdraw_score']=result['withdraw_score'].transform(lambda x:x**0.5)
result['withdraw_score']=(result['withdraw_score']*100).transform(lambda x:round(x,2))

result['revenue_score']=np.sum(result[eval_col2],axis=1)
result['revenue_score']=result['revenue_score'].transform(lambda x:x-x.min())
result['revenue_score']=result['revenue_score'].transform(lambda x:x/x.max())
result['revenue_score']=result['revenue_score'].transform(lambda x:x**0.5)
result['revenue_score']=(result['revenue_score']*100).transform(lambda x:round(x,2))

result['sharpe_score']=np.sum(result[eval_col3],axis=1)
result['sharpe_score']=result['sharpe_score'].transform(lambda x:x-x.min())
result['sharpe_score']=result['sharpe_score'].transform(lambda x:x/x.max())
result['sharpe_score']=result['sharpe_score'].transform(lambda x:x**0.5)
result['sharpe_score']=(result['sharpe_score']*100).transform(lambda x:round(x,2))

for risk_factor in [x*10 for x in range(1,10)]:
    result['%s%%_aggressive_score'%risk_factor]=result['withdraw_score']*(1-risk_factor/100)+result['revenue_score']*risk_factor/100
    result['%s%%_aggressive_score'%risk_factor]=result['%s%%_aggressive_score'%risk_factor]-result['%s%%_aggressive_score'%risk_factor].min()
    result['%s%%_aggressive_score'%risk_factor]=result['%s%%_aggressive_score'%risk_factor]/result['%s%%_aggressive_score'%risk_factor].max()
    result['%s%%_aggressive_score'%risk_factor]=(result['%s%%_aggressive_score'%risk_factor]*100).transform(lambda x:round(x,2))

log('outputing...')
# score_cols=['%s%%_aggressive_score'%(x*10) for x in range(1,10)]
# cols=['symbol', 'name','withdraw_score','revenue_score',
#         'week_revenue', 'month_revenue', '3_month_revenue', '6_month_revenue', 'year_revenue',
#         'week_max_withdraw', 'month_max_withdraw', '3_month_max_withdraw']+score_cols+['sharpe_score']
# result_t=result[cols].sort_values(by=['%s0%%_aggressive_score'%aggressive_rate],ascending=False)

for aggressive_rate in [3,6,9]:
    cols=['symbol','withdraw_score','revenue_score','sharpe_score','%s0%%_aggressive_score'%aggressive_rate,
            'week_revenue', 'month_revenue', '3_month_revenue',
            '6_month_revenue', 'year_revenue',
            'week_max_withdraw', 'month_max_withdraw',
            '3_month_max_withdraw', '6_month_max_withdraw',
            'week_sharpe', 'month_sharpe', '3_month_sharpe']
    result_t=result.sort_values(by=['%s0%%_aggressive_score'%aggressive_rate],ascending=False)
    result_t=result_t.head(20).sort_values('sharpe_score',ascending=False).head(first_n_funds*2)
    result_t.to_excel('res/%s_%s_result_filtered.xlsx'%(str(datetime.datetime.now().date()).replace('-',''),aggressive_rate),index=False)

    # display(result_t[['symbol','withdraw_score','revenue_score',
    #                 'sharpe_score','%s0%%_aggressive_score'%aggressive_rate]].reset_index(drop=True))
    display(result_t[cols].reset_index(drop=True))

Web Scraping and Funds Filtering/Filtering_and_Scoring.py


## Normal HTML Rendering
import requests
from bs4 import BeautifulSoup

html = request.get('https://askdjango.github.io/lv1/').Text
soup = BeautifulSoup(html, 'html.parser')

for tag in soup.select('#course_list > .course > a'):
    print(tag.text, tag['href'])

## Ajax Rendering
import requests

course_list = requests.get('https://askdjango.github.io/lv2/data.json').json()
for course in course_list:
    print(course['name'], course['url'])

## Javascript Rendering Crawling
## Later


import argparse
import os

import numpy as np
import tensorflow as tf

from tensorflow import keras

import flwr as fl

from myutils import get_client_at_index, get_data_for_client

from model import create_model, process_data

# Make TensorFlow logs less verbose
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"

# Define Flower client
class FemnistClient(fl.client.NumPyClient):
    def __init__(self, model, x_train, y_train, x_test, y_test):
        self.model = model
        self.x_train, self.y_train = x_train, y_train
        self.x_test, self.y_test = x_test, y_test

    def get_parameters(self):
        """Get parameters of the local model."""
        raise Exception("Not implemented (server-side parameter initialization)")

    def fit(self, parameters, config):
        """Train parameters on the locally held training set."""

        # Update local model parameters
        self.model.set_weights(parameters)

        # Get hyperparameters for this round
        batch_size: int = config["batch_size"]
        epochs: int = config["local_epochs"]

        # Train the model using hyperparameters from config
        history = self.model.fit(
            self.x_train, self.y_train, batch_size, epochs, validation_split=0.1
        )

        # Return updated model parameters and results
        parameters_prime = self.model.get_weights()
        num_examples_train = len(self.x_train)
        results = {
            "loss": history.history["loss"][0],
            "accuracy": history.history["accuracy"][0],
            "val_loss": history.history["val_loss"][0],
            "val_accuracy": history.history["val_accuracy"][0],
        }
        return parameters_prime, num_examples_train, results

    def evaluate(self, parameters, config):
        """Evaluate parameters on the locally held test set."""

        # Update local model with global parameters
        self.model.set_weights(parameters)

        # Get config values
        steps: int = config["val_steps"]

        # Evaluate global model parameters on the local test data and return results
        loss, accuracy = self.model.evaluate(self.x_test, self.y_test, 32, steps=steps)
        num_examples_test = len(self.x_test)
        return loss, num_examples_test, {"accuracy": accuracy}

def start_client(client_id, train_data, test_data):
    # Load and compile Keras model
    model = create_model()
    model.compile("adam", "categorical_crossentropy", metrics=["accuracy"])

    # Get processed client data that can be fed to model
    (x_train, y_train), (x_test, y_test) = process_data(train_data, test_data)
    print("client_id: {} ;(x_train, y_train), (x_test, y_test) = ({}, {}), ({}, {})".format(client_id, len(x_train), len(y_train), len(x_test), len(y_test)))

    # Start Flower client
    client = FemnistClient(model, x_train, y_train, x_test, y_test)
    fl.client.start_numpy_client("[::]:8080", client=client)

def main() -> None:
    # Parse command line argument 'cid'
    parser = argparse.ArgumentParser(description="Flower")
    parser.add_argument("--cid", type=int, choices=range(0, 100), required=True)
    args = parser.parse_args()

    # Get dataset and start client
    c_train_data, c_test_data = get_data_for_client(args.cid)
    start_client(args.cid, c_train_data, c_test_data)

if __name__ == "__main__":
    main()


import requests
import lxml
from bs4 import BeautifulSoup
from serpapi import GoogleSearch

from bs4 import BeautifulSoup
import requests, lxml

headers = {
    'User-agent':
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)"
        "Chrome/70.0.3538.102 Safari/537.36 Edge/18.19582"
}

html = requests.get('https://www.google.com/search?q=goldman sachs&hl=en', headers=headers)

def get_knowledge_graph():
    soup = BeautifulSoup(html.text, 'lxml')

    title = soup.select_one('#rhs .mfMhoc span').text
    subtitle = soup.select_one('.wwUB2c span').text
    try:
        snippet = soup.select_one('.zsYMMe+ span').text
    except:
        snippet = None

    print(f"{title}\n{subtitle}\n{snippet}\n")

    for result in soup.select(".rVusze"):
        key_element = result.select_one(".w8qArf").text
        if result.select_one(".kno-fv"):
            value_element = result.select_one(".kno-fv").text.replace(": ", "")
        else:
            value_element = None
        key_link = f'https://www.google.com{result.select_one(".w8qArf a")["href"]}'
        try:
            key_value_link = f'https://www.google.com{result.select_one(".kno-fv a")["href"]}'
        except:
            key_value_link = None

        print(f"{key_element}{value_element}\nkey_link: {key_link}\nkey_value_link: {key_value_link}")

def get_top_10_search():
    # The code to get the html contents here.

    soup = BeautifulSoup(html, 'html.parser')

    # Find all the search result divs
    divs = soup.select("#search div.g")
    for div in divs:
        # Search for a h3 tag
        results = div.select("h3")

        # Check if we have found a result
        if (len(results) >= 1):
            # Print the title
            h3 = results[0]
            print(h3.get_text())

search_list = ['Dell','Apple']
for search_key in search_list:
    html = requests.get('https://www.google.com/search?q='+search_key+'&hl=en', headers=headers)
    get_knowledge_graph()
    get_top_10_search()


from django.core.urlresolvers import reverse
from django.test import TestCase

class ViewTests(TestCase):

    def test_home_view(self):
        response = self.client.get(reverse('home'))
        self.assertEqual(response.status_code, 200)
        self.assertContains(response, 'Hello, world!')

    def test_about_view(self):
        response = self.client.get(reverse('about'))
        self.assertEqual(response.status_code, 200)
        self.assertContains(response, 'This is the about page')


"""
This should only be run ONCE unless the project needs to be created from scratch.
Used to create the initial Mailchimp subscriber table
11-5-2020
"""

from pathlib import Path
from rc.db.sqlite_3.oop import Database

HOME_DIR = Path(__file__).resolve().parent

def build_mailchimp_users_table(db):
    db.run_script("""DROP TABLE IF EXISTS 'user';""")
    create_query = \
        """CREATE TABLE user (id INTEGER PRIMARY KEY, email_id TEXT, email TEXT NOT NULL UNIQUE,
        status TEXT, rating INTEGER, created DATE, updated DATE, last_open DATE)"""
    db.create_table(create_query)

if __name__ == '__main__':
    database = Database(HOME_DIR.joinpath('db/test.db'))
    build_mailchimp_users_table(database)


import os
import xlwt
import lib
import classes
import methods
from methods import Create
from methods import View
from methods import Edit
from lib import settings
from lib import FormatSheet
from lib import PopulateSheet
from classes import CharacterClass
import pickle

if not os.path.exists('Excel/'):
    os.makedirs('Excel/')
if not os.path.exists('Characters/'):
    os.makedirs('Characters/')

settings.init()

def LoadCharacter():
    characterName = input("Which character do you want to load?")
    object = open('Characters/' + characterName, 'rb')
    character = pickle.load(object)
    print('{} {}'.format(character.name, character.characterclass.race))

progrun = True
while progrun == True:
    action = 0
    actioncheck = False
    while actioncheck == False:
        print('Would you like to:')
        print('1 - Create a new character?')
        print('2 - Edit a character?')
        print('3 - View your characters?')
        print('4 - View character details?')
        print('5 - Delete a character?')
        print('6 - Generate a sheet for an existing character?')
        print('7 - Generate a blank sheet?')
        print('8 - Exit?')
        print('')
        try:
            action = int(input('Pick one by number:  '))
        except:
            print('')
            print('Please pick one by its number.')
            print('')

        if action == 1 or action == 2 or action == 3 or action == 4 or action == 5 or action == 6 or action == 7 or action == 8:
            actioncheck = True
        else:
            print('')
            print('Please pick one by its number.')
            print('')

    if action == 1:
        methods.Create.CreateNew()
    elif action == 2:
        character = methods.Edit.pickCharacter()
        methods.Edit.editCharacterMenu(character)
    elif action == 3:
        methods.View.viewCharacters()
    elif action == 4:
        character = methods.Edit.pickCharacter()
        methods.View.viewDetails(character)
    #elif action == 5:
        # delete logic here
    elif action == 6:
        Database.pickCharacter(settings.book, settings.sheet)
    elif action == 7:
        FormatSheet.initFormat(settings.book, settings.sheet)
        PopulateSheet.initSheet(settings.book, settings.sheet)
        settings.book.save('Excel/BlankSheet.xls')
    elif action == 8:
        print('')
        print('Exiting...')
        progrun = False


import configparser

from openpyxl import load_workbook
from openpyxl.styles import Alignment, Font, Border, Protection, Fill

# сначала нужно посмотреть borrow_formatting_new.py, так как текущий модуль делает то же самое, только с ini-файлом.
# в ini-файле прописываются вручную нужные свойства у строк и столбцов, порядок их указания определяет приоритет настроек:
# например, если сначала у строки 1 выставилось выравнивание вправо, а у столбца 1 выравнивание влево, то у ячейки [1,1]
# будет выравнивание столбца, так как он 'лег' на ячейку последним
# настройки берутся из свойств объектов openpyxl (так как мы говорим про столбцы и строки, то надо смотреть объекты
# row_dimensions и column_dimensions -> https://docs-python.ru/packages/modul-openpyxl/razmer-stroki-stolbtsa/)

# ini-файл можно создать программно, для этого см. модуль make_config.py

def get_dict_from_str(dict_str):
    namespace = {}
    exec(f"result = dict({dict_str})", locals(), namespace)
    return namespace["result"]

def _init_openpyxl_style_with_json(prop_name, props_string):
    styles = {
        'alignment': Alignment,
        'font': Font,
        'border': Border,
        'protection': Protection,
        'fill': Fill
    }
    if prop_name not in styles:
        return props_string
    props = get_dict_from_str(props_string)
    return styles[prop_name](**props)

def _copy_props(_obj, _prop_dict):
    [setattr(_obj, prop, _init_openpyxl_style_with_json(prop, _prop_dict[prop])) for prop in _prop_dict.keys() if
     hasattr(_obj, prop)]

config_path = 'example.ini'
file_out = 'result.xlsx'

config = configparser.ConfigParser()
config.read(config_path)

wb = load_workbook(file_out)
ws = wb.active

unusual_rows = []
unusual_columns = []
for section in config.sections():
    prop_dict = config[section]
    if prop_dict['type'] == 'column':
        _copy_props(ws.column_dimensions[section], prop_dict)
        unusual_columns.append(section)
    elif prop_dict['type'] == 'row':
        _copy_props(ws.row_dimensions[int(section)], prop_dict)
        unusual_rows.append(int(section))

for row in ws.rows:
    for cell in row:
        if cell.row in unusual_rows:
            if cell.column_letter in unusual_columns:
                for section in reversed(config.sections()):
                    if section == str(cell.row) or section == cell.column_letter:
                        _copy_props(cell, config[section])
                        break
            else:
                _copy_props(cell, config[str(cell.row)])
        elif cell.column_letter in unusual_columns:
            _copy_props(cell, config[cell.column_letter])

wb.save(file_out)


import re
import tekore as tk
import pathlib
import pickle
import argparse
import inquirer

def get_config():
    conf_path  = pathlib.Path.home().joinpath('.spotify.cnf')
    return tk.config_from_file(conf_path.as_posix())

def get_user_token():
    config = get_config()
    token_path = pathlib.Path.home().joinpath('.spotify.token.pkl')

    if not token_path.exists():
        user_token = tk.prompt_for_user_token( *config, scope=tk.scope.every)
        with open(token_path.as_posix(), 'wb') as f:
            pickle.dump({'refresh': user_token.refresh_token}, f)
    else:
        with open(token_path.as_posix(), 'rb') as f:
            d = pickle.load(f)
            refresh_token = d['refresh']
        refresher = tk.RefreshingCredentials(*config)
        user_token = refresher.refresh_user_token(refresh_token)
    return user_token

def play_top_k_tracks(spotify, k=10):
    tracks = spotify.current_user_top_tracks(limit=k).items
    spotify.playback_start_tracks([t.id for t in tracks])

def init():
    conf = get_config()
    app_token = tk.request_client_token(*conf[:2])
    spotify = tk.Spotify(app_token)
    spotify.token = get_user_token()
    return spotify

def albums(spotify, args):
    format_str = "%s - %s"
    def artists_str(artists):
        st = "%-30s" % ("; ".join([a.name for a in artists])[:30])
        return st

    albums = [a.album for a in spotify.saved_albums(limit=20).items]
    questions = [
      inquirer.List('choosen',
                    message="What album do you want to hear?",
                    choices=[format_str % (artists_str(a.artists), a.name) for a in albums],
                ),
    ]
    answers     = inquirer.prompt(questions)
    context_uri = next( (a.uri for a in albums if format_str % (artists_str(a.artists), a.name) == answers['choosen']) )
    spotify.playback_start_context(context_uri)

def playlists(spotify, args):
    playlists = spotify.followed_playlists(limit=50).items
    questions = [
            inquirer.List('choosen',
                message="What playlist do you want to hear?",
                choices=[p.name for p in playlists]
                ),
        ]
    answers = inquirer.prompt(questions)
    context_uri = next( (p.uri for p in playlists if p.name == answers['choosen']) )
    spotify.playback_start_context(context_uri)

def status(spotify, args):
    pb = spotify.playback()
    print("%-15s" % "Playing:", pb.item.name, " by ", "; ".join([a.name for a in pb.item.artists]))
    print("%-15s" % "On device:", pb.device.name)

def next_track(spotify, args):
    spotify.playback_next()

def previous_track(spotify, args):
    spotify.playback_previous()

def analyze(spotify, args):
    playlist = spotify.followed_playlists(limit=1).items[0]
    track = spotify.playlist_items(playlist.id, limit=1).items[0].track
    name = f'"{track.name}" from {playlist.name}'
    if track.episode:
        print(f'Cannot analyse episodes!\nGot {name}.')
    elif track.track and track.is_local:
        print(f'Cannot analyse local tracks!\nGot {name}.')
    else:
        print(f'Analysing {name}...\n')
        analysis = spotify.track_audio_features(track.id)
        print(repr(analysis))

def play(spotify, args):
    spotify.playback_resume()

def pause(spotify, args):
    spotify.playback_pause()

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    subparsers = parser.add_subparsers()
    # Play pause
    p = subparsers.add_parser('play')
    p.set_defaults(func=play)

    p = subparsers.add_parser('pause')
    p.set_defaults(func=pause)

    p = subparsers.add_parser('next')
    p.set_defaults(func=next_track)

    p = subparsers.add_parser('previous')
    p.set_defaults(func=previous_track)

    p = subparsers.add_parser('analyze')
    p.set_defaults(func=analyze)

    p = subparsers.add_parser('albums')
    p.set_defaults(func=albums)

    p = subparsers.add_parser('playlists')
    p.set_defaults(func=playlists)

    p = subparsers.add_parser('status')
    p.set_defaults(func=status)

    spotify = init()

    args = parser.parse_args()
    args.func(spotify, args)


"""mysite URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/2.2/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.conf import settings
from django.contrib import admin
from django.urls import include, path
from django.views.generic.base import TemplateView

from zoo_checks import views

urlpatterns = [
    path("manage/", admin.site.urls, name="admin"),
    path("accounts/", include("allauth.urls")),
    path(
        "account_management/",
        TemplateView.as_view(template_name="account_management.html"),
        name="account_management",
    ),
    path("", views.home, name="home"),
    path("count/<slug:enclosure_slug>/", views.count, name="count"),
    path(
        "count/<slug:enclosure_slug>/<int:year>/<int:month>/<int:day>/",
        views.count,
        name="count",
    ),
    path(
        "tally_date_handler/<slug:enclosure_slug>",
        views.tally_date_handler,
        name="tally_date_handler",
    ),
    path(
        "edit_species_count/<slug:species_slug>/<slug:enclosure_slug>/<int:year>/<int:month>/<int:day>/",
        views.edit_species_count,
        name="edit_species_count",
    ),
    path(
        "edit_group_count/<group>/<int:year>/<int:month>/<int:day>/",
        views.edit_group_count,
        name="edit_group_count",
    ),
    path(
        "edit_animal_count/<animal>/<int:year>/<int:month>/<int:day>/",
        views.edit_animal_count,
        name="edit_animal_count",
    ),
    path("animal_counts/<animal>", views.animal_counts, name="animal_counts"),
    path("group_counts/<group>", views.group_counts, name="group_counts"),
    path(
        "species_counts/<slug:species_slug>/<slug:enclosure_slug>",
        views.species_counts,
        name="species_counts",
    ),
    path("upload/", views.ingest_form, name="ingest_form"),
    path("confirm_upload/", views.confirm_upload, name="confirm_upload"),
    path("export/", views.export, name="export"),
    # for django browser reload
    path("__reload__/", include("django_browser_reload.urls")),
]

if settings.DEBUG:
    try:
        import debug_toolbar

        urlpatterns = [path("__debug__/", include(debug_toolbar.urls))] + urlpatterns
    except ImportError:
        pass


import pandas as pd
import plotly.express as px
import plotly.graph_objects as go

import dash
from dash import dcc
from dash.dependencies import Input, Output
import os
import dash_bootstrap_components as dbc

global df, title, x_buy, x_sell, y_buy, y_sell
# inizialization variables
x_buy, x_sell, y_buy, y_sell = [],[],[],[]
last_save=0
last_cancel=0
value = 'BTCEUR'
title = value
df = pd.read_csv('./DATA/{}.csv'.format(value))
df.start = pd.to_datetime(df.start)
df = df.sort_values('start')
df.index = df.start
fig = px.line(x=df.index, y=df['close']
    ).add_traces(
    px.line(x=df.index, y=df['close']
    )
        .update_traces(marker_color="rgba(0,0,0,0)").data
)
fig.layout.height = 400

tab1_content = dbc.Card(
    dbc.CardBody(
        [
            dash.dcc.Graph(id="graph_1", figure=fig, ),
        ]
    ),
    className="mt-3",
)

tab2_content = dbc.Card(
    dbc.CardBody(
        [
            dash.dcc.Graph(id="graph_2", figure=fig),
        ]
    ),
    className="mt-3",
)

tabs = dbc.Tabs(
    [
        dbc.Tab(tab1_content, label="Buy "),
        dbc.Tab(tab2_content, label="Sell"),

    ]
)

# Build App

external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css']
app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP], suppress_callback_exceptions=True)
app.layout = dash.html.Div([
    dbc.Row([
        dbc.Row(dbc.Card(dash.html.H1('FastTradingLabel'),color='danger')),
        dbc.Col([

            dbc.Row( [
        dbc.Row(dash.html.H6('FTL is a tool to quickly label cryptocurrency data,'
                             ' choose the exchange and click on the chart to the right'
                             'the market entry and exit points you consider appropriate')),
        dbc.Row(dash.html.H4('Select exchange')),
        dbc.Row(
                            dcc.Dropdown(
                                options=list({'label':x[:-4],'value':x} for x in os.listdir('DATA')), id='select_cripto',

                            )
                    ),

        dbc.Row( id='n_operation_buy'),
        dbc.Row( id='n_operation_sell'),
        dbc.Button('Cancel', n_clicks=0,id='cancel'),
        dbc.Button('Save', n_clicks=0,id='save'),
    ],
    className="d-grid gap-2 col-6 mx-auto",),
            dbc.Row(),
            dbc.Row( id='saved'),
        ],align="end"),
        dbc.Col(tabs,id='tab', width=8)
    ]),
    dbc.Row(dash.html.Div(id="where"))
])

@app.callback(
    Output("where", "children"),
    Output("n_operation_buy", "children"),
    Output("n_operation_sell", "children"),
    Input("graph_1", "clickData"),
    Input("graph_2", "clickData"),
    Input("save", 'n_clicks'),
    Input("cancel", 'n_clicks')
)
def click(clickData1,clickData2,save,cancel):
    global x_buy, x_sell, y_buy, y_sell, last_cancel, last_save

    if cancel == last_cancel and save == last_save:
        if clickData1:
            x_buy.append(clickData1['points'][0]['x'])
            y_buy.append(clickData1['points'][0]['y'])

        if clickData2:
            x_sell.append(clickData2['points'][0]['x'])
            y_sell.append(clickData2['points'][0]['y'])

    if cancel > last_cancel:
        last_cancel += 1

        if x_buy != []:
            x_buy.pop()
        if x_sell != []:
            x_sell.pop()
        if y_buy != []:
            y_buy.pop()
        if y_sell != []:
            y_sell.pop()

    if save > last_save:

        last_save += 1
        df['BUY']=0
        df['SELL'] = 0
        df['BUY'].loc[list(set(x_buy))] = 1
        df['SELL'].loc[list(set(x_sell))] = 1
        if not os.path.exists('./LABELED_DATA'):
            os.mkdir('./LABELED_DATA')
        df.to_csv('./LABELED_DATA/{}.csv'.format(title), index=False)
        x_buy, x_sell, y_buy, y_sell = [], [], [], []

    trace = go.Candlestick(x=df.start,
                           open=df.open,
                           high=df.high,
                           low=df.low,
                           close=df.close)
    data = [trace]
    data.append(go.Scatter(x=x_buy, y=y_buy,
                               mode='markers',
                               name='buy',
                               marker_size=20,
                               marker_symbol="triangle-up",
                               marker_color="green"))
    data.append(go.Scatter(x=x_sell, y=y_sell,
                               mode='markers',
                               name='buy',
                               marker_size=20,
                               marker_symbol="triangle-down",
                               marker_color="red"))
    layout = {'title':title,'xaxis': {'rangeslider': {'visible': False}}}
    figure = {'data': data, 'layout': layout}

    return dash.dcc.Graph(figure=figure), dash.html.H4('Number of buy: {}'.format(str(len(set(x_buy))))),\
           dash.html.H4('Number of sell: {}\n'.format(str(len(set(x_sell)))))

@app.callback(
    (Output("tab", "children")),
    Input("select_cripto", 'value')
)
def drop(value):
    global df, title, x_buy, x_sell, y_buy, y_sell

    x_buy, x_sell, y_buy, y_sell = [], [], [], []
    ###### FIGURE TAB
    if not value:
        value = 'BTCEUR'
    else:
        value = value[:-4]
    title = value
    df = pd.read_csv('./DATA/{}.csv'.format(value))
    df.start = pd.to_datetime(df.start)
    df = df.sort_values('start')
    df.index = df.start
    fig = px.line(x=df.index, y=df['close']
                  ).add_traces(
        px.line(x=df.index, y=df['close']
                )
            .update_traces(marker_color="rgba(0,0,0,0)").data
    )
    fig.layout.height = 400
    tab1_content = dbc.Card(
        dbc.CardBody(
            [
                dash.dcc.Graph(id="graph_1", figure=fig),
            ]
        ),
        className="mt-3",
    )

    tab2_content = dbc.Card(
        dbc.CardBody(
            [
                dash.dcc.Graph(id="graph_2", figure=fig),
            ]
        ),
        className="mt-3",
    )

    tabs = dbc.Tabs(
        [
            dbc.Tab(tab1_content, label="Buy "),
            dbc.Tab(tab2_content, label="Sell"),

        ]
    )
    return tabs

# Run app and display result inline in the notebook
app.run_server()


from flask import Flask, render_template, request
import seaborn as sns
import plotly
import plotly.graph_objs as go
import json
from sqlalchemy import create_engine
import pymysql
import pandas as pd

app = Flask(__name__)

# Sumber data dari sql
sqlEngine = create_engine('mysql+pymysql://root:mysql123@127.0.0.1', pool_recycle = 3306)
dbConnection = sqlEngine.connect()
dfTips = pd.read_sql('SELECT * FROM flaskapp.tips', dbConnection)

#######################
## FUNCTION FOR PLOT ##
#######################

# Histogram dan Boxplot func
def category_plot(cat_plot = 'histoplot', cat_x = 'sex', cat_y = 'total_bill', estimator = 'count', hue = 'smoker'):

    if cat_plot == 'histoplot':
        data = []

        for val in dfTips[hue].unique(): # [No, Yes]
            hist = go.Histogram(

                    x = dfTips[dfTips[hue] == val][cat_x],
                    y = dfTips[dfTips[hue] == val][cat_y],
                    # histfunc = 'count',
                    name = val

                    )

            data.append(hist)

        title = 'HISTOGRAM'
    else:
        data = []

        for val in dfTips[hue].unique(): # [No, Yes]
            hist = go.Box(

                    x = dfTips[dfTips[hue] == val][cat_x],
                    y = dfTips[dfTips[hue] == val][cat_y],
                    name = val

                    )

            data.append(hist)

        title = "BOXPLOT"

    layout = go.Layout(

        title = title,

        title_x = 0.5,

        xaxis = dict(title = cat_x),

        yaxis = dict(title = cat_y),

        boxmode = 'group'

    )

    final = {'data' : data, 'layout' : layout}

    graphJSON = json.dumps(final, cls = plotly.utils.PlotlyJSONEncoder)

    return graphJSON

# Scatter Plot func
def scatter_plot(cat_x, cat_y):

    # Masukan data
    data_src = [

        go.Scatter(

            x = dfTips[cat_x],

            y = dfTips[cat_y],

            mode = 'markers'

        )

    ]

    # Pembuatan layout
    layout_src = go.Layout(

        title = 'SCATTER PLOT',

        title_x = 0.5,

        xaxis = dict(title = cat_x),

        yaxis = dict(title = cat_y)

    )

    # Gabungkan antara data dan layout dalam satu variabel
    # key 'data' dan 'layout' harus ada
    final = {'data' : data_src, 'layout' : layout_src}

    graphJSON = json.dumps(final, cls = plotly.utils.PlotlyJSONEncoder)

    return graphJSON

# Pie plot func
def pie_plot(cat):

    result = dfTips[cat].value_counts()

    labels = []
    values = []

    for i in result.iteritems():
        labels.append(i[0])
        values.append(i[1])

    data_src = [

        go.Pie(

            labels = labels,

            values = values,

        )

    ]

    layout_src = go.Layout(

        title = 'PIE',

        title_x = 0.5,

    )

    final = {'data' : data_src, 'layout' : layout_src}

    graphJSON = json.dumps(final, cls = plotly.utils.PlotlyJSONEncoder)

    return graphJSON

######################
# FUNCTION FOR ROUTE #
######################

@app.route('/')
def index():
    plot = category_plot()

    return render_template(
        'category.html',
        plot = plot,
        focus_plot = 'histoplot',
        focus_x = 'sex',
        focus_y = 'total_bill',
        focus_estimator = 'count'
    )

@app.route('/cat_fn')
def cat_fn():
    # Request untuk ambil data dari form
    cat_plot = request.args.get('cat_plot')
    cat_x = request.args.get('cat_x')
    cat_y = request.args.get('cat_y')
    estimator = request.args.get('estimator')
    hue = request.args.get('hue')

    if cat_plot == None and cat_x == None and cat_y == None and estimator == None and hue == None:
        cat_plot = 'histoplot'
        cat_x = 'sex'
        cat_y = 'total_bill'
        estimator = 'count'
        hue = 'smoker'

    plot = category_plot(cat_plot, cat_x, cat_y, estimator, hue)

    # List dropdown
    drop_plot_src = [('histoplot', 'Histogram'), ('boxplot', 'Boxplot')]
    drop_x_src = [('sex', 'Sex'), ('smoker', 'Smoker'), ('day', 'Day'), ('time', 'Time')]
    drop_y_src = [('total_bill', 'Total Bill'), ('tip', 'Tip'), ('size', 'size')]
    drop_est_src = [('count', 'Count'), ('sum', 'Sum'), ('avg', 'Average'), ('min', 'Minimum'), ('max', 'Maximum')]

    return render_template(
        'category.html',
        plot = plot,
        focus_plot = cat_plot,
        focus_x = cat_x,
        focus_y = cat_y,
        focus_estimator = estimator,
        focus_hue = hue,
        drop_plot = drop_plot_src,
        drop_x = drop_x_src,
        drop_y = drop_y_src,
        drop_est = drop_est_src,
        drop_hue = drop_x_src

    )

@app.route('/scat_fn')
def scat_fn():

    cat_x = request.args.get('cat_x')
    cat_y = request.args.get('cat_y')

    if cat_x == None and cat_y == None:
        cat_x = 'total_bill'
        cat_y = 'tip'

    plot = scatter_plot(cat_x, cat_y)

    return render_template('scatter.html', plot = plot, focus_x = cat_x, focus_y = cat_y)

@app.route('/pie_fn')
def pie_fn():

    cat = request.args.get('cat')

    if cat == None:
        cat = 'sex'

    plot = pie_plot(cat)

    return render_template('pie.html', plot = plot, focus_cat = cat)

if __name__ == '__main__':
    app.run(debug = True)


from starlette.requests import Request

from fastapi_jwt.schemas.jwt import AuthenticateSchema
from fastapi_jwt.services.jwt import JWTService
from fastapi_jwt.services.users import UserService

async def login(
        authenticate_data: AuthenticateSchema,
        jwt_service: JWTService,
        user_service: UserService,
) -> dict:
    user_id = await user_service.authenticate(**authenticate_data.model_dump())
    tokens = await jwt_service.create_auth_tokens(user_id)
    return tokens

async def refresh_access_token(
        request: Request,
        jwt_service: JWTService,
) -> dict:
    tokens = await jwt_service.refresh_auth_tokens(request.cookies.get('refresh_token'))
    return tokens

async def logout(
        request: Request,
        jwt_service: JWTService,
) -> None:
    await jwt_service.delete_refresh_token(request.cookies.get('refresh_token'))