# -*- coding: utf-8 -*-
# @Time     : 12/14/2021 19:59
# @Author   : Junyi
# @FileName: Agent.py
# @Software  : PyCharm
# Observing PEP 8 coding style
import pickle
import random
import numpy as np
from Landscape import Landscape

class Specialist:
    def __init__(self, N=None, landscape=None, coarse_landscape=None, state_num=4, expertise_amount=None):
        self.landscape = landscape
        self.coarse_landscape = coarse_landscape
        self.N = N
        self.state_num = state_num
        self.expertise_domain = np.random.choice(range(self.N), expertise_amount // 4, replace=False).tolist()
        self.expertise_representation = ["0", "1", "2", "3"]
        self.state = np.random.choice(range(self.state_num), self.N).tolist()
        self.state = [str(i) for i in self.state]  # state format: string
        self.coarse_state = self.state_2_coarse_state(state=self.state)  # will be the same as state, thus search accurately
        self.coarse_fitness = 0
        self.ave_fitness, self.max_fitness, self.min_fitness = 0, 0, 0
        # Mechanism: overlap with IM
        self.row_overlap = 0
        self.column_overlap = 0

        if not self.landscape:
            raise ValueError("Agent need to be assigned a landscape")
        if (self.N != landscape.N) or (self.state_num != landscape.state_num):
            raise ValueError("Agent-Landscape Mismatch: please check your N and state number.")
        if expertise_amount % 2 != 0:
            raise ValueError("Expertise amount needs to be a even number")
        if expertise_amount > self.N * 4:
            raise ValueError("Expertise amount should be less than {0}.".format(self.N * 4))

    def update_fitness(self):
        self.coarse_fitness = self.coarse_landscape.query_coarse_fitness(coarse_state=self.coarse_state)
        self.ave_fitness, self.max_fitness, self.min_fitness = self.landscape.query_potential_fitness(coarse_state=self.coarse_state)

    # def get_overlap_with_IM(self):
    #     influence_matrix = self.landscape.IM
    #     row_overlap, column_overlap = 0, 0
    #     for row in range(self.N):
    #         if row in self.expertise_domain:
    #             row_overlap += sum(influence_matrix[row])
    #     for column in range(self.N):
    #         if column in self.expertise_domain:
    #             column_overlap += sum(influence_matrix[:, column])
    #     self.column_overlap = column_overlap
    #     self.row_overlap = row_overlap

    # def align_default_state(self, state=None):
    #     for index in range(self.N):
    #         if index not in self.expertise_domain:
    #             self.state[index] = state[index]
    #     self.coarse_state = self.state_2_coarse_state(state=self.state)
    #     self.coarse_fitness = self.landscape.query_cog_fitness_partial(coarse_state=self.coarse_state, expertise_domain=self.expertise_domain)
    #     self.fitness, self.potential_fitness = self.landscape.query_cog_fitness_full(coarse_state=self.coarse_state)

    # def learn_from_pool(self, pool=None):
    #     exposure_state = pool[np.random.choice(len(pool))]
    #     cog_exposure_state = self.state_2_coarse_state(state=exposure_state)
    #     cog_fitness_of_exposure_state = self.landscape.query_cog_fitness_partial(coarse_state=cog_exposure_state, expertise_domain=self.expertise_domain)
    #     if cog_fitness_of_exposure_state > self.coarse_fitness:
    #         self.coarse_state = cog_exposure_state
    #         self.coarse_fitness = cog_fitness_of_exposure_state
    #         self.fitness, self.potential_fitness = self.landscape.query_cog_fitness_full(coarse_state=self.coarse_state)
    #         return True
    #     return False

    def search(self):
        next_coarse_state = self.coarse_state.copy()
        index = np.random.choice(self.expertise_domain)  # only select from the expertise domain,
        # thus will not change the unknown domain
        space = ["0", "1", "2", "3"]
        space.remove(self.coarse_state[index])
        next_coarse_state[index] = np.random.choice(space)
        next_coarse_fitness = self.coarse_landscape.query_coarse_fitness(coarse_state=next_coarse_state)
        if next_coarse_fitness > self.coarse_fitness:
            self.coarse_state = next_coarse_state
            self.coarse_fitness = next_coarse_fitness
            self.ave_fitness, self.max_fitness, self.min_fitness = self.landscape.query_potential_fitness(
                coarse_state=self.coarse_state)

    def coordinated_search(self, co_state=None, co_expertise_domain=None):
        # the focal agent's evaluation: whether to align with the teammate
        next_coarse_state = self.coarse_state.copy()
        for index in range(self.N):
            if index in co_expertise_domain:
                next_coarse_state[index] = co_state[index]
        next_coarse_fitness = self.landscape.query_cog_fitness_partial(coarse_state=next_coarse_state,
                                                    expertise_domain=self.expertise_domain)
        if next_coarse_fitness > self.coarse_fitness:
            self.coarse_state = next_coarse_state.copy()
            self.coarse_fitness = next_coarse_fitness
            self.fitness, self.potential_fitness = self.landscape.query_cog_fitness_full(coarse_state=self.coarse_state)

        # Proposal from the focal agent
        next_coarse_state = self.coarse_state.copy()
        index = np.random.choice(self.expertise_domain)  # only select from the expertise domain,
        # thus will not change the unknown domain
        space = ["0", "1", "2", "3"]
        space.remove(self.coarse_state[index])
        next_coarse_state[index] = np.random.choice(space)
        next_coarse_fitness = self.landscape.query_cog_fitness_partial(coarse_state=next_coarse_state, expertise_domain=self.expertise_domain)
        if next_coarse_fitness > self.coarse_fitness:
            self.coarse_state = next_coarse_state
            self.coarse_fitness = next_coarse_fitness
            self.fitness, self.potential_fitness = self.landscape.query_cog_fitness_full(coarse_state=self.coarse_state)

    def priority_search(self, co_state=None, co_expertise_domain=None):
        # learning from coupled agent
        next_coarse_state = self.coarse_state.copy()
        for index in range(self.N):
            if index in co_expertise_domain:
                next_coarse_state[index] = co_state[index]
            else:
                pass
        index = np.random.choice(self.expertise_domain)  # only select from the expertise domain,
        # thus will not change the unknown domain
        space = ["0", "1", "2", "3"]
        space.remove(self.coarse_state[index])
        next_coarse_state[index] = np.random.choice(space)
        next_coarse_fitness = self.landscape.query_cog_fitness_partial(coarse_state=next_coarse_state,
                                                                    expertise_domain=self.expertise_domain)
        if next_coarse_fitness > self.coarse_fitness:
            self.coarse_state = next_coarse_state
            self.coarse_fitness = next_coarse_fitness
            self.fitness, self.potential_fitness = self.landscape.query_cog_fitness_full(coarse_state=self.coarse_state)

    def state_2_coarse_state(self, state=None):
        state = state.copy()
        return [bit if i in self.expertise_domain else "*" for i, bit in enumerate(state)]
        # return state

    def cog_state_2_state(self, coarse_state=None):
        state = coarse_state.copy()
        return [np.random.choice(["0", "1", "2", "3"]) if bit == "*" else bit for bit in state]
        # return coarse_state

    def describe(self):
        print("Specialist of Expertise domain: ", self.expertise_domain)
        print("State: ", self.state)
        print("Coarse State: ", self.coarse_state)
        print("Coarse fitness: ", self.coarse_fitness)
        print("Average real fitness: ", self.ave_fitness)
        print("Max real fitness: ", self.max_fitness)
        print("Min real fitness: ", self.min_fitness)

if __name__ == '__main__':
    # Test Example
    from CogLandscape import CogLandscape
    import time
    t0 = time.time()
    search_iteration = 500
    N = 9
    K = 0
    state_num = 4
    expertise_amount = 32
    landscape = Landscape(N=N, K=K, state_num=state_num)
    specialist = Specialist(N=N, landscape=landscape, state_num=state_num, expertise_amount=expertise_amount)
    coarse_landscape = CogLandscape(landscape=landscape, expertise_domain=specialist.expertise_domain,
                                 expertise_representation=specialist.expertise_representation)
    specialist.coarse_landscape = coarse_landscape
    specialist.update_fitness()
    specialist.describe()
    ave_performance_across_time = []
    max_performance_across_time = []
    min_performance_across_time = []
    cog_performance_across_time = []
    for _ in range(search_iteration):
        specialist.search()
        print(specialist.coarse_state, specialist.ave_fitness)
        ave_performance_across_time.append(specialist.ave_fitness)
        max_performance_across_time.append(specialist.max_fitness)
        min_performance_across_time.append(specialist.min_fitness)
        cog_performance_across_time.append(specialist.coarse_fitness)
    # specialist.describe()
    import matplotlib.pyplot as plt
    import numpy as np
    x = np.arange(search_iteration)
    plt.plot(x, ave_performance_across_time, "r-", label="Ave")
    plt.plot(x, max_performance_across_time, "b-", label="Max")
    plt.plot(x, min_performance_across_time, "g-", label="Min")
    plt.plot(x, cog_performance_across_time, "k-", label="Cog")
    plt.title('Performance at N={0}, K={1}, Kn={2}'.format(N, K, expertise_amount))
    plt.xlabel('Iteration', fontweight='bold', fontsize=10)
    plt.ylabel('Performance', fontweight='bold', fontsize=10)
    # plt.xticks(x)
    plt.legend(frameon=False, fontsize=10)
    plt.savefig("S_performance.png", transparent=True, dpi=200)
    plt.show()
    plt.clf()
    t1 = time.time()
    print(time.strftime("%H:%M:%S", time.gmtime(t1-t0)))


import base64

import os

from base.db_handlers import save_message_to_db, get_name_from_db, add_photo_to_db_bot, save_photo_message_to_web_db, \
    update_text_in_message_in_web_db
from base.redis_handlers import send_message_to_bot
from bot.bot_handlers import get_telegram_photo_id, send_meeting_message_to_bot
from config.config_app import STATIC_PATH
from handlers.pydentic_models import MessageLoad, Event, MessageDetails
from logs.logs import my_logger
from utils.utils3 import generate_random_filename

async def save_image_to_server(message_id: int | None,
                               is_answer: bool,
                               user_id: int | None,
                               message_type: str,
                               text: str,
                               extension: str,
                               time=None):
    # text, user_id, message_type, extension,):
    my_logger.debug(f"start{user_id}")
    if message_type == 'message':
        my_logger.debug(f"end because it is a text message")
        return
    elif message_type == 'photo':
        my_logger.debug('it is a photo')
    elif message_type == 'document':
        my_logger.debug('it is a pdf document')
    else:
        my_logger.error(f'wrong message type {message_type}')
        return

    new_name = generate_random_filename(extension=extension)
    directory = STATIC_PATH + "/" + "/images" + "/" + str(user_id) + "/"
    if not os.path.exists(directory):
        os.makedirs(directory)
    file_name = directory + new_name
    binary_data = base64.b64decode(text)

    with open(file_name, 'wb') as f:
        f.write(binary_data)
    url = f"/static/images/{user_id}/{new_name}"
    return url

async def message_event_handler(data: MessageLoad, ws):
    """обрабатывает все message events от клиента"""
    my_logger.debug(f"start")
    data.name = await get_name_from_db(user_id=data.details.user_id)
    if data.details.message_type in ('photo', 'document'):
        my_logger.debug('start photo or document branch')
        filename = await save_image_to_server(**data.details.model_dump())
        data.details.text = filename
        await ws.send_str(data.model_dump_json())
        web_message_id = await save_photo_message_to_web_db(**data.details.model_dump())
        telegram_document_id = await get_telegram_photo_id(filename)
        photo_command = await add_photo_to_db_bot(photo_id=telegram_document_id,
                                                  is_answer=data.details.is_answer,
                                                  message_id=web_message_id)
        await update_text_in_message_in_web_db(web_message_id, text=photo_command)
        await send_meeting_message_to_bot(user_id=data.details.user_id,
                                          user_name=data.name,
                                          full_history=False)

    elif data.details.message_type in ('text'):
        my_logger.debug('start text brunch')
        message_id = await save_message_to_db(**data.details.model_dump())
        data.details.message_id = message_id
        await ws.send_str(data.model_dump_json())
        await send_meeting_message_to_bot(user_id=data.details.user_id,
                                          user_name=data.name,
                                          full_history=False)
    else:
        my_logger.error(f"Undefinded message type: {data.details.message_type}, should be photo,document or text")
        return


import csv
import os
import numpy as np
from argparse import ArgumentParser as ArgPar
from collections import namedtuple
import sys
sys.path.append('../')
import math
import shutil
import datetime
import traceback
import draw_graph
import write_gspread
import torch
import torch.nn as nn
import torch.backends.cudnn as cudnn
import torch.optim as optim
import datetime
from dataset4class import get_data
import dataset4class
from tqdm import tqdm
from wide_resnet_classification import WideResNet

from joblib import Parallel, delayed
import matplotlib.pyplot as plt

def get_arguments():
    argp = ArgPar()
    hp = {"batch_size": 128,
           "lr": 1.0e-2,
           "momentum": 0.9,
           "weight_decay": 5.0e-4,
           "width_coef1": 10,
           "width_coef2": 10,
           "width_coef3": 10,
           "n_blocks1": 4,
           "n_blocks2": 4,
           "n_blocks3": 4,
           "drop_rates1": 0.3,
           "drop_rates2": 0.3,
           "drop_rates3": 0.3,
           "lr_decay": 0.2,
           "num_of_class": 10}

    for name, v in hp.items():
        argp.add_argument("-{}".format(name), type = type(v), default = v)

    parsed_parameters = argp.parse_args()
    HyperParameters = {}

    for k in hp.keys():
        HyperParameters[k] =  eval("parsed_parameters.{}".format(k))

    return HyperParameters

def accuracy(y, target):
    pred = y.data.max(1, keepdim = True)[1]
    acc = pred.eq(target.data.view_as(pred)).cpu().sum()

    return acc

global pred_total, target_total
def accuracy4test(y, target_tensor, target, paths):
    global pred_total, target_total, now, epoch, counter, patience, all_epoch

    pred = y.data.max(1, keepdim = True)[1]
    acc = pred.eq(target_tensor.data.view_as(pred)).cpu().sum()
    pred_total.extend(pred.cpu())
    target_total.extend(target_tensor.cpu())

    if counter+1 >= patience or epoch == all_epoch-1: # or epoch%30==0:
        for i in range(len(target_tensor.cpu())):
            if not os.path.exists(dataset4class.dataset_directory  + dataset4class.dataset_folder[1:] + '/log/{0:%m%d}_{0:%H%M}/false_pred_{1}/{2}/{3}/'.format(now, epoch, target[i], np.array(pred.cpu()[i]))):
                os.makedirs(dataset4class.dataset_directory  + dataset4class.dataset_folder[1:] + '/log/{0:%m%d}_{0:%H%M}/false_pred_{1}/{2}/{3}/'.format(now, epoch, target[i], np.array(pred.cpu()[i])))
            splited_path = paths[i].split('/')
            shutil.copyfile(paths[i], dataset4class.dataset_directory  + dataset4class.dataset_folder[1:] + '/log/{0:%m%d}_{0:%H%M}/false_pred_{1}/{2}/{3}/'.format(now, epoch, target[i], np.array(pred.cpu()[i])) + splited_path[-1])
        draw_graph.plot_confusion_matrix(target_total, pred_total, dataset4class.test_classes.keys(), save_caption=dataset4class.dataset_folder, save_place=dataset4class.dataset_directory  + dataset4class.dataset_folder[1:] + '/log/{0:%m%d}_{0:%H%M}/false_pred_{1}/'.format(now, epoch))

    return acc, pred_total, target_total

def print_result(values):
    f1 = "  {"
    f2 = "}  "
    f_int = "{}:<20"
    f_float = "{}:<20.5f"
    f_vars = ""

    for i, v in enumerate(values):
        if type(v) == float:
            f_vars += f1 + f_float.format(i) + f2
        else:
            f_vars += f1 + f_int.format(i) + f2

    print(f_vars.format(*values))

def train(device, optimizer, learner, train_data, loss_func):
    train_acc, train_loss, n_train = 0, 0, 0
    lr = optimizer.param_groups[0]["lr"]
    learner.train()
    bar = tqdm(desc = "Training", total = len(train_data), leave = False)

    for data, target, paths in train_data:
        data, target = data.to(device), target.to(device)

        y = learner(data)
        loss = loss_func(y, target)
        optimizer.zero_grad() # clears the gradients of all optimized tensors.
        loss.backward()
        optimizer.step() # renew learning rate

        train_acc += accuracy(y, target)

        train_loss += loss.item() * target.size(0) # loss.item() is a loss num, but not a tensor. target.size(0) is batch size.
        n_train += target.size(0)

        bar.set_description("Loss: {0:.6f}, Accuracy: {1:.6f}".format(train_loss / n_train, float(train_acc) / n_train))
        previous_loss = train_loss / n_train
        bar.update()
    bar.close()

    return float(train_acc) / n_train, train_loss / n_train

def test(device, optimizer, learner, test_data, loss_func):
    global target_total, pred_total
    target_total = []
    pred_total = []
    test_acc, test_loss, n_test = 0, 0, 0
    bar = tqdm(desc = "Testing", total = len(test_data), leave = False)

    for data, target, paths in test_data:
        data, target_tensor = data.to(device), target.to(device)

        y = learner(data)
        loss = loss_func(y, target_tensor)

        acc, pred_total, target_total = accuracy4test(y, target_tensor, target, paths)
        test_acc += acc
        test_loss += loss.item() * target_tensor.size(0)
        n_test += target_tensor.size(0)

        bar.update()
    bar.close()

    draw_graph.plot_confusion_matrix(target_total, pred_total, dataset4class.test_classes.keys(), save_caption=dataset4class.dataset_folder, save_place=dataset4class.dataset_directory  + dataset4class.dataset_folder[1:] + '/log/{0:%m%d}_{0:%H%M}/'.format(now, now))

    return float(test_acc) / n_test, test_loss / n_test

global now
def main(learner):

    device = "cuda"
    learner = torch.nn.DataParallel(learner, device_ids=[0, 1]) # make parallel

    train_data, test_data = get_data(learner.module.batch_size)

    global now
    now = datetime.datetime.now()
    if not os.path.exists(dataset4class.dataset_directory + dataset4class.dataset_folder[1:] + '/log/{0:%m%d}_{0:%H%M}'.format(now, now)):
        os.makedirs(dataset4class.dataset_directory + dataset4class.dataset_folder[1:] + '/log/{0:%m%d}_{0:%H%M}'.format(now, now))

    learner = learner.to(device)
    cudnn.benchmark = True

    optimizer = optim.SGD( \
                        learner.parameters(), \
                        lr = learner.module.lr, \
                        momentum = learner.module.momentum, \
                        weight_decay = learner.module.weight_decay, \
                        nesterov = True \
                        )

    loss_func = nn.CrossEntropyLoss().cuda()

    milestones = learner.module.lr_step
    scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones = milestones, gamma = learner.module.lr_decay)

    rsl_keys = ["lr", "epoch", "TrainAcc", "TrainLoss", "TestAcc", "TestLoss", "Time"]
    rsl = []

    print_result(rsl_keys)

    global patience, counter, best_loss, epoch, all_epoch
    patience = 20 #set any adequate number here
    counter = 0
    earlystopper = 0
    best_loss = None
    all_epoch = learner.module.epochs

    for epoch in range(all_epoch):

        lr = optimizer.param_groups[0]["lr"]
        train_acc, train_loss = train(device, optimizer, learner, train_data, loss_func)

        learner.eval() # switch to test mode (make model not save the record of calculation)

        with torch.no_grad():
            test_acc, test_loss = test(device, optimizer, learner, test_data, loss_func)

            ### early stopping ###
            if best_loss is None:
                best_loss = test_loss
            elif test_loss > best_loss:
                counter += 1
                print("EarlyStopping: %i / %i" % (counter, patience))
                if counter >= patience:
                    print("EarlyStopping: Stop training")
                    earlystopper = 1
            else:
                best_loss = test_loss
                counter = 0
            ######################

        time_now = str(datetime.datetime.today())
        rsl.append({k: v for k, v in zip(rsl_keys, [lr, epoch + 1, train_acc, train_loss, test_acc, test_loss, time_now])})

        draw_graph.draw_graph(all_epoch, epoch, train_acc, train_loss, test_acc, test_loss, os.path.basename(dataset4class.dataset_folder), save_place=dataset4class.dataset_directory  + dataset4class.dataset_folder[1:] + '/log/{0:%m%d}_{0:%H%M}/'.format(now, now))

        hp_for_record= get_arguments()
        otherparams = []
        otherparams.append(hp_for_record["batch_size"])
        otherparams.append(hp_for_record["lr"])
        otherparams.append(hp_for_record["momentum"])
        otherparams.append(hp_for_record["weight_decay"])
        otherparams.append(hp_for_record["width_coef1"])
        otherparams.append(hp_for_record["width_coef2"])
        otherparams.append(hp_for_record["width_coef3"])
        otherparams.append(hp_for_record["n_blocks1"])
        otherparams.append(hp_for_record["n_blocks2"])
        otherparams.append(hp_for_record["n_blocks3"])
        otherparams.append(hp_for_record["drop_rates1"])
        otherparams.append(hp_for_record["drop_rates2"])
        otherparams.append(hp_for_record["drop_rates3"])
        otherparams.append(hp_for_record["lr_decay"])
        otherparams.append(time_now)

        save_place = dataset4class.dataset_directory  + dataset4class.dataset_folder[1:] + '/log/{0:%m%d}_{0:%H%M}/'.format(now, now)
        #write_gspread.update_gspread(dataset4class.dataset_folder, 'WRN', dataset4class.dataset_directory, now, train_acc, train_loss, test_acc, test_loss, epoch+1, learner.module.epochs, False, save_place, otherparams)

        print_result(rsl[-1].values())
        scheduler.step()

        torch.save(learner.module.state_dict(), dataset4class.dataset_directory + dataset4class.dataset_folder[1:] + '/log/{0:%m%d}_{0:%H%M}/'.format(now, now) + os.path.basename(dataset4class.dataset_folder) + '_{0:%m%d}_{0:%H%M}.pth'.format(now, now))

        if earlystopper == 1:
            #write_gspread.update_gspread(dataset4class.dataset_folder, 'WRN', dataset4class.dataset_directory, now, train_acc, train_loss, test_acc, test_loss, epoch+1, learner.module.epochs, True, save_place, otherparams)
            break

if __name__ == "__main__":
    hp_dict = get_arguments()
    hp_tuple = namedtuple("_hyperparameters", (var_name for var_name in hp_dict.keys() ) )
    hyperparameters = hp_tuple(**hp_dict)
    learner = WideResNet(hyperparameters)

    print("Start Training")
    print("")
    main(learner)


from __future__ import absolute_import

import io
import ssl
import errno
import socket
import struct
import threading

from .. import logs
from .. import utils
from .. import errors
from .. import param

from . import Connection, Transport

BACKLOG = socket.SOMAXCONN
CHUNK_SIZE = io.DEFAULT_BUFFER_SIZE
ACCEPT_TIMEOUT = 0.1

log = logs.get(__name__)

class TcpConnection(Connection):
    log = log

    def __init__(self, interface, sock, url, chunk_size=None):
        super(TcpConnection, self).__init__(interface, url)
        self._sock = sock
        self._chunk_size = chunk_size
        self.log.debug('connected: %s', self.url)

    def recv(self):
        return recv(self._sock, self._chunk_size)

    def send(self, data):
        try:
            return send(self._sock, data)
        except socket.error as e:
            raise errors.TransportError(e)

    def close(self):
        close(self._sock)
        self.log.debug('disconnected: %s', self.url)

class TcpTransport(Transport):
    _name_ = 'tcp'
    log = log
    Connection = TcpConnection

    @param('backlog', int, default=BACKLOG)
    @param('chunk_size', int, default=CHUNK_SIZE)
    @param('ssl_key', doc='server-side only')
    def __init__(self, url, timeout=None, backlog=None, chunk_size=None,
            ssl_cert=None, ssl_key=None):
        super(TcpTransport, self).__init__(url)
        url = self._url

        self._addr = (url.host, url.port)
        self._sock = None

        self.timeout = timeout
        self.backlog = backlog or BACKLOG
        self.chunk_size = chunk_size

        self._ssl_cert = ssl_cert
        self._ssl_key = ssl_key

        self._stop = threading.Event()
        self._stopped = threading.Event()

    def connect(self, client):
        s = socket.create_connection(self._addr, self.timeout)

        if self._ssl_cert:
            ctx = ssl.create_default_context()
            ctx.load_verify_locations(self._ssl_cert)
            hostname = socket.gethostbyaddr(self._addr[0])[0]
            s = ctx.wrap_socket(s, server_hostname=hostname)

        return self.Connection(client, s, self._url.netloc, self.chunk_size)

    def bind(self):
        self._sock = s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.settimeout(ACCEPT_TIMEOUT)
        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        s.bind(self._addr)
        s.listen(self.backlog)

    def serve(self, server):
        self.bind()
        self.log.info('listening: %s', self.url)

        s = self._sock

        ctx = None
        if self._ssl_cert:
            ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
            ctx.load_cert_chain(certfile=self._ssl_cert, keyfile=self._ssl_key)

        stop = self._stop
        stopped = self._stopped
        stop.clear()
        stopped.clear()

        try:
            while not stop.is_set():
                try:
                    sock, addr = s.accept()
                except socket.timeout:
                    continue
                sock.settimeout(self.timeout)

                try:
                    if ctx:
                        sock = ctx.wrap_socket(sock, server_side=True)
                except ssl.SSLError:
                    log.exception('ssl error')
                    continue

                utils.start_thread(self.handle, server, sock, addr)
        finally:
            stopped.set()

    def handle(self, server, sock, addr):
        addr = utils.url.format_addr(addr)
        with self.Connection(server, sock, addr, self.chunk_size) as con:
            server.handle(con)

    def stop(self):
        self._stop.set()

    def join(self, timeout=None):
        self._stopped.wait(timeout)

##
## sockio
##

def close(sock):
    try:
        sock.shutdown(socket.SHUT_RDWR)
    except (OSError, socket.error) as e:
        # ignore if not connected
        if e.errno not in (errno.ENOTCONN,):# errno.EBADF):
            raise
    sock.close()

def recv(sock, chunk_size=None):
    """Receives a block of data from the socket."""
    try:
        return b''.join(recviter(sock, chunk_size))
    except errors.ReceiveInterrupted:
        return b''

def recviter(sock, chunk_size=None):
    """Returns an iterator over the data received from the socket.

    Each chunk of data yielded will be equal to or less than
    *chunk_size*.

    Can raise a `ReceiveInterrupted` exception.
    """
    buf = b''.join(recvsize(sock, 4, chunk_size))
    data_len = struct.unpack('>I', buf)[0]
    for chunk in recvsize(sock, data_len, chunk_size):
        yield chunk

def recvsize(sock, size, chunk_size=None):
    """Returns an iterator over the chunks received from the socket.

    If the socket stops receiving data before *size* bytes have been read, a
    `ReceiveInterrupted` exception will be raised.
    """
    pos = 0
    chunk_size = min(size, chunk_size or CHUNK_SIZE)
    while pos < size:
        chunk = sock.recv(min(size-pos, chunk_size))
        if not chunk:
            raise errors.ReceiveInterrupted()
        pos += len(chunk)
        yield chunk

def send(sock, data):
    """Sends *data* over the socket."""
    data_len = len(data)
    size = struct.pack('>I', data_len)
    sock.sendall(size)
    sock.sendall(data)


"""
This script tests what accuracy can be gained on the credit approval dataset
"""

from sklearn.pipeline import make_pipeline, make_union, Pipeline
from sklearn.preprocessing import StandardScaler, Imputer, RobustScaler
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.svm import LinearSVC
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.dummy import DummyClassifier

from noxer.preprocessing import ColumnSelector, OneHotEncoder

from searchgrid import set_grid, build_param_grid

import pandas as ps
import numpy as np

from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.preprocessing import LabelBinarizer

class OneHotEncoder(BaseEstimator, TransformerMixin):
    """Wrapper around LabelBinarizer. Assumes that input X to fit and transform is a single
    column matrix of categorical values."""
    def fit(self, X, y=None):
        # create label encoder
        M = X[:, 0].astype('str')
        self.encoder = LabelBinarizer()
        self.encoder.fit(M)
        return self

    def transform(self, X, y=None):
        M = X[:, 0].astype('str')
        return self.encoder.transform(M)

data = ps.read_csv('titanic.csv')
data = data.replace('?', 'NaN')
data = data.as_matrix()

X = data[:, :-1]
y = data[:, -1].astype('int')

X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)

category = lambda idx: make_pipeline(ColumnSelector(idx), OneHotEncoder())
number = lambda idx: make_pipeline(ColumnSelector(idx), Imputer())

features = make_union(
    number(1), # Pclass
    category(3), # Gender
    number(4), # Age
    number(5), # SibSp
    category(6), # Parch
    number(8), # Fare
    category(10), # Embarked
)

estimator = Pipeline([
    ('features', features),
    ('scaler', StandardScaler()),
    ('model', LinearSVC())
])

models = [
        set_grid(
            DecisionTreeClassifier(),
            min_samples_split=[2 ** -i for i in range(1, 8)],
            max_depth=list(range(1, 21))
        ),

        set_grid(
            LinearSVC(),
            C=np.logspace(-6, 6, num=20)
        ),
]

estimator = set_grid(
    estimator,
    model=[

        set_grid(
            GradientBoostingClassifier(),
            n_estimators=[2 ** i for i in range(1, 11)],
            learning_rate=np.logspace(-4, 0, num=10)
        ),

    ],
    scaler=[
        StandardScaler()
    ]
)

param_grid = build_param_grid(estimator)

gsearch = GridSearchCV(
    estimator=estimator,
    param_grid=param_grid,
    verbose=1,
    cv=3,
    n_jobs=-1
)

model = gsearch
dummy = DummyClassifier(strategy='most_frequent')

model.fit(X_train, y_train)
dummy.fit(np.zeros((len(y_train), 1)), y_train)
print(model.best_params_)
print(model.score(X_test, y_test))
print(dummy.score(np.zeros((len(y_test), 1)), y_test))


from django.urls import path

from . import views

urlpatterns = [
    path("", views.welcome_page, name="welcome_page"),
    path("index", views.index, name="index"),
    path("login", views.login_view, name="login"),
    path("logout", views.logout_view, name="logout"),
    path("generar_pedido", views.generar_pedido_view, name="generar_pedido"),
    path("generar_turno", views.generar_turno_view, name="generar_turno"),
    path("editar_paciente", views.editar_paciente_view, name="editar_paciente"),
    path("turnos_med", views.turnos_med_view, name="turnos_med"),
    path("agregar_paciente", views.agregar_paciente_view, name="agregar_paciente"),
    path("turnos_hoy", views.turnos_hoy_view, name="turnos_hoy"),
    path("editar_turnos", views.editar_turnos_view, name="editar_turnos"),
    path("diagnosticar", views.diagnosticar_view, name="diagnosticar"),
    path("pacientes_med", views.pacientes_med_view, name="pacientes_med"),
    path("ver_pedidos", views.ver_pedidos_view, name="ver_pedidos"),
    path("reporte_1", views.reporte_1_view, name="reporte_1"),
    path("reporte_2", views.reporte_2_view, name="reporte_2"),
    path("reporte_3", views.reporte_3_view, name="reporte_3"),
    path("reporte_4", views.reporte_4_view, name="reporte_4"),
    path("taller", views.taller_view, name="taller"),
    path("ver_productos", views.ver_productos_view, name="ver_productos"),
    path("ver_pacientes", views.ver_pacientes_view, name="ver_pacientes"),
    path("nuevo_producto", views.nuevo_producto_view, name="nuevo_producto")
]


import evaluation_noNoise as ev
import matplotlib.pyplot as plt
import numpy as np
import os
import math
import sys
import event_view_plot as plt_3d
#from importlib import reload
#import mplhep

def event_count(event,clustering,count):
    true_cluster_num = np.unique(event.y).size
    predicted_cluster_num = np.unique(clustering).size
    count[0].append(true_cluster_num)
    count[1].append(predicted_cluster_num)
    return count

def count_hit_inCluster(event,clustering,acc):
    count = check_prediction(event,clustering)
    #assert count in range(len(count))
    #assert sorted(count[0].values(),reverse=True) in range(len(sorted(count[0].values(),reverse=True)))

    if len(count[0]) == 1 or len(count[1]) == 1 : return acc
    accuracy_1stcluster = sorted(count[1].values(),reverse=True)[0] / sorted(count[0].values(),reverse=True)[0]
    if len(sorted(count[1].values(),reverse=True))!=1 and len(sorted(count[0].values(),reverse=True)) !=1:
        accuracy_2ndcluster = sorted(count[1].values(),reverse=True)[1] / sorted(count[0].values(),reverse=True)[1]
    else : plt_3d.event_display(event,clustering)
    acc[0].append(accuracy_1stcluster)
    #acc[1].append(accuracy_2ndcluster)
    return acc

def check_prediction(event,clustering):
    count=[{},{}]
    type_hit_true = np.unique(event.y)
    type_hit_predicted = np.unique(clustering)
    for type_hit in type_hit_true:
        count[0][type_hit]=np.count_nonzero(event.y == type_hit) #count the number of hits of true label
    for type_hit in type_hit_predicted:
        count[1][type_hit]=np.count_nonzero(clustering == type_hit) #predicted label
    return count

def ilclabel(ax, text=r'$\itSimulation \; Preliminary$', x=.12, y=.88, dx=.12):
    ax.text(x, y, r'$\bfILD$', ha='left', va='bottom',transform=ax.figure.transFigure,fontsize=36)
    ax.text(x+dx, y, text,ha='left', va='bottom',transform=ax.figure.transFigure,fontsize=28)

def plot_stats(stats_numpy,hist_min,hist_max,key_name,num):
    fig = plt.figure(facecolor="white",figsize=[9,5])
    ax = fig.gca()
    #ax.hist(stats['eta_truth'])
    bins = np.linspace(hist_min, hist_max, 50)
    #epred_o_etruth = stats['n_pred_id'][sel] / stats['n_truth_id'][sel]
    ax.hist(stats_numpy, bins=bins, linewidth=2)
    average_hist = np.round(np.nanmean(stats_numpy),decimals=5)
    #average_hist = np.mean(stats_numpy)
    ax.text(0.85,0.85,f"average : {average_hist}")
    plt.show()
    #fig.savefig(f"GravPNG/{key_name}_{hist_min}_{hist_max}_{num}.png")

def plot_n_pred_in_truth(stats):
    fig = plt.figure(facecolor="white",figsize=[9,5])
    ax = fig.gca()
    #ax.hist(stats['eta_truth'])
    cat_name = {0: 'EM', 1: 'HAD', 2: 'MIP', 3: 'MIX'}

    ax.set_xlabel(r'$\Sigma E_{hit}^{pred} / \Sigma E_{hit}^{truth}$')
    ax.set_xlabel(r'$N_{hits}^{pred} / N_{hits}^{truth}$ (energy weighted)')
    ax.set_ylabel('A.U.')
    bins = np.linspace(0., 1.2, 50)

    #epred_o_etruth = stats['n_pred_id'][sel] / stats['n_truth_id'][sel]
    epred_o_etruth = stats['rate_pred']
    ax.hist(epred_o_etruth, bins=bins, linewidth=2)
    plt.show()

def scan_stats_plt_eachEvent(tbeta=.2,td=.5,nmax=4,yielder=None):
    stats = ev.Stats()
    if yielder is None :yielder = ev.TestYielder()
    count_cluster_inEvent=[[],[]]
    rate_accuracy = [[],[]]
    energy_rate_accuracy=[[],[]]
    for event,prediction,clustering,matches in yielder.iter_matches(tbeta,td,nmax):
        if(len(event.y)<10):continue
        check = ev.get_hit_matched_vs_unmatched_noStat_eachEvent(event,clustering,matches)
        if check == "y" : break
        elif check == "z" : continue

def scan_stats_plt(tbeta=.2,td=.5,nmax=4,yielder=None):
    stats = ev.Stats()
    if yielder is None :yielder = ev.TestYielder()
    count_cluster_inEvent=[[],[]]
    rate_accuracy = [[],[]]
    energy_rate_accuracy=[[],[]]
    for event,prediction,clustering,matches in yielder.iter_matches(tbeta,td,nmax):
        if(len(event.y)<10):continue
        check = ev.get_hit_matched_vs_unmatched_noStat(event,clustering,matches)
        if check == "y" : break
        elif check == "z" : continue

def scan_stats(tbeta=.2,td=.5,nmax=4,yielder=None):
    stats = ev.Stats()
    if yielder is None :yielder = ev.TestYielder()
    count_cluster_inEvent=[[],[]]
    rate_accuracy = [[],[]]
    energy_rate_accuracy=[[],[]]
    check = True
    for event,prediction,clustering,matches in yielder.iter_matches(tbeta,td,nmax):
        if(len(event.y)<10):continue
        #count_cluster_inEvent=event_count(event,clustering,count_cluster_inEvent)
        #rate_accuracy=count_hit_inCluster(event,clustering,rate_accuracy)
        #energy_rate_accuracy,rate_1st,rate_2nd=cal_energy_rate(event,clustering,energy_rate_accuracy)
        #if rate_1st <0.1:
        #    plt_3d.event_display(event,clustering)

def get_stats_test(tbeta=.2, td=.5, nmax=4, yielder=None):
    stats = ev.Stats()
    yielder = ev.TestYielder()
    for event, prediction, clustering,matches in yielder.iter_matches(tbeta,td,nmax):
        print(f"event : {type(event.y)}")
        print(f"clustering : {type(clustering)}")
        print(f"matches : {type(matches[0][0])}")

def get_stats(tbeta=.2, td=.5, nmax=4, yielder=None,accType="number"):
    stats = ev.Stats()
    if yielder is None:yielder = ev.TestYielder()
    count_cluster_inEvent = [[],[]]
    rate_accuracy = [[],[]]
    energy_rate_accuracy = [[],[]]
    check = True
    for event, prediction, clustering,matches in yielder.iter_matches(tbeta,td,nmax):
        #count_cluster_inEvent=event_count(event,clustering,count_cluster_inEvent)
        #rate_accuracy=count_hit_inCluster(event,clustering,rate_accuracy)
        #energy_rate_accuracy,rate_1st,rate_2nd = cal_energy_rate(event,clustering,energy_rate_accuracy)
        #if check == True :
        #    print(f"matches : {matches}")
        #    if len(matches[1]) > 2:check = False
        #stats.extend(ev.statistics_per_match(event,clustering,matches))
        if accType == "number":
            stats.extend(ev.get_hit_matched_vs_unmatched(event,clustering,matches))
        elif accType =="energy":
            stats.extend(ev.get_hit_matched_vs_unmatched_energy(event,clustering,matches))
        #stats.extend(ev.show_Stats(event,clustering,matches))
        #stats.add('confmat',ev.signal_to_noise_confusion_matrix(event,clustering,norm=True))
    return stats

def show_accNumber():
    stats=get_stats(nmax=20000,accType="number")
    plot_stats(stats['num_pred'],0,1,"num_pred",1)
    plot_stats(stats['rate_pred'],0,1,"rate_pred",1)

def show_accEnergy():
    stats=get_stats(nmax=20000,accType="energy")
    plot_stats(stats['num_pred_energy'],0,1,"num_pred_energy",1)
    plot_stats(stats['rate_pred_energy'],0.99,1,"rate_pred_energy",1)

def check_match():
    path="/home/tsumura/ILC/GravNet/pytorch_ILC_PFA/check_Grav/grav_check2.txt"
    if os.path.isfile(path) :
        with open(path,'w') as f:
            f.write('')
    stats = get_stats(nmax=20000)
    check = True
    for matched_pred,unmatched_pred in zip(stats['matched_pred_size'],stats['unmatched_pred_size']):
        for matched_pred in stats['matched_pred']:
            if check:
                print(f"matched pred :{matched_pred}")
                print(f"matched pred size :{matched_pred}")
                print(f"unmatched pred size :{unmatched_pred}")
                check = False
    print(f"hit unmatch pred : {stats['nhits_unmatched_pred'][0]}")
    ev.dump_stats('stats_%b%d',stats)
    nhit_energy_acc(stats)
    plot_stats(stats['rate_pred'],0.9,1.,"rate_pred",sys.argv[1])
    plot_stats(stats['rate_pred'],0.,1.,"rate_pred",sys.argv[1])
    plot_n_pred_in_truth(stats)
    nhit_truth(stats)

def check_scan_stats():
    stats=scan_stats(nmax=5)
    nhit_acc(stats)
    plot_stats(stats['sum_truth'])
    plot_n_pred_in_truth(stats)

def main():
    show_accNumber()
    #show_accEnergy()

if __name__=='__main__':
    main()


from fastaReader import readFastaMul
from timer import Timer
import character as ch
import os, shutil
from pathlib import Path

def trimDecision(path_fasta_file, exclusion_fraction, trim_fraction_tolerated, list_AA):
    """
    Management of gaps and ambiguous amino-acids:
    - exclusion_fraction: if the fraction of residus at a position in the sequence is > exclusion_fraction
                            --> remove this position from each sequence of the seed
    - trim_fraction_tolarated: if the fraction of positions removed is > trim_fraction_tolerated
                            --> remove the seed from de dataset
    """
    list_seq = readFastaMul(path_fasta_file)
    nbre_seq = len(list_seq)
    len_seq = len(list_seq[0][1])
    count_excluded = [0]*len_seq

    for _, seq in list_seq:
        for index_aa in range(0, len_seq):
            if seq[index_aa] not in list_AA:
                count_excluded[index_aa] += 1
    fraction_excluded = [count/nbre_seq for count in count_excluded]

    to_exclude = []
    for fraction in fraction_excluded:
        if fraction > exclusion_fraction:
            to_exclude.append(1)
        else:
            to_exclude.append(0)

    seed_trimed_kept = sum(to_exclude)/len_seq < trim_fraction_tolerated

    return seed_trimed_kept, to_exclude

def trimApplication(path_fasta_file, path_fasta_file_trimmed, seed_trimed_kept, to_exclude, count_seed_excluded):

    if seed_trimed_kept == False:
        count_seed_excluded += 1

    else:
        input_handle = open(path_fasta_file)
        output_handle = open(path_fasta_file_trimmed, "w")
        list_seq = readFastaMul(path_fasta_file)
        len_seq = len(list_seq[0][1])

        for l in input_handle:
            if l[0] == ">":
                output_handle.write(l)
                aa_counter = 0
            else:
                for elem in l[0:-1]:
                    if to_exclude[aa_counter] == 0:
                        output_handle.write(elem)
                    aa_counter += 1
                if aa_counter == len_seq:      # if the sequence is finished, go back to the line
                    output_handle.write(l[-1])
        output_handle.close()
        input_handle.close()
    return count_seed_excluded

def trimSave(path_folder_fasta, path_folder_fasta_trimmed, list_AA, exclusion_fraction, trim_fraction_tolerated):
    t = Timer()
    t.start()

    if os.path.isdir(path_folder_fasta_trimmed):
        shutil.rmtree(path_folder_fasta_trimmed)
    os.mkdir(path_folder_fasta_trimmed)

    path_folder_fasta = Path(path_folder_fasta)
    path_fasta_files = path_folder_fasta.iterdir()

    count_seed_excluded = 0

    for path_fasta_file in path_fasta_files:
        seed_trimed_kept, to_exclude = trimDecision(path_fasta_file, exclusion_fraction, trim_fraction_tolerated, list_AA)
        num_accession = os.path.basename(path_fasta_file).split(".")[0] + '.' + os.path.basename(path_fasta_file).split(".")[1]
        name_fasta_file_trimmed = num_accession + '.trim'
        path_fasta_file_trimmed = path_folder_fasta_trimmed + "/"+ name_fasta_file_trimmed
        count_seed_excluded = trimApplication(path_fasta_file, path_fasta_file_trimmed, seed_trimed_kept, to_exclude, count_seed_excluded)
    print("count_seed_excluded:", count_seed_excluded)

    t.stop("Compute and save the trimmed files")

if __name__ == '__main__':
    #path_folder_fasta = "/Users/pauline/Desktop/data/Pfam_fasta"
    #path_folder_fasta_trimmed = "/Users/pauline/Desktop/data/Pfam_fasta_trimmed"
    #list_AA = ch.characterList()
    #exclusion_fraction = 0.5
    #trim_fraction_tolerated = 0.5
    #trimSave(path_folder_fasta, path_folder_fasta_trimmed, list_AA, exclusion_fraction, trim_fraction_tolerated) # 207.76572 s

    # mini test
    path_folder_fasta = "Pfam_test"
    path_folder_fasta_trimmed = "Pfam_test_trimmed"
    list_AA = ch.characterList()
    exclusion_fraction = 0.5
    trim_fraction_tolerated = 0.5
    trimSave(path_folder_fasta, path_folder_fasta_trimmed, list_AA, exclusion_fraction, trim_fraction_tolerated) # 207.76572 s


import pandas as pd
import plotly.express as px
import plotly
import plotly.graph_objects as go
from plotly.subplots import make_subplots

def annotate_line(fig, title, date):
    fig.update_layout(
        title='%s for %s' % (title, date),
        xaxis_title="longitude",
        xaxis_showgrid=False,
        xaxis_range=[100.533, 100.594],
        yaxis_title="speed km/h",
        yaxis_range=[0, 90],
        yaxis_showgrid=False,
        xaxis_nticks=35,
        yaxis_nticks=35,
        autosize=True,
        # height = 600,
        # width = 1400,
        shapes=[
            dict(
                type='line',
                yref='paper', y0=0, y1=1,
                xref='x', x0=100.536, x1=100.536,
            ),
            dict(
                type='line',
                yref='paper', y0=0, y1=1,
                xref='x', x0=100.545, x1=100.545,
            ),
            dict(
                type='line',
                yref='paper', y0=0, y1=1,
                xref='x', x0=100.552, x1=100.552,
            ),
            dict(
                type='line',
                yref='paper', y0=0, y1=1,
                xref='x', x0=100.558, x1=100.558,
            ),
            dict(
                type='line',
                yref='paper', y0=0, y1=1,
                xref='x', x0=100.567, x1=100.567,
            ),
            dict(
                type='line',
                yref='paper', y0=0, y1=1,
                xref='x', x0=100.584, x1=100.584,
            ),
            dict(
                type='line',
                yref='paper', y0=0, y1=1,
                xref='x', x0=100.592, x1=100.592,
            ),
            # speed
            dict(
                layer="below",
                x0=100.533,
                x1=100.594,
                y0=0,
                y1=21,
                type='rect',
                fillcolor='#EF9A9A',
                line_width=0
            ),
            dict(
                layer="below",
                x0=100.533,
                x1=100.594,
                y0=21,
                y1=26,
                type='rect',
                fillcolor='#FFC300',
                line_width=0
            ),
            dict(
                layer="below",
                x0=100.533,
                x1=100.594,
                y0=26,
                y1=46,
                type='rect',
                fillcolor='#FFFF8D',
                line_width=0
            ),
            dict(
                layer="below",
                x0=100.533,
                x1=100.594,
                y0=46,
                y1=90,
                type='rect',
                fillcolor='#CCFF90',
                line_width=0
            ),

        ])

    fig.add_annotation(
        x=100.536,
        y=70,
        text="แยกศาลาแดง",
        showarrow=False,
        font=dict(
            family='Courier New, monospace',
            size=18,
        )
    )

    fig.add_annotation(
        x=100.545,
        y=70,
        text="แยกวิทยุ",
        showarrow=False,
        font=dict(
            family='Courier New, monospace',
            size=18,
        )
    )

    fig.add_annotation(
        x=100.551,
        y=70,
        text="แยกใต้ทางด่วน",
        showarrow=False,
        font=dict(
            family='Courier New, monospace',
            size=18,
        )
    )

    fig.add_annotation(
        x=100.558,
        y=70,
        text="แยกพระราม4",
        showarrow=False,
        font=dict(
            family='Courier New, monospace',
            size=18,
        )
    )

    fig.add_annotation(
        x=100.567,
        y=70,
        text="แยกเกษมราษฎร์",
        showarrow=False,
        font=dict(
            family='Courier New, monospace',
            size=18,
        )
    )

    fig.add_annotation(
        x=100.584,
        y=70,
        text="แยกกล้วยน้ำไท",
        showarrow=False,
        font=dict(
            family='Courier New, monospace',
            size=18,
        )
    )

    fig.add_annotation(
        x=100.592,
        y=70,
        text="แยกพระขโนง",
        showarrow=False,
        font=dict(
            family='Courier New, monospace',
            size=18,
        )
    )

def plot_line_csv(df, date):
    fig = px.line(df, x="lon3", y="speed_km/h", title='ค.เร็วเฉลี่ย(km/h) ทุกๆ100เมตร ',color = "direction",animation_frame="round_min")
    annotate_line(fig, 'Speed line', date)
    fig.layout.updatemenus[0].buttons[0].args[1]["frame"]["duration"] = 1500
    return fig

def plot_line(df, date):
    fig = px.line(df, x="lon3", y="speed_kph", title='ค.เร็วเฉลี่ย(km/h) ทุกๆ100เมตร ',color = "direction")
    annotate_line(fig, 'Speed line', date)
    return fig

def plot_line_pred(df, date):
    fig = px.line(df, x="lon3", y="speed_kph", title='ค.เร็วเฉลี่ย(km/h) ทุกๆ100เมตร ',color = "direction",animation_frame="time")
    annotate_line(fig, 'Speed line prediction', date)
    fig.layout.updatemenus[0].buttons[0].args[1]["frame"]["duration"] = 1500
    return fig


# -*- coding: utf-8 -*-

u"""
.. module:: models
"""

import logging
import os
# pylint: disable=unused-import
import uuid

from django.conf import settings
from django.contrib.auth.models import User
from django.db import models
from django.db.models import F
from django.utils import timezone

logger = logging.getLogger('volontulo.models')

class Organization(models.Model):
    u"""Model that handles ogranizations/institutions."""
    name = models.CharField(max_length=150)
    address = models.CharField(max_length=150)
    description = models.TextField()

    def __str__(self):
        u"""Organization model string reprezentation."""
        return self.name

class OffersManager(models.Manager):
    u"""Offers Manager."""

    def get_active(self):
        u"""Return active offers."""
        return self.filter(
            offer_status='published',
            action_status__in=('ongoing', 'future'),
            recruitment_status__in=('open', 'supplemental'),
        ).all()

    def get_for_administrator(self):
        u"""Return all offers for administrator to allow management."""
        return self.filter(offer_status='unpublished').all()

    def get_weightened(self, count=10):
        u"""Return all published offers ordered by weight.

        :param count: Integer
        :return:
        """
        return self.filter(
            offer_status='published').order_by('weight')[:count]

    def get_archived(self):
        u"""Return archived offers."""
        return self.filter(
            offer_status='published',
            action_status__in=('ongoing', 'finished'),
            recruitment_status='closed',
        ).all()

class Offer(models.Model):
    u"""Offer model."""

    OFFER_STATUSES = (
        ('unpublished', u'Unpublished'),
        ('published', u'Published'),
        ('rejected', u'Rejected'),
    )
    RECRUITMENT_STATUSES = (
        ('open', u'Open'),
        ('supplemental', u'Supplemental'),
        ('closed', u'Closed'),
    )
    ACTION_STATUSES = (
        ('future', u'Future'),
        ('ongoing', u'Ongoing'),
        ('finished', u'Finished'),
    )

    objects = OffersManager()
    organization = models.ForeignKey(Organization)
    volunteers = models.ManyToManyField(User)
    description = models.TextField()
    requirements = models.TextField(blank=True, default='')
    time_commitment = models.TextField()
    benefits = models.TextField()
    location = models.CharField(max_length=150)
    title = models.CharField(max_length=150)
    started_at = models.DateTimeField(blank=True, null=True)
    finished_at = models.DateTimeField(blank=True, null=True)
    time_period = models.CharField(max_length=150, default='', blank=True)
    status_old = models.CharField(
        max_length=30,
        default='NEW',
        null=True,
        unique=False
    )
    offer_status = models.CharField(
        max_length=16,
        choices=OFFER_STATUSES,
        default='unpublished',
    )
    recruitment_status = models.CharField(
        max_length=16,
        choices=RECRUITMENT_STATUSES,
        default='open',
    )
    action_status = models.CharField(
        max_length=16,
        choices=ACTION_STATUSES,
        default='ongoing',
    )
    votes = models.BooleanField(default=0)
    recruitment_start_date = models.DateTimeField(blank=True, null=True)
    recruitment_end_date = models.DateTimeField(blank=True, null=True)
    reserve_recruitment = models.BooleanField(blank=True, default=True)
    reserve_recruitment_start_date = models.DateTimeField(
        blank=True,
        null=True
    )
    reserve_recruitment_end_date = models.DateTimeField(
        blank=True,
        null=True
    )
    action_ongoing = models.BooleanField(default=False, blank=True)
    constant_coop = models.BooleanField(default=False, blank=True)
    action_start_date = models.DateTimeField(blank=True, null=True)
    action_end_date = models.DateTimeField(blank=True, null=True)
    volunteers_limit = models.IntegerField(default=0, null=True, blank=True)
    weight = models.IntegerField(default=0, null=True, blank=True)

    def __str__(self):
        u"""Offer string representation."""
        return self.title

    def set_main_image(self, is_main):
        u"""Set main image flag unsetting other offers images.

        :param is_main: Boolean flag resetting offer main image
        """
        if is_main:
            OfferImage.objects.filter(offer=self).update(is_main=False)
            return True
        return False

    def save_offer_image(self, gallery, userprofile, is_main=False):
        u"""Handle image upload for user profile page.

        :param gallery: UserProfile model instance
        :param userprofile: UserProfile model instance
        :param is_main: Boolean main image flag
        """
        gallery.offer = self
        gallery.userprofile = userprofile
        gallery.is_main = self.set_main_image(is_main)
        gallery.save()
        return self

    def create_new(self):
        u"""Set status while creating new offer."""
        self.offer_status = 'unpublished'
        self.recruitment_status = 'open'

        if self.started_at or self.finished_at:
            self.action_status = self.determine_action_status()

    def determine_action_status(self):
        u"""Determine action status by offer dates."""
        if (
                (
                    self.finished_at and
                    self.started_at < timezone.now() < self.finished_at
                ) or
                (
                    self.started_at < timezone.now() and
                    not self.finished_at
                )
        ):
            return 'ongoing'
        elif self.started_at > timezone.now():
            return 'future'
        else:
            return 'finished'

    def change_status(self, status):
        u"""Change offer status.

        :param status: string Offer status
        """
        if status in ('published', 'rejected', 'unpublished'):
            self.offer_status = status
            self.save()
        return self

    def unpublish(self):
        u"""Unpublish offer."""
        self.offer_status = 'unpublished'
        self.save()
        return self

    def publish(self):
        u"""Publish offer."""
        self.offer_status = 'published'
        Offer.objects.all().update(weight=F('weight') + 1)
        self.weight = 0
        self.save()
        return self

    def reject(self):
        u"""Reject offer."""
        self.offer_status = 'rejected'
        self.save()
        return self

    def close_offer(self):
        u"""Change offer status to close."""
        self.offer_status = 'unpublished'
        self.action_status = 'finished'
        self.recruitment_status = 'closed'
        self.save()
        return self

class UserProfile(models.Model):
    u"""Model that handles users' profiles."""

    user = models.OneToOneField(User)
    organizations = models.ManyToManyField(
        Organization,
        related_name='userprofiles',
    )
    is_administrator = models.BooleanField(default=False, blank=True)
    phone_no = models.CharField(
        max_length=32,
        blank=True,
        default='',
        null=True
    )
    uuid = models.UUIDField(default=uuid.uuid4, unique=True)

    def is_admin(self):
        u"""Return True if current user is administrator, else return False"""
        return self.is_administrator

    def is_volunteer(self):
        u"""Return True if current user is volunteer, else return False"""
        return not (self.is_administrator and self.organizations)

    def can_edit_offer(self, offer=None, offer_id=None):
        u"""Checks if the user can edit an offer based on its ID"""
        if offer is None:
            offer = Offer.objects.get(id=offer_id)
        return self.is_administrator or self.organizations.filter(
            id=offer.organization_id).exists()

    def get_avatar(self):
        u"""Return avatar for current user."""
        return UserGallery.objects.filter(
            userprofile=self,
            is_avatar=True
        )

    def clean_images(self):
        u"""Clean user images."""
        images = UserGallery.objects.filter(userprofile=self)
        for image in images:
            try:
                os.remove(os.path.join(settings.MEDIA_ROOT, str(image.image)))
            except OSError as ex:
                logger.error(ex)

            image.delete()

    def __str__(self):
        return self.user.email

class UserGallery(models.Model):
    u"""Handling user images."""
    userprofile = models.ForeignKey(UserProfile, related_name='images')
    image = models.ImageField(upload_to='profile/')
    is_avatar = models.BooleanField(default=False)

    def __str__(self):
        u"""String representation of an image."""
        return str(self.image)

class OfferImage(models.Model):
    u"""Handling offer image."""
    userprofile = models.ForeignKey(UserProfile, related_name='offerimages')
    offer = models.ForeignKey(Offer, related_name='images')
    path = models.ImageField(upload_to='offers/')
    is_main = models.BooleanField(default=False)
    created_at = models.DateTimeField(auto_now_add=True)

    def __str__(self):
        u"""String representation of an image."""
        return str(self.path)

class OrganizationGallery(models.Model):
    u"""Handling organizations gallery."""
    organization = models.ForeignKey(Organization, related_name='images')
    published_by = models.ForeignKey(UserProfile, related_name='gallery')
    path = models.ImageField(upload_to='gallery/')
    is_main = models.BooleanField(default=False, blank=True)

    def __str__(self):
        u"""String representation of an image."""
        return str(self.path)

    def remove(self):
        u"""Remove image."""
        self.remove()

    def set_as_main(self, organization):
        u"""Save image as main.

        :param organization: Organization model instance
        """
        OrganizationGallery.objects.filter(organization_id=organization.id)\
            .update(
                is_main=False
            )
        self.is_main = True
        self.save()

    @staticmethod
    def get_organizations_galleries(userprofile):
        u"""Get images grouped by organizations

        :param userprofile: UserProfile model instance
        """
        organizations = Organization.objects.filter(
            userprofiles=userprofile
        ).all()
        return {o.name: o.images.all() for o in organizations}

class Page(models.Model):
    """Static page model."""

    title = models.CharField(max_length=255)
    content = models.TextField()
    author = models.ForeignKey(UserProfile)
    published = models.BooleanField(default=False)
    modified_at = models.DateTimeField(auto_now=True)
    created_at = models.DateTimeField(auto_now_add=True)

    def __str__(self):
        return self.title


import json,subprocess,conf,multiprocessing,metadata
from sys import argv

def launch_pub(host,topic_count_map,sample_count,send_interval,run_id,payload,experiment_type):
  command_string='cd %s && ansible-playbook playbooks/experiment/publisher.yml  --limit %s\
    --extra-vars="topic_count_map=%s sample_count=%s send_interval=%s payload_size=%s zk_connector=%s experiment_type=%s send=1"'%\
    (metadata.ansible,host,str(topic_count_map).replace(" ",""),\
    sample_count,send_interval,payload,metadata.public_zk,experiment_type)
  subprocess.check_call(['bash','-c',command_string])

if __name__=="__main__":
  host_topic_map=json.loads(argv[1])
  sample_count=argv[2]
  send_interval=argv[3]
  run_id=argv[4]
  payload=argv[5]
  experiment_type=argv[6]

  processes=[]
  for host,topic_count_map in host_topic_map.items():
    processes.append(multiprocessing.Process(target=launch_pub,args=(host,topic_count_map,sample_count,send_interval,run_id,payload,experiment_type)))

  for p in processes:
    p.start()
  for p in processes:
    p.join()


import matplotlib.pyplot as plt
import numpy as np
from matplotlib.colors import ListedColormap
from sklearn.datasets import make_circles, make_classification, make_moons
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
from sklearn.ensemble import AdaBoostClassifier, RandomForestClassifier
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import GaussianNB
from sklearn.neighbors import KNeighborsClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier

h = .02
m = 200

names = [
    # "Nearest Neighbors",
    "Linear SVM",
    "RBF SVM",
    # "Gaussian Process",
    # "Decision Tree",
    # "Random Forest",
    "Neural Net",
    # "AdaBoost",
    # "Naive Bayes",
    # "QDA"
]

classifiers = [
    # KNeighborsClassifier(3),
    SVC(kernel="linear", C=0.025),
    SVC(gamma=2, C=1),
    # GaussianProcessClassifier(1.0 * RBF(1.0)),
    # DecisionTreeClassifier(max_depth=5),
    # RandomForestClassifier(max_depth=5, n_estimators=10, max_features=1),
    MLPClassifier(alpha=1, max_iter=1000),
    # AdaBoostClassifier(),
    # GaussianNB(),
    # QuadraticDiscriminantAnalysis()
]

X, y = make_classification(n_samples=m, n_features=2, n_redundant=0, n_informative=2, random_state=1, n_clusters_per_class=1)
rng = np.random.RandomState(2)
X += 2 * rng.uniform(size=X.shape)
linearly_separable = (X, y)

datasets = [
    make_moons(n_samples=m, noise=0.2, random_state=0),
    make_circles(n_samples=m, noise=0.2, factor=0.5, random_state=1),
    linearly_separable
]

figure = plt.figure(figsize=(12, 9))
i = 1

with plt.style.context('Solarize_Light2'):
    for ds_cnt, ds in enumerate(datasets):

        X, y = ds
        X = StandardScaler().fit_transform(X)  # 标准化
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.4, random_state=42)  # 训练集、测试集

        x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5
        y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5
        xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))

        ax = plt.subplot(len(datasets), len(classifiers) + 1, i)
        if ds_cnt == 0:
            ax.set_title("Input data")
        ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, edgecolors='k')
        ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, alpha=0.6, edgecolors='k')
        ax.set_xlim(xx.min(), xx.max())
        ax.set_ylim(yy.min(), yy.max())
        ax.set_xticks(())
        ax.set_yticks(())
        i += 1

        for name, clf in zip(names, classifiers):
            ax = plt.subplot(len(datasets), len(classifiers) + 1, i)

            clf.fit(X_train, y_train)  # 训练模型
            score = clf.score(X_test, y_test)  # 测试

            if hasattr(clf, "decision_function"):
                Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])
            else:
                Z = clf.predict_proba(np.c_[xx.ravel(), yy.ravel()])[:, 1]

            Z = Z.reshape(xx.shape)
            ax.contourf(xx, yy, Z, alpha=.8)

            ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, edgecolors='k')
            ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, edgecolors='k', alpha=0.6)

            ax.set_xlim(xx.min(), xx.max())
            ax.set_ylim(yy.min(), yy.max())
            ax.set_xticks(())
            ax.set_yticks(())
            if ds_cnt == 0:
                ax.set_title(name)
            ax.text(xx.max() - .3, yy.min() + .3, ('%.2f' % score).lstrip('0'), size=15, horizontalalignment='right')
            i += 1

plt.tight_layout()
plt.show()


def send_table_data_in_fragments_to_S3(aws_s3_bucket, aws_s3_folder, aws_s3_subfolder, project, dataset, table, except_columns, csv_fragment_field, csv_week_field, csv_name_field):
    """
    Sends to an Amazon Web Service S3 bucket the information of a BigQuery table in CSV format.
    The CSV files are partitioned by a chosen column of the BigQuery table specified in 'csv_fragment_field'

    Attributes
    ----------
    aws_s3_bucket : string
        Amazon Web Service S3 bucket name to store the CSV files
    aws_s3_folder : string
        Amazon Web Service S3 bucket folder name to store the CSV files
    aws_s3_subfolder : string
        Amazon Web Service S3 bucket subfolder name to store the CSV files
    project : string
        BigQuery datasource project name
    dataset : string
        BigQuery datasource dataset name
    table : string
        BigQuery datasource table name
    except_columns : comma separated strings (Ex: week_file, key, send_file_name)
        BigQuery table columns to exclude in the CSV files
    csv_fragment_field : string
        BigQuery table column chose to partition the CSV files
    csv_week_field : string
        BigQuery table column chose to generate a subfolder with the week name
    csv_name_field : string
        BigQuery table column chose to name the CSV files
    """
    import boto3  # 1.21.25
    import io
    from google.cloud import bigquery
    from airflow.models import Variable

    # Creacion cliente S3
    AWS_S3_CONFIG = Variable.get('aws_s3_credentials', deserialize_json=True)
    AWS_ACCESS_KEY_ID = AWS_S3_CONFIG['aws_access_key_id']
    AWS_SECRET_ACCESS_KEY = AWS_S3_CONFIG['aws_secret_access_key']
    try:
        S3_CLIENT = boto3.client(
            's3',
            aws_access_key_id=AWS_ACCESS_KEY_ID,
            aws_secret_access_key=AWS_SECRET_ACCESS_KEY
        )
    except Exception as err:
        raise Exception(f'FAIL: S3 client error, exception was {str(err)}')

    # Creacion cliente BigQuery
    try:
        BQ_CLIENT = bigquery.Client()
    except Exception as err:
        raise Exception(f'FAIL: BQ client error, exception was {str(err)}')

    # Lectura del campo utilizado para el folder de la semana (ya viene en la query)
    SQL_csv_week = f"""
        SELECT DISTINCT
            {csv_week_field}
        FROM `{project}.{dataset}.{table}`
    """
    csv_week_rows = BQ_CLIENT.query(SQL_csv_week).result()
    for csv_week_row in csv_week_rows:
        csv_week = csv_week_row[0]

    # Lectura de los valores distintos de la columna "fragment_column" (tomados para la iteracion)
    SQL_fragments = f"""
        SELECT DISTINCT
            {csv_fragment_field}
        FROM `{project}.{dataset}.{table}`
    """
    fragments_rows = BQ_CLIENT.query(SQL_fragments).result()

    # Comienzo de iteracion sobre los valores distintos de "fragment_column" y envio a S3
    print('---------- SENDING DATA TO S3 ----------')
    print(
        f'S3 location: {aws_s3_bucket}/{aws_s3_folder}/{aws_s3_subfolder}/{csv_week}')

    for fragments_row in fragments_rows:

        # Lectura del campo utilizado para el nombre del CSV (ya viene en la query y esta hasheado)
        SQL_csv_name = f"""
            SELECT DISTINCT
                {csv_name_field}
            FROM `{project}.{dataset}.{table}`
            WHERE {csv_fragment_field} = '{fragments_row[0]}'
        """
        csv_name_rows = BQ_CLIENT.query(SQL_csv_name).result()
        for csv_name_row in csv_name_rows:
            csv_name = csv_name_row[0]

        # Lectura de fragmento de la tabla
        SQL_table = f"""
            SELECT
                * EXCEPT({except_columns})
            FROM `{project}.{dataset}.{table}`
            WHERE {csv_fragment_field} = '{fragments_row[0]}'
        """
        SQL_table_dataframe = BQ_CLIENT.query(SQL_table).to_dataframe()

        # Envio de la data en formato CSV a un bucket en S3
        with io.StringIO() as csv_buffer:
            SQL_table_dataframe.to_csv(csv_buffer, index=False)
            response = S3_CLIENT.put_object(
                Bucket=aws_s3_bucket,
                Key=f'{aws_s3_folder}/{aws_s3_subfolder}/{csv_week}/{csv_name}',
                Body=csv_buffer.getvalue()
            )
            status = response.get("ResponseMetadata", {}).get("HTTPStatusCode")
            if status == 200:
                print(f"OK IN S3: CSV '{csv_name}'")
            else:
                raise Exception(f"FAIL IN S3: CSV '{csv_name}'")

def send_table_data_in_fragments_to_S3_operator(task_id, aws_s3_bucket, aws_s3_folder, aws_s3_subfolder, project, dataset, table, except_columns, csv_fragment_field, csv_week_field, csv_name_field):
    """
    Returns an Airflow PythonVirtualenvOperator using the function 'send_table_data_in_fragments_to_S3'
    Requires the install of the boto3 v1.21.25 package

    Attributes
    ----------
    task_id : string
        Name of the Airflow task
    aws_s3_bucket : string
        Amazon Web Service S3 bucket name to store the CSV files
    aws_s3_folder : string
        Amazon Web Service S3 bucket folder name to store the CSV files
    aws_s3_subfolder : string
        Amazon Web Service S3 bucket subfolder name to store the CSV files
    project : string
        BigQuery datasource project name
    dataset : string
        BigQuery datasource dataset name
    table : string
        BigQuery datasource table name
    except_columns : comma separated strings (Ex: week_file, key, send_file_name)
        BigQuery table columns to exclude in the CSV files
    csv_fragment_field : string
        BigQuery table column chose to partition the CSV files
    csv_week_field : string
        BigQuery table column chose to generate a subfolder with the week name
    csv_name_field : string
        BigQuery table column chose to name the CSV files
    """
    from airflow.operators.python_operator import PythonVirtualenvOperator
    return PythonVirtualenvOperator(
        task_id=task_id,
        python_callable=send_table_data_in_fragments_to_S3,
        op_kwargs=dict(
            aws_s3_bucket=aws_s3_bucket,
            aws_s3_folder=aws_s3_folder,
            aws_s3_subfolder=aws_s3_subfolder,
            project=project,
            dataset=dataset,
            table=table,
            except_columns=except_columns,
            csv_fragment_field=csv_fragment_field,
            csv_week_field=csv_week_field,
            csv_name_field=csv_name_field
        ),
        requirements=[
            "boto3==1.21.25"
        ]
    )


from selenium import webdriver
from selenium import *
from time import sleep
from selenium.webdriver.common.keys import Keys

from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

options = webdriver.ChromeOptions()
options.add_argument("start-maximized")
options.add_argument("disable-infobars")
options.add_argument("--disable-extensions")
options.add_argument('headless')
driver = webdriver.Chrome(options=options,executable_path="F:\Engg Tools Lab\chromedriver.exe")
driver.get("https://internshala.com/fresher-jobs")

close = driver.find_element_by_id("close_popup")
close.click()

def func(href_val):
    for link in href_val:
        driver.get(link)
        desc = driver.find_element_by_xpath('//div[@class="text-container"]')
        print(desc.text)

def answer():
    href_val=[]
    containers = driver.find_elements_by_xpath('//div[@class="container-fluid individual_internship visibilityTrackerItem "]')

    for items in containers:
        Post_name = items.find_element_by_xpath('.//div[@class="heading_4_5 profile"]')
        print(Post_name.text)

        Company_name = items.find_element_by_xpath('.//div[@class="heading_6 company_name"]')
        print(Company_name.text)

        atag = items.find_element_by_xpath('.//a[@class="view_detail_button"]')
        href_val.append(atag.get_attribute('href'))

    func(href_val)
answer()

driver.quit()


import sys

from sqlalchemy import Column, ForeignKey, Integer, String, Table
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship
from sqlalchemy import create_engine

Base = declarative_base()

class Restaurant(Base):
    __tablename__ = 'restaurant'
    id = Column(Integer, primary_key=True)
    name = Column(String(80), nullable=False)

class MenuItem(Base):
    __table__ = Table('menu_item', Base.metadata,
                      Column('id', Integer, primary_key=True),
                      Column('name', String(80), nullable=False),
                      Column('course', String(250)),
                      Column('description', String(250)),
                      Column('price', String(8)),
                      Column('restaurant_id', Integer, ForeignKey('restaurant.id')))

    restaurant = relationship(Restaurant)

engine = create_engine('sqlite:///restaurantmenu.db')

Base.metadata.create_all(engine)


import cv2
import numpy as np

p = [[255, 0, 0], [0, 255, 0], [0, 0, 255]]

def distance(a, b):
    x1 = (a[0] - b[0])**2
    x2 = (a[1] - b[1])**2
    x3 = (a[2] - b[2])**2

    out = (x1 + x2 + x3)**.5
    return out

def minDistance(pixel):
    p = [[255, 0, 0], [0, 255, 0], [0, 0, 255]]
    dist = []

    for i in range(3):
        dist.append(distance(pixel, p[i]))

    min = 10000
    minNum = -1
    for i in range(len(dist)):
        if dist[i] < min:
            min = dist[i]
            minNum = i

    return minNum

img = cv2.imread('input.png')
output1 = np.zeros((img.shape[0], img.shape[1], 3), dtype=np.uint8)
output2 = img

minDistance(img[50, 25])

for i in range(0, img.shape[0]):
    for j in range(0, img.shape[1]):
        num = minDistance(img[i, j])
        output1[i, j] = p[num]

cv2.imwrite("./output1.png", output1)

print(output2.shape[0])
center0 = output2.shape[0] / 2
center1 = output2.shape[1] / 2
for i in range(0, output2.shape[0]):
    if i >= center0 - 25 and i <= center0 + 25:
        for j in range(0, output2.shape[1]):
            if j >= center1 - 25 and j <= center1 + 25:
                output2[i, j] = [0, 0, 0]

cv2.imwrite("./output2.png", output2)


import sys
sys.path.append('/ldfssz1/ST_BI/USER/qindanhua/projects/st/stereopy/')

from stereo.tools.highly_variable_genes import HighlyVariableGenes
from stereo.algorithm.highly_variable_genes import highly_variable_genes_seurat_v3, highly_variable_genes_single_batch
from stereo.utils.hvg_utils import materialize_as_ndarray, get_mean_var, check_nonnegative_integers
# from stereo.io.reader import read_stereo

#
# path = '/ldfssz1/ST_BI/USER/qindanhua/projects/st/data/FP200000381BL_B2.bin1.Lasso.gem'
# data = read_stereo(path, 'bins', 200)
# df = data.to_df().reindex
# df.to_csv('/ldfssz1/ST_BI/USER/qindanhua/projects/st/data/exp.csv', index=False)
import pickle
# with open('/ldfssz1/ST_BI/USER/qindanhua/projects/st/data/test_data', 'wb') as w:
#     pickle.dump(data, w)

with open('/ldfssz1/ST_BI/USER/qindanhua/projects/st/data/b2_bin200.pk', 'rb') as r:
    data = pickle.load(r)

from anndata import AnnData
adata = AnnData(data.to_df())

def test_highly_variable_genes_basic(data):
    import numpy as np
    import pandas as pd
    hvg = HighlyVariableGenes(data, method='cell_ranger')
    hvg.fit()
    hvg.method = 'seurat_v3'
    hvg.fit()
    hvg.method = 'seurat'
    hvg.fit()

    # add group
    groups = pd.DataFrame({
        'group': np.random.binomial(3, 0.5, size=(len(data.cell_names)))
    }, index=data.cell_names)

    hvg = HighlyVariableGenes(data, groups=groups, method='cell_ranger', n_top_genes=200)
    hvg.fit()
    hvg.method = 'seurat_v3'
    hvg.fit()
    hvg.method = 'seurat'
    hvg.fit()
#
    adata.obs['batch'] = groups['group']
    adata.obs['batch'] = adata.obs['batch'].astype('category')
    import scanpy as sc
    sc.pp.highly_variable_genes(adata, flavor='cell_ranger', n_top_genes=200)

    mr = hvg.result[hvg.result['highly_variable']]
    mr_sc = adata.var[adata.var['highly_variable']]
    len(set(mr.index))
    len(set(mr.index))
    len(set(mr.index) & set(mr_sc.index))

# def test_single_batch():
#     import pandas as pd
#     matrix = pd.read_csv('/ldfssz1/ST_BI/USER/qindanhua/projects/st/data/exp_matrix.csv')
#     data = matrix.values()
#     highly_variable_genes_single_batch(data, method='seurat', n_top_genes=None)


import os
import glob
import json
from datetime import datetime
import openai
from dotenv import load_dotenv

load_dotenv()

openai.api_key = os.environ["OPENAI_API_KEY"]

SINGLE_SUMMARY_PROMPT = "The following is a transcription of a video for web developers called '{title}'. Write a thorough and detailed summary of the video, providing an account of the steps taken and topics covered. Include any names that would be useful for a keyword search: names of tools, specific commands or messages, etc. Use the first person point of view, using 'I' instead of referring to the speaker as 'the host' or 'the speaker'.\n\n---\nTRANSCRIPTION:\n\n{transcription}\n\n(END OF TRANSCRIPTION)"

FULL_SUMMARY_PROMPT = "The following are summaries of different parts of a video named '{title}'. Please create a comprehensive and detailed summary that combines all of the individual summaries provided, without omitting any information. The final summary should include all relevant details from each summary and provide a complete overview of the subject matter. It should include any names that would be useful in for a keyword search: names of tools, specific commands or messages, etc. Write it in first person point of view.\n\nSUMMARIES:\n\n{summary_string}"

if __name__ == '__main__':
    for file in glob.glob("chunks/*.json"):
        with open(file, "r") as f:
            chunks = json.load(f)

        file_name = file.replace("chunks/", "").replace(".json", "")
        output_dir = f"summaries/{file_name}"

        if not os.path.exists(output_dir):
            os.makedirs(output_dir)

        title = chunks[0]["title"]

        print(f"Processing {title}...")

        summaries = []

        for i, chunk in enumerate(chunks):
            single_summary_prompt_formatted = SINGLE_SUMMARY_PROMPT.format(title=title, transcription=chunk["text"])

            completion = openai.ChatCompletion.create(
                model="gpt-4",
                temperature=0.7,
                max_tokens=3000,
                messages= [{ "role": "user", "content": single_summary_prompt_formatted }]
            )

            summary = completion["choices"][0]["message"]["content"]
            summaries.append(summary)

            timestamp_str = datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
            with open(f"{output_dir}/{timestamp_str}_summary_{i}.txt", "w") as file:
                file.write(f"PROMPT:\n\n{single_summary_prompt_formatted}\n\n---\n\nSUMMARY {i}:\n\n{summary}")

            print(f"\tFinished summary {i}")

        summary_string = ""
        for s in summaries:
            summary_string += f"{s}\n\n"

        full_summary_prompt_formatted = FULL_SUMMARY_PROMPT.format(title=title, summary_string=summary_string)

        completion = openai.ChatCompletion.create(
            model="gpt-4",
            temperature=0.7,
            max_tokens=3000,
            messages= [{ "role": "user", "content": full_summary_prompt_formatted }]
        )

        final_summary = {
            "summary": completion["choices"][0]["message"]["content"],
            "title": title,
            "url": chunks[0]["url"]
        }

        timestamp_str = datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
        with open(f"{output_dir}/{timestamp_str}_final-summary-prompt.txt", "w") as file:
            file.write(full_summary_prompt_formatted)

        with open(f"{output_dir}/{timestamp_str}_final-summary.json", "w") as file:
            file.write(json.dumps(final_summary))

        print(f"\tFinished final summary")


#Setup Lib
from flask import Flask, render_template, redirect, url_for, request
import pandas as pd

app = Flask(__name__, static_url_path='/Static', static_folder='Static',template_folder='Template')

#Read Excel File
df= pd.read_excel('Excel_File/file.xlsx')

####################################################################################################################
#App Get all data and render them to Html file
@app.route('/', methods=("POST", "GET"))
def index():
    return render_template("Index.html",column_names=df.columns.values, row_data=list(df.values.tolist()))
####################################################################################################################

####################################################################################################################
# App add New Book
@app.route('/add', methods=["POST"])
def add():
    #Retrive The data
    if request.method=='POST':
        Book_Title   = request.form['Book_Title_add']
        Book_Url     = request.form['Book_Url_add']
        Author_Name  = request.form['Author_Name_add']
        Author_Url   = request.form['Author_Url_add']
        Country_Name = request.form['Country_Name_add']
        Country_URL  = request.form['Country_URL_add']

    #Add Data To DataFrame
    df.loc[len(df)+1] = [len(df)+1,Book_Title,Book_Url,Author_Name,Author_Url,Country_Name,Country_URL]

    #Save DataFrame
    df.to_excel('Excel_File/file.xlsx',index=False)

    return render_template("Index.html",column_names=df.columns.values, row_data=list(df.values.tolist()))
####################################################################################################################

####################################################################################################################
# App Update Book
@app.route('/update', methods=("POST", "GET"))
def update():
    if request.method=='POST':
        id_Book      = request.form['id_update']
        Book_Title   = request.form['Book_Title_update']
        Book_Url     = request.form['Book_Url_update']
        Author_Name  = request.form['Author_Name_update']
        Author_Url   = request.form['Author_Url_update']
        Country_Name = request.form['Country_Name_update']
        Country_URL  = request.form['Country_URL_update']

    #update Data To DataFrame
    df.loc[df.index==int(id_Book)-1,'index']=id_Book
    df.loc[df.index==int(id_Book)-1,'book_title']=Book_Title
    df.loc[df.index==int(id_Book)-1 , 'book_url'] = Book_Url
    df.loc[df.index==int(id_Book)-1 , 'author_name'] = Author_Name
    df.loc[df.index==int(id_Book)-1 , 'author_url'] = Author_Url
    df.loc[df.index==int(id_Book)-1 , 'country_name'] = Country_Name
    df.loc[df.index==int(id_Book)-1 , 'country_url'] = Country_URL

    #Save DataFrame
    df.to_excel('Excel_File/file.xlsx',index=False)
    return render_template("Index.html",column_names=df.columns.values, row_data=list(df.values.tolist()))
####################################################################################################################

####################################################################################################################
#App Delete Book
@app.route('/delete', methods=("POST", "GET"))
def delete():
    if request.method=='POST':
        id_Book= request.form['id_delete']
    df.drop(labels=int(id_Book)-1, axis=0, inplace=True)

    #Save DataFrame
    df.to_excel('Excel_File/file.xlsx',index=False)
    return render_template("Index.html",column_names=df.columns.values, row_data=list(df.values.tolist()))
####################################################################################################################

####################################################################################################################
#Run App
if __name__=='__main__':
    app.run(debug=True)
####################################################################################################################


import pandas as pd
import numpy as np
from sklearn.preprocessing import LabelBinarizer
from datetime import datetime

def preprocess(df, store):
    df = pd.merge(df, store, on='Store', how='left')

    df['Date'] = df['Date'].map(lambda x: datetime.strptime(x, "%Y-%m-%d"))
    # df['DaysSince20150731'] = df['Date'].map(lambda x: (datetime.strptime(x, "%Y-%m-%d") - datetime.strptime("2015-07-31", "%Y-%m-%d")).days)

    day2str = {1:'Mon', 2:'Tue', 3:'Wed', 4:'Thu', 5:'Fri', 6:'Sat', 7:'Sun'}
    for i in range(1, 8):
        df[day2str[i]] = (df['DayOfWeek'] == i).astype(float)

    df['StateHoliday'] = df['StateHoliday'].astype(str)
    for i in ['0', 'a', 'b', 'c']:
        df['StateHoliday_'+str(i)] = (df['StateHoliday'] == i).astype(float)

    for i in ['a', 'b', 'c', 'd']:
        df['StoreType_'+str(i)] = (df['StoreType'] == i).astype(float)

    for i in ['a', 'b', 'c']:
        df['Assortment_'+str(i)] = (df['Assortment'] == i).astype(float)

    df['PromoInterval'] = df['PromoInterval'].astype(str)
    for mon in ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sept', 'Oct', 'Nov', 'Dec']:
        df['PromoInterval_'+mon] = df['PromoInterval'].map(lambda x: float(mon in x))

    return df

def add_feature(df):
    df['Year'] = df['Date'].dt.year
    df['Month'] = df['Date'].dt.month
    df['Day'] = df['Date'].dt.day  # day in month
    df['WeekOfYear'] = df['Date'].dt.weekofyear

    # whether competition is currently open and it started how many months ago
    df['CompetitionOpenMonthsAgo'] = 12 * (df['Year'] - df['CompetitionOpenSinceYear']) + (df['Month'] - df['CompetitionOpenSinceMonth'])
    df['CompetitionOpen'] = df['CompetitionOpenMonthsAgo'].apply(lambda x: x if x > 0 else 0)

    # whether Promo2 is currently open and it started how many months ago
    df['Promo2OpenMonthsAgo'] = 12 * (df['Year'] - df['Promo2SinceYear']) + (df['WeekOfYear'] - df['Promo2SinceWeek']) / 4.0
    df['Promo2Open'] = df['Promo2OpenMonthsAgo'].apply(lambda x: x if x > 0 else 0)
    df.loc[df['Promo2SinceYear']==0, 'Promo2Open'] = 0

    # whether PromoInterval is currently open
    month2str = {1:'Jan', 2:'Feb', 3:'Mar', 4:'Apr', 5:'May', 6:'Jun', 7:'Jul', 8:'Aug', 9:'Sept', 10:'Oct', 11:'Nov', 12:'Dec'}
    def checkPromoInterval(x):
        mon = month2str[x['Month']]
        return float(x['PromoInterval_'+mon]==1)
    df['IsPromoIntervalMonth'] = df.apply(checkPromoInterval, axis=1)

    df['TueToFri'] = ((df['DayOfWeek'] > 1) & (df['DayOfWeek'] < 6)).astype(float)

    # whether it is currently open and it is currently holiday
    df['OpenHoliday'] = np.where((df['Open'] == 1) & ((df['SchoolHoliday'] == 1)|(df['StateHoliday_0'] != 1)), 2, 1)
    df.loc[df['Open']==0,'OpenHoliday'] = 0

    return df

def fill_and_drop(df, withoutOpen=False, isTest=False):
    df['CompetitionDistance'].fillna(1000000, inplace=True)
    df['CompetitionOpenMonthsAgo'].fillna(0, inplace=True)
    df['Promo2OpenMonthsAgo'].fillna(0, inplace=True)

    df.drop(columns=['DayOfWeek', 'StateHoliday', 'StoreType', 'Assortment', 'PromoInterval', 'Date'], inplace=True)
    df.drop(columns=['CompetitionOpenSinceYear', 'CompetitionOpenSinceMonth', 'Promo2SinceYear', 'Promo2SinceWeek'], inplace=True)

    if withoutOpen:
        df = df[df['Open']!=0]
        df.drop(columns=['Open'], inplace=True)

    if not isTest:
        df = df[df['Sales']>0]

    return df

def data_prepare(withoutOpen=False):
    train = pd.read_csv("data/train.csv")
    test = pd.read_csv("data/test.csv")
    store = pd.read_csv("data/store.csv")

    # there are NaN of Open in test, fill it 1
    test.fillna(1, inplace=True)

    df_train = preprocess(train, store)
    df_train = add_feature(df_train)
    df_train = fill_and_drop(df_train, withoutOpen)

    df_test = preprocess(test, store)
    df_test = add_feature(df_test)
    df_test = fill_and_drop(df_test, withoutOpen, True)

    X_train = df_train.drop(['Sales','Customers'], axis=1)
    y_train = df_train['Sales']
    X_test = df_test.drop(['Id'], axis=1)

    return X_train, y_train, X_test

if __name__ == "__main__":
    X_train, y_train, X_test = data_prepare(True)


# -*- coding:utf-8 -*-
# @Author: Jone Chiang
# @Date  : 2019/6/12 9:03
# @File  : db_utils
import re
import os
import calendar
import sqlite3
from datetime import datetime

cur_year = datetime.now().year
common_year = [-1, 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
leap_year = [-1, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]

def get_all_days(s):
    """
    '12月30日~1月1日'
    :param s:
    :return:
    """
    begin, end = s.split('~')
    begin_0, begin_1 = re.findall('\d+', begin)
    end_0, end_1 = re.findall('\d+', end)
    begin_0, begin_1 = int(begin_0), int(begin_1)
    end_0, end_1 = int(end_0), int(end_1)

    # print(begin, end)

    date = []

    if begin_0 == end_0:  # 起止日期在同一个月
        days = list(range(begin_1, end_1+1))
        for day in days:
            day = str(begin_0) + '月' + str(day) + '日'
            date.append(day)

    elif begin_0 > end_0:  # 跨年 begin_0 12月 end_0 1月
        days_of_last_month = list(range(begin_1, begin_1 + leap_year[begin_0] - begin_1 + 1))
        for day in days_of_last_month:
            day = str(begin_0) + '月' + str(day) + '日'
            date.append(day)

        days_of_cur_month = list(range(1, end_0 + 1))
        for day in days_of_cur_month:
            day = str(end_0) + '月' + str(day) + '日'
            date.append(day)

    elif begin_0 < end_0:  # 起止日期不在同一个月
        if calendar.isleap(cur_year):
            days_of_last_month = list(range(begin_1, begin_1 + leap_year[begin_0] - begin_1 + 1))
            for day in days_of_last_month:
                day = str(begin_0) + '月' + str(day) + '日'
                date.append(day)

            days_of_cur_month = list(range(1, end_0 + 1))
            for day in days_of_cur_month:
                day = str(end_0) + '月' + str(day) + '日'
                date.append(day)
        else:
            days_of_last_month = list(range(begin_1, begin_1 + common_year[begin_0] - begin_1 + 1))
            for day in days_of_last_month:
                day = str(begin_0) + '月' + str(day) + '日'
                date.append(day)

            days_of_cur_month = list(range(1, end_1 + 1))
            for day in days_of_cur_month:
                day = str(end_0) + '月' + str(day) + '日'
                date.append(day)
    # print(date)
    return date

def get_all_days_2(s):
    res = re.findall('\d+月\d+日', s)
    print(res)
    return res

class DbUtils:
    def __init__(self, db_path):
        self.db_path = db_path

    def get_con(self):
        self.con = sqlite3.connect(self.db_path)
        self.cur = self.con.cursor()

    def close_db(self):
        if self.cur:
            self.cur.close()
        if self.con:
            self.con.close()

    def create_tab_holiday(self, year):
        """
        新建holiday表
        :return:
        """
        self.get_con()
        holiday_sql = """
            create table holiday_{}(id INTEGER primary key autoincrement, holiday_name varchar(50), holiday_period varchar(200), paid_leave varchar(200), days_off_num integer(2))
        """.format(year)
        self.cur.execute(holiday_sql)
        self.close_db()

    def analyze_holiday_period(self, year):
        """
        分析holiday表中的数据: 放假时间
        :return:
        """
        self.get_con()
        holiday_period_sql = """
            select holiday_period from holiday_{}
        """.format(year)
        self.cur.execute(holiday_period_sql)
        result = []
        results = self.cur.fetchall()
        for res in results:
            res0 = get_all_days(res[0])
            result.extend(res0)
        # print(result)

        path = './db/holiday_period_{}.txt'.format(cur_year)
        if os.path.exists(path):
            os.remove(path)
        for i in result:
            with open('./db/holiday_period_{}.txt'.format(cur_year), 'a+', encoding='utf-8') as f:
                f.write(i+'\n')

        self.close_db()

    def analyze_paid_leave(self, year):
        """
        分析holiday表中的数据: 调休上班日期
        :param year:
        :return:
        """
        self.get_con()
        paid_leave_sql = """
            select paid_leave from holiday_{}
        """.format(year)
        self.cur.execute(paid_leave_sql)
        result = []
        results = self.cur.fetchall()
        for res in results:
            res0 = get_all_days_2(res[0])
            result.extend(res0)
        # print(result)

        path = './db/paid_leave_{}.txt'.format(cur_year)
        if os.path.exists(path):
            os.remove(path)

        for i in result:
            with open('./db/paid_leave_{}.txt'.format(cur_year), 'a+', encoding='utf-8') as f:
                f.write(i+'\n')

        self.close_db()

    def get_holiday(self):
        with open('./db/holiday_period_{}.txt'.format(cur_year), 'r', encoding='utf-8') as f:
            res = f.readlines()
            res = [i.strip() for i in res]

        print(res)
        return res

    def get_workday(self):
        with open('./db/paid_leave_{}.txt'.format(cur_year), 'r', encoding='utf-8') as f:
            res = f.readlines()
            res = [i.strip() for i in res]

        print(res)
        return res

if __name__ == '__main__':
    cur_year = datetime.now().year
    db_util = DbUtils('./db/holiday.db')
    # db_util.create_tab_holiday(cur_year)
    # db_util.analyze_holiday_period(cur_year)
    # db_util.analyze_paid_leave(cur_year)

    # get_all_days('2月27日~3月5日')
    # # get_all_days('5月29日(除夕）~6月2日')

    # get_all_days_2('2月2日（周六）、2月3日（周日）上班')

    db_util.get_workday()


"""empty message

Revision ID: 140b69c7c499
Revises: 11adbfe25cef
Create Date: 2018-07-20 23:13:10.243019

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '140b69c7c499'
down_revision = '11adbfe25cef'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('task', sa.Column('completed', sa.Boolean(), nullable=True))
    op.add_column('task', sa.Column('fin_date', sa.DateTime(), nullable=True))
    op.add_column('task', sa.Column('fin_order', sa.Integer(), nullable=True))
    op.create_index(op.f('ix_task_fin_date'), 'task', ['fin_date'], unique=False)
    op.create_index(op.f('ix_task_fin_order'), 'task', ['fin_order'], unique=False)
    op.drop_index('ix_task_status', table_name='task')
    op.drop_column('task', 'status')
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('task', sa.Column('status', sa.VARCHAR(length=16), autoincrement=False, nullable=True))
    op.create_index('ix_task_status', 'task', ['status'], unique=False)
    op.drop_index(op.f('ix_task_fin_order'), table_name='task')
    op.drop_index(op.f('ix_task_fin_date'), table_name='task')
    op.drop_column('task', 'fin_order')
    op.drop_column('task', 'fin_date')
    op.drop_column('task', 'completed')
    # ### end Alembic commands ###


from numpy.core.defchararray import lower
import json, pandas
import os.path
from Db.DbHandler import dbHandler

db = dbHandler.getInstance()

def process_message(body):
    if 'format' not in body or 'path' not in body or 'loadTo' not in body:
        print('missing one or more fields in body object')
        return False

    format = lower(body['format'])
    file = body['path']
    loadTo = body['loadTo']

    if os.path.exists(file):
        if format == 'csv':
            return load_csv_to_db(file, loadTo)
        elif format == 'json':
            return load_json_to_db(file, loadTo)
        else:
            print("error - received {0} format, invoice format must be csv or json".format(format))
            return False
    else:
        print("could not find given path")
        return False

def load_json_to_db(path, tablename):
    db.create_table(tablename)
    try:
        with open(path, encoding='utf-8-sig') as json_file:
            json_data = json.loads(json_file.read())
            query = 'insert into "{0}" values (?,?,?,?,?,?,?,?,?)'.format(tablename)
            for child in json_data:
                params = (child['InvoiceId'], child['CustomerId'], child['InvoiceDate'], child['BillingAddress'], child['BillingCity'], child['BillingState'], child['BillingCountry'], child['BillingPostalCode'], child['Total'])
                db.insert(query, params)
            json_file.close()
            print('invoices inserted successfully!')
            return True
    except Exception as e:
        print(e)
        return False

def load_csv_to_db(path, tablename):
    db.create_table(tablename)
    try:
        df = pandas.read_csv(path)
        df.to_sql(tablename, db.conn, if_exists='append', index=False)
        print('invoices inserted successfully!')
        return True
    except Exception as e:
        print(e)
        return False