# !/usr/bin/env python
# -*- coding: utf-8 -*-

# -------------------------------------------#
# author: sean lee                           #
# email: lxm_0828@163.com                    #
#--------------------------------------------#

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.nn.utils as utils
import torch.optim.lr_scheduler as lr_scheduler
from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence
from torch.autograd import Variable

import time
import jieba
import math
import numpy as np

from datetime import datetime

from nnclf.utils import *

class CNNBiGRU(object):
    def __init__(self, model_dir, **kwargs):
        self.model_dir = model_dir.rstrip(os.path.sep)
        self.model, self.start_epoch = Utils.load_previous_model(self.model_dir)

        self.kwargs = kwargs
        '''
        @description: 每次输入的批量数据进行训练，加快训练速度
        @notice: 批处理的大小，适用于大量数据，一般设置为2的n次方且不宜超过128，一般2，4最佳
        '''
        if 'batch_size' not in self.kwargs:
            self.kwargs['batch_size'] = 4
        '''
        @description: 隐藏层大小
        '''
        if 'hidden_size' not in self.kwargs:
            self.kwargs['hidden_size'] = 64
        '''
        @description: 训练回合数，总的训练次数=回合数x数据集大小
        '''
        if 'epoch' not in self.kwargs:
            self.kwargs['epoch'] = 3
        '''
        @description: 是否使用word2vec预训练词向量训练
        @notice: 为True时必须传入word2vec_model指定词向量路径
        '''
        if 'word2vec' not in self.kwargs:
            self.kwargs['word2vec'] = False
        '''
        @description: word2vec模型路径
        '''
        if 'word2vec_model' not in self.kwargs:
            self.kwargs['word2vec_model'] = None
        '''
        @description: 学习速率
        '''
        if 'lr_rate' not in self.kwargs:
            self.kwargs['lr_rate'] = 0.01
        '''
        @description: 学习速率是否自动进行梯度变化
        '''
        if 'autolr' not in self.kwargs:
            self.kwargs['autolr'] = True
        '''
        @description: 学习速率变化的次数
        '''
        if 'autolr_times' not in self.kwargs:
            self.kwargs['autolr_times'] = 3
        '''
        @description: 是否输出运行记录
        '''
        if 'print_log' not in self.kwargs:
            self.kwargs['print_log'] = False
        '''
        @description: 训练多少次时输出运行记录
        '''
        if 'log_interval' not in self.kwargs:
            self.kwargs['log_interval'] = 100

    def train(self, datas, labels, **kwargs):
        '''
        @description: retrain 是否重复训练，默认False，如果为默认情况则在先前模型基础上继续训练
        '''
        if 'retrain' in kwargs and kwargs['retrain'] is True:
            Utils.remove_models(self.model_dir)

        self.datas = datas
        self.labels = labels

        # 加载之前的模型和步数
        self.model, self.start_epoch = Utils.load_previous_model(self.model_dir)
        lang = Lang()
        all_labels = []
        for idx in range(len(datas)):
            data = datas[idx]
            if type(data) == str:
                data_list = jieba.lcut(data)
            lang.index_words(data)
            if labels[idx] not in all_labels:
                all_labels.append(labels[idx])

        if self.kwargs['word2vec'] is True:
            if self.kwargs['word2vec_model'] == None:
                raise ValueError('请填写word2vec模型存储路径')
            self.dataset = Dataset(datas, labels, lang, all_labels, batch_size=self.kwargs['batch_size'], word2vec=True, word2vec_model=self.kwargs['word2vec_model'])
            if self.model == None:
                self.kwargs['word_embedding'] = self.dataset.word_embedding
                self.kwargs['embed_size'] = self.dataset.embed_size
        else:
            self.dataset = Dataset(datas, labels, lang, all_labels, batch_size=self.kwargs['batch_size'], word2vec=False)

        param = self.kwargs
        param['dataset'] = self.dataset
        Utils.save_param(param, self.model_dir)

        if self.model == None:
            self.kwargs['input_size'] = lang.n_words
            self.kwargs['output_size'] = len(all_labels)

            self.model = CNNBiGRUNet(self.kwargs)

        self.criterion = torch.nn.NLLLoss()
        self.optimizer = torch.optim.Adam(self.model.parameters(), lr = self.kwargs['lr_rate'])

        self.n_epochs = len(labels) * self.kwargs['epoch']
        self.progress = ProgressBar(count = self.start_epoch, total = self.n_epochs+1)

        self.train_iter()

    def train_iter(self):
        self.log("开始训练时间：%s" % datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
        self.log("总共训练次数：%s" % self.n_epochs)

        self.model.train(True)
        start_time = time.time()

        if self.kwargs['autolr'] is True:
            #scheduler = lr_scheduler.ReduceLROnPlateau(self.optimizer, 'min', verbose=True, min_lr=0.000001)
            scheduler = lr_scheduler.StepLR(self.optimizer, step_size=self.n_epochs//self.kwargs['autolr_times'], gamma=0.1)

        runing_loss = 0
        for epoch in range(self.start_epoch, self.n_epochs+1):

            loss = 0

            if self.kwargs['autolr'] is True:
                scheduler.step()
                #self.log("learning_rate改变为：%s, 当前训练次数：%d" % (self.optimizer.param_groups[0].get("lr"), epoch))

            self.optimizer.zero_grad()

            input_variable, target_variable, inputs_len = self.dataset.random_train()
            hidden = self.model.init_hidden()
            output_pred = self.model(input_variable, inputs_len, hidden, training=True)
            loss += self.criterion(output_pred.view(target_variable.size(0), -1), target_variable)
            runing_loss += loss.data[0]
            if self.kwargs['print_log'] is True:
                self.progress.next()
                if epoch % self.kwargs['log_interval'] == 0:

                    _, pred_ids = torch.max(output_pred, dim=1)
                    corrects = (pred_ids.data == target_variable.data)
                    acc = torch.mean(corrects.float())
                    self.progress.log('epoch %d , loss: %.3f, acc %.3f' % (epoch, runing_loss/self.kwargs['log_interval'], acc))

                    runing_loss = 0
                else:
                    self.progress.log()

            if epoch % (self.n_epochs//10) == 0:
                Utils.save_model(self.model, epoch, self.model_dir)
                log = 'save model :%s' % datetime.now().strftime('%Y-%m-%d %H:%M:%S')
                self.log(log)
                if self.kwargs['print_log'] is True:
                    print(log)

            # update
            loss.backward()
            utils.clip_grad_norm(self.model.parameters(), max_norm=5.0)
            self.optimizer.step()

        # save model
        Utils.save_model(self.model, epoch, self.model_dir)
        time_count = time.time() - start_time
        self.log("结束训练时间： %s" % datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
        self.log("总共花费: %.2f小时, 最后loss: %f" % (time_count/3600, runing_loss))

    def predict(self, datas, k=1):
        param = Utils.load_param(self.model_dir)
        self.dataset = param['dataset']

        labels = []
        for data in datas:
            if type(data) == str:
                data = jieba.lcut(data)
            topv, topi = self.evaluate(data, k)
            labels.append([self.dataset.all_labels[x] for x in topi])
        return labels

    def predict_proba(self, datas, k=1):
        label_probas = []
        for data in datas:
            if type(data) == str:
                data = jieba.lcut(data)
            topv, topi = self.evaluate(data, k)
            arr = []
            for x in range(len(topv)):
                arr.append((self.dataset.all_labels[topi[x]], topv[x]))
            label_probas.append(arr)
        return label_probas

    def evaluate(self, data, k=1):
        if k > len(self.dataset.all_labels):
            k = len(self.dataset.all_labels)
        inp, inp_lens = self.dataset.evaluate_pad(data)
        input_tensor = torch.LongTensor(inp)
        input_variable = Variable(input_tensor, volatile=True).transpose(0, 1)
        output = self.model(input_variable, inp_lens, None)
        topv, topi = output.data.topk(k)
        return topv.squeeze(), topi.squeeze()

    def log(self, string):
        Utils.save_log(string, self.model_dir)

'''
 CNN-BiGRU网络
'''
class CNNBiGRUNet(nn.Module):
    def __init__(self, kwargs):
        super(CNNBiGRUNet, self).__init__()
        self.input_size = kwargs['input_size']
        self.hidden_size = kwargs['hidden_size']
        self.output_size = kwargs['output_size']

        self.n_layers = kwargs.get('n_layers', 1)
        self.batch_size = kwargs.get('batch_size', 1)
        self.embed_size = kwargs.get('embed_size', 100)
        self.kernel_num = kwargs.get('kernel_num', 64)
        self.kernel_sizes = kwargs.get('kernel_sizes', [1, 3, 5])
        self.dropout = kwargs.get('dropout', 0.1)

        self.embed = nn.Embedding(self.input_size, self.embed_size)

        if 'word_embedding' in kwargs:
            pretrained_weight = torch.from_numpy(kwargs['word_embedding'])
            self.embed.weight.data.copy_(pretrained_weight)

        Ci = 1
        Co = self.kernel_num
        Ks = self.kernel_sizes

        # conv2d卷积操作
        self.convs1 = [nn.Conv2d(Ci, Co, (K, self.embed_size), padding=(K//2, 0), stride=(1,1)) for K in Ks]
        self.batch_norm = nn.BatchNorm2d(Co)

        # CNNBiGRU num_directions = 2
        self.bigru = nn.GRU(self.embed_size, self.hidden_size, num_layers=self.n_layers, dropout=self.dropout, bidirectional=True, bias=True)
        self.hidden = self.init_hidden()

        # Linear
        L = len(Ks)*Co + self.hidden_size*2  # num_directions = 2

        self.h2o = nn.Linear(L, self.output_size)
        self.dropout = nn.Dropout(self.dropout)

    def init_hidden(self):
        # num_directions = 2
        return Variable(torch.zeros(self.n_layers * 2, self.batch_size, self.hidden_size))

    def forward(self, inp, inp_lens, hidden=None, training=False):
        embed = self.embed(inp) # N x batch_size x embed_size
        if training:
            embed = self.dropout(embed)

        inp = embed.view(len(inp), embed.size(1), -1)

        cnn = inp
        cnn = torch.transpose(cnn, 0, 1) # (batch_size, seq_len, embed_size)
        cnn = cnn.unsqueeze(1) # (batch_size, 1, seq_len, embed_size)
        cnn = [self.batch_norm(F.relu(conv(cnn)).squeeze(3)) for conv in self.convs1] # [(batch_size, seq_len, Co), ...]*len(Ks)
        cnn = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in cnn]
        cnn = torch.cat(cnn, 1) # 在1轴拼接 (seq_len, Co*len(Ks))

        # BiGRU
        bigru = inp.view(len(inp), inp.size(1), -1) # seq_len x batch_size x embed_size
        bigru = pack_padded_sequence(bigru, inp_lens)
        bigru, hn = self.bigru(bigru, hidden)
        bigru, output_lens = pad_packed_sequence(bigru)

        # bigru shape -> (seq_len, batch_size, 2*hidden_size)
        bigru = torch.transpose(bigru, 0, 1)    # (batch_size ,seq_len, 2*hidden_size)
        bigru = torch.transpose(bigru, 1, 2)     # (batch_size, 2*hidden_size, seq_len)
        bigru = F.max_pool1d(bigru, bigru.size(2)).squeeze(2)    # (batch_size, 2*hidden_sze)

        # CNN BiGRU
        cnn = torch.transpose(cnn, 0, 1) # (Co*len(Ks), batch_size)
        bigru = torch.transpose(bigru, 0, 1) #(2*hidden_size, batch_size)
        cnn_bigru = torch.cat((cnn, bigru), 0) # (Co*len(Ks) + 2*hidden_size, batch_size)
        cnn_bigru = torch.transpose(cnn_bigru, 0, 1) #(batch_size, Co*len(Ks)+2*hidden_size)

        output = self.h2o(F.tanh(cnn_bigru))
        if training:
            y = F.log_softmax(output)
        else:
            y = F.softmax(output)
        return y


"""
Evaluation scripts.

Released under Apache 2.0 License. See extended notice at the end of this file.

There are several functions defined here:
    1. logits_to_proba
    2. data_yielder
    3. split_yielder
    4. pr_roc_from_path
    5. sample_pics
    6. get_probabilities
"""

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.ticker import MultipleLocator

import tensorflow as tf

from sklearn.metrics import roc_curve, precision_recall_curve, auc

import os
import pickle

def logits_to_proba(logit):
    """
    Parameters: logit, logarithmic odds of probability p.
        By definition, logits are:
          logit(p) = log(p/(1-p)).
        logits are easily transformed into a probability:
          p = exp(logit)/(1 + exp(logit)).
        Logits are useful when training classifier models because they do not
        suffer from some of the numerical issues that the sigmoid function
        does. A key issue is saturation:  if \sigma(x) is the sigmoid, then
          (d/dx)\sigma(x) is very small for |x| >>0,
        leading to numerical underflow that prevents gradient descent methods
        from being effective in training classifiers.

    Values: probability of given logit
        p(logit) = exp(logit)/(1+exp(logit))
    """
    return np.exp(logit)/(1+np.exp(logit))

def sleep_classes(inputs, n_classes):
    """
    Convert 6-stage classification to n_classes.
    The six stages are:
        0: awake
        1: nrem1
        2: nrem2
        3: nrem3
        4: nrem4
        5: rem

    For our purposes, only makes sense to group these as follows:
        2 stage: sleep 0, wake 1
        3 stage: sleep 0, nrem 1, rem 2
        4 stage: sleep 0, light nrem 1, deep nrem 2, rem 3
    """
    if n_classes == 2:
        return np.where(inputs > 0, 1, 0)
    elif n_classes == 3:
        return np.piecewise(inputs, [inputs == 0,
                                     (inputs > 0) & (inputs < 5),
                                     inputs == 5], [0, 1, 2])
    elif n_classes == 4:
        return np.piecewise(inputs, [inputs == 0,
                                     (inputs > 0) & (inputs < 3),
                                     (inputs >= 3) & (inputs < 5),
                                     inputs == 5], [0, 1, 2, 3])
    else:
        raise ValueError("n_classes must be 2, 3 or 4.")

def data_yielder(data_path, neural_path = None, exclude = None, nn = True, nn_base_name = None):
    """
     Generator to create leave-one-out train/test splits.
     Upon initialization, loads the separate data pickles.
     Yields: (subject_number[i], spectrogram[i], time[i], psg labels[i], neural network[i])
        If exclude != None, then it's a list of subject numbers not to be yielded.
    """
    # Ensure uniform input
    data_path = data_path if data_path[-1] == "/" else data_path + "/"

    if neural_path is None:
        neural_path = data_path + "neural/"
    neural_path = neural_path if neural_path[-1] == "/" else neural_path + "/"
    pickle_path = data_path + "pickles/"

    # Load the separate numpy pickles into lists.
    spectros = []
    time = []
    psg = []
    nns = []

    if nn_base_name == None:
        nn_base_name = "_trained_cnn.h5"

    # next(os.walk)[0] is the top-level directory.
    # This is where the by-subject pickle folders should be.
    # next(os.walk)[1] is the list of files in next(os.walk)[0]
    #   (not folders, which is next(os.walk)[2])
    list_of_subjects = next(os.walk(pickle_path))[1]
    if exclude is not None:
        list_of_subjects = [s for s in list_of_subjects if s not in exclude]

    for s in list_of_subjects:

        with open(pickle_path + s + "/spectros.pickle", "rb") as f:
            spectros.append(pickle.load(f))
        with open(pickle_path + s + "/time.pickle", "rb") as f:
            time.append(pickle.load(f))
        with open(pickle_path + s + "/psg.pickle", "rb") as f:
            psg.append(pickle.load(f))
        if nn:
            nns.append(tf.keras.models.load_model(neural_path + s + nn_base_name))

    for i in range(len(list_of_subjects)):
        if nn:
            yield list_of_subjects[i], spectros[i], time[i], psg[i], nns[i]
        else:
            yield list_of_subjects[i], spectros[i], time[i], psg[i]

def split_yielder(data_path = None, exclude = None, with_time = True):
    """
    Yields: (subject number excluded from training split, training_split_data, testing_split_data)
    Where:
    training_split_data:
      [0] if with_time, a 2-tuple:
        [0] training spectrograms tensor, shape (*, 65, 73)
        [1] training time tensor, shape (*,)
        (!) if not with_time, just training spectrograms
      [1] training psg labels tensor, shape (*,)
    testing_split_data is similar.

    See data_yielder for an explanation of the parameters.
     """
    data_list = [d for d in data_yielder(data_path = data_path, exclude = exclude, nn = False)]

    for i, d in enumerate(data_list):
        if i == 0:
            skip_d = data_list[1:]
        else:
            skip_d = data_list[:i-1] + data_list[i:]

        train_spectro = np.concatenate([s[1] for s in skip_d])
        train_psg = np.concatenate([s[3] for s in skip_d])

        if with_time:
            train_time = np.concatenate([s[2] for s in skip_d])
            training_split_data = ((train_spectro, train_time), train_psg)
            testing_split_data = ((d[1], d[2]), d[3])
        else:
            training_split_data = (train_spectro, train_psg)
            testing_split_data = (d[1], d[3])

        yield (d[0], training_split_data, testing_split_data)

def eval_nn(subject_number, nn_base_name, with_time = True, data_path = None, neural_path = None):
    """
    Helper function to load one subject's data and the neural net training on the complementary data,
    then evaluate the neural network on the testing data. Returns a numpy array with shape
    (N, n_classes) where the subject's PSG has shape (N, 1) or (N, ).

    data_path and neural_path default to ../data/pickles and ../data/neural
    """
    # Set up paths
    if data_path is None:
        data_path = "data/pickles/"
    else:
        if not isinstance(data_path, str):
            raise ValueError("data_path must be a string.")

        if data_path[-1] != "/":
            data_path += "/"

    if neural_path is None:
        neural_path = "data/neural/"
    else:
        if not isinstance(neural_path, str):
            raise ValueError("neural_path must be a string.")

        if neural_path[-1] != "/":
            neural_path += "/"

    # Load the neural network
    try:
        nn = tf.keras.models.load_model(neural_path + subject_number + nn_base_name)
    except:
        print("There was an error opening the trained neural network.")
        return

    # last layer of network has shape (None, n_classes)
    n_classes = nn.output.shape[1]

    # Load the subject's data, throwing exceptions if the pickles are missing or unopenable.
    ## Load spectrograms
    try:
        with open(data_path + subject_number + "/spectros.pickle", "rb") as f:
            spectros = pickle.load(f)
    except:
        print("There was an error opening the pickled spectrograms for {}.".format(subject_number))
        return

    # Load time if we want that, and set up the testing data.
    if with_time is True:
        try:
            with open(data_path + subject_number + "/time.pickle", "rb") as f:
                time = pickle.load(f)
        except:
            print("There was an error opening the pickled times for {}.".format(subject_number))
            return

        testing_X = [spectros, time]

    else:
        testing_X = spectros

    return nn.predict(testing_X)

def pr_roc(subject_number, nn_base_name, data_path = None, neural_path = None, with_time = True, mode = "roc", pos_class = 0):
    """
    Helper function to load one subject's data and the neural net training on the complementary data,
    then evaluate the performance of this network via PR or ROC curve, depending on mode.

    When pos_class is an int, returns a tuple suitable for plotting, interpolation, and pointwise averaging:
        (false positive rate, true positive rate)
    OR
        (recall, precision)
    depending on if mode == "roc" or "pr", respectively.

    When pos_class is list<int>, returns a list with len(pos_class) elements.
        The elements correspond to applying this function with the various pos_classes in the list.
        E.g. if pos_class = [0,1] then returns
        [ pr_roc(..., pos_class = 0, ...), pr_roc(..., pos_class = 1, ...) ]

    """
    print(f"{mode} Subject number: {subject_number}")

    if data_path is None:
        data_path = "data/pickles/"
    elif data_path[-1] != "/":
        data_path += "/"

    if neural_path is None:
        neural_path = "data/neural/"
    elif neural_path[-1] != "/":
        neural_path += "/"

    outcome = eval_nn(subject_number, nn_base_name, with_time=with_time, data_path=data_path, neural_path=neural_path)

    n_classes = outcome.shape[-1] # (n_timestamps, n_classes)
    ## Load labels
    try:
        with open(data_path + subject_number + "/psg.pickle", "rb") as f:
            psg = pickle.load(f)

        psg = sleep_classes(psg, n_classes)

    except:
        print("There was an error opening the pickled PSG for {}.".format(subject_number))
        return

    # Evaluate the outcomes, either operating on a single class if pos_class is an integer, or once per class given.
    eval_fn = roc_curve if mode == "roc" else precision_recall_curve
    if isinstance(pos_class, int):
        x, y, _ = eval_fn(y_true = psg, y_score = outcome[:, pos_class], pos_label = pos_class)
        return (x, y)

    else: # isinstance(pos_class, list)
        xys = [
            eval_fn(
                psg,
                outcome[:, p], # this parameter name is different depending on curve type?
                pos_label = p
                )[:-1] # Only grab x and y, not thresholds
            for p in pos_class
            ]

        return xys

def pointwise_avg(xy_pairs):
    """
    Given a list of (x, y) pairs, returns the pointwise average of
    the y-values viewed as functions of their x-values.

    To accomplish this, we first take the refinement (union) of all the x's
    by what is essentially a merge sort + unique of these arrays.
    """
    if len(xy_pairs) == 1:
        return xy_pairs

    xs = [np.array(x) for x, y in xy_pairs]

    refined = xs[0]
    for x in xs[1:]:
        refined = np.hstack([refined, x])
        refined.sort()
        refined = np.unique(refined)

    # Interpolate to all be evaluated on the full refinement
    ys = [np.interp(refined, x, y) for x, y in xy_pairs]

    return (refined, np.mean(ys, axis=0))

def pr_roc_from_path(
        nn_base_name,
        pickle_path,
        neural_path,
        exclude = None,
        with_time = True,
        pos_class = 0,
        pos_class_name = None,
        title = None,
        label_names = None,
        saveto = None,
        from_logits = False):
    """
    Given:
      - nn_base_name (str): the base name of NNs,
      - pickle_path (str): the folder containing spectrogram and time pickles, sorted by subject
      - neural_path (str): the folder containing trained NNs to be evaluated
    Produce:
      - grid with two rows and len(pos_class) columns, with ROC in top row, PR in bottom row.
        -- pos_class (list or int): which class should be considered "positive" for PR/ROC?
            If a list, evaluates PR and ROC per class.
        Optionally, supports:
        -- pos_class_name (str or list): column headers.
            If pos_class is an int, this should be a str, otherwise a list.
        -- title (str): for the title if not None
        -- label_names (list or dict): a list matching pos_class, or a dict if pos_class is int
        -- saving PDF to disk, if saveto is not None.

    Optional parameters:
      - with_time (bool, default True): does the NN set require a (None, 1) shaped time input?
      - from_logits (bool, default False): If True, output of NN is logits. False, output is probabilities/softmax.

    """

    import re

    # Get list of NNs we have trained. These are the only ones we can evalutate!
    nns = next(os.walk(neural_path))[-1]
    subs = [re.search("[0-9]*", nn).group(0) for nn in nns]
    if exclude is not None:
        subs = [s for s in subs if s not in exclude]

    # Make all of the plots.
    # We want to make an ROC and a PR plot for each class in pos_class.
    # First make sure we have a list
    if isinstance(pos_class, int):
        pos_class = [pos_class]

    if isinstance(pos_class_name, str):
        pos_class_name = [pos_class_name]

    if len(pos_class) != len(pos_class_name):
        raise ValueError("pos_class and pos_class_name must be an int/string pair, or lists of the same length.")

    n_classes = len(pos_class)
    #fig, axs = plt.subplots(nrows=n_classes, ncols=2, figsize=(20, 10*n_classes)) # vertical format
    fig, axs = plt.subplots(nrows=2, ncols=n_classes, figsize=(10*n_classes, 20)) # horizontal format

    if isinstance(label_names, dict) and (n_classes == 1):
        label_names = [label_names]
    elif isinstance(label_names, dict) and (n_classes > 1):
        raise ValueError("label_names is a dict, but there is more than one class. Either make label_names a list of dicts, or use only one class.")

    plot_data = {
            'pr': [pr_roc(
                subject_number = s,
                nn_base_name = nn_base_name,
                data_path = pickle_path,
                neural_path = neural_path,
                with_time = with_time,
                mode="pr",
                pos_class = pos_class) for s in subs],

            'roc': [pr_roc(
                subject_number = s,
                nn_base_name = nn_base_name,
                data_path = pickle_path,
                neural_path = neural_path,
                with_time = with_time,
                mode="roc",
                pos_class = pos_class) for s in subs]
            }

    print("per-class (x,y) pairs created for PR and ROC curves.")

    for j in range(n_classes):
        print("class = ", j)
        ax_roc = axs[0, j]
        ax_pr = axs[1, j]

        ax_pr.grid(True)
        ax_roc.grid(True)

        ax_pr.margins(0.01)
        ax_pr.xaxis.set_major_locator(MultipleLocator(0.1))
        ax_pr.yaxis.set_major_locator(MultipleLocator(0.1))
        ax_roc.margins(0.01)
        ax_roc.xaxis.set_major_locator(MultipleLocator(0.1))
        ax_roc.yaxis.set_major_locator(MultipleLocator(0.1))

        if isinstance(label_names, list): # No negative reactions is label_names not correct.
            if label_names[j] is not None:
                xlabel_pr = "%s recall" % label_names["positive"]
                ylabel_pr = "%s precision" % label_names["positive"]

                xlabel_roc = "%s error rate" % label_names[j]["negative"]
                ylabel_roc = "%s accuracy" % label_names[j]["positive"]
        else:
            xlabel_pr = "Recall"
            ylabel_pr = "Precision"
            xlabel_roc = "False positive rate"
            ylabel_roc = "True positive rate"

        if pos_class_name is not None:
            ax_roc.set_title(pos_class_name[j], fontsize=20)

        ax_pr.set_xlabel(xlabel_pr, fontsize=12)
        ax_pr.set_ylabel(ylabel_pr, fontsize=12)
        ax_roc.set_xlabel(xlabel_roc, fontsize=12)
        ax_roc.set_ylabel(ylabel_roc, fontsize=12)

        # Plot all of the PR curves, plus their pointwise average curve
        # plot_data is a dict like:
        # {
        #   "pr": [..., (PR curve subject s and pos_class[0], ..., PR curve pos_class[-1]), ...]
        #   "roc": [..., (ROC curve subject s and pos_class[0], ..., ROC curve pos_class[-1]), ...]
        # }
        # So, we want index [i][j] for subject i, class j
        for i in range(len(subs)):
            ax_pr.plot(
                plot_data["pr"][i][j][0], # recall
                plot_data["pr"][i][j][1], # precision
                c = 'orange',
                alpha = 0.7)
            ax_roc.plot(
                plot_data["roc"][i][j][0], # FPR
                plot_data["roc"][i][j][1], # TPR
                c = 'orange',
                alpha = 0.7)

        # Set up 'no-skill' baseline for ROC
        ax_roc.plot([0, 1], '--', c='#000000')

        # Find and Plot the pointwise average
        avg_pr = pointwise_avg([plot_data["pr"][i][j] for i in range(len(subs))])
        avg_roc = pointwise_avg([plot_data["roc"][i][j] for i in range(len(subs))])

        ax_pr.plot(avg_pr[0], avg_pr[1], c = 'blue', alpha = 1)
        ax_roc.plot(avg_roc[0], avg_roc[1], c = 'blue', alpha = 1)

    if title is not None:
        fig.suptitle(title, fontsize=20)

    if saveto is not None:
        plt.savefig(saveto, bbox_inches='tight', pad_inches=0.1)

    return (fig, axs)

def get_probabilities(data_yielder, n_classes=2, from_logits=True):

    subjects = []
    evaluations = []
    psgs = []
    pr_rocs = []
    failed = []
    succeeded = []
    for subject, sp, time, psg, nn in data_yielder:
        print(subject)
        psg = sleep_classes(psg, n_classes)

        if from_logits:
            evaluations.append(logits_to_proba(nn.predict([sp, time])))
        else:
            evaluations.append(nn.predict([sp, time]))

        subjects.append(subject)
        psgs.append(psg)

    return subjects, psgs, evaluations

#
#"""
# Displays 3x3 grid of spectrograms, sampled randomly from a
# numpy tensor, Xs, whose axis-0 cross-sections are the spectrograms.
# Each spectrogram is labeled with the axis-0 index and class label from ys.
#
# - Xs should be the spectrograms, ys the PSG labels.
# -- Uses matplotlib.pyplot.pcolormesh, so this can be
#    used for displaying random axis-0 cross-sections of numpy 3-tensors.
# - ys should be the class labels
#"""
#def sample_pics(Xs, ys = None, with_maxmin = False):
#    import random
#
#    Xs_len = Xs.shape[0]
#    if with_maxmin:
#        color_min = Xs.min()
#        color_max = Xs.max()
#    else:
#        color_min = None
#        color_max = None
#
#    # Randomly select 9 indices for plotting the colormesh of their stft.
#    # Running this cell repeatedly gives a new subset each time.
#    inds = random.sample(range(Xs_len), k=9)
#
#    # mapping dict taking pairs (i,j) to our random indices
#    # Used in for loop to plot random subset of our data.
#    mapper = {
#        (0,0) : inds[0],
#        (0,1) : inds[1],
#        (0,2) : inds[2],
#        (1,0) : inds[3],
#        (1,1) : inds[4],
#        (1,2) : inds[5],
#        (2,0) : inds[6],
#        (2,1) : inds[7],
#        (2,2) : inds[8],
#    }
#
#    fig, axs = plt.subplots(ncols=3, nrows=3, figsize=(10,10))
#
#    for i in [0,1,2]:
#        for j in [0,1,2]:
#            index = mapper[(i,j)]
#            X = Xs[index]
#            axs[i][j].pcolormesh(X, vmin=color_min, vmax=color_max)
#            title = "Index: {}".format(index)
#            if ys:
#                title += ", Class: {}".format(ys[index])
#
#            axs[i][j].set_title(title)
#    plt.tight_layout()

"""
Copyright 2020 Eric Canton (ericcanton)

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
"""


##
import json
import os

from tqdm import tqdm

with open("./data/LCCC-base_train.json", 'r', encoding = 'utf-8') as f:
    m = json.load(f)
    print(len(m))
m
##
if os.path.exists("./data/train.jsonl"):
    # 存在，则删除文件
    os.remove("./data/train.jsonl")
if os.path.exists("./data/valid.jsonl"):
    # 存在，则删除文件
    os.remove("./data/valid.jsonl")
with open("./data/train.jsonl", 'a', encoding = 'utf-8') as f1:
    with open("./data/valid.jsonl", 'a', encoding = 'utf-8') as f2:
        for i in tqdm(range(len(m))):
            if i < int(len(m) * 0.8):
                f1.write(json.dumps(m[i], ensure_ascii = False) + "\n")
            else:
                f2.write(json.dumps(m[i], ensure_ascii = False) + "\n")


from django.http import HttpResponseRedirect
from django.shortcuts import render
from django.core.files.storage import FileSystemStorage
from upload_process.controller import execute_script
from django.contrib import messages

# Imaginary function to handle an uploaded file.
# from somewhere import handle_uploaded_file

def homePage(request):
    if request.method == 'POST':
        uploaded_file = request.FILES.getlist('sentFile')
        output_name = request.POST.get('GSheet')
        fs = FileSystemStorage()
        if uploaded_file:
            for f in uploaded_file:
                fs.save(f.name, f)

            result = execute_script(output_name)
            if result:
                messages.error(request, "Sheet Not Found")

    context = {}
    return render(request, 'home.html', context)


import ssl, json
from socket import *
from urllib.parse import urlencode, quote_plus
from select import epoll, EPOLLIN, EPOLLHUP

api_key = ""

placement_score = {
	1: 300,
	2: 150,
	3: 100,
	4: 75,
	5: 60,
	6: 50,
	7: 40,
	8: 30,
	9: 20,
	10: 10
}

base_killscore = 10
base_killmultiplier = 1

# account.c5afe..bbeb #DeathDeler

# https://api.pubg.com/shards/$platform/players?filter[playerNames]=$playerName

def request(url, options={}):
	s = socket()

	#if len(options):
	#	url += '?'
	#	for key, val in options.items():
	#		url += quote_plus(key) + '=' + quote_plus(val) + '&'
	#	url = url[:-1]

	print(url)
	hostname = 'api.pubg.com'
	context = ssl.create_default_context()

	with create_connection((hostname, 443)) as sock:
		with context.wrap_socket(sock, server_hostname=hostname) as s:
			poller = epoll()
			poller.register(s.fileno(), EPOLLIN | EPOLLHUP)

			r = f'GET {url} HTTP/1.1\r\n'
			r += f'Host: {hostname}\r\n'
			r += f'Authorization: Bearer {api_key}\r\n'
			r += 'Accept: application/vnd.api+json\r\n'
			r += '\r\n'

			s.send(bytes(r, 'UTF-8'))
			response = b''
			alive = True
			got_first = False
			while alive or got_first is False:
				alive = False
				for fileno, event in poller.poll(0.25):
					got_first = True
					alive = True
					data = s.recv(8192)
					if len(data) <= 0:
						alive = False
						break
					response += data

			headers = {}
			code = 0
			header, response = response.split(b'\r\n\r\n', 1)
			for index, item in enumerate(header.split(b'\r\n')):
				if index == 0:
					_, code, _ = item.split(b' ', 2)
					code = int(code)
				else:
					key, val = item.split(b':', 1)
					headers[key.decode('UTF-8').lower().strip()] = val.decode('UTF-8').strip()

			if 'transfer-encoding' in headers and headers['transfer-encoding'] == 'chunked':
				tmp = response
				block = b''
				data = b''
				while b'\r\n' in tmp:
					length, block = tmp.split(b'\r\n',1)
					if length == b'' and block == b'0\r\n\r\n':
						break

					try:
						if int(length, 16) > len(block):
							print('[Error] Parsing length issue: {} vs reality {}'.format(int(length, 16), len(block)))
							exit(1)
					except:
						print('Error@78:', length, block[:30])
						exit(1)

					#print(block)
					#print(block[:int(length, 16)])
					data += block[:int(length, 16)]
					#print(tmp[:20], '---', block[-10:])
					tmp = tmp[int(length, 16)+len(length)+4:]

				#print(data)
				return code, headers, data
			else:
				return code, headers, response

def getMatch(gid, mode=None, custom=False):
	code, headers, response = request(f'/shards/pc-eu/matches/{gid}')
#	print(json.dumps(headers, indent=4, sort_keys=True))
	with open('test.bin', 'wb') as fh:
		fh.write(response)

	data = json.loads(response)
	with open(f'./games/{gid}.json', 'w') as fh:
		fh.write(json.dumps(data, indent=4, sort_keys=True))

	if mode:
		if not mode in data['data']['attributes']['gameMode']:
			print(f'[Notice] Game {gid} did not match a {mode} game')
			return None

	if custom:
		if not data['data']['attributes']['isCustomMatch']:
			print(f'[Notice] Game {gid} is not a custom game.')
			return None

	return data

def parse_leaderboard(game_data):
	if not game_data: return None, None
	#print(json.dumps(game_data['included'], indent=4, sort_keys=True))
	players = {}
	teams = {}
	for player in game_data['included']:
#		try:
		if player['type'] == 'participant':
			players[player['id']] = player
			#print('Name: {}, Kills: {}, Placement: {}'.format(player['attributes']['stats']['name'],
			#												  player['attributes']['stats']['kills'],
			#												  player['attributes']['stats']['winPlace']))
		elif player['type'] == 'roster':
			teams[player['id']] = player
#		except:
#			print(json.dumps(player, indent=4, sort_keys=True))
#			exit(1)

	return players, teams

code, headers, response = request(f'/shards/pc-eu/players?filter[playerNames]=DeathDeler')
data = json.loads(response)

#{'type': 'match', 'id': '1f04e9db-968a-4179-9358-15a37370d5fc'}
#{'type': 'match', 'id': 'b5ccec6d-976e-4099-842d-24b278123736'}
#{'type': 'match', 'id': '2daaf7f1-6d92-4ef7-83f7-4fa2dae70b56'}
#{'type': 'match', 'id': '1024424a-7487-4387-94b8-b00b38da9606'}
for match in data['data'][0]['relationships']['matches']['data']:
	#getMatch(match['id'])
	match_data = getMatch(match['id'], mode='duo')

	players, teams = parse_leaderboard(match_data)#, custom=True))
	if not players or not teams:
		print(f' - {match["id"]}: Skipped...')
		continue

	with open(f'./stats/{match["id"]}.csv', 'w') as score:
		for team_id, team in teams.items():
			new_struct = []
			team_score = {'kills' : 0, 'winPlace' : 0, 'points' : 0}
			for player in team['relationships']['participants']['data']:
				if not player['id'] in players:
					print('[Error] Malformed player struct')

				team_score['kills'] += players[player['id']]['attributes']['stats']['kills']
				team_score['winPlace'] = players[player['id']]['attributes']['stats']['winPlace']
				new_struct.append(players[player['id']])

			team['relationships']['participants']['data'] = new_struct
			base_killscore
			if team_score['winPlace'] in placement_score:
				team_score['points'] += placement_score[team_score['winPlace']]
			if team_score['kills']:
				final_score = 0
				worth = base_killscore
				for i in range(team_score['kills']):
					final_score += worth
					worth += base_killmultiplier

				team_score['points'] += final_score
			team['ninjat_score'] = team_score

			tmp = team['relationships']['participants']['data']
			if len(tmp) == 1:
				tmp_players = [tmp[0]['attributes']['stats']['name']]
			else:
				tmp_players = [tmp[0]['attributes']['stats']['name'], tmp[1]['attributes']['stats']['name']]
			print('Team {} scored {} kills and {} placement. ({}).'.format(team_id,
						team['ninjat_score']['kills'],
						team['ninjat_score']['winPlace'],
						' & '.join(tmp_players)))

			score.write('{},{},{},{},{},{},{}\r\n'.format(match_data['data']['attributes']['createdAt'],
												match_data['data']['attributes']['titleId'],
												team_id,
												team['ninjat_score']['kills'],
												team['ninjat_score']['winPlace'],
												team['ninjat_score']['points'],
												' & '.join(sorted(tmp_players))))

#	print(json.dumps(teams, indent=4, sort_keys=True))
	#parse_leaderboard(getMatch('1a1033c4-dfb1-48ad-aa79-0e5610ac1098'))
	exit(1)

#print(json.dumps(, indent=4, sort_keys=True))
#print(json.dumps(data['data'][0]['relationships']['matches'], indent=4, sort_keys=True))


import logging

from pages.utils import rand_str, log_decorator

class TestChat:
    log = logging.getLogger("[ChatPage]")

    @log_decorator
    def test_chat(self, hello_page):
        """
            Set up:
                Sign-up\Sign-in as a user
            Steps:
                Open chat
                Send message
                Verify message
                Send one more message
                Verify messages
                """
        chat = hello_page.header.open_chat()
        message_1 = rand_str(10)
        chat.send_message(message_1)
        chat.verify_messages([message_1])
        message_2 = rand_str(14)
        chat.send_message(message_2)
        chat.verify_messages([message_1, message_2])


import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import time
from torchvision import models, utils
from torchvision import datasets
from torchvision import transforms
from collections import defaultdict
from tqdm import tqdm
import seaborn as sns
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from torch.utils.data import DataLoader, SubsetRandomSampler
import os
from sklearn.metrics import classification_report, confusion_matrix
# from torch.utils.tensorboard import SummaryWriter
from torch.cuda.amp import autocast, GradScaler
import warnings
import random

import cv2
from PIL import Image

warnings.filterwarnings('ignore')

if torch.cuda.is_available():
    DEVICE = torch.device('cuda:0')
    torch.backends.cudnn.benchmark = True
    torch.cuda.manual_seed(42)
    torch.cuda.empty_cache()
    # memory_summary = torch.cuda.memory_summary(device='cuda', abbreviated=False)
    # print(memory_summary)
else:
    DEVICE = torch.device('cpu')
    torch.manual_seed(42)

np.random.seed(100)

def calculate_accuracy(y_pred, y_test):
    y_pred_softmax = torch.log_softmax(y_pred, dim=1)
    _, y_pred_tags = torch.max(y_pred_softmax, dim=1)
    correct_pred = (y_pred_tags == y_test).float()
    acc = correct_pred.sum() / len(correct_pred)
    acc = torch.round(acc * 100)
    return acc

class MetricMonitor:
    def __init__(self, float_precision=3):
        self.float_precision = float_precision
        self.reset()

    def reset(self):
        self.metrics = defaultdict(lambda: {'val': 0, 'count': 0, 'avg': 0})

    def update(self, metric_name, val):
        metric = self.metrics[metric_name]

        metric['val'] += val
        metric['count'] += 1
        metric['avg'] = metric['val'] / metric['count']

    def __str__(self):
        return ' | '.join(
            [
                '{metric_name}: {avg:.{float_precision}f}'.format(
                    metric_name=metric_name, avg=metric['avg'], float_precision=self.float_precision
                )
                for (metric_name, metric) in self.metrics.items()
            ]
        )

def train(train_loader, net, criterion, optimizer, epoch, scaler, writer=None):
    metric_monitor = MetricMonitor()
    net.train()
    stream = tqdm(train_loader)
    train_accuracy = []
    train_loss = []
    for batch_index, (images, labels) in enumerate(stream, start=1):
        images = images.to(DEVICE, non_blocking=True)  # tensor size [batch_size, 3, img_w, img_h]
        labels = labels.to(DEVICE, non_blocking=True)  # tensor size [batch_size, ]
        # dataset visualization using tensorboard
        # img_grid = utils.make_grid(images)
        # writer.add_image('train_images', img_grid)
        # writer.add_graph(net, images)

        optimizer.zero_grad()
        with autocast():
            outputs = net(images)  # tensor size [batch_size, len(classes)]
            loss = criterion(outputs, labels)  # calculate loss
            accuracy = calculate_accuracy(outputs, labels)
            metric_monitor.update('Loss', loss.item())
            metric_monitor.update('Accuracy', accuracy.item())
            train_accuracy.append(accuracy.item())
            train_loss.append(loss.item())

        scaler.scale(loss).backward()
        scaler.step(optimizer)
        scaler.update()

        stream.set_description(
            'Epoch: {epoch}. Train.      {metric_monitor}'.format(epoch=epoch, metric_monitor=metric_monitor)
        )
    return train_loss, train_accuracy, scaler

def validate(val_loader, net, criterion, epoch):
    metric_monitor = MetricMonitor()
    net.eval()
    stream = tqdm(val_loader)
    val_accuracy = []
    val_loss = []
    with torch.no_grad():
        for i, (images, labels) in enumerate(stream, start=1):
            images = images.to(DEVICE, non_blocking=True)
            labels = labels.to(DEVICE, non_blocking=True)
            outputs = net(images)
            loss = criterion(outputs, labels)
            accuracy = calculate_accuracy(outputs, labels)

            metric_monitor.update('Loss', loss.item())
            metric_monitor.update('Accuracy', accuracy.item())
            val_accuracy.append(accuracy.item())
            val_loss.append(loss.item())
            stream.set_description(
                'Epoch: {epoch}. Validation. {metric_monitor}'.format(epoch=epoch, metric_monitor=metric_monitor)
            )
    return val_loss, val_accuracy

def test(test_loader, net, idx2class):
    metric_monitor = MetricMonitor()
    net.eval()
    stream = tqdm(test_loader)
    y_pred_list = []
    y_true_list = []
    test_accuracy = []
    with torch.no_grad():
        for x_batch, y_batch in stream:
            x_batch = x_batch.to(DEVICE, non_blocking=True)
            y_batch = y_batch.to(DEVICE, non_blocking=True)
            y_test_pred = net(x_batch)
            _, y_pred_tag = torch.max(y_test_pred, dim=1)
            y_pred_list.append(y_pred_tag.cpu().numpy().tolist())   # list shape: [_, batch_size]
            y_true_list.append(y_batch.cpu().numpy().tolist())
            accuracy = calculate_accuracy(y_test_pred, y_batch)
            metric_monitor.update('Accuracy', accuracy.item())
            test_accuracy.append(accuracy.item())
            stream.set_description(
                'Test.  {metric_monitor}'.format(metric_monitor=metric_monitor)
            )

    y_pred_array = []
    y_true_array = []
    for i in range(len(y_pred_list)):
        y_pred_array += y_pred_list[i]  # 1-dimension
        y_true_array += y_true_list[i]
    # print('y_pred_list: ', y_pred_array)
    # print('y_true_list: ', y_true_array)
    target_names = [v for k, v in idx2class.items()]

    print('classification_report')
    print(classification_report(y_true_array, y_pred_array, target_names=target_names))
    print('confusion_matrix')
    print(confusion_matrix(y_true_array, y_pred_array))
    # confusion_matrix_df = pd.DataFrame(confusion_matrix(y_true_array, y_pred_array)).rename(columns=idx2class, index=idx2class)
    # fig, ax = plt.subplots(figsize=(20, 18))
    # sns.heatmap(confusion_matrix_df, annot=True, ax=ax)
    # plt.savefig('Heatmap.eps', dpi=600, format='eps')
    # plt.show()
    return test_accuracy

def get_class_distribution(dataset, idx2class):
    count_dict = {k: 0 for k, v in dataset.class_to_idx.items()}
    for _, label_id in dataset:
        label = idx2class[label_id]
        count_dict[label] += 1
    print('count_dict: ', count_dict)
    return count_dict

def get_class_distribution_loaders(dataloader_obj, dataset_obj, idx2class):
    count_dict = {k: 0 for k, v in dataset_obj.class_to_idx.items()}
    if dataloader_obj.batch_size == 1:
        for _, label_id in dataloader_obj:
            y_idx = label_id.item()
            y_lbl = idx2class[y_idx]
            count_dict[str(y_lbl)] += 1
    else:
        for _, label_id in dataloader_obj:
            for idx in label_id:
                y_idx = idx.item()
                y_lbl = idx2class[y_idx]
                count_dict[str(y_lbl)] += 1
    return count_dict

def plot_from_dict(dict_obj, plot_title, legend_switch, **kwargs):
    data = pd.DataFrame.from_dict([dict_obj]).melt().rename(columns={'variable': 'stu_name', 'value': 'img_num'})
    if legend_switch:
        plot = sns.barplot(data=data, x='stu_name', y='img_num', hue='stu_name', dodge=False, **kwargs)
        plt.legend(bbox_to_anchor=(1.01, 1), loc='upper left', borderaxespad=0)
    else:
        plot = sns.barplot(data=data, x='stu_name', y='img_num', dodge=False, **kwargs)
    plot.set_xticklabels(plot.get_xticklabels(), rotation=45)
    plot.set_title(plot_title, fontsize=20)
    plot.set_ylabel('image number', fontsize=20)
    return plot

def loadtraindata(path):
    filename = path.split('/')[-1].split('_')[-1]
    print('path:', path)
    print('filename:', filename)

    image_transforms = {
        'train': transforms.Compose([
            transforms.RandomHorizontalFlip(p=0.5),
            transforms.RandomRotation(degrees=(-90, 90)),
            transforms.RandomVerticalFlip(p=0.5),
            transforms.Resize((224, 224)),
            transforms.ToTensor(),
            transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
        ]),
        'test': transforms.Compose([
            transforms.Resize((224, 224)),
            transforms.ToTensor(),
            transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
        ])
    }

    # method 1 for construct training dataset and validate dataset
    # draw whole dataset class distribution
    if os.path.exists(path + '/.DS_Store'):
        os.remove(path + '/.DS_Store')
    dataset = datasets.ImageFolder(path, transform=image_transforms['train'])
    idx2class = {v: k for k, v in dataset.class_to_idx.items()}
    # plt.figure(figsize=(18, 8))
    # plot_from_dict(get_class_distribution(dataset, idx2class), plot_title='Entire Dataset', legend_switch=1)
    # plt.savefig('Entire Dataset Distribution.eps', dpi=600, format='eps')
    print('classes: ', dataset.classes)  # all classes in dataset
    print('number of classes: ', len(dataset.classes), dataset.class_to_idx)
    print('dataset: ', type(dataset), dataset)
    dataset_size = len(dataset)
    dataset_indices = list(range(dataset_size))
    np.random.shuffle(dataset_indices)
    val_split_index = int(np.floor(0.7 * dataset_size))  # train 70%, val 20%, test 10%
    test_split_index = int(np.floor(0.9 * dataset_size))
    train_idx, val_idx, test_idx = dataset_indices[:val_split_index], dataset_indices[val_split_index:test_split_index], dataset_indices[test_split_index:]
    print('len: ', len(train_idx), len(val_idx), len(test_idx))
    train_sampler = SubsetRandomSampler(train_idx)
    val_sampler = SubsetRandomSampler(val_idx)
    test_sampler = SubsetRandomSampler(test_idx)
    # batch_size: number of iteration in each time
    # when use SubsetRandomSampler cannot use shuffle, shuffle: whether random sort in each time
    train_loader = DataLoader(dataset=dataset, batch_size=20, shuffle=False, sampler=train_sampler, num_workers=8, pin_memory=True)
    val_loader = DataLoader(dataset=dataset, batch_size=20, shuffle=False, sampler=val_sampler, num_workers=8, pin_memory=True)
    test_loader = DataLoader(dataset=dataset, batch_size=20, shuffle=False, sampler=test_sampler, num_workers=8, pin_memory=True)

    '''
    use new test dataset to test model accuracy, need to delete finally
    '''
    # test_dataset = 'dataset_test'
    # test_dataset = datasets.ImageFolder(test_dataset, transform=image_transforms['test'])
    # dataset_size = len(test_dataset)
    # dataset_indices = list(range(dataset_size))
    # np.random.shuffle(dataset_indices)
    # test_split_index = int(np.floor(0.1 * dataset_size))
    # test_idx = dataset_indices[:test_split_index]
    # test_sampler = SubsetRandomSampler(test_idx)
    # idx2class_test = {v: k for k, v in test_dataset.class_to_idx.items()}
    # test_loader = DataLoader(dataset=test_dataset, batch_size=20, shuffle=False, sampler=test_sampler, num_workers=8, pin_memory=True)

    print('len of training set: ', len(train_loader), train_loader)
    print('len of validation set: ', len(val_loader), val_loader)
    print('len of test set: ', len(test_loader), test_loader)

    # draw train, val, test dataset class distribution
    # fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(24, 8))
    # plot_from_dict(get_class_distribution_loaders(train_loader, dataset, idx2class), plot_title='Train Set', legend_switch=0, ax=axes[0])
    # plot_from_dict(get_class_distribution_loaders(val_loader, dataset, idx2class), plot_title='Val Set', legend_switch=1, ax=axes[1])
    # plt.savefig('Train and Validate Distribution.eps', dpi=600, format='eps')
    # plt.figure(figsize=(18, 8))
    # plot_from_dict(get_class_distribution_loaders(test_loader, dataset, idx2class), plot_title='Test Set', legend_switch=1)
    # plt.savefig('Test Distribution.eps', dpi=600, format='eps')
    # plt.show()

    # method 2 for construct training dataset and validate dataset
    # draw whole dataset class distribution
    # if os.path.exists(path + '/.DS_Store'):
    #     os.remove(path + '/.DS_Store')
    # dataset = datasets.ImageFolder(path)
    # idx2class = {v: k for k, v in dataset.class_to_idx.items()}
    # plt.figure(figsize=(18, 8))
    # plot_from_dict(get_class_distribution(dataset, idx2class), plot_title='Entire Dataset', legend_switch=1)
    # plt.savefig('Entire Dataset Distribution.eps', dpi=600, format='eps')
    # stu_class = os.listdir(path)
    # train_dataset = path + '/train'
    # val_dataset = path + '/validate'
    # test_dataset = path + '/test'
    # if os.path.exists(train_dataset):
    #     shutil.rmtree(train_dataset)
    # if os.path.exists(val_dataset):
    #     shutil.rmtree(val_dataset)
    # if os.path.exists(test_dataset):
    #     shutil.rmtree(test_dataset)
    # os.makedirs(train_dataset)
    # os.makedirs(val_dataset)
    # os.makedirs(test_dataset)
    # for i in range(len(stu_class)):
    #     file_path_one_stu = path + '/' + stu_class[i]
    #     file_list_one_stu = os.listdir(file_path_one_stu)
    #
    #     file_list_one_stu = sorted(file_list_one_stu)
    #     np.random.shuffle(file_list_one_stu)
    #     # print(file_list_one_stu)
    #     val_split_index = int(np.floor(0.7 * len(file_list_one_stu)))  # train 70%, val 20%, test 10%
    #     test_split_index = int(np.floor(0.9 * len(file_list_one_stu)))
    #     train_idx, val_idx, test_idx = file_list_one_stu[:val_split_index], file_list_one_stu[val_split_index:test_split_index], file_list_one_stu[test_split_index:]
    #     os.makedirs(train_dataset + '/' + stu_class[i])
    #     os.makedirs(val_dataset + '/' + stu_class[i])
    #     os.makedirs(test_dataset + '/' + stu_class[i])
    #     for file in os.listdir(file_path_one_stu):
    #         if file in train_idx:
    #             shutil.move(file_path_one_stu + '/' + file, train_dataset + '/' + stu_class[i])
    #         elif file in val_idx:
    #             shutil.move(file_path_one_stu + '/' + file, val_dataset + '/' + stu_class[i])
    #         else:
    #             shutil.move(file_path_one_stu + '/' + file, test_dataset + '/' + stu_class[i])
    #     shutil.rmtree(file_path_one_stu)
    #
    # train_dataset = datasets.ImageFolder(train_dataset, transform=image_transforms['train'])
    # val_dataset = datasets.ImageFolder(val_dataset, transform=image_transforms['train'])
    # test_dataset = datasets.ImageFolder(test_dataset, transform=image_transforms['test'])
    # train_loader = DataLoader(dataset=train_dataset, batch_size=20, shuffle=True, num_workers=8, pin_memory=True)
    # val_loader = DataLoader(dataset=val_dataset, batch_size=20, shuffle=True, num_workers=8, pin_memory=True)
    # test_loader = DataLoader(dataset=test_dataset, batch_size=20, shuffle=True, num_workers=8, pin_memory=True)
    #
    # print('len of training set: ', len(train_loader), train_loader)
    # print('len of validation set: ', len(val_loader), val_loader)
    # print('len of test set: ', len(test_loader), test_loader)
    #
    # # draw train, val, test dataset class distribution
    # fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(24, 8))
    # plot_from_dict(get_class_distribution(train_dataset, idx2class), plot_title='Train Set', legend_switch=0, ax=axes[0])
    # plot_from_dict(get_class_distribution(val_dataset, idx2class), plot_title='Val Set', legend_switch=1, ax=axes[1])
    # plt.savefig('Train and Validate Distribution.eps', dpi=600, format='eps')
    # plt.figure(figsize=(18, 8))
    # plot_from_dict(get_class_distribution(test_dataset, idx2class), plot_title='Test Set', legend_switch=1)
    # plt.savefig('Test Distribution.eps', dpi=600, format='eps')
    # plt.show()

    return train_loader, val_loader, test_loader, filename, tuple(dataset.classes), idx2class

def loadtestdata(test_dataset='dataset_test'):
    filename = test_dataset.split('/')[-1].split('_')[-1]
    print('path:', test_dataset)
    print('filename:', filename)

    image_transforms = {
        'test': transforms.Compose([
            transforms.Resize((224, 224)),
            transforms.ToTensor(),
            transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
        ])
    }

    # draw whole dataset class distribution
    if os.path.exists(test_dataset + '/.DS_Store'):
        os.remove(test_dataset + '/.DS_Store')
    dataset = datasets.ImageFolder(test_dataset, transform=image_transforms['test'])
    idx2class = {v: k for k, v in dataset.class_to_idx.items()}
    # plt.figure(figsize=(18, 8))
    # plot_from_dict(get_class_distribution(dataset, idx2class), plot_title='Entire Dataset', legend_switch=1)
    # plt.savefig('Entire Dataset Distribution.eps', dpi=600, format='eps')
    print('classes: ', dataset.classes)  # all classes in dataset
    print('number of classes: ', len(dataset.classes), dataset.class_to_idx)
    print('dataset: ', type(dataset), dataset)

    '''
    use new test dataset to test model accuracy, need to delete finally
    '''
    test_ratio = 1
    test_dataset = datasets.ImageFolder(test_dataset, transform=image_transforms['test'])
    dataset_size = len(test_dataset)
    dataset_indices = list(range(dataset_size))
    np.random.shuffle(dataset_indices)
    test_split_index = int(np.floor(test_ratio * dataset_size))
    test_idx = dataset_indices[:test_split_index]
    # test_sampler = SubsetRandomSampler(test_idx)
    # batch_size: number of iteration in each time
    # when use SubsetRandomSampler cannot use shuffle, shuffle: whether random sort in each time
    test_loader = DataLoader(dataset=test_dataset, batch_size=20, shuffle=False, num_workers=8, pin_memory=True)

    print('len of test set: ', len(test_loader), test_loader)

    return test_loader, filename, tuple(dataset.classes), idx2class

class Net(nn.Module):  # define net, which extends torch.nn.Module
    def __init__(self, class_num):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(3, 6, 5)  # convolution layer
        self.pool = nn.MaxPool2d(2, 2)  # pooling layer
        self.conv2 = nn.Conv2d(6, 16, 5)  # convolution layer
        self.fc1 = nn.Linear(16 * 5 * 5, 120)  # fully connected layer
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, class_num)  # output is class_num, class_num is the number of class in dataset

    def forward(self, x):  # feed forward

        x = self.pool(F.relu(self.conv1(x)))  # F is torch.nn.functional
        x = self.pool(F.relu(self.conv2(x)))
        x = x.view(x.shape[0],
                   -1)  # .view( ) is a method tensor, which automatically change tensor size but elements number not change

        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x

def trainandsave(path, epoches):
    train_loader, validate_loader, test_loader, filename, classes, idx2class = loadtraindata(path)

    # myNet
    # net = Net(len(classes))

    # alexNet
    # net = models.alexnet(pretrained=True, )
    # net.classifier[6] = nn.Linear(4096, len(classes))

    # googleNet:
    # net = models.googlenet(pretrained=True)
    # net.fc = nn.Linear(1024, len(classes))

    # resNet18
    # net = models.resnet18(pretrained=True, )
    # net.fc = nn.Linear(512 * models.resnet.BasicBlock.expansion, len(classes))

    # resNet50
    net = models.resnet50(pretrained=True, )
    net.fc = nn.Linear(512 * models.resnet.Bottleneck.expansion, len(classes))

    # resNet101
    # net = models.resnet101(pretrained=True, )
    # net.fc = nn.Linear(512 * models.resnet.Bottleneck.expansion, len(classes))

    # resnet152
    # net = models.resnet152(pretrained=True, )
    # net.fc = nn.Linear(512 * models.resnet.Bottleneck.expansion, len(classes))

    # densenet121
    # net = models.densenet121(pretrained=True, )
    # net.classifier = nn.Linear(1024, len(classes))

    # densenet161
    # net = models.densenet161(pretrained=True, )
    # net.classifier = nn.Linear(2208, len(classes))

    # vgg11 CUDA out of memory
    # net = models.vgg11(pretrained=True, )
    # net.classifier[6] = nn.Linear(4096, len(classes))

    # vgg11_bn CUDA out of memory
    # net = models.vgg11_bn(pretrained=True, )
    # net.classifier[6] = nn.Linear(4096, len(classes))

    # mobilenet_v3
    # net = models.mobilenet_v3_small(pretrained=False, num_classes=len(classes), )

    # net = models.mobilenet_v3_small(pretrained=True, )
    # net.classifier[3] = nn.Linear(1024, len(classes))

    # squeezenet1_1
    # net = models.squeezenet1_1(pretrained=True, )
    # net.classifier[1] = nn.Conv2d(512, len(classes), kernel_size=(1,1), stride=(1,1))

    # inception_v3
    # net = models.inception_v3(pretrained=True, aux_logits=True)
    # net.fc = nn.Linear(2048, len(classes))
    # net.AuxLogits = models.inception.InceptionAux(768, len(classes))

    # print(net)
    net.to(DEVICE)
    optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)  # learning rate=0.001
    criterion = nn.CrossEntropyLoss().to(DEVICE)  # loss function
    scaler = GradScaler()  # automatic mixed precision

    # training part
    for epoch in range(epoches):  # 10 epoch
        # each epoch train all images, so total train 10 times
        # writer = SummaryWriter('runs/iExam_20220127_0518_1')
        # train_loss, train_accuracy,scaler = train(train_loader, net, criterion, optimizer, epoch, scaler, writer)
        train_loss, train_accuracy, scaler = train(train_loader, net, criterion, optimizer, epoch, scaler)
        val_loss, val_accuracy = validate(validate_loader, net, criterion, epoch)

        # test_accuracy = test(test_loader, net, idx2class)
        # writer.add_scalar('loss/train', np.asarray(np.mean(train_loss)), epoch)
        # writer.add_scalar('acc/train', np.asarray(np.mean(train_accuracy)), epoch)
        # writer.add_scalar('loss/val', np.asarray(np.mean(val_loss)), epoch)
        # writer.add_scalar('acc/val', np.asarray(np.mean(val_accuracy)), epoch)

        textfile = open('tmp.txt', 'a')
        textfile.write('epoch_{0}_train_loss = ['.format(epoch))
        string = ', '.join(str(item) for item in train_loss)
        textfile.write(string + ']\n')
        textfile.write('epoch_{0}_train_accuracy = ['.format(epoch))
        string = ', '.join(str(item) for item in train_accuracy)
        textfile.write(string + ']\n')
        textfile.write('epoch_{0}_val_loss = ['.format(epoch))
        string = ', '.join(str(item) for item in val_loss)
        textfile.write(string + ']\n')
        textfile.write('epoch_{0}_val_accuracy = ['.format(epoch))
        string = ', '.join(str(item) for item in val_accuracy)
        textfile.write(string + ']\n')
        textfile.close()

    print('Finished Training')
    test_accuracy = test(test_loader, net, idx2class)
    # print('test accuracy:', test_accuracy)
    print('Finished Test')
    # textfile = open('tmp.txt', 'a')
    # textfile.write('Test_accuracy = [')
    # string = ', '.join(str(item) for item in test_accuracy)
    # textfile.write(string + ']\n')
    # textfile.close()

    # save net
    # torch.save(net, 'net_'+filename+'.pkl')  # save structure and parameter
    file_name = 'net_params_resnet50_Pretrain_10epoch_224x224_' + filename + '.pth'
    torch.save(net.state_dict(), file_name)  # only save parameter

    return file_name, classes, net

def another_dataset_test(net_path, test_dataset='dataset_test'):
    test_loader, filename, classes, idx2class = loadtestdata(test_dataset)

    # googleNet:
    # net = models.googlenet(pretrained=True)
    # net.fc = nn.Linear(1024, len(classes))

    # resNet18
    # net = models.resnet18(pretrained=True, )
    # net.fc = nn.Linear(512 * models.resnet.BasicBlock.expansion, len(classes))

    # resNet50
    net = models.resnet50(pretrained=True, )
    net.fc = nn.Linear(512 * models.resnet.Bottleneck.expansion, len(classes))

    # print(net)
    net.load_state_dict(torch.load(net_path))
    net.to(DEVICE)
    test_accuracy = test(test_loader, net, idx2class)
    # print('test accuracy:', test_accuracy)
    print('Finished Test')
    textfile = open('tmp.txt', 'a')
    textfile.write('Test_accuracy = [')
    string = ', '.join(str(item) for item in test_accuracy)
    textfile.write(string + ']\n')
    textfile.close()

def image_test(net_path, classes):
    # googleNet:
    # net = models.googlenet(pretrained=True)
    # net.fc = nn.Linear(1024, len(classes))

    # resNet18
    # net = models.resnet18(pretrained=True, )
    # net.fc = nn.Linear(512 * models.resnet.BasicBlock.expansion, len(classes))

    # resNet50
    net = models.resnet50(pretrained=True, )
    net.fc = nn.Linear(512 * models.resnet.Bottleneck.expansion, len(classes))

    # alexNet
    # net = models.alexnet(pretrained=True, )
    # net.classifier[6] = nn.Linear(4096, len(classes))

    net.load_state_dict(torch.load(net_path))
    net.to(DEVICE)
    net.eval()

    data_transform = transforms.Compose([
        transforms.Resize((224, 224)),  # reszie image to 224*224
        transforms.ToTensor(),  # each pixel to tensor
        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
    ])
    img_pred_lst = []
    img_pred_score = []
    metric_monitor = MetricMonitor()
    print('classes:',classes)
    stream = tqdm(image_sampler())
    for name in stream:
        image = cv2.imread(name)
        image = Image.fromarray(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
        # image.show()
        image = data_transform(image)  # change PIL image to tensor
        # print('before:', image.shape, image)
        image = image.view(-1, 3, 224, 224)  # change 3-dimensional to 4-dimensional for input
        # print('after:',image.shape, image)

        output = net(image.to(DEVICE, non_blocking=True))
        y_pred_softmax = torch.log_softmax(output, dim=1)
        _, y_pred_tag = torch.max(y_pred_softmax, dim=1)
        tmp = y_pred_tag.cpu().detach().numpy()
        if classes[tmp[0]] != name.split('\\')[1].split('_')[0]:
            img_pred_score.append(0)
            metric_monitor.update('Accuracy', 0)
        else:
            img_pred_score.append(1)
            metric_monitor.update('Accuracy', 1)
        img_pred_lst.append(classes[tmp[0]])
        stream.set_description(
            'Test.  {metric_monitor}'.format(metric_monitor=metric_monitor)
        )

            # print(y_pred_tag.cpu().detach().numpy())
#     print(img_pred_lst)
    img_compare = zip(img_pred_lst, stream)
    # print('img_compare', tuple(img_compare))
    print('sum: ', sum(img_pred_score), 'len is: ', len(img_pred_score))

def image_sampler(image_path='dataset_fusion'):
    img_path_lst = []
    return_lst = []
    for root, dirs, files in os.walk(image_path):
        for name in files:
            img_path_lst.append(os.path.join(root, name))

    for num in range(len(img_path_lst)//5):
        while True:
            tmp = random.randint(0, len(img_path_lst))
            if img_path_lst[tmp] in return_lst:
                continue
            else:
                return_lst.append(img_path_lst[tmp])
                break
    # print(return_lst)
    return return_lst

if __name__ == '__main__':
    path = 'marked_image_5min'
    before = time.asctime(time.localtime(time.time()))
    print(before)
    # trainandsave(path, epoches=10)
    after = time.asctime(time.localtime(time.time()))
    print(after)


from tests.base_test import BaseTest
from views.bus import update_live_locations, update_services_stops, delete_services_stops
from database import db_session
from models.vehicle import Vehicle
from models.stop import Stop
from models.service import Service
import json

class BusTest(BaseTest):

    def create_dummy_data(self):
        # Create dummy vehicle
        test_vehicle = Vehicle()
        test_vehicle.vehicle_id = 5
        test_vehicle.destination = "North Pole"
        test_vehicle.speed = 250
        test_vehicle.heading = 10
        test_vehicle.latitude = 1.111111
        test_vehicle.longitude = 1.22222
        test_vehicle.service_name = "xxx"
        db_session.add(test_vehicle)

        # Create dummy stop
        test_stop = Stop()
        test_stop.name = "North Pole"
        test_stop.latitude = 1.22222
        test_stop.longitude = 1.3333
        db_session.add(test_stop)

        # Create dummy service
        test_service = Service()
        test_service.name = "xxx"
        test_service.description = "best service 2014"
        test_service.service_type = "DAY"
        test_service.route = "[{'destination':'Gyle Centre','points':[{'latitude':55.93879,'longitude':-3.104563},{'latitude':55.93862,'longitude':-3.105503}]}]"
        test_service.stops.append(test_stop)
        db_session.add(test_service)

        db_session.commit()
        db_session.flush()

    def setUp(self):
        # Keep the original setUp functionality as well
        super(BusTest, self).setUp()
        self.create_dummy_data()

    def test_get_vehicles(self):
        # First request triggers the update of the database -> vehicle gone
        response = self.app.post('/get/vehicles', data={"services[]": ["10"]})
        self.assertEquals(response.status_code, 200)

        # Need to make vehicle again, and request for it, so that data won't need to be updated yet
        self.create_dummy_data()
        response = self.app.post('/get/vehicles', data={"services[]": ["xxx"]})
        self.assertEquals(response.status_code, 200)

        response_json = response.data
        response_data = json.loads(response_json)
        vehicles = response_data["vehicles"]
        self.assertEquals(len(vehicles), 1)
        test_vehicle = vehicles[0]
        self.assertEquals(test_vehicle["speed"], 250)

    def test_get_stop_names(self):
        # First request triggers the update of the database -> North Pole gone
        response = self.app.post('/get/stop_names', data={"stop_name": "Nor"})
        self.assertEquals(response.status_code, 200)

        # Need to make North Pole again, and request for it, so that data won't need to be updated yet
        self.create_dummy_data()
        response = self.app.post('/get/stop_names', data={"stop_name": "North Pole"})
        self.assertEquals(response.status_code, 200)
        response_json = response.data
        response_data = json.loads(response_json)
        stop_names = response_data["stop_names"]
        self.assertEquals(len(stop_names), 1)
        test_stop = stop_names[0].decode('utf-8')
        self.assertEquals(test_stop, "North Pole")

    def test_get_route_and_stops(self):
        # First request triggers the update of the database -> North Pole gone
        response = self.app.post('/get/route_and_stops', data={"service": "xxx"})
        self.assertEquals(response.status_code, 200)

        # Need to make North Pole again, and request for it, so that data won't need to be updated yet
        self.create_dummy_data()
        response = self.app.post('/get/route_and_stops', data={"service": "xxx"})
        self.assertEquals(response.status_code, 200)
        response_json = response.data
        response_data = json.loads(response_json)
        service = response_data["service"]
        self.assertEquals(service["description"], "best service 2014")

    def test_delete_service_stop(self):

        # Get all services initially
        q = db_session.query(Service)
        services = q.filter(Service.name == "xxx").all()
        self.assertEquals(len(services), 1)

        # Get all stops to test the association as well
        q = db_session.query(Stop)
        stops = q.filter(Stop.services.contains(services[0])).all()
        self.assertEquals(len(stops), 1)

        # Test the delete function for stops and services
        delete_services_stops()

        # Get all services initially
        q = db_session.query(Service)
        services = q.filter(Service.name == "xxx").all()
        self.assertEquals(len(services), 0)

        # Get all stops to test the association as well
        q = db_session.query(Stop)
        stops = q.filter().all()
        self.assertEquals(len(stops), 0)

    def test_update_service_stops(self):

        # Clear the tables initially then update it with all new data. This might take a few seconds.
        delete_services_stops()
        update_services_stops()

        # Assume that service 5 exists
        q = db_session.query(Service)
        services = q.filter(Service.name == "5").all()
        self.assertEqual(len(services), 1)

        # Assume that every service should have more than 1 stops
        q = db_session.query(Stop)
        stops = q.filter(Stop.services.contains(services[0])).all()
        self.assertGreater(len(stops), 1)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#2-10-3-7.py
import base64
import random
import math

def enCodeBase64(myStr):
    return base64.b64encode(myStr.encode("utf8"))

def deCodeBase64(myStr):
    return base64.b64decode(myStr).decode("utf8")

strTxt="""
1978年出现了著名的RSA算法，它通常是先生成一对RSA密钥，
其中之一是保密密钥，由用户保存；
另一个为公开密钥，可对外公开，甚至可在网络服务器中注册。
为提高保密强度，RSA密钥至少为500位长，一般推荐使用1024位。
这就使加密的计算量很大。
"""

def getPrimeNumbers(minNum,maxNum):

    for n in range(minNum,maxNum):
        for i in range(2,int(math.sqrt(n))+1):
            if  (n%i==0):
                break
        else:
            pN=n
            break
    return pN

def getGreatestCommonDivisor(a,b):
    while a != 0:
        a, b = b % a, a
    return b

def getAnotherGcd(x):
    minN=random.randint(100,x)
    for i in range(minN,x):
        print("=",end="")
        if (getGreatestCommonDivisor(i,x)==1):
            print(f">\n找到互质数:{i}")
            break
    return i

def getModuloInverse(e,r):
    d=r-1
    for i in range(2,r):
        if i%100000==0: print(f"{i/r*100}%")
        if ((e*i)%r==1):
            d=i
            print("\n找到模逆元")
            break
    return d

def getSegBytes(strTxt,maxMesLen):
    encodeBytesStr=enCodeBase64(strTxt)
    resultSegBytes=[len(encodeBytesStr)]
    for i in range(0,len(encodeBytesStr),maxMesLen):
        segBytes=encodeBytesStr[i:i+maxMesLen]
        resultSegBytes.append(int.from_bytes(segBytes,'little'))
    return resultSegBytes

def putSegBytes(segBytes,maxMesLen):
    strBytes=[]
    strLen=segBytes[0]
    for data in segBytes[1:]:
        strBytes.append(data.to_bytes(length=maxMesLen,byteorder='little'))

    return bytes.join(b"",strBytes)[0:strLen]

def encryptBytes(segBytesList,e,N):
    encryedBytes=[]
    for data in segBytesList:
        encryedBytes.append(pow(data,e)% N)
    return encryedBytes

def decryptBytes(encryedSegBytesList,d,N):
    decryedBytes=[]
    for data in encryedSegBytesList:
        decryedBytes.append(pow(data,d)% N)
    return decryedBytes

#p,q
minN=random.randint(10, 200)
maxN=random.randint(minN+50, minN+100)
p=getPrimeNumbers(minN,maxN)
minN=random.randint(200, 300)
maxN=random.randint(minN+50, minN+100)
q=getPrimeNumbers(minN,maxN)
print(p,q)
#
N=p*q
#r
r=(p-1)*(q-1)
#e
e=getAnotherGcd(r)
#e关于r的模逆元d
d=getModuloInverse(e,r)
#公钥，私钥
pubKey=(N,e)
priKey=(N,d)
print(f"\n公钥：({N},{e})\n私钥：({N},{d})")

encodeStr=enCodeBase64(strTxt)

lenN=len(str(N))
maxChar="~"#ascii=126
maxTestChar=""
segMessageLen=0
for i in range(0,lenN):
    maxTestChar+=maxChar
    if (int.from_bytes(bytes(maxTestChar,encoding='ascii'),'little')>N):
        segMessageLen=i
        break
print(N,maxTestChar,int.from_bytes(bytes("~"*segMessageLen,encoding='ascii'),'little'))
print(strTxt)
print(encodeStr)
segBytesStr=getSegBytes(strTxt,segMessageLen)
print(segBytesStr)
encryptedMessList=encryptBytes(segBytesStr,e,N)
print(encryptedMessList)
decryptedMessList=decryptBytes(encryptedMessList,d,N)
print(decryptedMessList)
resStr=putSegBytes(decryptedMessList,segMessageLen)
print(resStr)
decodeDecryedStr=deCodeBase64(resStr)
print(decodeDecryedStr)


import requests
from bs4 import BeautifulSoup

# res = requests.get('https://www.goodreads.com/author/on_goodreads')
# with open('author_data/authors.html', 'a') as f:
#    f.write(res.text)
# soup = BeautifulSoup(res.text, 'lxml')

with open('author_data/authors.html', 'r') as f:
    res = f.read()
soup = BeautifulSoup(res, 'lxml')

f.close()

links = []

for data in soup.find_all('div', class_="elementList"):
    link = 'https://www.goodreads.com' + data.a['href']
    links.append(link)

for i in range(len(links)):
    res = requests.get(links[i])

    with open(f"author_data/{i+1}.html", 'w') as f:
        f.write(res.text)

    print(links[i].split('.')[3], 'saved')

f.close()


from pyasn1_modules import pem, rfc2437, rfc2459
from pyasn1.codec.der import decoder, encoder
from hashlib import sha256
import os
import binascii
import argparse
import sys
import base64

#Usage: name_certificate.pem [key_filename]

root = False
if(len(sys.argv) > 2):
    root = True

#Extract the certificate
substrate = pem.readPemFromFile(open(sys.argv[1]))
certType = rfc2459.Certificate()
cert = decoder.decode(substrate, asn1Spec = certType)[0]
tbs = cert.getComponentByName("tbsCertificate") # Dump the TBS

if(root):
    # If the root certificate we have to change the public key with the one prodused by dkg funcionality
    f = open(sys.argv[2],"r")
    x = int(f.readline())
    y = int(f.readline())
    tbs['subjectPublicKeyInfo']['subjectPublicKey'] = format("0x%02x" % 0x04) + format("%064x" %  x) + format("%064x" %  y)
    f = open(sys.argv[1][:-4]+".der", "wb")
    cert.setComponentByName("tbsCertificate",tbs)
    f.write(encoder.encode(cert))
    f.close()

hash = sha256(encoder.encode(tbs))
res = hash.digest()
f = open("signer/msg","w")
f.write("msg: \"" + base64.b64encode(res).decode()+ "\"")
f.close()


import numpy as np
import math
import matplotlib.pyplot as plt

def TCSF(frequency):
    S = np.abs(148.7 * ((1 + 2j * math.pi * frequency * 0.00267) ** (-15) - 0.882 * (
                1 + 2j * math.pi * frequency * 1.834 * 0.00267) ** (-16)))
    return S

if __name__ == "__main__":
    x = np.linspace(0,120,1000)
    y = np.zeros(x.shape)
    for i in range(len(x)):
        y[i] = TCSF(x[i])
    plt.figure(figsize=(6,6))
    plt.plot(x, y)
    plt.title('TCSF')
    plt.xlabel('Frequency (Hz)')
    plt.ylabel('Contrast Sensitivity')
    plt.savefig('TCSF.png')
    plt.close()


#!/usr/bin/env python
# coding: utf-8

# # Run Simulations
#
# Given an encoded dataset of targets and queries, run simulations.

# In[1]:

import numpy as np
import pandas as pd

import primo.models

import cupyck

from tqdm import tqdm
import os

# In[2]:

#これは謎
#hosts = [
#    ("localhost", 2046),
#]
#client = cupyck.Client(hosts)
cupyck_sess = cupyck.GPUSession(max_seqlen=200, nblocks=2048, nthreads=512)
simulator = primo.models.Simulator(cupyck_sess)

cnn_model = input("model name:")
file=input("file:")
# In[3]:
dataset_name = input("dataset:")+"/"
import os
data_dir = ''
data_dir = data_dir+dataset_name

tar_name=input("dna sequences file name:")
que_name=tar_name
out_name=input("output file name:")
target_seqs = pd.read_hdf(os.path.join(data_dir,'targets/{}/{}/{}.h5'.format(cnn_model,file,tar_name)))
query_seqs = pd.read_hdf(os.path.join(data_dir,'queries/{}/{}/{}.h5'.format(cnn_model,file,que_name)))
total_store = pd.DataFrame()
if not os.path.exists(data_dir+'simulation/{}'.format(cnn_model)):
    os.mkdir(data_dir+'simulation/{}'.format(cnn_model))
for que_in in range(len(query_seqs.index)):
    pairs = (target_seqs
    .rename(columns={'FeatureSequence':'target_features'})
    .assign(query_features = query_seqs.iloc[int(que_in)].FeatureSequence)
    )
    print("label {}:".format(int(que_in)))

# In[5]:

# 4,000 here is just a memory-management batch size so that each progress chunk reports period of time.
    split_size = 4000
    nsplits = len(pairs) / split_size
    splits = np.array_split(pairs, nsplits)

# In[6]:

    result_store = pd.DataFrame()
    try:
        for split in tqdm(splits, leave=False):
            results = simulator.simulate(split)
            result_store=pd.concat([result_store,results[['duplex_yield']]],axis=0)
    finally:
        total_store=pd.concat([total_store,result_store.rename(columns={"duplex_yield":"label_{}".format(que_in)})], axis=1)
        del result_store

total_store[total_store>1.0] = 1
total_store.to_hdf(os.path.join(data_dir,'simulation/{}/{}.h5'.format(cnn_model, out_name)), key='df', mode='w')
# In[ ]:


from IPython.display import display
import os
import wfdb
import numpy as np
from tqdm import tqdm
import cnibp.preprocessing.utils.signal_utils as su

'''
find_available_data(root_path)

< input >
    root_path : folder path containing person
< output >
    ABPBVP_list : list of people that have both channel; BVP, ABP
'''

def find_available_data(root_path):
    print('\n***MIMICdataset.find_available_data() called***')
    print(root_path)
    person_list = []
    for (path, dirs, files) in os.walk(root_path):
        for d in dirs:
            if len(d) == 3:
                person_list.append(d)

    available_list = []
    for person in person_list:
        channel_cnt = 0
        p = root_path + person
        for (path, dirs, files) in os.walk(p):
            for file in files:
                if file.split('.')[-1] == 'abp' or file.split('.')[-1] == 'ple':
                    channel_cnt += 1
                    if channel_cnt == 2:
                        available_list.append(person)
                        continue
    available_list = sorted(available_list)
    return available_list

'''
def find_idx(path)

< input >
    path : data path
< output >
    channel : index list of ABP, BVP
    flag : 2=[ABP, BVP], 1 = missing either of two
'''

def find_idx(path):
    record = wfdb.rdrecord(path)
    ABP_channel = [i for i in range(len(record.sig_name)) if (record.sig_name[i] == 'ABP')]
    PPG_channel = [i for i in range(len(record.sig_name)) if (record.sig_name[i] == 'PLETH')]
    # print('channel :', len(channel))
    # flag = len(channel)
    return ABP_channel, PPG_channel

'''
def read_record(path, sampfrom=0, sampto=None)

< input >
    path : data path
    sampfrom : starting point of slice
    sampto : end point of slice
< output >
    record : record of [ABP, BVP] channel

wfdb.io.rdrecord()
Read a WFDB record and return the signal and record descriptors
as attributes in a Record or MultiRecord object.
'''

''' 바로 return.p_signal 하도록 변경 '''

# TODO -> Record.init_value를 추가로 return 해주는데, 뒤에 함수들 전부 수정
def read_record(path, sampfrom=0, sampto=None):
    # channel = ['ABP', 'PLETH']
    ABP_channel, PPG_channel = find_idx(path)
    ABP_record = wfdb.rdrecord(path, channels=ABP_channel, sampfrom=sampfrom, sampto=sampto)
    PPG_record = wfdb.rdrecord(path, channels=PPG_channel, sampfrom=sampfrom, sampto=sampto)
    # wfdb.plot_wfdb(record=record, title='Record from PhysioNet MIMIC Dataset')
    # display(ABP_record.__dict__)
    return ABP_record.p_signal, PPG_record.p_signal

def data_aggregator(model_name, read_path, chunk_size, samp_rate):
    print('MIMIC-I dataset selected')
    available_list = find_available_data(read_path)
    print('Number of total ICU patient :', len(available_list))
    print(' >>', available_list)
    total_data = np.empty((1, 2))
    total_cnt = 0
    print('Number of selected ICU patient :', len(available_list))
    used_file_cnt = 0
    # TODO ''' use total data after solving the problem of get_diastolic() '''
    for a in available_list[:1]:
        p = read_path + a
        for (path, dirs, files) in os.walk(p):
            for file in tqdm(files):
                if (len(file.split('/')[-1]) == 12) and (file.split('.')[-1] == 'hea'):
                    data_path = (p + '/' + file).strip('.hea')
                    used_file_cnt += 1
                    total_data = np.append(total_data, read_record(data_path), axis=0)

    sig_total = total_data[1:]
    # for reshaping in signal_utils
    relength = (len(sig_total) // chunk_size) * chunk_size
    sig_total = sig_total[:relength]
    print('np.shape(sig_total) :', np.shape(sig_total))

    abp, ple, dsm = su.signal_slicing(model_name, sig_total, chunk_size, samp_rate, fft=True)
    return ple, abp, dsm


import nltk
import csv
import glob
import numpy as np
import pandas as pd
from nltk.corpus import wordnet as wn
from nltk.corpus import sentiwordnet as swn
from test import Bert_pre
nltk.download('sentiwordnet')
nltk.download('wordnet')
nltk.download('omw-1.4')

def read_csv(path):
    oa_pairs = pd.read_csv(path, encoding= 'unicode_escape')
    print(oa_pairs.shape)
    print(oa_pairs)
    return oa_pairs

def cal_w_synsets_s_score(word): # opinion modifier may contains multiple words
    word = word.split()
    aspect_score_list = []
    for w in word:
        denominator = 0
        synsets_score = []
        w_list = list(swn.senti_synsets(w))
        if not w_list:
            if  w == word[-1]:
                return 0
            continue
        for i in range(len(w_list)):
            syn_pos = w_list[i].pos_score()
            syn_neg = w_list[i].neg_score()
            try:
                syn_score = (syn_pos - syn_neg) / (i + 1)
                synsets_score.append(syn_score)
                # print(synsets_score)
            except:
                pass
            denominator +=  1 / (i + 1)
        # print('denominator: ',denominator)
        w_score = sum(synsets_score) / denominator
        # print('w_score: ',w_score)
        aspect_score_list.insert(0, w_score)
    # print(aspect_score_list)
    while len(aspect_score_list)>1:
        if aspect_score_list[0] > 0 and aspect_score_list[1] < 0:
            score = aspect_score_list[0] - (aspect_score_list[0] * abs(aspect_score_list[1]))
        else:
            score = (-1) * (abs(aspect_score_list[0]) + (1 - abs(aspect_score_list[0])) * abs(aspect_score_list[1]))
        aspect_score_list[0] = score
        aspect_score_list.pop(1)
    return aspect_score_list[0] # return sentiment score of a aspect(base on sentiwordnet)

def check_cluster_belongs(oa_pairs):
    init_center_ws = [['acting','chemistry','performanece','charm','comedian'], ['direction','director','filmmaker','screenplay','sequence','script','editing','lines'],
    ['music','audio','vocals'], ['story','plot'], ['effect','scene','efficacy','scenery','photography','camera','budget']]
    c1 = oa_pairs[oa_pairs['cluster'] == 0]
    c2 = oa_pairs[oa_pairs['cluster'] == 1]
    c3 = oa_pairs[oa_pairs['cluster'] == 2]
    c4 = oa_pairs[oa_pairs['cluster'] == 3]
    c5 = oa_pairs[oa_pairs['cluster'] == 4]
    clusters_list = [list(c1['aspect']), list(c2['aspect']), list(c3['aspect']),
    list(c4['aspect']), list(c5['aspect'])]
    cluster_belong_to_aspect = {'acting': [], 'direction': [], 'music': [], 'story': [], 'effect': []}
    for c in clusters_list:
        for init in init_center_ws:
            for i in init:
                if i in c:
                    cluster_belong_to_aspect[init_center_ws[init_center_ws.index(init)][0]].append(clusters_list.index(c))
                    break
    return cluster_belong_to_aspect

class Opinions_sentiment(object):
    print('opinion')
    def run_opinions_sentiment(self):
        path = '.\movies_aspect_clusterd_result_0423\*.csv'
        file_names = glob.glob(path)
        # file = '.\movies_oa_pairs\Kingpin 1996.csv'
        for file in file_names:
            file_name = file.split('\\')[-1]
            title = file_name.rsplit(' ', 1)[0]
            print('run_opinions_sentiment')
            clustered_oa_pairs = read_csv(file)
            cluster_belong_to_aspect = check_cluster_belongs(clustered_oa_pairs)
            all_clus_score = []
            for i in range(5):  # for each cluster
                aspects_score = []
                opinions = clustered_oa_pairs.loc[clustered_oa_pairs.cluster == i, 'DynamicColumn_1':clustered_oa_pairs.columns[-1]].values.flatten().tolist()
                opinions = [x for x in opinions if pd.isnull(x) == False]
                print(opinions)
                for w in opinions:
                    aspects_score.append(cal_w_synsets_s_score(w))
                # print(len(aspects_score))
                all_clus_score.append(np.array(aspects_score).sum() / len(aspects_score))
            # print(all_clus_score)
            each_aspect_score = [0,0,0,0,0]
            c_value = list(cluster_belong_to_aspect.values())
            for cluster in c_value:
                for c in cluster:
                    each_aspect_score[c_value.index(cluster)] =  each_aspect_score[c_value.index(cluster)] + all_clus_score[c]
                if each_aspect_score[c_value.index(cluster)] != 0:
                    each_aspect_score[c_value.index(cluster)] = each_aspect_score[c_value.index(cluster)] / len(cluster)

            movie_csv = self.path.split('/')[-1]
            title = movie_csv.rsplit(' ', 1)[0]
            with open('all_movies_aspect_sentiment_score_0423.csv','a', newline = '') as f:
                each_aspect_score.insert(0, title)
                writer = csv.writer(f)
                writer.writerow(each_aspect_score)
            story_clusters_Index = cluster_belong_to_aspect['story']
            story_opinions = clustered_oa_pairs.loc[clustered_oa_pairs['cluster'].isin(story_clusters_Index), 'DynamicColumn_1':clustered_oa_pairs.columns[-1]].values.flatten().tolist()
            story_opinions = opinions = [x for x in opinions if pd.isnull(x) == False]
            bert_pre = Bert_pre(story_opinions)

            # get sentiment score of each movie through BERT multi-label calssification model.
            with open('bert_predict_sentiment_score_0502.csv','a', newline = '') as f:
                bert_pre.insert(0, title)
                writer = csv.writer(f)
                writer.writerow(bert_pre)

if __name__ == '__main__':
    objName = Opinions_sentiment()
    objName.run_opinions_sentiment()


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

# Try to change the value of alpha
def mf_sgd(R, K=64, alpha=1e-4, beta=1e-2, iterations=50):
    """
    :param R: user-item rating matrix
    :param K: number of latent dimensions
    :param alpha: learning rate
    :param beta: regularization parameter
    """
    num_users, num_items = R.shape

    # Initialize user and item latent feature matrice
    P = np.random.normal(scale=1. / K, size=(num_users, K))
    Q = np.random.normal(scale=1. / K, size=(num_items, K))

    # Initialize the biases
    b_u = np.zeros(num_users)
    b_i = np.zeros(num_items)
    b = np.mean(R[np.where(R != 0)])

    # Create a list of training samples
    samples = [
        (i, j, R[i, j])
        for i in range(num_users)
        for j in range(num_items)
        if R[i, j] > 0
    ]

    # Perform stochastic gradient descent for number of iterations
    training_loss = []

    for iters in range(iterations):
        np.random.shuffle(samples)

# My code start
        loss = 0
# My code end

        for i, j, r in samples:
            """
            TODO:
            In this for-loop scope,
            you need to (1)update "b_u"(vector of rating bias for users) and "b_i"(vector of rating bias for items)
            and (2)update user and item latent feature matrices "P", "Q"
            """

# My code start
            # Comment out pred = 0
            # pred = 0

            # Calculate prediction and error
            hat_r = b + b_u[i] + b_i[j] + np.dot(P[i, :], Q[j, :])
            d_ij = r - hat_r

            # Calculate gradients
            gradient_b_u = -d_ij + (beta * b_u[i])
            gradient_b_i = -d_ij + (beta * b_i[j])
            gradient_P = -d_ij * Q[j, :] + (beta * P[i, :])
            gradient_Q = -d_ij * P[i, :] + (beta * Q[j, :])

            # Update all variables
            b_u[i] -= alpha * gradient_b_u
            b_i[j] -= alpha * gradient_b_i
            P[i, :] -= alpha * gradient_P
            Q[j, :] -= alpha * gradient_Q

            # Calculate loss
            loss += ((r - b - b_u[i] - b_i[j] - np.dot(P[i, :], Q[j, :])) ** 2)

        # Calculate loss
        loss /= len(samples)
        loss = loss ** 0.5

        temp = [iters, loss]
        training_loss.append(temp)

    pred = np.dot(P, Q.transpose())
# My code end

    return pred, b, b_u, b_i, training_loss

def plot_training_loss(training_loss):
    x = [x for x, y in training_loss]
    y = [y for x, y in training_loss]
    plt.figure(figsize=(16, 4))
    plt.plot(x, y)
    plt.xticks(x, x)
    plt.xlabel("Iterations")
    plt.ylabel("Root Mean Square Error")
    plt.grid(axis="y")
    plt.savefig("training_loss.png")
    plt.show()

if __name__ == "__main__":
    data = pd.read_csv('ratings.csv')
    table = pd.pivot_table(data, values='rating', index='userId', columns='movieId', fill_value=0)
    R = table.values

    pred, b, b_u, b_i, loss = mf_sgd(R,iterations=50)
    print("P x Q:")
    print(pred)
    print("Global bias:")
    print(b)
    print("User bias:")
    print(b_u)
    print("Item bias:")
    print(b_i)
    plot_training_loss(loss)