# -*- coding: utf-8 -*-
import numpy as np
import pandas as pd
import h5py
import random
import glob
from sklearn.model_selection import StratifiedKFold
import os
from ushuffle import Shuffler, shuffle
import pdb

"""
sequence to matrix
"""

class GeneRateOneHotMatrix():

    def __init__(self):

        self.OriSeq = None
        self.Seqlabel = None
        self.SeqMatrix = None
        self.TrainX = None
        self.TestX = None
        self.TrainY = None
        self.TestY = None
        self.TrainID = None
        self.TestID = None
        self.Trainlist = []
        self.Testlist = []
        self.seq_pos_matrix_out = None

    def mkdir(self, path):
        """
        :param path:
        :return:
        """
        isExists = os.path.exists(path)
        if not isExists:
            os.makedirs(path)
            return (True)
        else:
            return False

    def seq_to_matrix(self,seq, seq_matrix, seq_order):
        '''
        change target 3D tensor according to sequence and order
        :param seq: Input single root sequence
        :param seq_matrix: Input initialized matrix
        :param seq_order: This is the first sequence
        :return:
        '''
        for i in range(len(seq)):
            if ((seq[i] == 'A') | (seq[i] == 'a')):
                seq_matrix[seq_order, i, 0] = 1
            if ((seq[i] == 'C') | (seq[i] == 'c')):
                seq_matrix[seq_order, i, 1] = 1
            if ((seq[i] == 'G') | (seq[i] == 'g')):
                seq_matrix[seq_order, i, 2] = 1
            if ((seq[i] == 'T') | (seq[i] == 't')):
                seq_matrix[seq_order, i, 3] = 1
        return seq_matrix

    def genarate_matrix_for_train(self, seq_shape, seq_series):
        """
        generate a large matrix composed of sequences.
        :param shape: (seq number, sequence_length, 4)
        :param seq_series: A file in dataframe format composed of seq
        :return:seq
        """
        seq_matrix = np.zeros(seq_shape)
        for i in range(seq_series.shape[0]):
            seq_tem = seq_series[i]
            seq_matrix = self.seq_to_matrix(seq_tem, seq_matrix, i)
        return seq_matrix

    def cross_validation(self, number_of_folds, total_number, random_seeds=233):
        """
        This function is used to generate the index of n-fold cross validation
        :param number_of_folds:
        :param total_number:
        :param random_seeds:
        :return:
        """
        x = np.zeros((total_number,), dtype=np.int)
        split_iterator = StratifiedKFold(n_splits=number_of_folds, random_state=random_seeds, shuffle=True)
        split_train_index_and_test_index_list = [
            (train_index, test_index)
            for train_index, test_index in split_iterator.split(x, x)
            ]
        return (split_train_index_and_test_index_list)

    def split_dataset(self, split_index_list, fold, data_x, data_y, data_id=None):
        """
        Return training set and test set as needed
        :param split_index_list:Index of each fold returned by cross_validation
        :param fold:
        :param data_id:
        :param data_x:X
        :param data_y:Y
        :return:
        """
        id_train = data_id[split_index_list[fold][0].tolist()]
        x_train = data_x[split_index_list[fold][0].tolist()]
        y_train = data_y[split_index_list[fold][0].tolist()]
        id_test = data_id[split_index_list[fold][1].tolist()]
        x_test = data_x[split_index_list[fold][1].tolist()]
        y_test = data_y[split_index_list[fold][1].tolist()]
        return [x_train, y_train, id_train, x_test, y_test, id_test]

    def StoreTrainSet(self, rootPath):
        """
        Store multi-fold  data
        :param rootPath:
        :param ValNum: Generate valNum different validation data sets
        :param RandomSeeds:
        :param allData:
        :return:
        """
        self.mkdir(rootPath)
        training_path = rootPath + "/training_set.hdf5"
        test_path = rootPath + "/test_set.hdf5"

        f_train = h5py.File(training_path)
        f_test = h5py.File(test_path)

        f_train.create_dataset("sequences", data=self.TrainX)
        f_train.create_dataset("labs", data=self.TrainY)
        f_train.create_dataset("seq_idx", data=self.TrainID)
        f_train.close()
        f_test.create_dataset("sequences", data=self.TestX)
        f_test.create_dataset("labs", data=self.TestY)
        f_test.create_dataset("seq_idx", data=self.TestID)
        f_test.close()

    def k_mer_shuffle(self,seq_shape, seq_series, k=2):
        """
        Return a shuffled negative matrix
        :param seq_series:sequence list
        :param seq_shape:
        :param k:kshuffle
        :return:
        """
        seq_shuffle_matrix = np.zeros(seq_shape)

        for i in range(seq_shape[0]):
            seq = seq_series[i]
            shuffler = Shuffler(seq, k)
            seqres = shuffler.shuffle()
            seq_shuffle_matrix = self.seq_to_matrix(seqres, seq_shuffle_matrix, i)

        return seq_shuffle_matrix

    def runSeveral(self, SeqPath, OutputDir, SaveData=False):
        """
        Call this function to directly generate data, seq format requirements, odd-numbered lines are names,
        even-numbered lines are corresponding sequences
        :return:
        """
        allFileFaList = glob.glob(SeqPath)
        for allFileFa in allFileFaList:
            AllTem = allFileFa.split("/")[-1].split(".")[0]
            output_dir = OutputDir + AllTem + "/"
            SeqLen = 0
            ChipSeqlFileFa = pd.read_csv(allFileFa, sep='\t', header=None, index_col=None)
            seq_positive_series = np.asarray(ChipSeqlFileFa[1::2]).reshape(np.asarray(ChipSeqlFileFa[1::2]).shape[0], )
            seq_positive_shape = seq_positive_series.shape[0]
            for i in range(seq_positive_shape):
                SeqLen = max(SeqLen, len(seq_positive_series[i]))
            seq_positive_name = np.asarray(ChipSeqlFileFa[::2]).reshape(seq_positive_shape, )

            self.seq_pos_matrix_out = self.genarate_matrix_for_train((seq_positive_shape, SeqLen, 4),
                                                           seq_positive_series)
            seq_pos_label_out = np.ones(seq_positive_shape, )

            seq_negative_name = seq_positive_name
            seq_neg_matrix_out = self.k_mer_shuffle((seq_positive_shape, SeqLen, 4), seq_positive_series)
            seq_neg_label_out = np.zeros(seq_positive_shape, )

            seq = np.concatenate((self.seq_pos_matrix_out, seq_neg_matrix_out), axis=0)
            label = np.concatenate((seq_pos_label_out, seq_neg_label_out), axis=0)
            id_tem = np.concatenate((seq_positive_name, seq_negative_name), axis=0)
            index_shuffle = range(seq_positive_shape + seq_positive_shape)
            random.shuffle(index_shuffle)
            seq_matrix_out = seq[index_shuffle, :, :]
            label_out = label[index_shuffle]
            id_out = id_tem[index_shuffle].astype("string_")
            outData = [seq_matrix_out, label_out, id_out]
            self.GeneRateTrain(allData=outData, ValNum=10, RandomSeeds=233)
            if SaveData:
                self.StoreTrainSet(rootPath=output_dir)

    def GeneRateTrain(self, allData, ValNum=10, RandomSeeds=233):
        """

        """
        dataNum = allData[1].shape[0]
        split_train_index_and_test_index_list = self.cross_validation(number_of_folds=ValNum, total_number=dataNum,
                                                                 random_seeds=RandomSeeds)
        i = 0
        outDataTem = self.split_dataset(split_train_index_and_test_index_list, fold=i, data_x=allData[0], data_y=allData[1],
                                   data_id=allData[2])

        self.TrainX = outDataTem[0]
        self.TestX = outDataTem[3]
        self.TrainY = outDataTem[1]
        self.TestY = outDataTem[4]
        self.TrainID = outDataTem[2]
        self.TestID = outDataTem[5]

    def runSimple(self, SeqPath, OutputDir, SaveData=False):
        """
        Call this function to directly generate data, here is a separate data set, including Training and test
        seq format requirements, odd-numbered lines are names, even-numbered lines are corresponding sequences
        :return:
        """
        SeqLen = 0
        ChipSeqlFileFa = pd.read_csv(SeqPath, sep='\t', header=None, index_col=None)
        self.OriSeq = np.asarray(ChipSeqlFileFa[1::2]).reshape(np.asarray(ChipSeqlFileFa[1::2]).shape[0], )
        seq_positive_shape = self.OriSeq.shape[0]
        for i in range(seq_positive_shape):
            SeqLen = max(SeqLen, len(self.OriSeq[i]))
        seq_positive_name = np.asarray(ChipSeqlFileFa[::2]).reshape(seq_positive_shape, )

        self.seq_pos_matrix_out = self.genarate_matrix_for_train((seq_positive_shape, SeqLen, 4),
                                                       self.OriSeq)
        seq_pos_label_out = np.ones(seq_positive_shape, )

        seq_negative_name = seq_positive_name
        seq_neg_matrix_out = self.k_mer_shuffle((seq_positive_shape, SeqLen, 4), self.OriSeq)
        seq_neg_label_out = np.zeros(seq_positive_shape, )

        seq = np.concatenate((self.seq_pos_matrix_out, seq_neg_matrix_out), axis=0)
        label = np.concatenate((seq_pos_label_out, seq_neg_label_out), axis=0)
        id_tem = np.concatenate((seq_positive_name, seq_negative_name), axis=0)
        index_shuffle = range(seq_positive_shape + seq_positive_shape)
        random.shuffle(index_shuffle)
        self.SeqMatrix = seq[index_shuffle, :, :]
        self.Seqlabel = label[index_shuffle]
        id_out = id_tem[index_shuffle].astype("string_")
        outData = [self.SeqMatrix, self.Seqlabel, id_out]
        self.GeneRateTrain(allData=outData, ValNum=10, RandomSeeds=233)
        if SaveData:
            self.StoreTrainSet(rootPath=OutputDir)

if __name__ == '__main__':
    base = {0:"A",1:"C",2:"G",3:"T"}
    allFileFaList = glob.glob("./chip-seqFa/*fa")
    OutputDir = "./chip-seqFa/TrainingData/"
    GeneRateOneHotMatrixTest = GeneRateOneHotMatrix()

    GeneRateOneHotMatrixTest.runSeveral("./chip-seqFa/*fa", OutputDir, SaveData=True)

    GeneRateOneHotMatrixTest.runSimple(allFileFaList[0], OutputDir, SaveData=True)


import recordlinkage.config as cf
from recordlinkage.config import (is_int, is_bool, is_text,
                                  is_instance_factory, is_one_of_factory,
                                  get_default_val, is_callable)

pairs_type_doc = """
: str
    Specify the format how record pairs are stored. By default, record
    pairs generated by the toolkit are returned in a
    pandas.MultiIndex object ('multiindex' option).

    Valid values: 'multiindex'
"""

with cf.config_prefix('indexing'):
    cf.register_option('pairs', 'multiindex', pairs_type_doc,
                       validator=is_one_of_factory(['multiindex']))


# coding: utf-8
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
from torch.autograd import Function, Variable

class EmbLoss_v1(nn.Module):
    def __init__(self, feature_dim=4, loss_weight=1.0):
        super(EmbLoss_v1, self).__init__()
        self.feature_dim = feature_dim
        self.loss_weight = loss_weight
        self.delta_v = 0.5
        self.delta_d = 1.5
        self.weights = (1.0, 1.0)

    def forward_single(self, emb, instance, kernel, training_mask, bboxes):
        training_mask = (training_mask > 0.5).long()
        kernel = (kernel > 0.5).long()
        instance = instance * training_mask
        instance_kernel = (instance * kernel).view(-1)
        instance = instance.view(-1)
        emb = emb.view(self.feature_dim, -1)

        unique_labels, unique_ids = torch.unique(instance_kernel, sorted=True, return_inverse=True)
        num_instance = unique_labels.size(0)
        if num_instance <= 1:
            return 0

        emb_mean = emb.new_zeros((self.feature_dim, num_instance), dtype=torch.float32)
        for i, lb in enumerate(unique_labels):
            if lb == 0:
                continue
            ind_k = instance_kernel == lb
            emb_mean[:, i] = torch.mean(emb[:, ind_k], dim=1)

        l_agg = emb.new_zeros(num_instance, dtype=torch.float32)  # bug
        for i, lb in enumerate(unique_labels):
            if lb == 0:
                continue
            ind = instance == lb
            emb_ = emb[:, ind]
            dist = (emb_ - emb_mean[:, i:i + 1]).norm(p=2, dim=0)
            dist = F.relu(dist - self.delta_v) ** 2
            l_agg[i] = torch.mean(torch.log(dist + 1.0))
        l_agg = torch.mean(l_agg[1:])

        if num_instance > 2:
            emb_interleave = emb_mean.permute(1, 0).repeat(num_instance, 1)
            emb_band = emb_mean.permute(1, 0).repeat(1, num_instance).view(-1, self.feature_dim)
            # print(seg_band)

            mask = (1 - torch.eye(num_instance, dtype=torch.int8)).view(-1, 1).repeat(1, self.feature_dim)
            mask = mask.view(num_instance, num_instance, -1)
            mask[0, :, :] = 0
            mask[:, 0, :] = 0
            mask = mask.view(num_instance * num_instance, -1)
            # print(mask)

            dist = emb_interleave - emb_band
            dist = dist[mask > 0].view(-1, self.feature_dim).norm(p=2, dim=1)
            dist = F.relu(2 * self.delta_d - dist) ** 2
            l_dis = torch.mean(torch.log(dist + 1.0))
        else:
            l_dis = 0

        l_agg = self.weights[0] * l_agg
        l_dis = self.weights[1] * l_dis
        l_reg = torch.mean(torch.log(torch.norm(emb_mean, 2, 0) + 1.0)) * 0.001
        loss = l_agg + l_dis + l_reg
        return loss

    def forward(self, emb, instance, kernel, training_mask, bboxes, reduce=True):
        loss_batch = emb.new_zeros((emb.size(0)), dtype=torch.float32)

        for i in range(loss_batch.size(0)):
            loss_batch[i] = self.forward_single(emb[i], instance[i], kernel[i], training_mask[i], bboxes[i])

        loss_batch = self.loss_weight * loss_batch

        if reduce:
            loss_batch = torch.mean(loss_batch)

        return loss_batch


import argparse
import math
import random
import statistics
import time
from collections import (
    defaultdict,
)
from pathlib import (
    Path,
)

import gym
import numpy as np
import pandas as pd
import plotly.express as px
from gym import (
    wrappers,
)
from tensorflow.keras.models import (
    load_model,
    save_model,
)
from tensorflow.keras.utils import (
    plot_model,
)
from tqdm import (
    tqdm,
)

from bc.bc_model import *
from bc.driving_data import *
from common.consts import *
from common.driving_utils import *
from common.utils import (
    load_pkl,
    print_step,
    save_pkl,
)

def run():
    run_bc_model(
        "checkpoints/decent_bc",
        1,
        False,
    )

def check():
    model = load_model(
        "checkpoints/bc"
    )
    (
        imgs,
        features,
        labels,
    ) = (
        load_bc_data()
    )
    n = len(
        imgs
    )
    for _ in range(
        10
    ):
        i = random.randrange(
            0,
            n,
        )
        fig = px.imshow(
            imgs[
                i
            ],
            title=f"{INDEX2STRING[int(labels[i])]}",
        )
        fig.write_image(
            f"imgs/model_validation/{i}.png"
        )
        # fig.show()

def resume():
    model = load_model(
        "checkpoints/bc"
    )
    (
        imgs,
        features,
        labels,
    ) = (
        load_bc_data()
    )
    train_model(
        model,
        imgs,
        features,
        labels,
        40,
    )

# ======= Train =========

def train():
    model = (
        get_bc_model()
    )
    plot_model(
        model,
        "imgs/model.png",
    )
    (
        imgs,
        features,
        labels,
    ) = (
        load_bc_data()
    )
    train_model(
        model,
        imgs,
        features,
        labels,
        40,
    )

# ======= Evaluate =========

def evaluate():
    (
        x,
        y,
    ) = (
        load_bc_data()
    )
    m = load_model(
        "checkpoints/bc/"
    )
    m.evaluate(
        x,
        y,
    )
    pred = m.predict(
        x
    )
    pred = np.argmax(
        pred,
        axis=1,
    )
    print(
        sum(
            y
            == pred
        )
        / len(
            y
        )
    )

# ======= Generate Data =======

def generate():
    human_generate_data(
        False
    )

def gather():
    gather_data_balanced()

def human():
    human_race()

CHOICES = [
    "run",
    "check",
    "resume",
    "train",
    "evaluate",
    "generate",
    "gather",
    "human",
]
CHOICE2FUNC = {
    "run": run,
    "check": check,
    "resume": resume,
    "train": train,
    "evaluate": evaluate,
    "generate": generate,
    "gather": gather,
    "human": human,
}

if (
    __name__
    == "__main__"
):
    parser = (
        argparse.ArgumentParser()
    )
    parser.add_argument(
        "action",
        type=str,
        choices=CHOICES,
    )
    args = (
        parser.parse_args()
    )
    CHOICE2FUNC[
        args.action
    ]()

# TODO simplify data by setting the speed automatically
# TODO get rl agent working!

# env = gym.make("CarRacing-v0")
# env.reset()
# print(env.action_space)

"""
Car Racing
Actions: [steer, gas, break] (floats)
          steer (left) -1  0 1 (right)
          gas
          break
Observations:
    92x92x3 image
    greens: (102, 204, 102), (104, 229, 104)
    grey: (102-107, 102-107, 102-107)
"""


__author__ = "V.A. Sole - ESRF"
__contact__ = "sole@esrf.fr"
__license__ = "MIT"
__copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
import sys
import time
import numpy
import logging
from PyMca5.PyMcaGui import PyMcaQt as qt
from PyMca5.PyMcaGui.misc import CalculationThread

try:
    from . import PCAWindow
    PCA = True
    MDP = PCAWindow.MDP
except ImportError:
    PCA = False
    MDP = False

_logger = logging.getLogger(__name__)

class PCADialog(qt.QDialog):
    def __init__(self, parent=None, rgbwidget=None, selection=False):
        qt.QDialog.__init__(self, parent)
        self.setWindowTitle("PCA calculation dialog")
        self.mainLayout = qt.QVBoxLayout(self)
        self.calculateButton = qt.QPushButton(self)
        self.calculateButton.setAutoDefault(False)
        self.calculateButton.setText("Perform PCA")
        self.showLastButton = qt.QPushButton(self)
        self.showLastButton.setAutoDefault(False)
        self.showLastButton.setText("Last Results")
        self.mainLayout.addWidget(self.calculateButton)
        self.mainLayout.addWidget(self.showLastButton)
        self._data = None
        self.pcaWindow = PCAWindow.PCAWindow(parent=None,
                                             rgbwidget=rgbwidget,
                                             #selection=True,
                                             selection=selection,
                                             colormap=True,
                                             #imageicons=True,
                                             imageicons=selection,
                                             standalonesave=True)
        self.pcaWindow.setDefaultColormap(0, logflag=False)
        self.pcaParametersDialog = PCAWindow.PCAParametersDialog(self)
        self.pcaParametersDialog.nPC.setMaximum(11)
        self.pcaParametersDialog.nPC.setValue(10)
        self.pcaParametersDialog.hide()
        self.pcaParametersDialogInitialized = False
        self.pcaWindow.hide()

        #connections
        self.calculateButton.clicked.connect(self._calculateSlot)
        self.showLastButton.clicked.connect(self._showLastSlot)

    def sizeHint(self):
        return qt.QSize(int(4 * qt.QDialog.sizeHint(self).width()),
                        qt.QDialog.sizeHint(self).height())

    def _calculateSlot(self):
        if self._data is None:
            msg = qt.QMessageBox(self)
            msg.setWindowTitle("No data")
            msg.setIcon(qt.QMessageBox.Information)
            msg.setText("No data to perform calculation")
            msg.exec()
            return

        if not self.pcaParametersDialogInitialized:
            self.pcaParametersDialog.nPC.setMaximum(self._spectrumLength)
            self.pcaParametersDialog.nPC.setValue(
                min(10, self._spectrumLength))
            ddict = {'options': self._binningOptions,
                     'binning': 1,
                     'method': 0}
            self.pcaParametersDialog.setParameters(ddict)
            self.pcaParametersDialogInitialized = True
        ret = self.pcaParametersDialog.exec()
        if ret:
            t0 = time.time()
            pcaParameters = self.pcaParametersDialog.getParameters()
            self.pcaParametersDialog.close()
            function = pcaParameters['function']
            binning = pcaParameters['binning']
            npc = pcaParameters['npc']
            mask = pcaParameters.get('mask', None)
            kw = pcaParameters.get('kw', {})
            data = self._data
            old_shape = self._data.shape
            if mask is not None:
                if mask.sum() < npc:
                    msg = qt.QMessageBox(self)
                    msg.setWindowTitle("Not enough data")
                    msg.setIcon(qt.QMessageBox.Information)
                    msg.setText("Number of components too high")
                    msg.exec()
                    return
            if _logger.getEffectiveLevel() == logging.DEBUG:
                images, eigenvalues, eigenvectors = function(data,
                                                             npc,
                                                             binning=binning,
                                                             mask=mask,
                                                             **kw)
            else:
                try:
                    threadResult = self._submitThread(function,
                                                         data,
                                                         npc,
                                                         binning=binning,
                                                         mask=mask,
                                                         **kw)
                    if type(threadResult) == type((1,)):
                        if len(threadResult):
                            if hasattr(threadResult[0], "startswith"):
                                if threadResult[0].startswith("Exception"):
                                    raise Exception(threadResult[1],
                                                threadResult[2])
                    images, eigenvalues, eigenvectors = threadResult
                except Exception:
                    if isinstance(data, numpy.ndarray):
                        self._data.shape = old_shape
                    msg = qt.QMessageBox(self)
                    msg.setIcon(qt.QMessageBox.Critical)
                    msg.setText("%s" % sys.exc_info()[1])
                    msg.exec()
                    return
            if isinstance(self._data, numpy.ndarray):
                self._data.shape = old_shape
            _logger.debug("PCA Elapsed = %s", time.time() - t0)
            methodlabel = pcaParameters.get('methodlabel', "")
            imagenames = None
            vectornames = None
            if " ICA " in methodlabel:
                nimages = images.shape[0]
                imagenames = []
                vectornames = []
                itmp = int(nimages / 2)
                for i in range(itmp):
                    imagenames.append("ICAimage %02d" % i)
                    vectornames.append("ICAvector %02d" % i)
                for i in range(itmp):
                    imagenames.append("Eigenimage %02d" % i)
                    vectornames.append("Eigenvector %02d" % i)
            self.pcaWindow.setPCAData(images,
                                      eigenvalues,
                                      eigenvectors,
                                      imagenames=imagenames,
                                      vectornames=vectornames)
            self.pcaWindow.show()
            self.pcaWindow.raise_()

    def _showLastSlot(self):
        self.pcaWindow.show()
        self.pcaWindow.raise_()

    def setData(self, data=None, spectrumindex=-1):
        if type(data) == type([]):
            #assume is an image list
            if data[0].dtype not in [numpy.float32, numpy.float64]:
                dtype = numpy.float64
            else:
                dtype = data[0].dtype
            self._spectrumLength = len(data)
            self._shape = data[0].shape
            n = 1
            for shape in self._shape:
                n *= shape
            self._binningOptions = [1]
            if len(self._shape) == 1:
                self._data = numpy.zeros((self._shape[0],
                                          self._spectrumLength), dtype)
                for i in range(self._spectrumLength):
                    self._data[:, i] = data[i][:]
            elif len(self._shape) == 2:
                self._data = numpy.zeros((self._shape[0],
                                          self._shape[1],
                                          self._spectrumLength), dtype)
                for i in range(self._spectrumLength):
                    self._data[:, :, i] = data[i][:, :]
        else:
            self._shape = data.shape
            self._data = data
            self._spectrumLength = self._shape[spectrumindex]
            self._binningOptions = [1]
            for number in [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19]:
                if (self._spectrumLength % number) == 0:
                    self._binningOptions.append(number)
        if self.pcaParametersDialog is not None:
            value = self.pcaParametersDialog.nPC.value()
            self.pcaParametersDialog.nPC.setMaximum(self._spectrumLength)
            self.pcaParametersDialog.nPC.setValue(min(value,
                                                      self._spectrumLength))

    def setSpectrum(self, x, y, legend=None):
        return self.pcaParametersDialog.setSpectrum(x, y, legend=legend)

    def _submitThread(self, function, *var, **kw):
        message = "Please Wait: PCA Going On"
        thread = CalculationThread.CalculationThread(
                                      calculation_method=function,
                                      calculation_vars = var,
                                      calculation_kw = kw,
                                      expand_vars=True,
                                      expand_kw=True)
        thread.start()
        CalculationThread.waitingMessageDialog(thread,
                                              message=message,
                                              parent=self,
                                              modal=True,
                                              update_callback=None,
                                              frameless=True)
        threadResult = thread.getResult()
        self.raise_()
        return threadResult

if __name__ == "__main__":
    _logger.setLevel(logging.DEBUG)
    import os
    from PyMca5.PyMcaIO import EdfFile
    app = qt.QApplication([])
    app.lastWindowClosed.connect(app.quit)
    d = PCADialog()
    if len(sys.argv) < 2:
        fileList = [r"D:\DATA\ICA\mix1.edf",
                    r"D:\DATA\ICA\mix2.edf",
                    r"D:\DATA\ICA\mix3.edf"]
    else:
        fileList = []
        for i in range(1, len(sys.argv)):
            fileList.append(sys.argv[i])
    imageList = []
    for fname in fileList:
        print(fname)
        if not os.path.exists(fname):
            print("File name %s does not exists" % fname)
            break
        edf = EdfFile.EdfFile(fname)
        data = edf.GetData(0)
        edf = None
        imageList.append(data)
    if len(imageList):
        d.setData(imageList)
    d.show()
    app.exec()


from sqlalchemy import Table, Column, Integer, String
from db.base import metadata

favorite_deals = Table(
    "favorite_deals",
    metadata,
    Column("id", Integer, primary_key=True, autoincrement=True),
    Column("user_id", Integer),
    Column("deal_id", Integer),
)


import string
import random
import requests
from datetime import datetime
class TestChatPost:
    url = 'http://localhost:8080'
    postUrl = 'http://localhost:8080/api/posts'
    message = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(20))

    #Check for
    def test_post_200(self):
        response = requests.get(self.url)

        assert response.status_code == 200

    def test_post_400(self):
        #Send invalid post request to ../api/posts
        invalidPostRequest = requests.post(self.postUrl, json={
        "invalid1": str(datetime.now()),
        "invalid2": 'client',
        "invalid3": f"{self.message}",
        "invalid4": str(datetime.now())})

        assert invalidPostRequest.status_code == 400

    def test_post_404(self):
        #Invalid url
        invalidPostUrl = 'http://localhost:8080/api/invalid'

        #Send post request to ../api/posts
        response = requests.post(invalidPostUrl, json={
        "id": str(datetime.now()),
        "user": 'client',
        "message": f"{self.message}",
        "ts": str(datetime.now())})

        assert response.status_code == 404


from bs4 import BeautifulSoup
from os import listdir
from os.path import isfile, join

# считать токены
tokens_file = open('task2_output/tokens.txt')
index_file = open('task3_output/index.txt', 'w')

directory = 'task1_pages/'

filenames = listdir(directory)
filenames.remove('pages_archive.zip')
filenames.sort(key=lambda x: int(x.replace('.html', '')))

texts = [BeautifulSoup(open(directory + f).read(), 'html.parser').get_text().lower() for f in filenames if isfile(join(directory, f))]

index = []

for token in tokens_file.read().splitlines():
    i = 0
    numbers_of_texts = " "
    for text in texts:
        i = i + 1
        if token in text:
            numbers_of_texts += (str(i) + " ")
    index_file.write(token + numbers_of_texts + '\n')

index_file.close()


from flask import Flask, render_template
from werkzeug.exceptions import HTTPException

from views.login import login
from views.index import index
from views.detail import detail

app = Flask(__name__, static_url_path = "", static_folder = "static", template_folder = "templates")

@app.errorhandler(HTTPException)
def not_found_error(error):
    background_color = ""
    tmp = error.code // 100

    if tmp == 4:
        background_color = "#0090e7"
    elif tmp == 5:
        background_color = "#e85f8e"

    return render_template("error-page.html", return_data = {
        "description" : error.description,
        "status_code" : error.code,
        "background_color" : background_color
    }), error.code

if __name__ == "__main__":
    app.register_blueprint(login.bp)
    app.register_blueprint(index.bp)
    app.register_blueprint(detail.bp)

    app.config["SAVE_DIR_PATH"] = "/home/universe/Desktop/git/proxify/logs/"

    app.run(debug=True, host='0.0.0.0', port=8081)


import numpy as np
from skimage.metrics import peak_signal_noise_ratio, structural_similarity
from .meter import AvgMeter
from .func import normalize_zero_to_one
import torch

def check_sanity(im1, im2):
    assert type(im1) == type(im2), "Mismatched type for the two images."
    assert im1.shape == im2.shape, "Mismatched shape for the two images."

def check_batch_sanity(im1, im2):
    check_sanity(im1, im2)
    assert im1.ndim == 4 # ! What if it's more than 4 dimension?
    assert isinstance(im1, torch.Tensor), "Input batch should be a tensor."

def slice_psnr(gt, pred, maxval= None): # This is only for a single pair of images (gt, pred), especially in ndarray format. For a batch? Not for now, see the below.
    check_sanity(gt, pred)
    if type(pred) is torch.Tensor:
        gt, pred = gt.detach().cpu().numpy(), pred.detach().cpu().numpy()
    maxval = gt.max() if maxval is None else maxval
    return peak_signal_noise_ratio(gt, pred, data_range=maxval)

def slice_ssim(gt, pred, maxval = None): # the in-built function has been already applied channel-wise, now we need focus on the batch-size, even it's just 1.
    check_sanity(gt, pred)
    if type(pred) is torch.Tensor:
        gt, pred = gt.detach().cpu().numpy(), pred.detach().cpu().numpy()
    color = True if pred.shape[1] == 3 else False
    maxval = gt.max() if maxval is None else maxval
    return structural_similarity(gt, pred, data_range=maxval, multichannel=color)

def batch_psnr(gt, pred, maxval=None): # (N, 1, H, W) / (N, H, W, 1) ---> (H, W) Suitable only for gray images of ndim=4 (Tensor)
    check_batch_sanity(gt, pred)
    gt, pred = gt.detach().cpu().numpy(), pred.detach().cpu().numpy()
    batch_size = gt.shape[0]
    psnr = 0.0
    for i in range(batch_size):
        max_val = gt[i].max() if maxval is None else maxval
        psnr += peak_signal_noise_ratio(gt[i].squeeze(), pred[i].squeeze(), data_range=max_val)
    return psnr / batch_size

def batch_ssim(gt, pred, maxval=None):
    color_flag = False
    check_batch_sanity(gt, pred)
    gt = gt.permute(0, 2, 3, 1)
    pred = pred.permute(0, 2, 3, 1)
    color_flag = True if pred.shape[-1] == 3 else False
    gt, pred = gt.detach().cpu().numpy(), pred.detach().cpu().numpy()
    batch_size = gt.shape[0]
    ssim = 0.0
    for i in range(batch_size):
        max_val = gt[i].max() if maxval is None else maxval
        ssim += structural_similarity(gt[i].squeeze(), pred[i].squeeze(), data_range=max_val, multichannel=color_flag)
    return ssim / batch_size

class Metrics:
    def __init__(self, batch_flag=True):
        self.psnr_meter = AvgMeter()
        self.ssim_meter = AvgMeter()
        if batch_flag:
            self.metric_funcs= {'PSNR':batch_psnr, 'SSIM':batch_ssim}
        else:
            self.metric_funcs= {'PSNR':slice_psnr, 'SSIM':slice_ssim}
        self.stats = {metric: None for metric in self.metric_funcs}

    def push(self, target, recons):
        for metric, func in self.metric_funcs.items():
            self.stats[metric]= func(target, recons)
        self.psnr_meter.update(self.stats['PSNR'])
        self.ssim_meter.update(self.stats['SSIM'])
    def reset(self):
        self.psnr_meter.reset()
        self.ssim_meter.reset()
        self.stats = {metric: None for metric in self.metric_funcs}

"""
def ssim(prediction, target):
    import cv2
    C1 = (0.01 * 255)**2
    C2 = (0.03 * 255)**2
    img1 = prediction.astype(np.float64)
    img2 = target.astype(np.float64)
    kernel = cv2.getGaussianKernel(11, 1.5)
    window = np.outer(kernel, kernel.transpose())
    mu1 = cv2.filter2D(img1, -1, window)[5:-5, 5:-5]  # valid
    mu2 = cv2.filter2D(img2, -1, window)[5:-5, 5:-5]
    mu1_sq = mu1**2
    mu2_sq = mu2**2
    mu1_mu2 = mu1 * mu2
    sigma1_sq = cv2.filter2D(img1**2, -1, window)[5:-5, 5:-5] - mu1_sq
    sigma2_sq = cv2.filter2D(img2**2, -1, window)[5:-5, 5:-5] - mu2_sq
    sigma12 = cv2.filter2D(img1 * img2, -1, window)[5:-5, 5:-5] - mu1_mu2
    ssim_map = ((2 * mu1_mu2 + C1) *
                (2 * sigma12 + C2)) / ((mu1_sq + mu2_sq + C1) *
                                       (sigma1_sq + sigma2_sq + C2))
    return ssim_map.mean()

def calculate_ssim(target, ref):
    '''
    calculate SSIM
    the same outputs as MATLAB's
    img1, img2: [0, 255]
    '''
    img1 = np.array(target, dtype=np.float64)
    img2 = np.array(ref, dtype=np.float64)
    if not img1.shape == img2.shape:
        raise ValueError('Input images must have the same dimensions.')
    if img1.ndim == 2:
        return ssim(img1, img2)
    elif img1.ndim == 3:
        if img1.shape[2] == 3:
            ssims = []
            for i in range(3):
                ssims.append(ssim(img1[:, :, i], img2[:, :, i]))
            return np.array(ssims).mean()
        elif img1.shape[2] == 1:
            return ssim(np.squeeze(img1), np.squeeze(img2))
    else:
        raise ValueError('Wrong input image dimensions.')

def psnr(target, ref):
    img1 = np.array(target, dtype=np.float32)
    img2 = np.array(ref, dtype=np.float32)
    diff = img1 - img2
    psnr = 10.0 * np.log10(255.0 * 255.0 / np.mean(np.square(diff)))
    return psnr

def calculate_psnr(target, ref):
    img1 = np.array(target, dtype=np.float64)
    img2 = np.array(ref, dtype=np.float64)
    if not img1.shape == img2.shape:
        raise ValueError('Input images must have the same dimensions.')
    if img1.ndim == 2:
        return psnr(img1, img2)
    elif img1.ndim == 3:
        if img1.shape[2] == 3:
            psnrs = []
            for i in range(3):
                psnrs.append(psnr(img1[...,i], img2[..., i]))
            return np.array(psnrs).mean()
        elif img1.shape[2] == 1:
            return psnr(img1.squeeze(), img2.squeeze())
    else:
        raise ValueError('Wrong input image dimensions.')
"""

if __name__ == '__main__':
    import PIL.Image as Image
    target = Image.open("./asylum/metrics/101085_gray.png")
    noisy = Image.open("./asylum/metrics/101085_noisy.png")
    target_np = np.array(target)
    noisy_np = np.array(noisy)
    print('-' * 10)
    print(f"Shape: target: {target_np.shape}; noisy: {noisy_np.shape}")
    print(f"Max: target: {target_np.max()}; noisy: {noisy_np.max()}")
    print(f"Min: target: {target_np.min()}; noisy: {noisy_np.min()}")

    # print(f"PSNR: {calculate_psnr(target_np, noisy_np)}")
    # print(f"SSIM: {calculate_ssim(target_np, noisy_np)}")

    metric_slice = Metrics(batch_flag=False)
    metric_slice.push(target_np, noisy_np)
    print(metric_slice.psnr_meter.avg)
    print(metric_slice.ssim_meter.avg)

    target_np = np.expand_dims(target_np, 0)
    noisy_np = np.expand_dims(noisy_np, 0)
    target_np = np.expand_dims(target_np, -1)
    noisy_np = np.expand_dims(noisy_np, -1)

    print('-'*10)
    print(f"Shape: target: {target_np.shape}; noisy: {noisy_np.shape}")
    print(f"Max: target: {target_np.max()}; noisy: {noisy_np.max()}")
    print(f"Min: target: {target_np.min()}; noisy: {noisy_np.min()}")

    metrics_batch = Metrics(batch_flag=True)
    metrics_batch.push(target_np, noisy_np)
    print(metrics_batch.psnr_meter.avg)
    print(metrics_batch.ssim_meter.avg)

    # target_np = normalize_zero_to_one(target_np)
    # noisy_np = normalize_zero_to_one(noisy_np)

    # print(f"Shape: target: {target_np.shape}; noisy: {noisy_np.shape}")
    # print(f"Max: target: {target_np.max()}; noisy: {noisy_np.max()}")
    # print(f"Min: targ:et: {target_np.min()}; noisy: {noisy_np.min()}")

    # # print(f"PSNR: {calculate_psnr(target_np, noisy_np)}")
    # # print(f"SSIM: {calculate_ssim(target_np, noisy_np)}")

    # metrics_batch = Metrics()
    # metrics_batch.push(target_np, noisy_np)
    # print(metrics_batch.psnr_meter.avg)
    # print(metrics_batch.ssim_meter.avg)
    # print('----------------------------------------')

    # target_np = np.clip(target_np * 255, 0, 255)
    # noisy_np = np.clip(noisy_np * 255, 0, 255)

    # print(f"Shape: target: {target_np.shape}; noisy: {noisy_np.shape}")
    # print(f"Max: target: {target_np.max()}; noisy: {noisy_np.max()}")
    # print(f"Min: target: {target_np.min()}; noisy: {noisy_np.min()}")

    # # print(f"PSNR: {calculate_psnr(target_np, noisy_np)}")
    # # print(f"SSIM: {calculate_ssim(target_np, noisy_np)}")

    # # target_np = np.expand_dims(target_np, -1)
    # # noisy_np = np.expand_dims(noisy_np, -1)

    # metrics_batch = Metrics()
    # metrics_batch.push(target_np, noisy_np)
    # print(metrics_batch.psnr_meter.avg)
    # print(metrics_batch.ssim_meter.avg)


from collections import Counter

def get_data(file):
    with open(file, encoding='utf-8') as f:
        data = f.read()
        inputs = data.splitlines()
    return inputs

diagnostic = get_data('Day3Data.txt')

def part_one(diaglist):
  gamma = ''
  epsilon = ''

  for i in range(len(diaglist[0])):
    common = Counter([x[i] for x in diaglist])

    if common['0'] > common['1']:
      gamma += '0'
      epsilon += '1'
    else:
      gamma += '1'
      epsilon += '0'

  return (int(gamma,2)*int(epsilon,2))

print(part_one(diagnostic))

def part_two_oxygen(diaglist):
  oxrating = ''

  for i in range(len(diaglist[0])):
    common = Counter([x[i] for x in diaglist])

    if common['0'] > common['1']:
      diaglist = [x for x in diaglist if x[i] == '0']
    else:
      diaglist = [x for x in diaglist if x[i] == '1']

    oxrating = diaglist[0]
  return int(oxrating,2)

def part_two_C02(diaglist):
  C02rating = ''

  for i in range(len(diaglist[0])):
    common = Counter([x[i] for x in diaglist])

    if common['0'] > common['1']:
      diaglist = [x for x in diaglist if x[i] == '1']
    else:
      diaglist = [x for x in diaglist if x[i] == '0']

    if diaglist:
      C02rating = diaglist[0]

  return int(C02rating,2)

print(part_two_C02(diagnostic)*part_two_oxygen(diagnostic))


import logging
import sys
import traceback
import openpyxl
import requests
import geocoding
import os

LOG_FORMAT = "%(asctime)s %(name)s %(levelname)s %(pathname)s %(message)s "  # 配置输出日志格式
DATE_FORMAT = '%Y-%m-%d  %H:%M:%S %a '  # 配置输出时间的格式，注意月份和天数不要搞乱了
logging.basicConfig(level=logging.INFO,
                    format=LOG_FORMAT,
                    datefmt=DATE_FORMAT,
                    filename=r"./logs/poiSpider.log"  # 有了filename参数就不会直接输出显示到控制台，而是直接写入文件
                    )

class PoiSpider:
    def __init__(self, key, polygon_points, type_code, keywords):
        # 接口地址
        self.spider_url = 'https://restapi.amap.com/v3/place/polygon'
        # 高德api key
        self.api_key_list = key

        # POI 大类列表 参考：https://lbs.amap.com/api/webservice/download 的 POI分类编码
        self.type_code_list = str.split(type_code, '|')
        self.polygon_points = polygon_points
        self.keywords = keywords
        # self.city_code = area_code
        self.fileName = 'POI爬取.xlsx'
        self.workbook = None
        self.res_list = []

    def get_poi(self):
        """
        获取POI信息
        """
        # 获取大类
        for code in self.type_code_list:
            self.get_json(poi_type=code, page=1)

    def get_json(self, poi_type, page):
        """
        发送请求获取结果集
        :param poi_type: 字典类型
        :param page: 当前页数
        :return:
        """
        try:
            url_params = {
                'polygon': self.polygon_points,
                # 'city': self.city_code,
                'key': self.api_key_list,
                'offset': 20,
                'page': page,
                'types': poi_type
            }
            if self.keywords:
                url_params.keywords = self.keywords
            print('请求 %s %d 页数据开始' % (poi_type, page))

            requests.adapters.DEFAULT_RETRIES = 5
            s = requests.session()
            s.keep_alive = False
            res = requests.get(self.spider_url, headers={'Connection': 'close'}, params=url_params)
            if res.status_code == 200:
                res_dict = res.json()
                print(res_dict['count'])
                # 获取POI的信息
                pois = res_dict['pois']
                for poi in pois:
                    # 避免值出现空[]等错误，进行判断后再加入列表
                    poi_list = [
                        poi['id'] if "id" in poi and poi['id'] else '',
                        poi['name'] if "name" in poi and poi['name'] else '',
                        poi['type'] if "type" in poi and poi['type'] else '',
                        self.change_to_wgs84(poi['location']) if "location" in poi and poi['location'] else '',
                        poi['tel'] if "tel" in poi and poi['tel'] else '',
                        poi['pname'] if "pname" in poi and poi['pname'] else '',
                        poi['cityname'] if "cityname" in poi and poi['cityname'] else '',
                        poi['adname'] if "adname" in poi and poi['adname'] else '',
                        poi['address'] if "address" in poi and poi['address'] else ''
                    ]
                    self.res_list.append(poi_list)

                if page * url_params.get('offset') < int(res_dict['count']):
                    self.get_json(poi_type=poi_type, page=page + 1)
                else:
                    self.save_poi(poi_type)
                return 0
            else:
                self.save_poi(poi_type)
                print('接口请求异常！错误信息为：%s' % res.json())
                print("当前正在请求的接口参数为：{page: %s, types: %s}  !" % (url_params.get('page'), url_params.get('types')))
                print("输入任意字符程序关闭...")
                input()
                return -1
        except Exception as e:
            logging.error("程序异常====> %s", e.args)
            logging.error("异常信息====> %s", traceback.format_exc())
            print('程序出现异常！具体可以查看日志')
        finally:
            self.save_poi(poi_type)

    def change_to_wgs84(self, coord):
        """
        执行坐标转换
        :param coord:坐标
        :return:
        """
        if coord:
            lon = float(coord.split(',')[0])
            lat = float(coord.split(',')[1])
            lonlat = geocoding.gcj02_to_wgs84(lon, lat)
            return str(lonlat[0]) + ',' + str(lonlat[1])
        else:
            return ''

    def save_poi(self, type):
        """
        保存POI信息
        :param type: POI类型
        :return:
        """
        try:
            # 创建文件
            if os.path.isfile(self.fileName):
                self.workbook = openpyxl.load_workbook(self.fileName)
                if type not in self.workbook.sheetnames:
                    self.workbook.create_sheet(type)
                    table = self.workbook[type]
                    # 设置表头
                    table.append(['id', 'name', 'type', 'location', 'tel', 'pname', 'cityname', 'adname', 'address'])
                else:
                    table = self.workbook[type]
            else:
                self.workbook = openpyxl.Workbook()
                table = self.workbook.active
                table.title = type

                # 设置表头
                table.append(['id', 'name', 'type', 'location', 'tel', 'pname', 'cityname', 'adname', 'address'])
                self.workbook.save(self.fileName)

            # 循环插入数据
            for item in self.res_list:
                if item:
                    table.append(item)
            # 保存文件
            self.workbook.save(self.fileName)
            self.workbook.close()  # 对程序中只读的workbook
        except Exception as e:
            logging.error("程序异常====> %s", e.args)
            logging.error("异常信息====> %s", traceback.format_exc())
            print('程序出现异常！具体可以查看日志')
        finally:
            # 清空
            self.res_list = []
        return ''

if __name__ == "__main__":
    try:
        print("1.请输入高德API key")
        key = input()
        if key is None or key == '':
            print("输入参数错误，输入任意字符退出...")
            input()
            exit(0)
        print("2.请输入爬取范围坐标串，经度在前，纬度在后，坐标对用'|'分割。经纬度小数点后不得超过6位,首尾坐标对需相同")
        polygon_points = input()
        if polygon_points is None or polygon_points == '':
            print("参数错误！爬取的范围未输入！任意字符退出...")
            temp = input()
            sys.exit(0)
        print("3.请输入分类代码，多个类型用'|'分割")
        type_code = input()
        if type_code is None or type_code == '':
            print("参数错误！分类代码未输入！任意字符退出...")
            temp = input()
            sys.exit(0)
        print("4.请输入关键字，多个关键字用“|”分割;如果不需要，回车进入下一步")
        keywords = input()
        poi_spider = PoiSpider(key, polygon_points, type_code, keywords)
        poi_spider.get_poi()
        print("爬取完毕！输入任意字符程序关闭...")
        input()
        sys.exit(0)

    except Exception as e:
        logging.error("程序异常====> %s", e.args)
        logging.error("异常信息====> %s", traceback.format_exc())
        print("程序出现异常，输入任意字符退出...")
        input()
        sys.exit(0)


from tkinter import *
import sqlite3

root = Tk()
root.title('DATABASE GUI')
root.geometry("550x650")

conn = sqlite3.connect('address_book.db')
c = conn.cursor()

# button function
def submit():
    conn = sqlite3.connect('address_book.db')
    c = conn.cursor()
    # insert into tables
    c.execute("INSERT INTO addresses VALUES (:f_name, :l_name, :address, :city, :state, :zipcode)",
              {
                  'f_name': 'f_name.get()',
                  'l_name': 'l_name.get()',
                  'address': 'address.get()',
                  'city': 'city.get()',
                  'state': 'state.get()',
                  'zipcode': 'zipcode.get()'
              })

    conn.commit()
    conn.close()
    # Clear the textboxes
    f_name.delete(0, END)
    l_name.delete(0, END)
    address.delete(0, END)
    city.delete(0, END)
    state.delete(0, END)
    zipcode.delete(0, END)

def query():
    conn = sqlite3.connect('address_book.db')
    c = conn.cursor()
    c.execute("SELECT *, oid FROM addresses")
    records = c.fetchall()
    print(records)
    print_records = ''
    for record in records[0]:
        print_records += str(record) + "\n"
    query_label = Label(root, text=print_records).grid(row=8, column=0, columnspan=2)

    conn.commit()
    conn.close()

# create textboxes
f_name = Entry(root, width=30).grid(row=0, column=1, padx=20)
l_name = Entry(root, width=30).grid(row=1, column=1, padx=20)
address = Entry(root, width=30).grid(row=2, column=1, padx=20)
city = Entry(root, width=30).grid(row=3, column=1, padx=20)
state = Entry(root, width=30).grid(row=4, column=1, padx=20)
zipcode = Entry(root, width=30).grid(row=5, column=1, padx=20)
# creating textbox labels
f_name_label = Label(root, text="First name").grid(row=0, column=0)
l_name_label = Label(root, text="Last name").grid(row=1, column=0)
address_label = Label(root, text="Address").grid(row=2, column=0)
city_label = Label(root, text="City").grid(row=3, column=0)
state_label = Label(root, text="State").grid(row=4, column=0)
zipcode_label = Label(root, text="Zipcode").grid(row=5, column=0)
# submit button
submit_btn = Button(root, text="ADD RECORD TO DATABASE", command=submit, bg="orange", fg="white")
submit_btn.grid(row=6, column=0, columnspan=2, pady=10, padx=10, ipadx=100)
# query button
query_btn = Button(root, text="Show records", command=query)
query_btn.grid(row=7, column=0, columnspan=2, pady=10, padx=10, ipadx=137)

root.mainloop()


from typing import Any

from ssz.exceptions import DeserializationError, SerializationError
from ssz.sedes.basic import BasicSedes

class UInt(BasicSedes[int, int]):
    def __init__(self, num_bits: int) -> None:
        if num_bits % 8 != 0:
            raise ValueError("Number of bits must be a multiple of 8")
        self.num_bits = num_bits
        super().__init__(num_bits // 8)

    def serialize(self, value: int) -> bytes:
        if value < 0:
            raise SerializationError(
                f"Can only serialize non-negative integers, got {value}"
            )

        try:
            return value.to_bytes(self.size, "little")
        except OverflowError:
            raise SerializationError(
                f"{value} is too large to be serialized in {self.size * 8} bits"
            )

    def deserialize(self, data: bytes) -> int:
        if len(data) != self.size:
            raise DeserializationError(
                f"Cannot deserialize length {len(data)} byte-string as uint{self.size*8}"
            )
        return int.from_bytes(data, "little")

    def get_sedes_id(self) -> str:
        return f"{self.__class__.__name__}{self.num_bits}"

    def __hash__(self) -> int:
        return hash((hash(UInt), self.num_bits))

    def __eq__(self, other: Any) -> bool:
        return isinstance(other, UInt) and other.num_bits == self.num_bits

uint8 = UInt(8)
uint16 = UInt(16)
uint32 = UInt(32)
uint64 = UInt(64)
uint128 = UInt(128)
uint256 = UInt(256)


from django.contrib import admin

# Register your models here.
from .models import Price, Product, ProductTag , PaymentHistory

class PriceAdmin(admin.StackedInline):
    model = Price

@admin.register(Product)
class ProductAdmin(admin.ModelAdmin):
    inlines = (PriceAdmin,)

    class Meta:
        model = Product

admin.site.register(ProductTag)
admin.site.register(Price)
admin.site.register(PaymentHistory)


import pandas as pd
import numpy as np
import psycopg2
import random
import math
import matplotlib.pyplot as plt
from sklearn.ensemble import RandomForestRegressor
import statsmodels.api as sm
from statsmodels.tsa.arima_model import ARIMA, ARIMAResults
from statsmodels.tsa.arima_process import ArmaProcess
from numpy.linalg import LinAlgError
from functools import reduce
import warnings
warnings.filterwarnings("ignore")

def arima_range(df, alph, day, i):
    """calculates confidence interval of arima predictions"""
    res = sm.tsa.arma_order_select_ic(df.diff(periods=1)[1:], ic=['aic', 'bic']
                                      , trend='nc')
    (ar, ma) = res.aic_min_order
    try:
        model = ARIMA(df.iloc[(i - int(day * 24 * 12)): i], order=(ar, 1, ma))\
                      .fit(method='css')
        forecast, stderr, conf = model.forecast(steps=int(day * 24 * 12 + 1),
                                                alpha=alph)
        low = conf[int(day * 24 * 12)][0]
        high = conf[int(day * 24 * 12)][1]
        return high, low
    except LinAlgError:
        pass

def reduc(l):
    """calculates returns of all the trades"""
    return reduce(lambda x, y: x * y, [a for (a, b, c) in l]), len(l), l

def next_i(df, i):
    """returns the next i"""
    return df[i: i + 1].values[0]

def get_perc(df, i, num_day, perc):
    """calculates the percentile if a given range"""
    return np.percentile(df[(i - 12 * 24 * num_day): i], perc)

def check_instance(df, bound, i, num_day, perc):
    """if it's a number, divide by 2, else get the percentage"""
    return bound / 2 if isinstance(bound, float) else get_perc(df, i, num_day,
                                                               perc)

def update_bounds(pre, df, i, num_day, perc_high, perc_low):
    """update the trading decision bounds"""
    if (i < 12 * 24 * num_day):
        hbound = get_perc(pre, len(pre), num_day, perc_high)
        lbound = get_perc(pre, len(pre), num_day, perc_low)
    else:
        lbound = get_perc(df, i, num_day, perc_low)
        try:
            hbound, x = arima_range(df, 0.9, num_day, i)
            flag = 0
        except (TypeError, ValueError) as e:
            hbound = get_perc(df, i, num_day, perc_high)
            flag = 1
        if flag == 0:
            hbound = check_instance(df, hbound, i, num_day, perc_high)
        if hbound < lbound:
            hbound = get_perc(df, i, num_day, perc_high)
    return hbound, lbound

res = []

def make_money(df, pre, i=0, n_day=2, perc_hi=80, perc_low=30, bailperc=0.05):
    """run simulation to see the profit potential of given df"""
    if i == len(df) - 1:
        return reduc(res)
    # set bounds
    hbound, lbound = update_bounds(pre, df, i, n_day, perc_hi, perc_low)
    # find the buying point to sell away
    j = 0
    while (i < len(df)) and (next_i(df, i) < hbound):
        i += 1
        j += 1
        if i == len(df) - 1:
            return reduc(res)
        if j > n_day * 12 * 24:
            hbound, lbound = update_bounds(pre, df, i, n_day, perc_hi, perc_low)
            j = 0
    res.append((1 + next_i(df, i), i, 1))
    bail = df[i: i + 1].values[0] + bailperc
    # find the buying point to sell back
    j = 0
    while (i < len(df)) and (next_i(df, i) < bail):
        if df[i: i + 1].values[0] < lbound:
            res.append((1 - next_i(df, i), i, 2))
            make_money(df, pre, i=i)
            return reduc(res)
        i += 1
        j += 1
        if j > n_day * 12 * 24:
            hbound, lbound = update_bounds(pre, df, i, n_day, perc_hi, perc_low)
            j = 0

def create_row(df, start_num, ptile, inday=2, pday=2):
    """create random forest friendly format"""
    tar = df[(start_num + 48 * inputday):(start_num + 48 * (inday + pday))]
    target = tar.quantile(ptile)
    return df[start_num: (start_num + 24 * inputday)], target

def to_fit_rf(df, day=0, perc=0.8):
    """Fit random forest"""
    X, y = [], []
    for i in range(48 * day, 48 * (day + 1)):
        a, b = create_row(df, i, perc)
        X.append(list(a))
        y.append(b)
    y = np.array(y)
    X = np.array(X)
    model = RandomForestRegressor(n_estimators=500, min_samples_split=8,
                                  min_samples_leaf=4, random_state=1)
    model.fit(X, y)
    return model

def to_predict_rf(df, day=1, prec=0.8):
    """predict random forest model"""
    modelu = to_fit_rf(df, day - 1, perc)
    Xpred = []
    for i in range(48 * day, 48 * (day + 1)):
        a, b = create_row(df, i, perc)
        Xpred.append(list(a))
    Xpred = np.array(Xpred)
    high = model.predict(Xpred).mean()
    modeld = to_fit_rf(df, day - 1, 1 - perc)
    Xpred = []
    for i in range(48 * day, 48 * (day + 1)):
        a, b = create_row(df, i, 1 - perc)
        Xpred.append(list(a))
    Xpred = np.array(Xpred)
    low = model.predict(Xpred).mean()
    return high, low


import torch
import torch.nn as nn
import torch.optim as optim
import torchvision
from torch.utils.tensorboard import SummaryWriter
import random

from per_channel_unet.unet import YUVUNet, UNet
from per_channel_unet.preprocess import preprocess, resize, postprocess

import sys

def main():
    if not torch.cuda.is_available():
        raise Exception("CUDA is not available")

    writer = SummaryWriter()

    if False:
        y_model = UNet(n_channels=1, tile_size=256).to('cuda')
        u_model = UNet(n_channels=1, tile_size=256).to('cuda')
        v_model = UNet(n_channels=1, tile_size=256).to('cuda')
    else:
        y_model = torch.load('y_wow_batch_2_5400.model').to('cuda')
        u_model = torch.load('u_wow_batch_2_5400.model').to('cuda')
        v_model = torch.load('v_wow_batch_2_5400.model').to('cuda')

    lr = 1e-6
    idx = 0

    criterion = nn.MSELoss(reduction='sum')
    y_optimizer = optim.SGD(y_model.parameters(), lr=lr, momentum=0.9)
    u_optimizer = optim.SGD(u_model.parameters(), lr=lr, momentum=0.9)
    v_optimizer = optim.SGD(v_model.parameters(), lr=lr, momentum=0.9)

    for t in range(10000):
        data = torchvision.datasets.ImageFolder(root='G:/workspace/cheap-dlss-torch/media/wow', transform=preprocess)
        data_loader = torch.utils.data.DataLoader(data, batch_size=1, shuffle=True, num_workers=2)

        pool = []
        pool_count = 0
        pool_exponent = 0

        for (image_batch, _) in data_loader:
            loss_mean = 0
            loss_var = 0

            window_idx = 0

            if 2 ** pool_exponent > pool_count:
                pool.append(image_batch)
                pool_count += 1
                continue

            while True:
                image_batch = random.choice(pool)

                lowres_batch = resize(image_batch).to('cuda')
                answer_batch = image_batch.to('cuda')

                y_lowres, u_lowres, v_lowres = torch.chunk(lowres_batch, chunks=3, dim=-3)
                y_answer, u_answer, v_answer = torch.chunk(answer_batch, chunks=3, dim=-3)

                y_pred = y_model(y_lowres, y_answer)
                u_pred = u_model(u_lowres, u_answer)
                v_pred = v_model(v_lowres, v_answer)

                # y_pred = torch.clamp(y_pred, min=0.0, max=1.0)
                # u_pred = torch.clamp(u_pred, min=0.0, max=1.0)
                # v_pred = torch.clamp(v_pred, min=0.0, max=1.0)

                y_loss = criterion(y_pred, y_answer)
                u_loss = criterion(u_pred, u_answer)
                v_loss = criterion(v_pred, v_answer)

                y_optimizer.zero_grad()
                u_optimizer.zero_grad()
                v_optimizer.zero_grad()

                y_loss.backward()
                u_loss.backward()
                v_loss.backward()

                y_optimizer.step()
                u_optimizer.step()
                v_optimizer.step()

                writer.add_scalar('y_wow_batch_2_loss_overfit', y_loss.item(), idx)
                writer.add_scalar('u_wow_batch_2_loss_overfit', u_loss.item(), idx)
                writer.add_scalar('v_wow_batch_2_loss_overfit', v_loss.item(), idx)

                loss_val = y_loss.item() + u_loss.item() + v_loss.item()
                writer.add_scalar('wow_batch_2_loss_overfit', loss_val, idx)

                loss_mean += loss_val
                loss_var += loss_val * loss_val
                window_idx += 1

                if window_idx > 100:
                    loss_mean /= window_idx
                    loss_var /= window_idx
                    loss_var -= loss_mean * loss_mean
                    window_idx = 0

                    print(idx, len(pool), loss_mean, loss_var)

                    torch.save(y_model, f'y_wow_batch_2_{idx}.model')
                    torch.save(y_model, f'u_wow_batch_2_{idx}.model')
                    torch.save(y_model, f'v_wow_batch_2_{idx}.model')

                    if loss_mean < 20 or loss_var < 10:
                        pred = torch.cat([y_pred, u_pred, v_pred], -3)
                        writer.add_image('wow_image_batch_pred',
                                         torch.clamp(postprocess(pred[0]), min=0.0, max=1.0),
                                         global_step=idx,
                                         dataformats='CHW')
                        writer.add_image('wow_image_batch_answer',
                                         torch.clamp(postprocess(answer_batch[0]), min=0.0, max=1.0),
                                         global_step=idx,
                                         dataformats='CHW')
                        break
                    else:
                        loss_mean = 0
                        loss_var = 0

                idx += 1

            print(idx, loss_val)

            if pool_exponent < 9:
                pool_exponent += 1
            else:
                pool_exponent = 0
            pool_count = 0
            pool.clear()

if __name__ == '__main__':
    main()


import sqlite3
import datetime

from io import BytesIO
from typing import List
from nonebot import get_bot
from nonebot.adapters.onebot.v11.message import MessageSegment, Message
from nonebot.adapters.onebot.v11.event import GroupMessageEvent

from Utils.ImageUtils import get_water_card

conn = sqlite3.connect("./Resources/db/water.db")
water_db = conn.cursor()

class WaterInfo:

    def __init__(self) -> None:
        self.time:datetime.datetime = datetime.datetime.now()
        self.format_time:str = self.time.strftime("%Y-%m-%d")
        self.format_time_aps:str = (self.time+datetime.timedelta(days=-1)).strftime("%Y-%m-%d")

    async def getRank(self, group_id:int, symbol:bool) -> BytesIO:
        table = self.format_time_aps if symbol else self.format_time
        member_info:List[tuple[int,str,int]] = []
        bot = get_bot()
        for _ in water_db.execute(f"""SELECT qq,water FROM "{table}" WHERE group_id={group_id} ORDER BY water desc LIMIT 15""").fetchall():
            try:
                info = await bot.get_group_member_info(group_id=group_id,user_id=_[0])
            except:
                continue
            else:
                member_info.append((_[0], info['nickname'], _[1]))
        return await get_water_card(member_info)

class WaterInfoSub(WaterInfo):

    def __init__(self, event:GroupMessageEvent) -> None:
        super().__init__()
        self.group:int = event.group_id
        self.qq:int = event.user_id
        self.water:int
        try:
            self.water = water_db.execute(f"""SELECT water FROM "{self.format_time}" WHERE (group_id={self.group} AND qq={self.qq})""").fetchall()[0][0]
        except:
            self.water = 0

    def createTable(self) -> None:
        water_db.execute(f"""CREATE TABLE "{self.format_time}" (
                "group_id" integer NOT NULL,
                "qq" integer NOT NULL,
                "water" integer NOT NULL,
                "last_time" TEXT NOT NULL,
                PRIMARY KEY ("group_id", "qq"));""")

    def add(self) -> None:
        try:
            water_db.execute(f"""REPLACE INTO "main"."{self.format_time}" ("group_id", "qq", "water", "last_time") VALUES ({self.group}, {self.qq}, {self.water+1}, '{self.format_time}')""")
        except:
            self.createTable()
            self.add()
        conn.commit()

    def getPersonalWater(self, *qq:int) -> Message:
        if qq:
            self.qq = qq[0]
            try:
                self.water = water_db.execute(f"""SELECT water FROM "{self.format_time}" WHERE (group_id={self.group} AND qq={self.qq})""").fetchall()[0][0]
            except:
                self.water = 0
        return (
        MessageSegment.text("好耶！那就让我来康康宁有多氵\n")
        +MessageSegment.text("查询对象：")+MessageSegment.at(self.qq)+MessageSegment.text("\n")
        +MessageSegment.text(f"吹氵次数：{self.water}\n")
        +MessageSegment.text(f"起始时间：00:00:00\n")
        +MessageSegment.text(f"终止时间：{datetime.datetime.now().strftime('%H:%M:%S')}"))


from . import views
from django.urls import path

app_name = 'todoapp'

urlpatterns = [
    path('', views.fun_add1, name="show1"),
   # path('add', views.fun_add1, name="add1"),
    path('todo_det', views.fun_det3, name="todo_det1"),
    path('del/<int:taskID>/', views.fun_del, name="del1"),
    path('edit/<int:taskID>/', views.fun_edit, name="edit1"),
    path('lsTaskView/', views.lsView1.as_view(), name="lsView1"),
    path('dtTaskView/<int:pk>/', views.dtView1.as_view(), name="dtView1"),
    path('upTaskView/<int:pk>/', views.updat1.as_view(), name="updView1"),
    path('delTaskView/<int:pk>/', views.clsDel1.as_view(), name="delView1"),

]


from sklearn.model_selection import GridSearchCV
from tensorflow.keras.wrappers.scikit_learn import KerasRegressor
from tensorflow.keras import layers
from tensorflow.keras.models import Sequential
from tensorflow.keras.optimizers import Adam
import tensorflow as tf

# Load the data
mnist = tf.keras.datasets.mnist
(X_train, y_train), (X_test, y_test) = mnist.load_data()

# Define the model architecture
def create_model(learning_rate=0.01, num_filters=32, kernel_size=3, activation='relu'):
    model = Sequential([
        layers.Conv2D(num_filters, kernel_size, activation=activation, input_shape=(28, 28, 1)),
        layers.MaxPooling2D(),
        layers.Flatten(),
        layers.Dense(10, activation='softmax')
    ])
    optimizer = Adam(lr=learning_rate)
    model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])
    return model

# Create a KerasRegressor object
model = KerasRegressor(build_fn=create_model, verbose=0)

# Define the hyperparameters to tune
param_grid = {
    'learning_rate': [0.01, 0.1, 0.5],
    'num_filters': [16, 32, 64],
    'kernel_size': [3, 5, 7],
    'activation': ['relu', 'sigmoid']
}

# Use GridSearchCV to find the best hyperparameters
grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=3)
grid_result = grid.fit(X_train, y_train)

# Print the best hyperparameters and accuracy
print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))


import pandas as pd
import numpy as np
from scipy.optimize import minimize
from sklearn.metrics import log_loss
import os

### finding the optimum weights

def log_loss_func(weights, predictions, test_y):
	''' scipy minimize will pass the weights as a numpy array '''
	final_prediction = 0
	for weight, prediction in zip(weights, predictions):
			final_prediction += weight*prediction

	return log_loss(test_y, final_prediction)

def find_weight(models, test_x, test_y):
	#the algorithms need a starting value, right not we chose 0.5 for all weights
	#its better to choose many random starting points and run minimize a few times
	predictions = []
	for i, model in enumerate(models):
		predictions.append(model.get_marginal(test_x[i]))

	starting_values = [0.5]*len(predictions)

	#adding constraints  and a different solver as suggested by user
	cons = ({'type':'eq','fun':lambda w: 1-sum(w)})
	#our weights are bound between 0 and 1
	bounds = [(0,1)]*len(predictions)

	res = minimize(log_loss_func, starting_values, args=(predictions, test_y), method='SLSQP', bounds=bounds, constraints=cons)

	print('Ensamble Score: {best_score}'.format(best_score=res['fun']))
	print('Best Weights: {weights}'.format(weights=res['x']))
	return res['x']


import numpy as np
import pandas as pd
from sklearn.metrics import roc_auc_score
from tqdm import tqdm

def compute_auc_scores(X, target, verbose=True):
    auc_scores = []
    if verbose:
        iterator = tqdm(X, total=X.shape[1])
    else:
        iterator = X
    for var in iterator:
        auc = compute_auc(X[var], target)
        auc_scores.append({'var': var, 'auc': auc})
    return (
        pd.DataFrame(auc_scores)
          .sort_values(by='auc')
    )

def compute_random_auc_scores(X, target, n_iters=1000, verbose=True):
    if verbose:
        iterator = tqdm(range(n_iters))
    else:
        iterator = range(n_iters)
    return [
        sample_auc(X, target)
        for _ in iterator
    ]

def sample_auc(X, target):
    var = np.random.choice(X.columns)
    values = X[var].sample(X.shape[0]).values
    return compute_auc(values, target)

def compute_auc(x, target):
    auc = roc_auc_score(target, x)
    if auc < 0.5:
        auc = roc_auc_score(target, -x)
    return auc


from setuptools import setup, find_packages

with open("README.md", "r") as fh:
    long_description = fh.read()

setup(
    name="gptbase",
    version="0.3.0",
    author="callmexss",
    author_email="callmexss@126.com",
    description="A package for simplified interaction with OpenAI's GPT-3 and GPT-4 models.",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/callmexss/gptbase",
    packages=find_packages(include=["gptbase"]),
    install_requires=[
        "openai==1.1.1",
        "rich==13.0.1",
        "tiktoken==0.5.1",
        "click==8.0.3",
        "iso-639==0.4.5",
    ],
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent",
    ],
    python_requires=">=3.10",
    entry_points={
        "console_scripts": [
            "gptbase-chat=gptbase.chat:chat",
        ],
    },
)


import numpy as np
import joblib
import random

def get_datasets(filepath, *, test_percent=0.1):
    episodes = joblib.load(filepath)
    cut = int(len(episodes) * test_percent)
    assert cut > 1

    trainset = Dataset(episodes[cut:])
    testset = Dataset(episodes[:cut])

    return trainset, testset

class Dataset(object):
    def __init__(self, episodes):

        self._episodes = episodes
        self._size = len(self._episodes)

    @property
    def episodes(self):
        return self._episodes

    @property
    def size(self):
        return self._size

    def next_episode(self):
        """Return next episode examples from this data set."""

        i = random.randint(0, self._size - 1)

        images, labels = self._episodes[i]
        images = np.array(images).astype(float)
        labels = np.array(labels).astype(float)
        return images, labels

    def next_episode_batch(self, batch_size):
        batch_images = []
        batch_labels = []
        for _ in range(batch_size):
            images, labels = self.next_episode()
            batch_images.append(images)
            batch_labels.append(labels)
        return batch_images, batch_labels


# 1
# https://leetcode.com/problems/01-matrix/

# https://leetcode.com/problems/01-matrix/discuss/1369741/C%2B%2BJavaPython-BFS-DP-solutions-with-Picture-Clean-and-Concise-O(1)-Space
# 그래도 조금 끄적여봣다가 포기...그래도 30분은 끄적여보면서 이것저것 시도..
# 패턴 조금 안듯...
# 문제부터 이해하는데 오래 걸렸다...또르르
from collections import deque
from typing import List
class Solution:
    def updateMatrix(self, mat: List[List[int]]) -> List[List[int]]:
        m, n = len(mat), len(mat[0])
        DIR = [0, 1, 0, -1, 0]

        q = deque([])
        for r in range(m):
            for c in range(n):
                if mat[r][c] == 0:
                    q.append((r, c))
                else:
                    mat[r][c] = -1  # Marked as not processed yet!

        while q:
            r, c = q.popleft()
            for i in range(4):
                nr, nc = r + DIR[i], c + DIR[i + 1]
                if nr < 0 or nr == m or nc < 0 or nc == n or mat[nr][nc] != -1: continue
                mat[nr][nc] = mat[r][c] + 1
                q.append((nr, nc))
        return mat

Leetcode Algorithms/Day9 (BFS,DFS)/542. 01 Matrix.py


import sys
import asyncio
from qasync import QEventLoop, asyncSlot, Queue
from PyQt5 import QtCore, QtWidgets
import pyqtgraph as pg
import numpy as np
from time import time
import sounddevice as sd

class OptionViz(QtCore.QObject):
    def __init__(self, app):
        super().__init__()
        self.app = app
        self.p = pg.plot()
        self.p.setWindowTitle("pyqtgraph example: PlotSpeedTest")
        self.p.setRange(QtCore.QRectF(0, -10, 5000, 20))
        self.p.setLabel("bottom", "Index", units="B")
        self.curve = self.p.plot()

        self.data = np.random.normal(size=(50, 5000))
        self.ptr = 0
        self.lastTime = time()
        self.fps = None

        q_in = Queue()

        def callback(indata, frame_count, time_info, status):
            loop.call_soon_threadsafe(q_in.put_nowait, (indata.copy(), status))
        # end callbac()

        stream = sd.InputStream(callback=callback, channels=1, device=2)

        timer = QtCore.QTimer(self)
        timer.timeout.connect(self.update)
        timer.start(0)

    @asyncSlot()
    async def update(self):
        self.curve.setData(self.data[self.ptr % 10])
        self.ptr += 1
        now = time()
        dt = now - self.lastTime
        self.lastTime = now
        if self.fps is None:
            fps = 1.0 / dt
        else:
            s = np.clip(dt * 3.0, 0, 1)
            self.fps = self.fps * (1 - s) + (1.0 / dt) * s

        self.p.setTitle("%0.2f fps" % fps)

if __name__ == "__main__":
    app = QtWidgets.QApplication(sys.argv)
    loop = QEventLoop(app)
    asyncio.set_event_loop(loop)
    viz = OptionViz(app)
    loop.run_forever()
# end if