import numpy as np
from PIL import Image
from scipy.signal import convolve2d
import matplotlib.pyplot as plt

import pickle
import DataHandler

import logging
logger = logging.getLogger('root.' + __name__)
logger.addHandler(logging.NullHandler())

def flipped(matrix):
    """
    Flip matrix horizontally and vertically. Used for flipping kernels.
    :param matrix: numpy.ndarray object
    :return: flipped matrix
    """
    result = np.ndarray(matrix.shape, dtype=matrix.dtype)
    for i in range(matrix.size):
        x = int(i / matrix.shape[1])
        y = i % matrix.shape[1]
        result[x][y] = matrix[matrix.shape[0] - x - 1][matrix.shape[1] - y - 1]
    return result

def sigmoid(x):
    """
    Simple sigmoid function. Warning: when overflow problems are encountered use sigmoid2 instead.
    :param x: numerical value or numpy.ndarray
    :return: sigmoid(x)
    """
    return 1 / (1 + np.exp(-x))

def sigmoid2(x):
    """More stable but slower version of sigmoid function
    :param x: ndarray
    :return: ndarray of sigmoids
    """
    if type(x) != 'np.ndarray' or (x.max() < 30 and x.min() > -30):
        return sigmoid(x)
    res = np.ndarray(x.shape)
    for i in range(0, x.shape[0]):
        for j in range(0, x.shape[1]):
            if x[i][j] > 30:
                res[i][j] = 1
            elif x[i][j] < -30:
                res[i][j] = 0
            else:
                res[i][j] = 1 / (1 + np.exp(-x[i][j]))
    return res

class CCRBM:
    """
    Convolutional Continuous Restricted Boltzmann Machine class.
    This class provides data structure for an CCRBM as well as methods used for training, testing and monitoring
    performance.
    """

    def __init__(self, size_v, size_h, filters_no, conv_kernel, typeB='scalar', typeC='matrix'):
        """
        :param size_v: vertical size of input image
        :param size_h: horizontal size of input image
        :param filters_no: how many feature maps
        :param conv_kernel: size of convolutional kernel (tuple)
        :param typeB: scalar or matrix version of feature map biases?
        :param typeC: scalar or matrix version of visible layer bias?
        """
        # RBM parameters
        self.insize_v = size_v
        self.insize_h = size_h
        self.filters_no = filters_no
        self.conv_kernel = conv_kernel
        # neurons, weigths and biases
        self.v = np.ndarray((size_v, size_h), dtype=np.float32)  # int32?
        self.h = [np.ndarray((size_v - conv_kernel[0] + 1, size_h - conv_kernel[1] + 1),
                             dtype=np.int8) for i in range(filters_no)]
        self.W = [np.random.normal(0, 0.01, conv_kernel) for i in range(filters_no)]

        if typeB not in ('scalar', 'matrix') or typeC not in ('scalar', 'matrix'):
            raise ValueError('Wrong input arguments. typeB and typeC must be either \'scalar\' or \'matrix\'')
        self.typeB = typeB
        self.typeC = typeC

        if self.typeB == 'scalar':
            self.b = [np.random.normal(0, 0.01) for i in range(filters_no)]
        else:
            self.b = [np.random.normal(0, 0.01, (size_v - conv_kernel[0] + 1, size_h - conv_kernel[1] + 1)) for i in
                      range(filters_no)]

        if self.typeC == 'scalar':
            self.c = np.random.normal(0, 0.01)
        else:
            self.c = np.random.normal(0, 0.01, (size_v, size_h))

        self.dh = DataHandler.DataHandler()
        self.imgInfo = None

        self.iterations = 0
        self.mse = []
        logger.info('Created CCRBM. {}'
                    .format(self))

    def __str__(self):
        res = 'v shape: ({}, {}), filters_no: {}, conv_kernel: {}, typeB: {}, typeC: {}'.format(self.insize_v,
                                                                                self.insize_h,
                                                                                self.filters_no,
                                                                                self.conv_kernel,
                                                                                self.typeB,
                                                                                self.typeC)
        return res

    def sample_h_given_v(self):
        """
        Sample hidden layer values from visible layer values.
        """
        for feature_map in range(self.filters_no):
            tmp = convolve2d(self.v, flipped(self.W[feature_map]), mode='valid') + self.b[feature_map]
            self.h[feature_map] = np.random.binomial(1, sigmoid2(tmp))

    def sample_v_given_h(self):
        """
        Sample visible layer values from hidden layers values/
        """
        tmp = np.zeros((self.insize_v, self.insize_h))
        for feature_map in range(self.filters_no):
            tmp += convolve2d(self.h[feature_map], self.W[feature_map])
        tmp += self.c
        self.v = np.random.normal(tmp, 0.01)

    def prob_h_given_v(self):
        """
        Calculate activations probabilities for hidden layer given v.
        """
        for feature_map in range(self.filters_no):
            self.h[feature_map] = sigmoid2(
                convolve2d(self.v, flipped(self.W[feature_map]), mode='valid') + self.b[feature_map])

    def prob_v_given_h(self):
        """
        Calculate activations probabilities for visible layer given h
        """
        tmp = np.zeros((self.insize_v, self.insize_h))
        for feature_map in range(self.filters_no):
            tmp += convolve2d(self.h[feature_map], self.W[feature_map])
        self.v = tmp + self.c

    def batchMSE(self, batchSize=None, steps=3, sample=False):
        """
        Mean Squared Error calculated over test set
        :param batchSize: how many images? All test set by default
        :param steps: how many Gibbs steps before calculating MSE
        :param sample: sample values if True, takes probabilies otherwise
        :return: Mean Squared Error over images from testset
        """
        if self.dh.train is None:
            raise ValueError('Data handler was not initialised, no source for images')
        if batchSize is None:
            batchSize = self.dh.te_size
        mse = 0
        for i in range(batchSize):
            self.loadImage(i, dataset='test')
            v0 = np.copy(self.v)
            for j in range(steps):
                if sample:
                    self.sample_h_given_v()
                    self.sample_v_given_h()
                else:
                    self.prob_h_given_v()
                    self.prob_v_given_h()
            mse += ((self.v - v0) ** 2).mean()
        return mse / batchSize

    def contrastiveDivergence(self, iterations, lrate, momentum, kGibbsSteps=1, batchSize=10, monitor=10):
        """
        Contrastive divergence - 1 implemented with mini batch. Perform given number of iterations to train
        CCRBM with given learning rate. Use provided batchSize. Monitor MSE every X steps using monitor parameter.
        :param iterations: how many iterations (how many mini-batches)
        :param lrate: learning hyperparameter
        :param batchSize: how many images in mini-batch
        :param monitor: track MSE every X iterations
        """
        # bshape = (self.insize_v - self.conv_kernel[0] + 1, self.insize_h - self.conv_kernel[1] + 1)
        # cshape = (self.insize_v, self.insize_h)

        print('Starting Contrastive Divergence with following parameters:\n' \
              'iterations = {}, learnig rate = {}, momentum = {}, k = {}, batch size = {}, monitor = {}'.format(iterations,
                                                                    lrate, momentum, kGibbsSteps, batchSize, monitor))
        logger.info('Contrastive Divergence called for CCRBM: {}'.format(self) +
                 'iterations = {}, learnig rate = {}, momentum = {}, k = {}, batch size = {}, monitor = {}'.format(iterations,
                                                                    lrate, momentum, kGibbsSteps, batchSize, monitor))
        imgcounter = 0

        dW_old = [0 for i in range(self.filters_no)]
        db_old = [0 for i in range(self.filters_no)]
        dc_old = 0

        for it in range(self.iterations, self.iterations + iterations):

            dW = [np.zeros(shape=self.W[0].shape, dtype=np.float32) for i in range(self.filters_no)]
            db = [0 for i in range(self.filters_no)]
            dc = 0

            for batchidx in range(batchSize):
                if imgcounter == self.dh.tr_size:
                    print('All dataset has been used, staring from 0 again.')
                    imgcounter = 0

                self.loadImage(imgcounter)
                imgcounter += 1

                v0 = np.copy(self.v)
                # print('MSE before update: {}'.format(self.msError(image)))

                pH0 = [sigmoid2(convolve2d(self.v, flipped(self.W[k]), mode='valid') + self.b[k]) for k in
                       range(self.filters_no)]
                grad0 = [convolve2d(self.v, flipped(pH0[k]), mode='valid') for k in range(self.filters_no)]
                self.h = [np.random.binomial(1, pH0[k]) for k in range(self.filters_no)]

                self.sample_v_given_h()
                for i in range(kGibbsSteps-1):
                    self.sample_h_given_v()
                    self.sample_v_given_h()

                pH1 = [sigmoid2(convolve2d(self.v, flipped(self.W[k]), mode='valid') + self.b[k]) for k in
                       range(self.filters_no)]
                grad1 = [convolve2d(self.v, flipped(pH1[k]), mode='valid') for k in range(self.filters_no)]

                for k in range(self.filters_no):
                    dW[k] += (grad0[k] - grad1[k])
                    if self.typeB == 'scalar':
                        db[k] += (pH0[k] - pH1[k]).sum()
                    else:
                        db[k] += (pH0[k] - pH1[k])
                if self.typeC == 'scalar':
                    dc += (v0 - self.v).sum()
                else:
                    dc += (v0 - self.v)

            for k in range(self.filters_no):
                self.W[k] += (lrate / batchSize) * dW[k] + dW_old[k] * momentum
                self.b[k] += (lrate / batchSize) * db[k] + db_old[k] * momentum
                dW_old[k] = (lrate / batchSize) * dW[k] + dW_old[k] * momentum
                db_old[k] = (lrate / batchSize) * db[k] + db_old[k] * momentum

            self.c += (lrate / batchSize) * dc + dc_old * momentum
            dc_old = (lrate / batchSize) * dc + dc_old * momentum

            if not it % monitor:
                if not self.mse:
                    self.mse.append((it, self.batchMSE(steps=1)))
                elif self.mse[-1][0] != it:
                    self.mse.append((it, self.batchMSE(steps=1)))
                print('Iter: {}   MSE: {}'.format(*self.mse[-1]))
                logger.info('Iter: {}   MSE: {}'.format(*self.mse[-1]))
        self.iterations += iterations
        self.mse.append((self.iterations, self.batchMSE(steps=1)))
        print('Iter: {}   MSE: {}'.format(*self.mse[-1]))
        logger.info('Iter: {}   MSE: {}'.format(*self.mse[-1]))

    def persistantCD(self, iterations, lrate, momentum, pcdSteps=5, monitor=10):
        """
        Persistant contrastive divergence - 1 implemented with mini batch. Perform given number of iterations to train
        CCRBM with given learning rate. Weights update every pscSteps steps. Monitor MSE every X steps using monitor parameter.
        :param iterations: how many iterations (how many mini-batches)
        :param lrate: learning hyperparameter
        :param pcdSteps: how many PSC steps for one training example
        :param monitor: track MSE every X iterations
        """
        # bshape = (self.insize_v - self.conv_kernel[0] + 1, self.insize_h - self.conv_kernel[1] + 1)
        # cshape = (self.insize_v, self.insize_h)
        # mse = []
        print('Starting Persistant Contrastive Divergence with following parameters:\n' \
              'iterations = {}, learning rate = {}, momentum = {}, pcd steps = {}, monitor = {}'.format(iterations, lrate, momentum, pcdSteps,
                                                                                monitor))
        logger.info('Persistant Contrastive Divergence called for CCRBM: {}'.format(self) +
                 'iterations = {}, learning rate = {}, momentum = {}, pcd steps = {}, monitor = {}'.format(iterations, lrate, momentum, pcdSteps,
                                                                                monitor))
        dW_old = [0 for i in range(self.filters_no)]
        db_old = [0 for i in range(self.filters_no)]
        dc_old = 0
        imgcounter = 0
        for it in range(self.iterations, self.iterations + iterations):

            dW = [np.zeros(shape=self.W[0].shape, dtype=np.float32) for i in range(self.filters_no)]
            db = [0 for i in range(self.filters_no)]
            dc = 0

            if imgcounter == self.dh.tr_size:
                print('All dataset has been used, staring from 0 again.')
                imgcounter = 0

            self.loadImage(imgcounter)
            imgcounter += 1

            for pcd in range(pcdSteps):
                if pcd == 0:
                    v0 = np.copy(self.v)
                # print('MSE before update: {}'.format(self.msError(image)))

                pH0 = [sigmoid2(convolve2d(v0, flipped(self.W[k]), mode='valid') + self.b[k]) for k in
                       range(self.filters_no)]
                grad0 = [convolve2d(v0, flipped(pH0[k]), mode='valid') for k in range(self.filters_no)]
                self.h = [np.random.binomial(1, pH0[k]) for k in range(self.filters_no)]

                self.sample_v_given_h()

                pH1 = [sigmoid2(convolve2d(self.v, flipped(self.W[k]), mode='valid') + self.b[k]) for k in
                       range(self.filters_no)]
                grad1 = [convolve2d(self.v, flipped(pH1[k]), mode='valid') for k in range(self.filters_no)]

                # print('W:{} grad0:{} grad1:{}'.format(self.W[0].shape, grad0[0].shape, grad1[0].shape))
                for k in range(self.filters_no):
                    # if k ==1 and pcd == 0 : print('Iter {} delta.mean(k=1): {}, W.mean(k=1) : {}'.format(iter, delta.mean(), self.W[k].mean()))
                    dW[k] += (grad0[k] - grad1[k])
                    if self.typeB == 'scalar':
                        db[k] += (pH0[k] - pH1[k]).sum()
                    else:
                        db[k] += (pH0[k] - pH1[k])
                if self.typeC == 'scalar':
                    dc += (v0 - self.v).sum()
                else:
                    dc += (v0 - self.v)

            for k in range(self.filters_no):
                self.W[k] += (lrate / pcdSteps) * dW[k] + dW_old[k] * momentum
                self.b[k] += (lrate / pcdSteps) * db[k] + db_old[k] * momentum
                dW_old[k] = (lrate / pcdSteps) * dW[k] + dW_old[k] * momentum
                db_old[k] = (lrate / pcdSteps) * db[k] + db_old[k] * momentum
                self.W[k] += (lrate / pcdSteps) * dW[k]
                self.b[k] += (lrate / pcdSteps) * db[k]
            self.c += (lrate / pcdSteps) * dc
            dc_old = (lrate / pcdSteps) * dc + dc_old * momentum

            if not it % monitor:
                if not self.mse:
                    self.mse.append((it, self.batchMSE(steps=1)))
                elif self.mse[-1][0] != it:
                    self.mse.append((it, self.batchMSE(steps=1)))
                print('Iter: {}   MSE: {}'.format(*self.mse[-1]))
                logger.info('Iter: {}   MSE: {}'.format(*self.mse[-1]))
        self.iterations += iterations
        self.mse.append((self.iterations, self.batchMSE(steps=1)))
        print('Iter: {}   MSE: {}'.format(*self.mse[-1]))
        logger.info('Iter: {}   MSE: {}'.format(*self.mse[-1]))

    def loadV(self, image):
        """
        Load visible layer providing an image.
        :param image: Image to be loaded to self.v
        """
        if image.shape != self.v.shape:
            logger.error('[loadV] Size of provided image does not match v layer size!')
            raise ValueError
        self.v = image
        self.imgInfo = None

    def loadImage(self, imgNo, dataset='train'):
        """
        Load image from data handler to visible layer
        :param imgNo: number of image to be loaded
        :param dataset: 'test' or 'train' set to be used
        """
        if dataset == 'train':
            image = self.dh.train[imgNo]
        elif dataset == 'test':
            image = self.dh.test[imgNo]
        else:
            logger.error('[loadImage] Only \'test\' or \'train\' datasets can be used')
            raise ValueError

        if image.shape != self.v.shape:
            logger.error('[loadImage] Size of provided image does not match v layer size!')
            raise ValueError
        self.v = image
        self.imgInfo = (dataset, imgNo)

    def displayV(self, normalize=True, retImage=False):
        """
        Display visible layer. Use normalize=True when using images from self.dh.
        :param normalize: Use if displaying images from self.dh
        :param retImage: if True will return V as PIL.Image. If False, will display V
        """
        if normalize:
            if self.imgInfo is not None:
                if self.imgInfo[0] == 'train':
                    keys = ('means_tr', 'std_tr')
                elif self.imgInfo[0] == 'test':
                    keys = ('means_te', 'std_te')
                else:
                    logger.error('[displayV] Normalization parameters were not provided.')
                    raise ValueError
                im = Image.fromarray(
                    self.v * self.dh.normalParams[keys[1]] + self.dh.normalParams[keys[0]][self.imgInfo[1]])
            else:
                print('Normalization parameters were not provided, displaying visible layer without normalization')
                im = Image.fromarray(self.v)
        else:
            im = Image.fromarray(self.v)

        if not retImage:
            im.show()
        else:
            return im

    def displayFilters(self, fshape=None, itpl=False, howmany=None):
        """
        Display filters of CCRBM.
        :param fshape: tuple, grid size. i.e. for 40 filters can be (8, 5)
        :param itpl: use bilinear interpolation or display raw pixels
        """
        fig = plt.figure()
        if fshape is None:
            tmp = np.ceil(np.sqrt(self.filters_no))
            fshape = [tmp, tmp]
            while fshape[0] * (fshape[1] - 1) >= self.filters_no:
                fshape[1] -= 1

        plt.subplot(fshape[0], fshape[1], 1)
        for i in range(len(self.W) if howmany is None else howmany):
            plt.subplot(fshape[0], fshape[1], i + 1)
            if itpl:
                plt.imshow(self.W[i], cmap='gray', interpolation='bilinear')

            else:
                plt.imshow(self.W[i], cmap='gray')
            # plt.title('# ' + str(i + 1))
            plt.xticks([])
            plt.yticks([])

        fig.show()

    def displayC(self):
        """
        Display value of self.C or show as image if typeC is 'matrix'.
        """
        if self.typeC == 'scalar':
            print('C layer is a scalar! c = ' + str(self.c))
            return
        tmp = np.copy(self.c)
        tmp -= tmp.min()
        tmp = tmp * 255 / tmp.max()
        Image.fromarray(tmp).show()

    def plotMSE(self):
        """
        Plot mean squared error as a function of iterations.
        """
        if not self.mse:
            print('MSE list is empty!')

        f = plt.figure()
        plt.plot([arg[0] for arg in self.mse], [arg[1] for arg in self.mse])
        f.show()

    def saveToFile(self, filename):  # TODO dont save datahandler images with CCRBM
        """
        Save CCRBM to file.
        :param filename: file name
        """
        with open(filename, 'wb') as f:
            pickle.dump(self, f)
        logger.info('Saved CCRBM {} to file: {}'.format(self, filename))

    def present(self, imgno=0):
        self.loadImage(imgno)
        self.displayV()

        self.sample_h_given_v()
        self.sample_v_given_h()
        self.displayV()

        self.loadImage(imgno)
        self.prob_h_given_v()
        self.prob_v_given_h()
        self.displayV()

        self.displayFilters()
        self.plotMSE()

def getRbm(imsize1=64, imsize2=64, filters=40, cfilter=(5, 5), loadData=True):
    """
    Get CCRBM, initialize DataHandler with brainweb data and normalize this data.
    Used for tests.
    :param imsize1: size_v
    :param imsize2: size_h
    :param filters: filters no
    :param cfilter: conv kernel
    :param loadBWdata: load and normalize brainweb data?
    :return: CCRBM object
    """
    rbm = CCRBM(imsize1, imsize2, filters, cfilter)
    if loadData:
        rbm.dh.readnpy(resize=True, shape=(imsize1, imsize2))
        rbm.dh.normalize()
    return rbm

def loadFromFile(filename):
    """
    Load CCRBM from file.
    :param filename: file name
    :return: CCRBM object
    """
    with open(filename, 'rb') as f:
        rbm = pickle.load(f)
    logger.info('Loaded CCRBM: {} from file: {}'.format(rbm, filename))
    return rbm

if __name__ == '__main__':
    print('What can I do for you?')


import sys, os, stat, glob, importlib.util, importlib, subprocess

this_path = os.path.dirname(os.path.abspath(__file__))

sys.path.append(os.path.join(this_path, "..", "infra"))
import infra

def import_file(filepath):
    name = os.path.splitext(os.path.split(filepath)[1])[0]
    spec = importlib.util.spec_from_file_location(name, filepath)
    mod = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(mod)
    return mod

class App:
    def __init__(self, icon_file):
        self.main_func = None
        if os.path.exists(icon_file):
            self.icon = infra.Sprite(icon_file)
        else:
            self.icon = None

    def start(self):
        if self.main_func is None:
            self.main_func = self.get_main()
        self.main_func(sys.argv)

class PyApp(App):
    def __init__(self, main_mod_file, icon_file):
        super().__init__(icon_file)
        self.main_mod_file = main_mod_file

    def get_main(self):
        mod = import_file(self.main_mod_file)
        return mod.main

PICO8_ICON_PATH = os.path.join(this_path, "..", "pico8", "icon.png")

if sys.platform == "linux":
    PICO8_EXE = os.path.join(this_path, "../pico8/pico-8/pico8_dyn")
else:
    PICO8_EXE = os.path.join(this_path, r"..\pico8\pico-8\pico-8_0.2.3_windows\pico-8\pico8.exe")

def start_pico8(args):
    os.environ["LD_PRELOAD"] = "/home/pi/led_console/sdl_disp_inject/sdl_disp_inject.so"
    cmd = [PICO8_EXE] + args
    print(' '.join(cmd))
    subprocess.call(cmd)

class Pico8App(App):
    def __init__(self, cart_file, icon_file):
        super().__init__(icon_file)
        self.cart_file = cart_file

    def start(self):
        start_pico8(['-run', self.cart_file])

class Pico8SploreApp(App):
    def __init__(self):
        super().__init__(PICO8_ICON_PATH)

    def start(self):
        start_pico8(['-splore'])

BUTTONS_IN_LINE = 4
BTN_X_PITCH = 32
BTN_Y_PITCH = 33
BTN_WIDTH = 29
BTN_HEIGHT = 30

BTN_COLOR = 0x888888
BTN_SEL_COLOR = 0xdddddd

class State(infra.BaseHandler):
    def __init__(self, inf, disp):
        self.enable_players_menu = False
        self.disp = disp
        self.joys = inf.get_joystick_state()
        self.inf = inf

        self.apps = []
        games = os.path.join(this_path, "..", "games")
        for path in glob.glob(os.path.join(games, "*")):
            if stat.S_ISDIR(os.stat(path).st_mode):
                mainpy = os.path.join(path, "main.py")
                if os.path.exists(mainpy):
                    self.apps.append(PyApp(mainpy, os.path.join(path, "icon.png")))

        self.apps.append(Pico8SploreApp())
        picos = os.path.join(this_path, "..", "pico8", "games")
        for modfile in glob.glob(os.path.join(picos, "*.py")):
            mod = import_file(modfile)
            self.apps.append(Pico8App(os.path.join(picos, mod.CART), os.path.join(picos, mod.ICON)))

        cur = []
        self.grid = [cur]  # list of lists
        for app in self.apps:
            cur.append(app)
            if len(cur) == 4:
                cur = []
                self.grid.append(cur)

        self.selected_coord = infra.Vec2i(0, 0)
        self.rep_filter_p1 = infra.JoyRepeatFilter()
        self.rep_filter_p2 = infra.JoyRepeatFilter()

    def draw(self):
        self.disp.pixels.fill(0)
        for y, line in enumerate(self.grid):
            for x, item in enumerate(line):
                rx = x * BTN_X_PITCH
                ry = y * BTN_Y_PITCH
                is_selected = self.selected_coord.x == x and self.selected_coord.y == y
                self.inf.draw.round_rect(rx + 1, ry + 1, BTN_WIDTH, BTN_HEIGHT, BTN_SEL_COLOR if is_selected else BTN_COLOR)
                if item.icon is not None:
                    item.icon.blit_to(self.disp.pixels, rx + 3, ry + 3)

                if is_selected:
                    self.inf.draw.round_rect(rx, ry, BTN_WIDTH + 2, BTN_HEIGHT + 2, BTN_SEL_COLOR, True)
        self.disp.refresh()

    def process_joy(self, joy, rep_filter):
        if not rep_filter.check(joy):
            return
        started = self.selected_coord.copy()
        sel_line_len = len(self.grid[self.selected_coord.y])
        if joy.x > 0:
            self.selected_coord.x = (self.selected_coord.x + 1) % sel_line_len
        elif joy.x < 0:
            self.selected_coord.x = (self.selected_coord.x - 1 + sel_line_len) % sel_line_len
        if joy.y != 0:
            if joy.y > 0 and self.selected_coord.y + 1 < len(self.grid):
                self.selected_coord.y += 1
            if joy.y < 0 and self.selected_coord.y > 0:
                self.selected_coord.y -= 1
            # adjust x if we reached a partial line
            sel_line_len = len(self.grid[self.selected_coord.y])
            if self.selected_coord.x >= sel_line_len:
                self.selected_coord.x = sel_line_len - 1
        return self.selected_coord.equals(started)

    def on_joy_event(self, eventObj):
        if eventObj.event == infra.JOY_BTN_A:
            app = self.grid[self.selected_coord.y][self.selected_coord.x]
            app.start()

    def step(self):
        if not self.process_joy(self.joys.p1, self.rep_filter_p1):
            self.process_joy(self.joys.p2, self.rep_filter_p2)

def main(args):
    inf = infra.infra_init(args)
    disp = inf.get_display()

    state = State(inf, disp)

    while True:
        if not inf.handle_events(state):
            break
        state.step()
        state.draw()

if __name__ == "__main__":
    sys.exit(main(sys.argv))


import numpy as np
import torch
from torch.utils.data import Dataset, DataLoader
import pandas as pd
import os
import glob as glob
import skimage.io as io
from  skimage.transform import resize
# import albumentations as A
# from albumentations.pytorch import ToTensorV2

from src.config import CLASSES, RESIZE_TO, TRAIN_DIR, VALID_DIR, TEST_DIR, BATCH_SIZE
from src.utils import collate_fn, get_transform

class MaskedPolypDataset(Dataset):
    def __init__(self, root, width, height, transforms=None):
        self.transforms = transforms
        self.root = root
        self.dir_images = os.path.join(self.root, "Original")
        self.dir_masks = os.path.join(self.root, "GroundTruth")
        self.width = width
        self.height = height

        self.all_images = list(sorted(glob.glob(os.path.join(self.dir_images, "*.tif"))))
        self.all_masks = list(sorted(glob.glob(os.path.join(self.dir_masks, "*.tif"))))

    def __getitem__(self, index):
        # Get image name and path
        img_path = self.all_images[index]
        mask_path = self.all_masks[index]
        img = io.imread(img_path)
        img = resize(img, (self.height, self.width))
        img = img.astype(np.float32) / 255.0

        mask_img = io.imread(mask_path)
        mask_img = resize(mask_img, (self.height, self.width))
        mask_img[mask_img < 1.] = 0.
        mask = mask_img.astype(np.uint8)

        # Get the boumding box coordinates for each mask
        num_objs = len(np.unique(mask))
        boxes = []
        for i in range(1, num_objs):
            pos = np.where(mask == i)
            xmin = np.min(pos[1])
            xmax = np.max(pos[1])
            ymin = np.min(pos[0])
            ymax = np.max(pos[0])
            box = (xmin, ymin, xmax, ymax)
            boxes.append(box)
        boxes = np.asarray(boxes, dtype=np.float32)
        # print(f'boxes shape: {boxes.shape}')
        # print(f'boxes shape: {boxes.shape}')

        labels = torch.ones((1,), dtype=torch.int64)
        image_id = torch.tensor([index])
        iscrowd = torch.zeros((1,), dtype=torch.int64)

        if self.transforms is not None:
            transformed = self.transforms(
                image=img,
                bboxes=boxes,
                masks=[mask],
                labels=[labels]
            )

        target = {}
        target["boxes"] = transformed["bboxes"]
        target["labels"] = transformed["labels"]
        target["masks"] = transformed["masks"]
        target["image_id"] = image_id
        # target["area"] = area
        target["iscrowd"] = iscrowd

        # img = np.transpose(img, (2, 0, 1))
        img = transformed["image"]

        return img, target

    def __len__(self):
        return len(self.all_images)

    # def get_transform(self, train=True):
    #     bbox_p = A.BboxParams(
    #     format='pascal_voc',
    #     min_visibility=0.1,
    #     min_area=128,
    #     label_fields=['labels'])

    #     if train:
    #         return A.Compose(
    #                 [
    #                     # TODO: Add more transformations and see what works best.
    #                     A.Flip(0.5),
    #                     A.RandomRotate90(0.5),
    #                     A.MotionBlur(p=0.5),
    #                     A.MedianBlur(blur_limit=3, p=0.1),
    #                     A.Blur(blur_limit=3, p=0.1),
    #                     ToTensorV2(p=1.0),
    #                 ],
    #                 bbox_params=bbox_p,
    #             )

    #     else:
    #         return A.Compose([
    #             ToTensorV2(p=1.0),
    #         ], bbox_params=bbox_p)


from tkinter import * # библиотека
import requests # библиотека
def get_crypt(): # функция
    """ДАННАЯ ФУНКЦИЯ ВЫПОЛНЯЕТ ПАРСИНГ КРИПТОВАЛЮТЫ С САЙТА COINMARKETCAP"""
    global info # переменная
    global metka # переменная
    metka.destroy() # удаляет метку при повторном получаении информации о криптовалюте
    lst=[] # Объявляет список
    request=requests.get(url='https://api.coinmarketcap.com/data-api/v3/cryptocurrency/listing?start=1&limit=10000').json() # API coinmarketcap в формате json
    for crypto in request['data']['cryptoCurrencyList']:
        if crypto.get('name').lower() == info.get().lower(): # сравнение поля ввода и названия криптовалюты
            info_name=crypto['name'] # название криптовалюты
            info_price=crypto['quotes'][0]['price'] # цена криптовалюты
            lst.append(info_name) # Добавляю в список название криптовалюты
            lst.append(info_price) # Добавляю в список цену криптовалюты
            metka=Label(root, text=f'Название: {lst[0]}\n' \
                                   f'Цена: {lst[1]:.3f}$', bg='#FFFFFF', fg='#060606', font=('Times New Romans', 14, 'bold')) # Метка с названием и ценой введеной пользователем криптовалюты
            metka.place(x=170,y=15)

""" Эти строки предназначены для создания окна приложения и его компонентов"""
root=Tk()
root.resizable(height=False,width=False)
root.geometry('500x500')
root.title('Парсер')
root.iconphoto(True, PhotoImage(file=('icon.png')))
root['bg']='#686868'
metka=Label(root)
info=StringVar()
vvod=Entry(root, font=('Times New Roman', 16, 'bold'), textvariable=info,justify='center', bg='#0004FF', fg='#FFFFFF')
vvod.place(x=145,y=250)
knpars=Button(root,text='Получить информацию', fg='#FFFFFF', bg='#FF0000', font=('Calibri', 12, 'bold'), command=get_crypt)
knpars.place(x=167,y=350)
root.mainloop()


import sys
import nltk.data
from nltk.tokenize import sent_tokenize

tokenizer = nltk.data.load('tokenizers/punkt/japanese.pickle')
SAMPLE_SIZE = 1024

sample = sys.stdin.read(SAMPLE_SIZE)
while sample:

    tokenized_sents = tokenizer.tokenize(sample)

    for sentence in tokenized_sents:
        sys.stdout.write(sentence + '\n')

    sample = sys.stdin.read(SAMPLE_SIZE)


import torch
from torch import nn
import torch.nn.functional as F
from torch.nn import BatchNorm1d
from torch.nn import Sequential, Linear, ReLU
from torch_geometric.nn import GINConv, global_add_pool, global_mean_pool

class GIN(torch.nn.Module):

    def __init__(self, dim_features, dim_target, model_configs, dataset_configs):
        super(GIN, self).__init__()

        self.dropout = model_configs['dropout']
        self.embeddings_dim = [model_configs['hidden_units'][0]] + model_configs['hidden_units']
        self.no_layers = len(self.embeddings_dim)
        self.first_h = []
        self.nns = []
        self.convs = []
        self.linears = []

        train_eps = model_configs['train_eps']
        if model_configs['aggregation'] == 'sum':
            self.pooling = global_add_pool
        elif model_configs['aggregation'] == 'mean':
            self.pooling = global_mean_pool

        for layer, out_emb_dim in enumerate(self.embeddings_dim):

            if layer == 0:
                self.first_h = Sequential(Linear(dim_features, out_emb_dim), BatchNorm1d(out_emb_dim), ReLU(),
                                    Linear(out_emb_dim, out_emb_dim), BatchNorm1d(out_emb_dim), ReLU())
                self.linears.append(Linear(out_emb_dim, dim_target))
            else:
                input_emb_dim = self.embeddings_dim[layer-1]
                self.nns.append(Sequential(Linear(input_emb_dim, out_emb_dim), BatchNorm1d(out_emb_dim), ReLU(),
                                      Linear(out_emb_dim, out_emb_dim), BatchNorm1d(out_emb_dim), ReLU()))
                self.convs.append(GINConv(self.nns[-1], train_eps=train_eps))  # Eq. 4.2

                self.linears.append(Linear(out_emb_dim, dim_target))

        self.nns = torch.nn.ModuleList(self.nns)
        self.convs = torch.nn.ModuleList(self.convs)
        self.linears = torch.nn.ModuleList(self.linears)  # has got one more for initial input

        self.task_type = dataset_configs["task_type"]
        self.multiclass_num_classes = dataset_configs["multiclass_num_classes"] if self.task_type == 'Multi-Classification' else None

        self.classification = self.task_type == 'Classification'
        if self.classification:
            self.sigmoid = nn.Sigmoid()
        self.multiclass = self.task_type == 'Multiclass-Classification'
        if self.multiclass:
            self.multiclass_softmax = nn.Softmax(dim=2)
        self.regression = self.task_type == 'Regression'
        if self.regression:
            self.relu = nn.ReLU()
        assert not (self.classification and self.regression and self.multiclass)

    def forward(self, data):
        x, edge_index, batch = data.x, data.edge_index, data.batch

        out = 0

        for layer in range(self.no_layers):
            if layer == 0:
                x = self.first_h(x)
                out += F.dropout(self.pooling(self.linears[layer](x), batch), p=self.dropout)
            elif layer == self.no_layers - 1:
                x = self.convs[layer-1](x, edge_index)
                out = x
            else:
                # Layer l ("convolution" layer)
                x = self.convs[layer-1](x, edge_index)
                out += F.dropout(self.linears[layer](self.pooling(x, batch)), p=self.dropout, training=self.training)
        return out

    def forward(self, data):
        x, edge_index, batch = data.x, data.edge_index, data.batch

        out = 0

        for layer in range(self.no_layers):
            if layer == 0:
                x = self.first_h(x)
                out += F.dropout(self.pooling(self.linears[layer](x), batch), p=self.dropout)
            else:
                # Layer l ("convolution" layer)
                x = self.convs[layer-1](x, edge_index)
                out += F.dropout(self.linears[layer](self.pooling(x, batch)), p=self.dropout, training=self.training)

        # Don't apply sigmoid during training b/c using BCEWithLogitsLoss
        if self.classification and not self.training:
            x = self.sigmoid(out)
        if self.multiclass:
            x = x.reshape((x.size(0), -1, self.multiclass_num_classes)) # batch size x num targets x num classes per target
            if not self.training:
                x = self.multiclass_softmax(x) # to get probabilities during evaluation, but not during training as we're using CrossEntropyLoss

        return out


from unittest.mock import MagicMock, patch

import pytest

from app.domain.entities import Tank
from app.domain.enums import Direction
from app.domain.utils import Size, Vector

class _ModulePatch:
    _PATH = "app.domain.entities.tank"
    BULLET_FACTORY = f"{_PATH}.BulletFactory"

class TestsTank:
    @patch(_ModulePatch.BULLET_FACTORY, new_callable=MagicMock)
    def test__post_init(self, factory):
        tank_obj = MagicMock()

        Tank.__post_init__(tank_obj)

        factory.assert_called_once_with(tank_obj._bullet_schema)
        tank_obj._bullet_factory = factory.return_value

    def test__get_bullet(self):
        bullet_schema = MagicMock()
        tank_obj = MagicMock(_bullet_schema=bullet_schema)

        bullet = Tank.get_bullet(tank_obj)

        tank_obj._get_bullet_position.assert_called_once_with(bullet_schema.size)
        tank_obj._bullet_factory.create.assert_called_once_with(
            position=tank_obj._get_bullet_position.return_value,
            direction=tank_obj.direction,
        )
        assert bullet == tank_obj._bullet_factory.create.return_value

    @pytest.mark.parametrize(
        "tank_location,tank_size,bullet_size,direction,expected",
        [
            (Vector(0, 0), Size(10, 10), Size(2, 2), Direction.DOWN, Vector(4, -3)),
            (Vector(10, 20), Size(10, 10), Size(2, 2), Direction.UP, Vector(14, 31)),
            (Vector(20, 10), Size(10, 10), Size(2, 2), Direction.LEFT, Vector(17, 14)),
            (Vector(40, 50), Size(10, 10), Size(2, 2), Direction.RIGHT, Vector(51, 54)),
        ],
    )
    def test__get_bullet_location(
        self, tank_location, tank_size, bullet_size, direction, expected
    ):
        tank_obj = MagicMock(
            direction=direction, position=tank_location, size=tank_size
        )

        actual = Tank._get_bullet_position(tank_obj, bullet_size)

        assert actual == expected


import os
import sys
import collections
import numpy as np
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split

class scoring():
    # This class includes all helper functions that can be used to assess performance of culling technique

    def __init__(self, featureNames):
        self.featureNames = featureNames

    def f1_score(self, currDistribution):
        """
        Calculate f1 score using the distribution of blinks (Positive) versus wire movements (Negative)
        """

        tp = currDistribution[0][-1] / currDistribution[0][0]
        fp = currDistribution[2][-1] / currDistribution[2][0]
        fn = (currDistribution[0][0] - currDistribution[0][-1]) / currDistribution[0][0]

        return tp / (tp + (0.5 * (fp + fn)))

    def accuracy(self, currDistribution):
        """
        Calculate the accuracy using the distribution of blinks (Positive) versus wire movements (Negative)
        """
        return (currDistribution[0][-1] + currDistribution[2][0] - currDistribution[2][-1]) / (currDistribution[0][0] + currDistribution[2][0])

    def performance(self, curr_cullInfo, currFinalLabelsCull, currFinalFeaturesCull):
        """
        Given the culling steps, take the original data and return the performance of the entire culling process
        """

        currDistribution = {key: [] for key in list((collections.Counter(currFinalLabelsCull)).keys())}
        classDistribution = collections.Counter(currFinalLabelsCull)
        for key in currDistribution.keys():
            currDistribution[key].append(classDistribution[key])

        for cullInfoInd in range(len(curr_cullInfo)):
            featureCull, expressions, values = curr_cullInfo[cullInfoInd]

            colInd = list(self.featureNames).index(featureCull)
            classDistribution = collections.Counter(currFinalLabelsCull)

            assert len(expressions) == len(values), f"Invalid cullInfo step. Expected values to match {len(expressions)} number of expressions, got {len(values)} values."
            for setInd in range(len(expressions)):
                mask = eval("currFinalFeaturesCull[:, colInd] " + expressions[setInd] + str(values[setInd]))

                # Apply the mask
                currFinalLabelsCull = currFinalLabelsCull[mask]
                currFinalFeaturesCull = currFinalFeaturesCull[mask]

        classDistribution = collections.Counter(currFinalLabelsCull)

        for key in currDistribution.keys():
            if key in classDistribution:
                currDistribution[key].append(classDistribution[key])
            else:
                currDistribution[key].append(0)

        return self.accuracy(currDistribution), self.f1_score(currDistribution)

class machineLearning():
    # This class includes all helper functions useful for feature selection using machine learning

    def getTrainingCurves(self, featureNames, performMachineLearning, standardizedFeatures_Cull, selectedFeatures, allFinalLabels, modelTypes, errorBars, modelInd):
        """
        Plots the training and testing performance of the ML model with respect to the number of features used in the model
        """
        selectedFeatures_Cull = performMachineLearning.modelControl.getSpecificFeatures(featureNames, selectedFeatures, standardizedFeatures_Cull)
        Training_Data, Testing_Data, Training_Labels, Testing_Labels = train_test_split(selectedFeatures_Cull, allFinalLabels, test_size=0.2, shuffle= True, stratify=allFinalLabels)

        avg_train_scores = []
        avg_test_scores = []

        std_train_scores = []
        std_test_scores = []

        # Train the model and save the accuracy

        # For each selected feature
        for i in range(1, len(selectedFeatures) + 1):

            # Get and split data for all features up to the currently selected feature
            Training_Data, Testing_Data, Training_Labels, Testing_Labels = train_test_split(selectedFeatures_Cull[:, :i], allFinalLabels, test_size=0.2, shuffle= True, stratify=allFinalLabels)
            modelPerformance = performMachineLearning.modelControl.modelClasses[modelInd].trainModel(Training_Data, Training_Labels, Testing_Data, Testing_Labels, selectedFeatures[0:i])

            # If we want to include error bars
            if errorBars:
                train_scores = []
                test_scores = []

                # Get standard deviation by generating 100 models using the same data, with different split
                for j in range(100):
                    Training_Data, Testing_Data, Training_Labels, Testing_Labels = train_test_split(selectedFeatures_Cull[:, :i], allFinalLabels, test_size=0.2, shuffle= True, stratify=allFinalLabels)
                    modelPerformance = performMachineLearning.modelControl.modelClasses[modelInd].trainModel(Training_Data, Training_Labels, Testing_Data, Testing_Labels, selectedFeatures[0:i], imbalancedData = True)
                    train_scores.append(performMachineLearning.modelControl.modelClasses[modelInd].scoreModel(Training_Data, Training_Labels, imbalancedData = True))

                    test_scores.append(modelPerformance)

                # append the average score across all 100 models to avg_train and avg_test
                avg_train_scores.append(np.mean(train_scores))
                avg_test_scores.append(np.mean(test_scores))

                # append the standard deviation across all 100 models
                std_train_scores.append(np.std(train_scores))
                std_test_scores.append(np.std(test_scores))

            else:
                # If we do not want to include error bars, we only generate 1 model, and do not keep track of standard deviation

                avg_train_scores.append(performMachineLearning.modelControl.modelClasses[modelInd].scoreModel(Testing_Data, Testing_Labels, imbalancedData = True))
                avg_test_scores.append(modelPerformance)

        plt.figure()

        if errorBars: # plot learning curve with errorbars using the standard deviation scores
            plt.errorbar(np.arange(1, len(selectedFeatures)+1), avg_train_scores, yerr=std_train_scores, marker='o', capsize=5)
            plt.errorbar(np.arange(1, len(selectedFeatures)+1), avg_test_scores, yerr=std_test_scores, marker='o', capsize=5)
        else: # otherwise, just plot the learning curve with respect to the number of features used to generate the model
            plt.plot(np.arange(1, len(selectedFeatures)+1), avg_train_scores, marker='o')
            plt.plot(np.arange(1, len(selectedFeatures)+1), avg_test_scores, marker='o')

        # formatting plot
        plt.xlabel('Number of Selected Features')
        plt.ylabel('Model Performance (Accuracy)')
        plt.legend(["training", "testing"])
        plt.title('Learning Curves: ' + modelTypes[modelInd])
        plt.show()

        return avg_train_scores, avg_test_scores

class gridSearch():
    # This class includes all helper functions useful for finding the optimal bounds for culling datapoints
    def __init__(self, featureNames, allFinalFeaturesCull, allFinalLabelsCull):
        self.featureNames = featureNames
        self.allFinalFeaturesCull = allFinalFeaturesCull
        self.allFinalLabelsCull = allFinalLabelsCull
        self.scoring = scoring(featureNames)

    def simplify(self, curr_cullInfo):
        """
        Given the current culling steps, remove unnecessary bounds (when the lower bound is already the minimum value of all
        datapoints, or the upper bound is the maximum value of all datapoints)
        """

        new_cullInfo = []
        # For each step in the culling pipeline
        for cull_step in curr_cullInfo:
            feature = cull_step[0]
            colInd = list(self.featureNames).index(feature)
            features = self.allFinalFeaturesCull[:,colInd]
            currmax = max(features)
            currmin = min(features)

            # If the value of the lower bound is the current minimum of all datapoints
            if cull_step[2][0] == currmin:
                # this culling step is not necessary, and we can simplify
                new_cullInfo.append((feature, ["<"], [cull_step[2][1]]))
            # If the value of the upper bound is the current maximum of all datapoints
            elif cull_step[2][1] == currmax:
                # this culling step is not necessary, and we can simplify.
                new_cullInfo.append((feature, [">"], [cull_step[2][0]]))
            else:
                # Otherwise, both boundaries are kept.
                new_cullInfo.append(cull_step)

        return new_cullInfo

    def eog_pipeline_formatted(self, cullInfo):
        """
        Puts cullInfo in this format:

        if not 0.008 < blinkDuration < 0.5:
            if debugBlinkDetection: print("\t\tBad Blink Duration:", blinkDuration, xData[peakInd])
            return [None]

        Which is how the EOG pipeline code is formatted in eogAnalysis.py (can be pasted after line 713).
        """

        print("---------- Code for Culling Bad Blinks in eogAnalysis.py ----------")

        for step in cullInfo:
            line = "if not "
            if '>' in step[1]:
                line += str(step[2][step[1].index('>')]) + " < "
            line += str(step[0][:-4])
            if '<' in step[1]:
                line += " < " + str(step[2][step[1].index('<')])
            line += ":"
            print(line)
            print("    " + "if debugBlinkDetection: print(\"\t\tBad " + str(step[0][:-4]) + ":\", " + str(step[0][:-4]) + ", xData[peakInd])")
            print("    return [None]")

        print("-------------------------------------------------------------------")

    def individual_search(self, curr_selectedFeatures):
        """
        Finds bound that optimizes accuracy for for each feature independently.
        and returns a list of all of the bounds found for the features.
        """
        bins = 50
        cullInfo = []
        # For each selected feature
        for feature in curr_selectedFeatures:

            # Get all data of given feature
            colInd = list(self.featureNames).index(feature)
            features = list(self.allFinalFeaturesCull[:,colInd])

            # Find maximum and minimum of all data
            currmax = max(features)
            currmin = min(features)

            # We only want to check bins number of boundaries. Calculate how much we need to increment by to assess the whole range of values
            delta = (max(features) - min(features)) / bins

            cullInfo.append((feature, [">"], [currmin]))

            scores = []

            # Loop over each possible lower bound, and compute the performance if the bound were to be placed at that value.
            for i in range(bins):
                cullInfo[-1] = (feature, [">"], [currmin + (i * delta)])
                curr_accuracy, curr_f1 = self.scoring.performance(cullInfo, self.allFinalLabelsCull, self.allFinalFeaturesCull)
                scores.append(curr_accuracy)

            # The lower bound is the bound that recieves the best performance score
            lower_bound = currmin + (np.argmax(scores) * delta)

            scores = []
            # Loop over each possible upper bound
            for i in range(bins):
                # Only loop until we have reached the lower bound, as the upper bound must be greater than the lower bound
                if currmax - (i * delta) <= lower_bound:
                    break
                cullInfo[-1] = (feature, [">", "<"], [lower_bound, currmax - (i * delta)])
                curr_accuracy, curr_f1 = self.scoring.performance(cullInfo, self.allFinalLabelsCull, self.allFinalFeaturesCull)
                scores.append(curr_accuracy)
            # The upper bound is the bound that recieves the best performance score
            upper_bound = currmax - (np.argmax(scores) * delta)

            cullInfo[-1] = (feature, [">", "<"], [lower_bound, upper_bound])

        # return all of the bounds,
        return cullInfo, self.scoring.performance(cullInfo, self.allFinalLabelsCull, self.allFinalFeaturesCull)

    def bfs(self, curr_selectedFeatures):
        """
        Performs a breadth-first search, exploring possible bounds for all of the features.
        Attempts to maximize the total performance across all features.
        """

        # Initialize visited and queue
        visited = []
        queue = []
        best_accuracy = 0
        best_cullInfo = []

        # Set number of bins for discretization of feature values
        bins = 15
        deltas = []
        curr_cullInfo = []

        # For each selected feature, find the amount we need to increment/decrement by to reach its "neighbor". We want to check bounds closest to the current bound.
        for feature in curr_selectedFeatures:
            colInd = list(self.featureNames).index(feature)
            features = list(self.allFinalFeaturesCull[:,colInd])

            currmax = max(features)
            currmin = min(features)
            deltas.append((max(features) - min(features)) / bins)

            # Initialize the culling to at first, not cull any datapoints.
            curr_cullInfo.append((feature, [">", "<"], [currmin, currmax]))

        # Queue the current culling step
        queue.append(curr_cullInfo)
        visited.append(str(curr_cullInfo))

        while queue: # Creating loop to visit each node
            # Pop a culling pipeline off of the queue
            currCullInfo = queue.pop(0)
            curr_accuracy, curr_f1_score = self.scoring.performance(currCullInfo, self.allFinalLabelsCull, self.allFinalFeaturesCull)

            # If this culling pipeline has the best performance so far, continue to explore its "neighbors".
            if curr_accuracy >= best_accuracy:
                best_accuracy = curr_accuracy
                best_cullInfo = currCullInfo

                # For each selected feature
                for feature_ind in range(len(curr_selectedFeatures)):
                    newCullInfo = np.copy(currCullInfo)
                    # Create a new culling pipeline where the culling step that includes the feature has a slightly stricter lower bound (increases by delta value)
                    newCullInfo[feature_ind] = (curr_selectedFeatures[feature_ind], [">", "<"], [currCullInfo[feature_ind][2][0] + deltas[feature_ind], currCullInfo[feature_ind][2][1]])
                    # If this pipeline has not been visited, add it to the queue. We want to visit this eventually
                    if str(newCullInfo) not in visited:
                        queue.append(newCullInfo)
                        visited.append(str(newCullInfo))

                    newCullInfo = np.copy(currCullInfo)
                    # Create a new culling pipeline where the culling step that includes the feature has a slightly stricter upper bound (decreases by delta value)
                    newCullInfo[feature_ind] = (curr_selectedFeatures[feature_ind], [">", "<"], [currCullInfo[feature_ind][2][0], currCullInfo[feature_ind][2][1] - deltas[feature_ind]])
                    # If this pipeline has not been visited, add it to the queue. We want to visit this eventually
                    if str(newCullInfo) not in visited:
                        queue.append(newCullInfo)
                        visited.append(str(newCullInfo))

        return best_cullInfo, best_accuracy


import numpy as np
import gym
import matplotlib.pyplot as plt

if __name__ == "__main__":
    ENV = gym.make("FrozenLake-v0")
    DISCOUNT = 0.9
    EPSILON = 0.1
    POSSIBLE_ACTIONS = [0, 1, 2, 3]
    STATE_SPACE = list(range(16))
    ESTIMATES = {}
    RETURNS = {}
    PAIRS_VISITED = {}

    for state in STATE_SPACE:
        for action in POSSIBLE_ACTIONS:
            ESTIMATES[(state, action)] = 0
            RETURNS[(state, action)] = 0
            PAIRS_VISITED[(state, action)] = 0

    POLICY = {}
    for state in STATE_SPACE:
        POLICY[state] = np.random.choice(POSSIBLE_ACTIONS)

    NUM_EPISODES = 100000
    STATUS_INTERVAL = 5000

    for i in range(NUM_EPISODES):
        states_actions_returns = []
        if i % STATUS_INTERVAL == 0:
            print("starting episode", i)
        observation = ENV.reset()
        memory = []
        done = False
        while not done:
            action = POLICY[observation]
            observation_, reward, done, info = ENV.step(action)
            memory.append((observation, action, reward))
            observation = observation_

        memory.append((observation, action, reward))
        returns = 0
        last = True # start at t = T - 1
        for state, action, reward in reversed(memory):
            if last:
                last = False
            else:
                states_actions_returns.append((state, action, returns))
            returns = DISCOUNT * returns + reward

        states_actions_returns.reverse()
        states_and_actions = []
        for state, action, returns in states_actions_returns:
            if (state, action) not in states_and_actions:
                PAIRS_VISITED[(state, action)] += 1
                RETURNS[(state, action)] += (
                    (1 / PAIRS_VISITED[(state, action)])
                    * (returns - RETURNS[(state, action)])
                )
                ESTIMATES[(state, action)] = RETURNS[(state, action)]
                states_and_actions.append((state, action))
                values = np.array(
                    [ESTIMATES[(state, a)] for a in POSSIBLE_ACTIONS]
                )
                best = np.random.choice(np.where(values == values.max())[0])
                rand = np.random.random()
                if rand < 1 - EPSILON:
                    POLICY[state] = POSSIBLE_ACTIONS[best]
                else:
                    POLICY[state] = np.random.choice(POSSIBLE_ACTIONS)
    NUM_GAMES = 1000
    REWARDS = np.zeros(NUM_GAMES)
    EPISODE_REWARDS = 0
    for i in range(NUM_GAMES):
        observation = ENV.reset()
        done = False
        while not done:
            action = POLICY[observation]
            observation_, reward, done, info = ENV.step(action)
            observation = observation_
            EPISODE_REWARDS += reward
        REWARDS[i] = EPISODE_REWARDS
    print(EPISODE_REWARDS / NUM_GAMES)
    plt.plot(REWARDS)
    plt.show()


import os
from argparse import ArgumentParser

import datasets
import yaml

from togepi.data_processors.tokenizers.tokenizer import TogepiTokenizer
from togepi.utils.utils import set_seed

def main(config_path, hf_dataset_path, path_to_store_hf_tokenizer):
    set_seed(seed=42)

    with open(config_path, 'r') as fp:
        config = yaml.safe_load(fp)

    hf_dataset = datasets.load_from_disk(dataset_path=hf_dataset_path)
    tokenizer = TogepiTokenizer(**config['tokenizer']['args'])
    tokenizer.train(hf_dataset['train'])
    tokenizer.save(path_to_store_hf_tokenizer)

if __name__ == '__main__':
    parser = ArgumentParser(description='build tokenizer using a huggingface dataset')
    parser.add_argument('--config_path', type=str, help='path to config file')
    parser.add_argument('--hf_dataset_path', type=str, help='path to the huggingface dataset', default=os.getcwd())
    parser.add_argument('--path_to_store_hf_tokenizer', type=str, help='path to store the tokenizer',
                        default=os.getcwd())

    args = parser.parse_args()

    main(config_path=args.config_path, hf_dataset_path=args.hf_dataset_path,
         path_to_store_hf_tokenizer=args.path_to_store_hf_tokenizer)


# get the data needed to plot the TEP with a quiver plot on top of it

import numpy as np
#import pickle
import os
import pandas as pd
import csv
from scipy.interpolate import interp1d

import sys
sys.path.insert(0,'../../functions') # so I can import the functions
from cycle_funcs import calc_fitness, calc_dfitness, sample_wsV_dimorph

# =====================================================================================================================

# model parameters
# ---

# which parameter set to find the isoclines for (comment out ones I've done)
suffix = '_1'

'''
# default parameter values
ID = 0
ngrid = 31 # number of grid points in log space along the resident1 axis
'''
# high dispersal cost parameter values
ID = 17
ngrid = 31 # number of grid points in log space along the resident1 axis

# where results will be stored
dir_results = '../../results/circular/'

# default algorithm parameters
params = {
        'tol_res': 1e-10,       # the euclidean distance in final resident structure before say it's equilibriated
        'tol_res_dimorph': 1e-6,# the euclidean distance in final dimorphic resident structure before say it's equilibriated
        'tol_mut': 1e-10,       # the euclidean distance in final mutant structure before say it's equilibriated
        'delta_m': 1e-9,        # step size to estimate gradients; default heuristic sqrt(machine error) * approx value
        }

# read in the info from the sing_strat_x.csv file and repopulate params dictionary
# ---

par_names = ['suffix', 'layout', 'f', 'c', 'r', 'KL', 'h', 'KS_mean', 'p_cat'] # names of missing parameter values

fname = dir_results + 'sing_strat' + suffix + '.csv'
df = pd.read_csv(fname)
ss_res = df.iloc[ID] # the particular row we want

for par_name in par_names:

    params[par_name] = ss_res[par_name]

# read in the isoclines to define the TEP region
# ---

fname = dir_results + 'isocline' + suffix + '_' + str(ID) + '.csv'
df = pd.read_csv(fname)
df = df.sort_values(by=['m_mut'])

# grab the isoclines
m_iso1v = df['m_res'].values
m_iso2v = df['m_mut'].values

# flip all points below the diagonal so that we define the TEP region
region = [ (m_mut, m_res) if m_res > m_mut else (m_res, m_mut) for m_res, m_mut in zip(m_iso1v, m_iso2v) ]

# make sure the first entry is 0,0
if region[0] != (0,0):
    region += [(0,0)] + region

# make sure the last entry is 0,y_intercept
if region[-1][1] == 1:
    region[-1] = (0, ss_res['y_intercept'])
else:
    region += [(0, ss_res['y_intercept'])]

# split the region boundary into an upper and lower bound on resident 2
# ---

# find the extremal point of the region in the resident 1 dimension, the bulging out to the right of the PIP graph
m_iso1V, m_iso2V = zip(*region)
extremal_res1 = max(m_iso1V)
extremal_idx = m_iso1V.index( extremal_res1 )
extremal_res2 = m_iso2V[extremal_idx]

# split the region boundary into two lines, one for a lower bound on res2, and one for an upper bound
line_lo = [ (m_res1, m_res2) for m_res1, m_res2 in region if m_res2 <= extremal_res2 ]
line_hi = [ (m_res1, m_res2) for m_res1, m_res2 in region if m_res2 >= extremal_res2 ]

# create functions that will return the lower and upper bound on res2 for a given res1
m_res1V, m_res2V = zip(*line_lo)
f_lo = interp1d(m_res1V, m_res2V)
m_res1V, m_res2V = zip(*line_hi)
f_hi = interp1d(m_res1V, m_res2V)

# create a grid along the resident 1 dimension, and find the isocline at each point along that grid
# ---

# do it in log space
pwrV = np.linspace(-6, np.log10(extremal_res1), ngrid)[:-1]
m_res1V = [ 10**pwr for pwr in pwrV ]

# if the csv file doesn't exist yet, create it, and include the resident 1 = 0 point in the grid
# ---

fname = dir_results + 'dimorph_isocline' + suffix + '_' + str(ID) + '.csv'
if not os.path.isfile(fname):

    # add the zero point to our search
    m_res1V = [0] + m_res1V

    # write the column headers
    with open(fname, "w", newline="") as ftarget:
        writer = csv.writer(ftarget)
        writer.writerow( ['m_res1', 'm_res2', 'dfit2'] )

# for each resident 1 strategy, find where the resident 2 mutant invasion fitness gradient equals 0
# (I know that this is an attractor)
# ---

for m_res1 in m_res1V:

    m_res2_hi_bnd = f_hi([m_res1])[0]
    m_res2_lo_bnd = f_lo([m_res1])[0]

    # find where resident 2 invasion fitness gradient goes positive
    # ---

    # initialise
    m_res2_lo = m_res2_hi_bnd
    dfit2_lo = -1   # at this point, the resident-2 fitness gradient is negative
    nL = None

    print('find where gradient goes positive')

    while dfit2_lo < 0:

        print(m_res2_lo)

        # update
        m_res2_hi = m_res2_lo
        dfit2_hi = dfit2_lo
        nL_hi = nL

        # halve distance to lower bound for the new low estimate
        m_res2_lo = (m_res2_lo_bnd + m_res2_hi) / 2

        # find the fitness gradient here
        wsV, nL = sample_wsV_dimorph(m_res1, m_res2_lo, params, return_nT=True, nL=nL)
        dfit2_lo = calc_dfitness(m_res2_lo, wsV, params)

    nL_lo = nL

    print('m_res2_lo = ' + str(m_res2_lo))

    # use bisection method to find the root
    # ---

    print('find root')

    tol_dfit = 1e-7 # maximum derivative that we'll accept as being close enough to 0
    dfit2_mid = dfit2_lo
    while abs(dfit2_mid) > tol_dfit:

        m_res2_mid = (m_res2_lo + m_res2_hi) / 2
        nL_mid = [ (nL_lo[0]+nL_hi[0])/2 , (nL_lo[1]+nL_hi[1])/2 ]
        wsV_mid, nL_mid = sample_wsV_dimorph(m_res1, m_res2_mid, params, return_nT=True, nL=nL_mid)
        dfit2_mid = calc_dfitness(m_res2_mid, wsV_mid, params)

        if np.sign(dfit2_lo) == np.sign(dfit2_mid):
            nL_lo = nL_mid
            wsV_lo = wsV_mid
            dfit2_lo = dfit2_mid
            m_res2_lo = m_res2_mid
        else:
            nL_hi = nL_mid
            wsV_hi = wsV_mid
            dfit2_hi = dfit2_mid
            m_res2_hi = m_res2_mid

        print(m_res2_mid)

    # write this isocline point to the csv
    # ---

    with open(fname, "a", newline="") as ftarget:

        writer = csv.writer(ftarget)
        writer.writerow([m_res1, m_res2_mid, dfit2_mid])


import json
import random

with open('data/LLMScience.json',encoding='utf-8') as f:
    datas = json.load(f)

datas = datas[:200]

# datas = random.sample(datas, 5)

prompt = """Question: {}
A: {}
B: {}
C: {}
D: {}
E: {}
Answer: {}"""

# 筛选出所有答案为A,B,C,D,E的数据
data_A = [item for item in datas if item['answer'] == 'A']
data_B = [item for item in datas if item['answer'] == 'B']
data_C = [item for item in datas if item['answer'] == 'C']
data_D = [item for item in datas if item['answer'] == 'D']
data_E = [item for item in datas if item['answer'] == 'E']

# 检查每个答案是否都至少有一个数据
if not all([data_A, data_B, data_C, data_D, data_E]):
    print("数据中不包含所有的答案至少一次!")
else:
    sampled_data = []
    sampled_data.append(random.choice(data_A))
    sampled_data.append(random.choice(data_B))
    sampled_data.append(random.choice(data_C))
    sampled_data.append(random.choice(data_D))
    sampled_data.append(random.choice(data_E))

random.shuffle(sampled_data)

total_few_shot = ""
for data in sampled_data:
    question = data['prompt']
    A = data['A']
    B = data['B']
    C = data['C']
    D = data['D']
    E = data['E']
    answer = data['answer']
    total_few_shot+=prompt.format(question,A,B,C,D,E,answer)+"\n\n"

print(total_few_shot)


import json
#  результат матча
file_name1 = "processed_data (1).json"
#  исходники для матчера
file_name2 = "input_data.json"
with open(file_name1, 'r+', encoding='utf-8') as file1:
    with open(file_name2, 'r', encoding='utf-8') as file2:
        print(type(file1))
        file1 = json.load(file1)
        file2 = json.load(file2)
        for i in ["mismatched_pairs"]:

            for mismatch_pair in file1[i]:
                event_id1 = mismatch_pair["event1"]["event_id"]
                event_id2 = mismatch_pair["event2"]["event_id"]
                sport = mismatch_pair["sport"]
                date_event = mismatch_pair["date"]
                flag_event1 = 0
                flag_event2 = 0
                for date_events in file2:
                    if flag_event1 and flag_event2:
                        break
                    if date_event != date_events["date"]:
                        continue
                    for event_serch in date_events["data"]:
                        if event_serch["sport"] != sport:
                            continue
                        if event_id1 == event_serch["event1"]["event_id"]:
                            flag_event1 = 1
                            mismatch_pair["event1"]["true_pair"] = event_serch["event2"]
                        if event_id2 == event_serch["event2"]["event_id"]:
                            flag_event2 = 1
                            mismatch_pair["event2"]["true_pair"] = event_serch["event1"]

        with open('new_result.json', 'w') as fp:
            json.dump(file1, fp)
# unmatched_results, mismatched_pairs


import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf
from sklearn import metrics
import pandas as pd
import pickle

from keras.optimizers import Adam
from keras.models import Model
from keras.layers import Dense
from keras.layers import Input
from keras.layers import Flatten
from keras.layers import Conv2D
from keras.layers.normalization.batch_normalization import BatchNormalization
from keras.layers import Dropout

def define_CNN(in_shape, n_keypoints):

    in_one = Input(shape=in_shape)
    conv_one_1 = Conv2D(16, kernel_size=(3, 3), activation='relu', strides=(1, 1), padding = 'same')(in_one)
    conv_one_1 = Dropout(0.3)(conv_one_1)
    conv_one_2 = Conv2D(32, kernel_size=(3, 3), activation='relu', strides=(1, 1), padding = 'same')(conv_one_1)
    conv_one_2 = Dropout(0.3)(conv_one_2)

    conv_one_2 = BatchNormalization(momentum=0.95)(conv_one_2)

    fe = Flatten()(conv_one_2)
    # dense1
    dense_layer1 = Dense(512, activation='relu')(fe)
    dense_layer1 = BatchNormalization(momentum=0.95)(dense_layer1)
    # # dropout

    # dropout
    dense_layer1 = Dropout(0.4)(dense_layer1)

    out_layer = Dense(n_keypoints, activation = 'linear')(dense_layer1)

    # model
    model = Model(in_one, out_layer)
    opt = Adam(lr=0.001, beta_1=0.5)

    # compile the model
    model.compile(loss='mse', optimizer=opt, metrics=['mae', 'mse', 'mape', tf.keras.metrics.RootMeanSquaredError()])
    return model

def main():
    # #load the feature and labels, 24066, 8033, and 7984 frames for train, validate, and test
    # featuremap_train = np.load('feature/featuremap_train.npy')
    # featuremap_validate = np.load('feature/featuremap_validate.npy')
    # featuremap_test = np.load('feature/featuremap_test.npy')

    # labels_train = np.load('feature/labels_train.npy')
    # labels_validate = np.load('feature/labels_validate.npy')
    # labels_test = np.load('feature/labels_test.npy')
    # remove the arms up one
    training_data_file_names = ["arms-up.bin", "t-pose.bin", "arms-down.bin"]
    # training_data_file_names = ["squat.bin"]
    # fig = plt.figure()
    # ax = fig.add_subplot(projection='3d')
    for name in training_data_file_names:
        with open(f"training_data/{name}", "rb") as f:
            training_features = pickle.load(f)
            for i in range(500):
                ax.clear()
                # ax.scatter(landmark.x, landmark.y, landmark.z)
                x = []
                y = []
                z = []
                for landmark in training_features[i].landmark:
                    x.append(landmark.x)
                    y.append(0)
                    z.append(landmark.y)
                    # ax.scatter(landmark.x, landmark.y, landmark.z)
                ax.scatter(x, y, z)
                ax.set_xlabel("X")
                ax.set_ylabel("Y")
                ax.set_zlabel("Z")
                ax.axes.set_xlim3d(left=-0.5, right=0.5)
                ax.axes.set_ylim3d(bottom=-0.5, top=0.5)
                ax.axes.set_zlim3d(bottom=1, top=0)

                plt.pause(0.05)

    # plt.show()
    #         # print(training_features[0].landmark[0])

    # # Initialize the result array
    # paper_result_list = []

    # # define batch size and epochs
    # batch_size = 128
    # epochs = 150

    # # load model
    # keypoint_model = tf.keras.models.load_model("model/MARS.h5")
    # # Repeat i iteration to get the average result
    # # for i in range(10):
    # # instantiate the model
    # keypoint_model = tf.keras.models.load_model("model/MARS.h5")

    # # save and print the metrics
    # # score_train = keypoint_model.evaluate(featuremap_train, labels_train,verbose = 1)
    # # print('train MAPE = ', score_train[3])
    # # score_test = keypoint_model.evaluate(featuremap_test, labels_test,verbose = 1)
    # # print('test MAPE = ', score_test[3])
    # print(len(featuremap_test))
    # result_test = keypoint_model.predict(np.array([featuremap_test[2735]]))

    # x = result_test[0][0:19]
    # y = result_test[0][19:38]
    # z = result_test[0][38:57]
    # # print(pts)

    # fig = plt.figure()
    # ax = fig.add_subplot(projection='3d')
    # ax.scatter(x, y, z)
    # ax.set_xlabel('X Label')
    # ax.set_ylabel('Y Label')
    # ax.set_zlabel('Z Label')
    # plt.show()

    # # df = pd.DataFrame(result_test)
    # # print(df)
    # # print(featuremap_test)
    # # print(featuremap_test[0])
    # # print(result_test)
    # # print(result_test[0])
    # exit()

    # # instantiate the model
    # # keypoint_model = define_CNN(featuremap_train[0].shape, 57)

    # # # # initial maximum error
    # # # score_min = 10
    # # history = keypoint_model.fit(featuremap_train, labels_train,
    # #                             batch_size=batch_size, epochs=epochs, verbose=1,
    # #                             validation_data=(featuremap_validate, labels_validate))
    # # result_test = keypoint_model.predict(featuremap_test)

if __name__ == "__main__":
    main()


from selenium import webdriver
import time

browser = webdriver.Chrome('C://Users//20143//Desktop//chromedriver.exe')
browser.get("https://basecamp.robolink.com/cwists/category#products=%5B1%5D&selected_sort_by=alphabetical")

time.sleep(5)
button = browser.find_element_by_xpath('//*[@id="activity-search-results"]/div/div[2]/button')
button.click()
time.sleep(5)

cards = browser.find_elements_by_class_name("card__img-wrap")
links = []
for card in cards:
    links.append(card.get_attribute('href'))
for link in links:
    browser.get(link)
    time.sleep(5)

    buttons = browser.find_elements_by_class_name('cpy-clip-txtarea')
    if len(buttons) == 0:
        continue

    title = browser.find_element_by_css_selector('.titlebar__title.u-flex')
    subtitle = browser.find_elements_by_css_selector('.step-content-h3.u-hide-print')
    title_name = title.get_attribute("textContent")

    try:
        wf = open(title_name[:2] + ".txt", "w", encoding='UTF-8')

        wf.write(title_name + "\n\n\n")
        for i in subtitle:
            wf.write(i.get_attribute("textContent")+"\n")
        wf.write("\n\n\n")
        for i in buttons:
            wf.write(i.get_attribute("textContent")+"\n\n\n\n\n")
        wf.write("\n\n\n")
        wf.close()
    except:
        print("FU"+title_name)

    time.sleep(1)
    print(link)

time.sleep(5)
browser.quit()


import numpy as np
import pandas as pd

from sqlalchemy import create_engine, MetaData, Table, Column, Integer, \
                       String, Float, DateTime, ForeignKeyConstraint, ForeignKey,\
                       Enum, UniqueConstraint, Boolean
from geoalchemy2 import Geometry

import website.models as m

# Pandas to human readable mapping
type_mappings = {
    'int': 'integer',
    'float': 'float',
    'datetime': 'datetime',
    'object': 'string'
}

# Human readable to alchemy mapping
alchemy_types = {
    'integer': Integer,
    'float': Float,
    'datetime': DateTime,
    'string': String
}

def to_sql(df, datatypes, table_name, schema, geospatial_columns=None):
    """
    Create a database table based on a DataFrame and load it with data

    Parameters:
    df (pandas.DataFrame) - The DataFrame the table will be generated based on.
                            The data found in this DataFrame will be loaded into the table
    datatypes (list) - A list of SQLAlchemy column datatypes
    table_name (str) - The name the table will be given
    schema (str) - The schema the table will be created into
    geospatial_columns(list) - A list of geospatial columns of the type returned
                               by get_geospatial_columns()

    Returns:
    table - The SQLAlchemy table object that was generated
    """
    create_table(df, datatypes, table_name, schema, geospatial_columns)
    table = getattr(m.Base.classes, table_name)
    insert_df(df, table, geospatial_columns)
    return table

def create_table(df, datatypes, table_name, schema, geospatial_columns=None):
    """
    Create a database table based on a DataFrame

    Parameters:
    df (pandas.DataFrame) - The dataframe the table will be generated for
    datatypes (list) - A list of SQLAlchemy column datatypes
    table_name (str) - The name the table will be given
    schema (str) - The schema the table will be created into
    geospatial_columns (list) - A list of geospatial columns of the type returned
                                from get_geospatial_columns()

    Returns:
    table - The generated SQLAlchemy table object
    """
    datatypes = get_alchemy_types(datatypes)
    columns = [Column('id', Integer, primary_key=True)]
    for i, c in enumerate(df.columns):
        columns.append(
            Column(c, datatypes[i])
        )
    if geospatial_columns is not None:
        for c in geospatial_columns:
            if c['type'] == 'latlon':
                columns.append(
                    Column(c['name'], Geometry('POINT', srid=c['srid']))
                )
    table = Table(table_name, m.m, *columns, schema=schema)
    m.m.create_all(m.engine)
    m.refresh()
    return table

def insert_df(df, table, geospatial_columns=None):
    """
    Load a DataFrame into an autogenerated database table

    Arguments:
    table - The SQLAlchemy table object into which data will be loaded
    geospatial_columns (list) - A list of geospatial columns found in the dataset.
                                Should be of the form returned by get_geospatial_columns()

    Returns:
    Nothing
    """
    insert_dict = df.to_dict('records')
    for row in insert_dict:
        for c in row:
            if pd.isnull(row[c]):
                row[c] = None
        if geospatial_columns is not None:
            for c in geospatial_columns:
                row[c['name']] = 'SRID=%s;POINT(%s %s)' % (c['srid'], row[c['lon_col']], row[c['lat_col']])
    m.engine.execute(
        table.__table__.insert(),
        insert_dict
    )
    return

def get_geospatial_columns(table_uuid):
    """
    Get a list of geospatial column definitions from the geospatial_columnns table
    for a given table

    Parameters:
    table_uuid (str) - The uuid of an autogenerated database table

    Retruns:
    columns (list) - A list of geospatial column definitions where each element
                     is of the type returned by parse_geospatial_column_string()
    """
    session = m.get_session()
    res = session.query(m.GEOSPATIAL_COLUMNS.column_definition).filter(
        m.GEOSPATIAL_COLUMNS.dataset_uuid == table_uuid
    )
    columns = []
    for col in res:
        columns.append(parse_geospatial_column_string(col[0]))
    session.close()
    return columns

def parse_geospatial_column_string(geospatial_column_string):
    """
    Convert a geospatial column definition String into a dictionary to be used
    by the database generation functions.

    Parameters:
    geospatial_column_string (str) - A string defining a geospatial column.
                                     probably either returned from the upload_file page
                                     or pulled from the column_definition column of the
                                     geospatial_columns table.
        example: name=geom&lat_col=LATITUDE&lon_col=LONGITUDE&srid=4326&type=latlon

    Returns:
    geospatial_column (dict) - A dictionary containing all the information foud in the defintion string
    """
    # For each geospatial column, create a dictionary using fields as keys to store values
    for column in geospatial_column_string.split(','):
        # Create the dictionary
        geospatial_column = {'column_definition': column}

        for field in column.split('&'):
            field = field.split('=')
            # "exampleone=7&exampletwo=8" -> {"exampleone":7, "exampletwo":8}
            geospatial_column[field[0]] = field[1]

        # Append the dictionary to geospatial_columns (for the to_sql function)
    return geospatial_column

def get_alchemy_types(mapped_types):
    """
    Get SQLAlchemy column datatype objects from a list of human readable datatypes.
    This converts a verified list human readable types from the frontend to objects
    for database generation on the backend

    Parameters:
    mapped_types (list) -  a list of human readable datatypes like those generated
                           by get_readable_types_from_dataframe().

    Returns:
    rt (list) - A list of SQLAlchemy column datatype objects
    """
    rt = []
    for t in mapped_types:
        rt.append(alchemy_types[t])
    return rt

def get_readable_types_from_dataframe(df):
    """
    Get human readable datatypes for the columns in a pandas.DataFrame. This is
    used to help the user verify that the system is auto-generating the correct
    database column types

    Parameters:
    df (pandas.DataFrame) - The DataFrame the human readable datatype list will
                            be generated from.

    Returns:
    readable_types (list) - a list of human readable datatypes
    """
    readable_types = []
    for d in df.dtypes:
        readable_types.append(convert_type(d))
    return readable_types

def convert_type(dtype):
    """
    Convert a pandas dtype to a human readable database type.

    Parameters:
    dtype - a dtype from the column of a pandas.Dataframe or a pandas.Series

    Returns:
    type_mapping (str) - the human readable equivalent of the dtype
    """
    d = str(dtype)
    for t in type_mappings:
        if t in d:
            return type_mappings[t]
    return None


# Author : Tim Destan
# Feature based clusterer

from collections import defaultdict
from qbcommon import all_pairs_symmetric
from featurespace import *

import logging
logger = logging.getLogger("FeatureClusterer")

class ScoreTypes(object):
  DISTANCE = "DISTANCE"
  SIMILARITY = "SIMILARITY"

class GuidGenerator(object):
  """
  Generates globally unique identifiers.
  """
  def __init__(self):
    self.next = 0

  def new_id(self):
    """
    Returns a new ID
    """
    self.next += 1
    return self.next

class AgglomerativeCluster(object):
  """
  Agglomerative clusterer based on a feature distance function that takes
  features as vectors from names to values.
  """
  def __init__(self, compositeRecords, featureSets, scoreFunction,
      threshold=None, scoreType=ScoreTypes.DISTANCE, baseDistanceCache={},
      guidGenerator=GuidGenerator(), globalClusters={}):
    """
    Constructor

    :param compositeRecords: Collection of composite records to classify.
    :param featureSets: List of featuresets. Each base record is
      interpreted as an index into this list.
    :param threshold: Optional distance threshold for when to stop merging clusters.
    :param scoreFunction: A distance function to compute the distance between two
      feature vectors. Should be a distance metric.
    :param baseDistanceCache: A cache of distances between base records.
    """
    self.baseFeatureArray = featureSets
    self.scoreFunction = scoreFunction

    self.onMerge = lambda slf, thresh: ()

    self.informative_features = defaultdict(float)

    if scoreType == ScoreTypes.DISTANCE:
      self.scoreIsBetter = lambda score, best: score < best
    elif scoreType == ScoreTypes.SIMILARITY:
      self.scoreIsBetter = lambda score, best: score > best
    else:
      raise ValueError("Unknown Score type: %s" % repr(scoreType))

    if threshold is None:
      logger.warn("Threshold should probably be specified.. Defaulting to infinity..")
      self.threshold = InfiniteFeatureComparisonResult()
    else:
      self.threshold = ConstantValueFeatureComparisonResult(threshold)

    # Define two maps, one from records to cluster identifiers (which we will
    # start with each record in a cluster with its own identifier) and a second
    # from cluster identifiers to the records contained therein.
    #
    self.b2c = {}
    self.c2b = {}

    # Presently unused
    self.globalClusters = globalClusters

    self.guidGenerator = guidGenerator

    for compositeRecord in compositeRecords:
      clusterIndex = self._newClusterId()
      self.c2b[clusterIndex] = compositeRecord
      for baseRecord in compositeRecord:
        self.b2c[baseRecord] = clusterIndex

    # Cache distances between two clusters and between base recs.
    #
    if baseDistanceCache:
      self.baseDistanceCache = baseDistanceCache
    else:
      self.baseDistanceCache = {}
    self.clusterDistanceCache = {}

  def _newClusterId(self):
    """Gets a new cluster ID unique to this object"""
    return self.guidGenerator.new_id()

  def _baseDistance(self, b1, b2):
    """
    Computes the distance between two base records

    :param b1: one record
    :param b2: another one
    """
    if b1 > b2:
      b1, b2 = b2, b1
    distance = self.baseDistanceCache.get((b1,b2), None)
    if distance is None:
      distance = self.scoreFunction(\
        self.baseFeatureArray[b1], self.baseFeatureArray[b2])
      self.baseDistanceCache[(b1,b2)] = distance
    return distance

  def distance(self, c1, c2):
    """
    Computes the difference between the clusters with the given identifiers.

    :param c1: a cluster identifier
    :param c2: another cluster identifier

    :returns: The distance between the two identified clusters.
    """
    # Concrete class should provide implementation.
    #
    raise NotImplemented

  def mergeClusters(self,c1,c2):
    """
    Merges the two argument clusters.

    :param c1: A cluster identifier
    :param c2: Another cluster identifier.
    """
    combinedBaseRecords = (self.c2b[c1] | self.c2b[c2])
    # Remove these two clusters.
    del self.c2b[c1]
    del self.c2b[c2]
    cNew = self._newClusterId()
    self.c2b[cNew] = combinedBaseRecords
    for baseRecord in combinedBaseRecords:
      self.b2c[baseRecord] = cNew

  def mergeNearestClusters(self):
    """
    Finds the two clusters at the minimum distance and merges them together.

    :returns: True if we merged any clusters.
    """
    if len(self.c2b) <= 1:
      # Cannot merge if there is only 1 cluster remaining.
      #
      logger.debug("Only one cluster left, terminating clustering.")
      return False

    bestDistance = None
    cbest1, cbest2 = None, None

    for c1,c2 in all_pairs_symmetric(self.c2b.keys()):
      distance = self.distance(c1,c2)
      if (not bestDistance) or self.scoreIsBetter(distance, bestDistance):
        bestDistance = distance
        cbest1, cbest2 = c1, c2

    #assert(cbest1 is not None and cbest2 is not None)

    if self.scoreIsBetter(self.threshold, bestDistance):
      logger.debug("Best score %g passed score threshold %g, terminating clustering." % \
        (bestDistance.total(), self.threshold.total()))
      return False
    contrib = bestDistance.feature_contributions()
    for feat in contrib:
      self.informative_features[feat] += contrib[feat]
    # This line is just too much output, makes debug level unusuable. Occasionally
    # uncommented when something in this class needs to be carefully debugged.
    #
    #logger.debug("Merging " + repr(self.c2b[cbest1]) + " and " + repr(self.c2b[cbest2]) + \
    #  " with score " + repr(bestDistance.total()) + ".")
    self.mergeClusters(cbest1, cbest2)
    self.onMerge(self, bestDistance)
    return True

  def cluster(self):
    """Clusters the records and returns the resulting clusters"""
    logger.debug("Beginning feature based clustering on %d clusters." % len(self.c2b))
    # Merge the two nearest clusters until we can't.
    #
    while self.mergeNearestClusters():
      pass
    logger.debug("After clustering, there are now %d clusters remaining." % len(self.c2b))
    return self.c2b.values()

class MinDistanceAgglomerativeCluster(AgglomerativeCluster):
  """
  Agglomerative clusterer that compares two clusters by the distance between
  their closest two points.
  """
  def distance(self, c1, c2):
    """
    Computes the difference between the clusters with the given identifiers.

    :param c1: a cluster identifier
    :param c2: another cluster identifier

    :returns: The distance between the two identified clusters.
    """
    if c1 > c2:
      c1, c2 = c2, c1
    clusterDistance = self.clusterDistanceCache.get((c1,c2), None)
    if clusterDistance is None:
      # Find the minimum distance between any two pairs in the clusters.
      #
      minDistance = InfiniteFeatureComparisonResult()
      for b1 in self.c2b[c1]:
        for b2 in self.c2b[c2]:
          baseDistance = self._baseDistance(b1, b2)
          if baseDistance < minDistance:
            minDistance = baseDistance
      clusterDistance = minDistance
      self.clusterDistanceCache[(c1,c2)] = clusterDistance
    return clusterDistance

class MaxDistanceAgglomerativeCluster(AgglomerativeCluster):
  """
  Agglomerative clusterer that compares two clusters by the distance between
  their farthest two points.
  """
  def distance(self, c1, c2):
    """
    Computes the difference between the clusters with the given identifiers.

    :param c1: a cluster identifier
    :param c2: another cluster identifier

    :returns: The distance between the two identified clusters.
    """
    if c1 > c2:
      c1, c2 = c2, c1
    clusterDistance = self.clusterDistanceCache.get((c1,c2), None)
    if clusterDistance is None:
      # Find the maximum distance between any two pairs in the clusters.
      #
      maxDistance = ConstantValueFeatureComparisonResult(0.0)
      for b1 in self.c2b[c1]:
        for b2 in self.c2b[c2]:
          baseDistance = self._baseDistance(b1, b2)
          if baseDistance > maxDistance:
            maxDistance = baseDistance
      clusterDistance = maxDistance
      self.clusterDistanceCache[(c1,c2)] = clusterDistance
    return clusterDistance

class AverageDistanceAgglomerativeCluster(AgglomerativeCluster):
  """
  Agglomerative clusterer that compares two clusters by the average distance
  between the points in those clusters.
  """
  def distance(self, c1, c2):
    """
    Computes the difference between the clusters with the given identifiers.

    :param c1: a cluster identifier
    :param c2: another cluster identifier

    :returns: The distance between the two identified clusters.
    """
    if c1 > c2:
      c1, c2 = c2, c1
    clusterDistance = self.clusterDistanceCache.get((c1,c2), None)
    if clusterDistance is None:
      totalDistance = FeatureComparisonResult() # 0.0
      count = 0
      for b1 in self.c2b[c1]:
        for b2 in self.c2b[c2]:
          totalDistance = totalDistance.add(self._baseDistance(b1, b2))
          count += 1
      if count == 0:
        clusterDistance = FeatureComparisonResult() # 0.0
      else:
        clusterDistance = totalDistance.normalize(count)
      self.clusterDistanceCache[(c1,c2)] = clusterDistance
    return clusterDistance


from typing import Optional, Dict, Any
from django.contrib.auth.models import AnonymousUser

from users.models import JustfixUser

class GraphQLStaticRequest:
    """
    This represents a GraphQL request made on behalf of front-end
    code that is trying to generate static content--such as a PDF
    or email text--and therefore may not have access to an
    actual Django HttpRequest. For example, it may be running in
    a worker process or from a Django management command.

    Because we've already written all our GraphQL mutations to
    expect a Django HttpRequest as the GraphQL context, however,
    our best way to accomodate this use case (without doing a *lot*
    of refactoring) is to create a tiny subset of the HttpRequest
    interface that only our static content-related GraphQL endpoints
    will need to access.
    """

    def __init__(
        self,
        user: Optional[JustfixUser] = None,
        session: Optional[Dict[str, Any]] = None,
    ):
        if user is None:
            user = AnonymousUser()

        # This corresponds to HttpRequest.user.
        self.user = user

        # This corresponds to HttpRequest.session.
        self.session: Dict[str, Any] = session or {}


import os
import sys
import logging
import json

config_logger = logging.getLogger("main")

class Configuration():

  @property
  def DEBUG(self):
    if "WINGDB_ACTIVE" in os.environ:
      return False
    else:
      return True

  @property
  def TESTING(self):
    return False

  @property
  def DATABASE_URI(self):
    if "WINGDB_ACTIVE" in os.environ:
      return False
    else:
      return True

  @classmethod
  def configfile(self,config_filename="ldap.json"):
    try:
      config_logger.debug("retrieving %s configuration"% config_filename)
      with open(self.config_path('%s'%config_filename), "r") as f:
        x = f.read()
      if config_filename.endswith("json"):
        config=json.loads(x)
      else:
        auth_logger.error("extension unknown")
      return config
    except:
      config_logger.error("failed to retrieve %s configuration"%config_filename)

  @staticmethod
  def config_path(config_name):
    try:
      if getattr(sys, 'frozen', False):
        application_path = os.path.dirname(sys.executable)
      elif __file__:
        application_path = os.path.dirname(__file__)

      confpath = application_path + '/..' +'/config/' + config_name
      return confpath
    except:
      config_logger.error("failed to get config path")


from datetime import date
from decimal import Decimal

from django.test import TestCase

from charcoallog.investments.forms import InvestmentDetailsForm
from charcoallog.investments.models import NewInvestmentDetails

class InvestmentDetailsFormTest(TestCase):
    def test_form_has_fields(self):
        """ Form must have 3 fields. The third field is in test_basicdata_form"""
        form = InvestmentDetailsForm()
        self.assertSequenceEqual(
            ['date', 'money', 'kind', 'tx_op',
             'brokerage', 'which_target', 'segment',
             'tx_or_price', 'quant'],
            list(form.fields)
        )

class InvestmentDetailSave(TestCase):
    def setUp(self):
        data = dict(
            user_name='teste',
            date='2018-09-19',
            money='10.00',
            kind='tesouro',
            tx_op=0.00,
            brokerage='A',
            which_target='selic',
            segment='2023',
            tx_or_price=0.00,
            quant=0.00,
        )
        self.form = InvestmentDetailsForm(data)

    def test_is_valid(self):
        self.assertTrue(self.form.is_valid())

    def test_save(self):
        """
        .save() param must exists. That's why the test.
        """
        self.form.save('teste')

        qs = NewInvestmentDetails.objects.all()
        expected = [
            (qs[0].which_target, 'selic'),
            (qs[0].segment, '2023'),
            (qs[0].tx_or_price, Decimal('0.00')),
            (qs[0].quant, 0.00),
            (qs[0].user_name, 'teste'),
            (qs[0].date, date(2018, 9, 19)),
            (qs[0].money, Decimal('10.00')),
            (qs[0].kind, 'tesouro'),
            (qs[0].tx_op, Decimal('0.00')),
            (qs[0].brokerage, 'A'),
        ]

        for obj, v in expected:
            with self.subTest():
                self.assertEqual(obj, v)


import pandas as pd
import matplotlib
import matplotlib.pyplot as plt
import os

directory = os.fsencode('../stations/')
l = []
for s in os.listdir(directory):
	s = s.decode('ascii')
	if s.startswith('augusta'):
		l.append(s)
l.sort()
files = []
for i in range(len(l)):
	files.append(pd.read_csv('../stations/'+l[i], delimiter = ";"))

merged_file = pd.concat(files, axis=0, ignore_index = True)
file_clean = merged_file.replace(to_replace = '--', value='0')
file_clean['Data rilevazione'] = file_clean[file_clean.columns[2:4]].apply(
    lambda x: ','.join(x.astype(str)),
    axis=1)
df = file_clean.drop(['Ora rilevazione', 'Grandezza', 'Stazione'], axis=1)
df['Valore'] = df['Valore'].apply(lambda x: x.replace(',','.')).astype(float)

def dry_days(df):
	l = []
	c = 0
	for i in range(len(df)):
		if df['Valore'][i] == 0.0:
			c+=1
		else:
			l.append(c)
			c = 0
			continue
	l.append(c)
	return round(max(l)/144)

def wet_hours(df):
	l = []
	c = 0
	for i in range(len(df)):
		if df['Valore'][i] != 0.0:
			c+=1
			continue
		else:
			if c != 0:
				l.append(c)
				c = 0
			continue
	l.append(c)
	return round(max(l)/6)

dry_days = dry_days(df)
wet_hours = wet_hours(df)
years = [0, 52560, 105121, 157682, 210387, 262948, 315509, 368070, 420775, 473336, 525897, 578458, 631163, 683725]
df_years = []
for i in range(len(years)-1):
	df_years.append(df.iloc[years[i]:years[i+1]+1])
for i in range(len(df_years)):
	year = df_years[i]['Data rilevazione'].loc[df_years[i]['Data rilevazione'].index[5]][6:10]
	df_years[i]['Valore'].plot()
	plt.text(0.25, 0.96,'Max:'+ str(round(df['Valore'].max(),2)), fontsize=10,bbox = dict(facecolor = 'green', alpha = 0.5), transform=plt.gcf().transFigure)
	plt.text(0.02, 0.96,'Max dry days:'+ str(dry_days), fontsize=10,bbox = dict(facecolor = 'orange', alpha = 0.5), transform=plt.gcf().transFigure)
	plt.text(0.02, 0.9,'Max wet hours:'+ str(wet_hours), fontsize=10,bbox = dict(facecolor = 'orange', alpha = 0.5), transform=plt.gcf().transFigure)
	plt.xlabel('time line - 10 min intervals')
	plt.ylabel('rain - mm')
	plt.title(str(file_clean['Stazione'].loc[file_clean['Stazione'].index[5]])+' ' +year)
	plt.ylim(top=22)
	plt.savefig('results/'+str(file_clean['Stazione'].loc[file_clean['Stazione'].index[5]])+year+'.png')