import os
import numpy as np
import glob
import cv2, json
import argparse
from sklearn.utils import shuffle
from keras.optimizers import SGD, Adam
from keras.models import Sequential
from keras.layers import Dropout, Dense
from keras.callbacks import EarlyStopping, ModelCheckpoint
from keras.preprocessing import image
from keras.preprocessing.image import ImageDataGenerator
from keras import applications
from keras.applications.inception_v3 import InceptionV3
from keras.applications.inception_v3 import preprocess_input
from keras.layers import LSTM
from keras.models import load_model, Model
from sequence_data_generator import FramesSeqGenerator, FeaturesSeqGenerator
from data_generator import DataGenerator
from keras import backend as K

# BATCH_SIZE = 32
# SEED = 42

def extract_cnn_features(raw_data_dir, features_dir):
    # base_model = InceptionV3(include_top=False, weights='imagenet', input_shape=(224, 224, 3))
    # print("Base model shape")
    # print(base_model.input_shape)

    model = InceptionV3(weights='imagenet', include_top=True)
    base_model = Model(inputs=model.input, outputs=model.get_layer('avg_pool').output)
    _, height, width, channels = base_model.input_shape

    frames = FramesSeqGenerator(raw_data_dir, BATCH_SIZE, 20, height, width, channels, False)
    for i, video in frames.videos_list.iterrows():
        video_name = video.frames_dir.split('/')[-1]
        label = video.label
        print(f'Video label = {label}')
        features_path = f'{features_dir}/{label}/{video_name}.npy'
        X, y = frames.generate(video)
        print(f'X shape = {X.shape}')
        features = base_model.predict(X)
        print(f'features shape = {features.shape}')

        if not os.path.exists(f'{features_dir}/{label}'):
            os.makedirs(f'{features_dir}/{label}')
        np.save(features_path, features)
    return

def extract_cnn_features_all(raw_data_dir, features_dir):
    model = InceptionV3(weights='imagenet', include_top=True)
    base_model = Model(inputs=model.input, outputs=model.get_layer('avg_pool').output)
    _, height, width, channels = base_model.input_shape

    print(base_model.input_shape)

    signs = sorted([name for name in os.listdir(raw_data_dir) if os.path.isdir(os.path.join(raw_data_dir, name))])
    for sign in signs:
        print("\nSign = {0}\n-------------------".format(sign))
        videos = sorted([name for name in os.listdir(os.path.join(raw_data_dir, sign)) if os.path.isdir(os.path.join(raw_data_dir, sign, name))])

        # For each video we will save the cnn features computed on each image
        for video in videos:
            print("Video = {0}".format(video))
            features_path = f'{features_dir}/{sign}/{video}.npy'
            if not os.path.exists(f'{features_dir}/{sign}'):
                os.makedirs(f'{features_dir}/{sign}')

            pathname = os.path.join(raw_data_dir, sign, video, '*.jpg')
            imagenames = sorted(glob.glob(pathname))
            featuresList = []
            for imagename in imagenames:
                img = image.load_img(imagename, target_size=(height, width))
                img_data = image.img_to_array(img)
                img_data = np.expand_dims(img_data, axis=0)
                img_data = preprocess_input(img_data)
                inceptionv3_feature = base_model.predict(img_data)
                featuresList.append(inceptionv3_feature)
            stackedFeatures = np.vstack(featuresList)
            np.save(features_path, stackedFeatures)
            print("{0} images processed\n".format(len(featuresList)))

def train_lstm(features_path, epochs, num_features, num_classess):
    model = Sequential()
    model.add(LSTM(num_features, dropout=0.2, input_shape=(20, num_features), return_sequences=True))
    model.add(LSTM(num_features * 1, return_sequences=False))
    model.add(Dense(num_classess, activation='softmax'))
    optimizer = Adam(lr=1e-4)
    model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])
    # callbacks = [EarlyStopping(monitor='val_loss', patience=10, verbose=0), ModelCheckpoint('video_1_LSTM_1_1024.h5', monitor='val_loss', save_best_only=True, verbose=0)]
    callbacks = [ModelCheckpoint('video_1_LSTM_1_1024.h5', monitor='val_loss', save_best_only=True, verbose=0)]

    train_features = FeaturesSeqGenerator(features_path + '/train', BATCH_SIZE, model.input_shape[1:])
    val_features = FeaturesSeqGenerator(features_path + '/test', BATCH_SIZE, model.input_shape[1:])
    model.fit_generator(generator=train_features,
            validation_data=val_features,
            epochs=int(epochs),
            workers=1,
            use_multiprocessing=False,
            verbose=1,
            callbacks=callbacks)
    return model

def train_lstm_jittered(features_path, sequencelength, epochs, num_features, num_classess):
    model = Sequential()
    model.add(LSTM(2048, return_sequences=False,input_shape=(sequencelength, num_features),dropout=0.5))
    model.add(Dense(512, activation='relu'))
    model.add(Dropout(0.5))
    model.add(Dense(num_classess, activation='softmax'))
    optimizer = Adam(lr=1e-4)
    metrics = ['accuracy']
    model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=metrics)

    callbacks = [ModelCheckpoint('video_LSTM.h5', monitor='val_loss', save_best_only=True, verbose=1)]

    data_path, batch_size, featureLength, shuffle = 'cnn_features/train', 32, 2048, True
    train_gen = DataGenerator(data_path,
                            batch_size=batch_size,
                            featureLength=featureLength,
                            seqLength=sequencelength,
                            shuffle=shuffle)

    data_path, batch_size, featureLength, shuffle = 'cnn_features/test', 16, 2048, False
    val_gen = DataGenerator(data_path,
                            batch_size=batch_size,
                            featureLength=featureLength,
                            seqLength=sequencelength,
                            shuffle=shuffle)

    model.fit_generator(generator=train_gen,
            validation_data=val_gen,
            epochs=int(epochs),
            workers=1,
            use_multiprocessing=False,
            verbose=1,
            callbacks=callbacks)
    return model

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Train CNN+LSTM model for short video snippets')

    parser.add_argument('--data-path', default='processed_data', required=False, help='path to the train and test directories')
    parser.add_argument('--lstm-epochs', default=100, required=False, help='number of LSTM epochs')
    parser.add_argument('--reload-cnn-features', default=True, required=False, help='re-extract CNN features from frames')
    parser.add_argument('--features-path', default='cnn_features', required=False, help='extracted CNN features path')
    args = vars(parser.parse_args())
    print(args)
    if args.get('data_path') is None:
        print('Please provide path to video frames')
        exit()

    # GPU configuration
    # K.tensorflow_backend._get_available_gpus()
    # config = tf.ConfigProto( device_count = {'GPU': 200, 'CPU': 4} )
    # sess = tf.Session(config=config)
    # keras.backend.set_session(sess)

    # extract_cnn_features(f'{args["data_path"]}/train', f'{args["features_path"]}/train')
    # extract_cnn_features(f'{args["data_path"]}/test', f'{args["features_path"]}/test')

    # extract_cnn_features_all(f'{args["data_path"]}/train', f'{args["features_path"]}/train')
    # extract_cnn_features_all(f'{args["data_path"]}/test', f'{args["features_path"]}/test')

    # model = train_lstm(args['features_path'], args['lstm_epochs'], num_features, num_classes)

    num_features = 2048
    num_classes = len(os.listdir(f'{args["features_path"]}/train'))
    seqLength = 10
    model = train_lstm_jittered(args['features_path'], seqLength, args['lstm_epochs'], num_features, num_classes)

    # model.save("final_model.h5")


import numpy as np
from scipy.special import erf as ERF
from scipy.stats import norm as NORM
try:
    from surrogate import MultiSurrogates
except:
    from .surrogate import MultiSurrogates
import _csupport as CS
from evoalgos.performance import FonsecaHyperVolume as FH
try:
    from BO_base import BayesianOptBase
except:
    from .BO_base import BayesianOptBase

class MultiSurrogate(BayesianOptBase):
    '''
    Base class for multi-surrogate approaches.
    '''

    def __init__(self, func, \
                    n_dim, n_obj, lower_bounds, upper_bounds, \
                    obj_sense=[-1, -1], args = (), kwargs={}, X=None, Y=None, \
                    kern=None, ref_vector=None):
        '''
        This constructor creates the multi-surrogate base class.

        Parameters.
        -----------
        func (method): the method to call for actual evaluations
        n_dim (int): number of decision space dimensions
        n_obj (int): number of obejctives
        lower_bounds (np.array): lower bounds per dimension in the decision space
        upper_boudns (np.array): upper bounds per dimension in the decision space
        obj_sense (np.array or list): whether to maximise or minimise each
                objective (ignore this as it was not tested).
                keys.
                    -1: minimisation
                     1: maximisation
        args (tuple): the required arguments for the objective function
        kwargs (dictionary): dictionary of keyword argumets to pass to the
                            objective function.
        X (np.array): initial set of decision vectors.
        Y (np.array): initial set of objective values (associated with X).
        kern (GPy kernel): kernel to be used with Gaussian process.
        ref_vector (np.array): reference vector in the objective space.
        '''
        super().__init__(func, n_dim, n_obj, lower_bounds, upper_bounds, \
                                obj_sense, args, kwargs, X, Y,\
                                ref_vector=ref_vector)
        self.kernels = kern
        self.hpv = FH(self.ref_vector)

    def get_toolbox(self, xtr, skwargs, cfunc=None, \
                        cargs=(), ckwargs={}, verbose=True):
        """
        Generate a DEAP toolbox for the infill criterion optimiser.

        Parameters.
        -----------
        xtr (np.array): traing decision vectors.
        skwargs (dict): options for infill criterion calculation; varies with
                        technique.
        cfunc (function): cheap constraint function.
        cargs (tuple): argumetns for constraint function.
        ckwargs (dict): keyword arguments for constraint function.
        verbose (bool): whether to print more comments.

        Returns a DEAP toolbox.
        """
        self.budget = skwargs.get('budget', 250)
        self.update(xtr) # update various parameters of multi-surrogates
        return self.init_deap(self.scalarise, cfunc=cfunc, cargs=cargs, \
                        ckwargs=ckwargs)

    def build_models(self, xtr, ytr, kernels):
        '''
        Build multiple surrogate models.

        Parameters.
        -----------
        xtr (np.array): training decision vectors.
        ytr (np.array): training objective vectors.
        kernels (list): GPy kernels to be used with multiple Gaussian processes.

        Returns multiple regression models.
        '''
        self.xtr = xtr
        self.ytr = ytr
        models = MultiSurrogates(xtr, ytr, kernels)
        return models

    def update(self, x_new):
        '''
        Update a range of setting for multi-surrogate.

        Parameters.
        -----------
        x_new (np.array): decision vector.

        Updates attributes, but returns nothing.
        '''
        # set/update xtr and ytr
        self.xtr = x_new
        self.ytr = self.m_obj_eval(x_new)
        assert self.xtr.shape[0] == self.ytr.shape[0] # check for smae amount of
                                                      # data
        yt, comp_mat = self.get_dom_matrix(self.ytr)
        self.comp_mat = comp_mat
        # update budget count
        self.b_count = self.budget - len(self.xtr) - 1
        # update + optimise models
        self.models = self.build_models(self.xtr, self.ytr, \
                            [kern.copy() for kern in self.kernels])
        # current pf
        self.pfr_inds = self.get_front(self.ytr, self.comp_mat)
        # current hv
        self.current_hv = self.current_hpv()
        # epsilon for sms-ego
        n_pfr = len(self.pfr_inds)
        c = 1 - (1/ 2**self.n_obj)
        self.epsilon = (np.max(self.ytr, axis=0) - np.min(self.ytr, axis=0))\
                        /(n_pfr + (c * self.b_count))

class MPoI(MultiSurrogate):
    '''
    Multi-Surrogate Minimum Probability of Improvement infill Criterion.
    '''

    def __init__(self, func, \
                    n_dim, n_obj, lower_bounds, upper_bounds, \
                    obj_sense=[-1, -1], args = (), kwargs={}, X=None, Y=None, \
                    kern=None, ref_vector=None):
        '''
        Simple constructor for invoking parent class.

        Parameters.
        -----------
        func (method): the method to call for actual evaluations
        n_dim (int): number of decision space dimensions
        n_obj (int): number of obejctives
        lower_bounds (np.array): lower bounds per dimension in the decision space
        upper_boudns (np.array): upper bounds per dimension in the decision space
        obj_sense (np.array or list): whether to maximise or minimise each
                objective (ignore this as it was not tested).
                keys.
                    -1: minimisation
                     1: maximisation
        args (tuple): the required arguments for the objective function
        kwargs (dictionary): dictionary of keyword argumets to pass to the
                            objective function.
        X (np.array): initial set of decision vectors.
        Y (np.array): initial set of objective values (associated with X).
        kern (GPy kernel): kernels to be used with Gaussian process.
        ref_vector (np.array): reference vector in the objective space.
        '''
        super().__init__(func, n_dim, n_obj, lower_bounds, upper_bounds, \
                                obj_sense, args, kwargs, X, Y,\
                                kern=kern, ref_vector=ref_vector)

    def scalarise(self, x, cfunc=None, cargs=(), ckwargs={}):
        '''
        Calculate the minimum probability of improvement compared to current
        Pareto front. Refer to the paper for full details.

        parameters:
        -----------
        x (np.array): decision vectors.
        cfunc (function): cheap constraint function.
        cargs (tuple): argument for constraint function.
        ckwargs (dict): keyword arguments for constraint function.

        Returns scalarised cost.
        '''
        n_sols = x.shape[0]
        if cfunc is not None:
            if not cfunc(x, *cargs, **ckwargs):
                return np.zeros((1,1))  # penalise for constraint violation
        yp, stdp = self.models.predict(x)
        y = self.ytr[self.pfr_inds]
        res = np.zeros((yp.shape[0], 1))
        sqrt2 = np.sqrt(2)
        for i in range(yp.shape[0]):
            m = (yp[i] - y)/(sqrt2 * stdp[i])
            pdom = 1 - np.prod(0.5 * (1 + ERF(m)), axis=1)
            res[i] = np.min(pdom)
        return res

class SMSEGO(MultiSurrogate):
    '''
    Multi-surrogate SMS-EGO. Note that this is inspired from GPareto package in
    R.
    '''

    def __init__(self, func, \
                    n_dim, n_obj, lower_bounds, upper_bounds, \
                    obj_sense=[-1, -1], args = (), kwargs={}, X=None, Y=None, \
                    kern=None, ref_vector=None):
        '''
        Simple constructor for invoking parent class.

        Parameters.
        -----------
        func (method): the method to call for actual evaluations
        n_dim (int): number of decision space dimensions
        n_obj (int): number of obejctives
        lower_bounds (np.array): lower bounds per dimension in the decision space
        upper_boudns (np.array): upper bounds per dimension in the decision space
        obj_sense (np.array or list): whether to maximise or minimise each
                objective (ignore this as it was not tested).
                keys.
                    -1: minimisation
                     1: maximisation
        args (tuple): the required arguments for the objective function
        kwargs (dictionary): dictionary of keyword argumets to pass to the
                            objective function.
        X (np.array): initial set of decision vectors.
        Y (np.array): initial set of objective values (associated with X).
        kern (GPy kernel): kernels to be used with Gaussian process.
        ref_vector (np.array): reference vector in the objective space.
        '''
        super().__init__(func, n_dim, n_obj, lower_bounds, upper_bounds, \
                                obj_sense, args, kwargs, X, Y,\
                                kern=kern, ref_vector=ref_vector)
        self.gain = - NORM.ppf(0.5 * (0.5**(1/self.n_obj)))

    def compare_add_solution(self, y, ytest, obj_sense):
        '''
        Compare and add a solution to the data set given its not dominated.

        Parameters.
        -----------
        y (np.array): current Pareto front objective vectors.
        y_test (np.array): candidate for adding to the archive.

        Returns latest Pareto front.
        '''
        result = np.ones(y.shape[0])
        for i in range(y.shape[0]):
            result[i] = CS.compare_solutions(y[i], ytest, self.obj_sense)
            if result[i] == 0:
                return y
        inds = np.where(result==3)[0]
        try:
            return np.concatenate([y[inds], ytest])
        except ValueError:
            print("Likely error in y: ", y[inds])
            return ytest

    def penalty(self, y, y_test):
        '''
        Penalty mechanism in the infill criterion. Penalise if dominated by the
        current front.

        Parameters.
        -----------
        y (np.array): current Pareto front elements.
        y_test (np.array): tentative solution.

        Returns a penalty value.
        '''
        yt = y_test - (self.epsilon * self.obj_sense)
        l = [-1 + np.prod(1 + y_test - y[i]) \
                if CS.compare_solutions(y[i], yt, self.obj_sense) == 0\
                else 0 for i in range(y.shape[0])]
        return (max([0, max(l)]))

    def scalarise(self, x, cfunc=None, cargs=(), ckwargs={}):
        '''
        S-metric infill criterion.

        parameters:
        -----------
        x (np.array): decision vectors.
        cfunc (function): cheap constraint function.
        cargs (tuple): argument for constraint function.
        ckwargs (dict): keyword arguments for constraint function.

        Returns scalarised cost.
        '''
        n_sols = len(x)
        if cfunc is not None:
            if not cfunc(x, *cargs, **ckwargs):
                return np.ones((1,1))*-100 # heavily penalise for constraint
                                           # violation
        # predictions
        yp, stdp = self.models.predict(x)
        # lower confidence bounds
        yl = yp - (self.gain * np.multiply(self.obj_sense, stdp))
        pen = self.penalty(self.ytr[self.pfr_inds], yl)
        if pen > 0:
            return np.array([-pen])
        # new front
        yn = self.compare_add_solution(self.ytr[self.pfr_inds], yl, self.obj_sense)
        return np.array([self.hpv.assess_non_dom_front(yn) - self.current_hv])


import pandas as pd
import numpy as np
import statsmodels.api as sm
import matplotlib
import matplotlib.pyplot as plt

def TAR(y, threshold, lags, save_path=None):
    """
    Objective: Using OLS, estimate a threshold autoregressive model.
    :param y: Dataframe with pandas DatetimeIndex.
    :param threshold: (float) cutoff level separating the two regimes.
    :param lags: (int) number of lags to include in AR model.
    :param save_path: (str) of location to save images or None to turn off saving.
    :return:
            -results: list object containing OLS results for the regions below and above, respectively.
            -residuals: Series object containing timeseries of residuals.
            predict: Series object containing timeseries of predicted values.
    """
    y = y.dropna()
    # X = pd.DataFrame(np.zeros((len(y), lags)))
    X = pd.DataFrame()
    # print(X)
    # print('a')
    for i in range(1, lags+1):
        lag = y.shift(i)
        # print(lag)
        # X.iloc[:, i-1] = lag
        X = pd.concat([X, lag], axis=1)
        # print(X)
    # print(X)
    X = X.dropna()
    # print(X)
    X = sm.add_constant(X)
    # print(X)

    # Regression for values below the threshold
    less = X[X.iloc[:, 1] < threshold]
    # print('b')
    out1 = sm.OLS(y.loc[less.index], less).fit()
    # print('c')
    print('Regression results for values BELOW threshold:')
    print(out1.summary())

    # Regression for values above the threshold
    greater = X[X.iloc[:, 1] >= threshold]
    out2 = sm.OLS(y.loc[greater.index], greater).fit()
    print('Regression results for values ABOVE threshold:')
    print(out2.summary())

    # Residuals
    resid1 = out1.resid
    resid2 = out2.resid
    residuals = pd.concat([resid1, resid2], axis=0)

    # Results
    results = [out1, out2]

    # ACF plot of residuals
    fig1, ax1 = plt.subplots(figsize=(36, 10))
    sm.graphics.tsa.plot_acf(residuals, ax=ax1)
    ax1.set_title('ACF of Residuals')
    ax1.set_xlabel('Lags')
    ax1.set_ylabel('Autocorrelation')
    fig1.show()

    if save_path is not None:
        fig1.savefig(save_path + 'residual_acf.png')

    # Predicted values
    predict1 = out1.predict(less)
    predict2 = out2.predict(greater)
    predict = pd.concat([predict1, predict2], axis=0).sort_index()

    # create predicted values plot
    fig2, ax2 = plt.subplots(figsize=(36, 10))
    ax2.plot(y, label='Actual', linewidth=2)
    ax2.plot(predict, label='Predicted', linestyle='--', linewidth=2)
    ax2.set_title('Actual vs Predicted Values')
    ax2.set_xlabel('Time')
    ax2.set_ylabel('Value')
    ax2.legend()
    fig2.show()

    if save_path is not None:
        fig2.savefig(save_path + 'predicted_values.png')

    return results, residuals, predict


"""
See `tts_calculation_without_bootstrapping` for a detailed explanation
on how to use the functions defined here.
"""

import numpy as np
import pandas as pd
from ps_from_rawdf import bv_prob, restrict_to_bvn
from tqdm import tqdm

def tts(prob: float, dt: float, desired: float = 0.99, unit: str = "log10") -> float:
    """Time to solution given the acquired probability, desired probability, and the time taken

    Args:
        prob (float): acquired probability
        dt (float): time for a single iteration
        desired (float, optional): desired probability. Defaults to 0.99.
        unit (str, optional): seconds or log10 seconds . Defaults to "log10".

    Returns:
        float: Time to solution
    """
    if prob == 0:
        return np.nan  # infinite TTS
    if unit == "default":
        return dt * (np.log(1 - desired) / np.log(1 - prob))
    if unit == "log10":
        return np.log10(tts(prob, dt, unit="default"))

def tts_on_bvn_row(row_data: pd.Series, seq: str, bv_durations: dict) -> float:
    """Given a row from a dataframe of success probabilities and the sequence
    we are concerned with, and the bv_durations compute the sucess probability
    for that particular entry.

    Args:
        row_data (pd.Series): a row from a dataframe of success probabilities
        seq (str): name of the DD sequence
        bv_durations (dict): dictionary representing durations for various BV-n

    Returns:
        float: Time to solution
    """
    bitstring = row_data.name  # get the bitstring like 00000
    no_of_1s = bitstring.count("1")
    duration = bv_durations[no_of_1s]
    prob = row_data[seq]
    # dur = row_data['circuit duration']
    return tts(prob, duration)

def tts_df(bv_n: pd.DataFrame, bv_durations: dict) -> pd.DataFrame:
    """Time to solution given success probabilities and corresponding durations

    Args:
        bv_n (pd.DataFrame): Success probabilities for all the oracles
        bv_durations (dict): Durations for all the oracles

    Returns:
        pd.DataFrame: Same dataframe as `bv_n` but probabilities are replaced with TTS
    """
    seqs = list(bv_n.columns)
    bv_n_tts = pd.DataFrame()
    for seq in seqs:
        bv_n_tts[seq] = bv_n.apply(
            lambda row: tts_on_bvn_row(row, seq, bv_durations), axis=1
        )
    return bv_n_tts

def tts_average(bv_n: pd.DataFrame, bv_durations: dict) -> pd.Series:
    """Averate Time to solution given a dataframe of TTS for each oracle

    Args:
        bv_n (pd.DataFrame): Dataframe with sucess probabilities for each oracle
        bv_durations (dict):  Durations for all the oracles

    Returns:
        pd.Series: Average TTS for each DD sequence
    """
    bv_n_tts = tts_df(bv_n, bv_durations)
    return bv_n_tts.apply(bv_prob)

def tts_average_for_all_n(bv_ns: list, bv_durations: dict) -> pd.DataFrame:
    """Given all the success probabilities and durations (for each oracle in all BV-n)
    return the average TTS for each BV-n

    Args:
        bv_ns (list): list of pd.DataFrame, each representing success probabilities for
            all oracles for a specific n
        bv_durations (dict): Durations for all the oracles

    Returns:
        pd.DataFrame: rows = n, columns = DD Sequence, entries are average TTS
    """

    tts_dataframe = pd.DataFrame()

    for n in range(0, len(bv_ns)):
        bv_n = bv_ns[n]
        tts_n = pd.DataFrame(
            tts_average(bv_n, bv_durations), columns=[n + 1]
        ).transpose()
        tts_dataframe = pd.concat([tts_dataframe, tts_n])

    return tts_dataframe

def tts_from_rawdf(rawdf: pd.DataFrame, bv_durations: dict) -> pd.DataFrame:
    """Time to solution for all sequences and BV-n given the rawdf and the duration.
    This is a composite function, that combines all the other functions in this script.

    Args:
        rawdf (pd.DataFrame): dataframes with prefix rawdf
        bv_durations (dict): Durations for all the oracles

    Returns:
        pd.DataFrame: rows = n, columns = DD Sequence, entries are average TTS
    """
    seqs = list(rawdf.keys()[0:3])  # first 3 elements are the sequences
    N = len(rawdf)  # 27
    ps_df_lst = []
    for n in tqdm(range(1, N)):
        ps_df_lst.append(restrict_to_bvn(rawdf, seqs, n=n))
    tts_all = tts_average_for_all_n(ps_df_lst, bv_durations)
    return tts_all


import torch
import string
import os
import torch.nn as nn

class TransformerModel(nn.Module):
    def __init__(self, chars, embed_size, heads, num_layers, hidden_size, sequence_length, lr, epochs, checkpoint_interval, clip_value):
        super(TransformerModel, self).__init__()
        self.chars = chars
        self.vocab_size = len(chars)
        self.embed_size = embed_size
        self.heads = heads
        self.num_layers = num_layers
        self.hidden_size = hidden_size
        self.sequence_length = sequence_length
        self.lr = lr
        self.epochs = epochs
        self.checkpoint_interval = checkpoint_interval
        self.clip_value = clip_value

        self.embedding = nn.Embedding(self.vocab_size, embed_size)
        self.transformer = nn.Transformer(embed_size, nhead=heads, num_encoder_layers=num_layers, num_decoder_layers=num_layers)
        self.fc = nn.Linear(embed_size, self.vocab_size)

    def forward(self, x):
        x = self.embedding(x)
        x = self.transformer(x, x)
        x = self.fc(x)
        return x

def char_to_index(char, model):
    return model.chars.index(char)

def index_to_char(index, model):
    return model.chars[index]

def sample_with_temperature(logits, temperature):
    probs = torch.nn.functional.softmax(logits / temperature, dim=0)
    return torch.multinomial(probs, 1).item()

# Load checkpoint
checkpoint_path = "model_checkpoint.pth"
if not os.path.exists(checkpoint_path):
    print("Checkpoint file not found. Please train the model first.")
    exit()

checkpoint = torch.load(checkpoint_path)
hyperparameters = checkpoint['hyperparameters']

# First, we create a dummy list of chars to instantiate the model
dummy_chars = ['a']  # This is just a placeholder

model = TransformerModel(
    chars=checkpoint['chars'],
    embed_size=hyperparameters['embed_size'],
    heads=hyperparameters['heads'],
    num_layers=hyperparameters['num_layers'],
    hidden_size=hyperparameters['hidden_size'],
    sequence_length=hyperparameters['sequence_length'],
    lr=hyperparameters['lr'],
    epochs=hyperparameters['epochs'],
    checkpoint_interval=hyperparameters['checkpoint_interval'],
    clip_value=hyperparameters['clip_value']
)

# Now load the state dict.
model.load_state_dict(checkpoint['model_state_dict'])
model.eval()

print("Chat with the model! Type 'exit' to end the chat.")
temperature = 0.5
output_length = 200

while True:
    input_text = input("> ").lower()
    if input_text == 'exit':
        break

    char_indices = [char_to_index(char, model) for char in input_text]  # Pass model to char_to_index
    input_tensor = torch.tensor([char_indices])

    # Generate additional characters after the user input
    for _ in range(output_length):
        output = model(input_tensor)
        char_idx = sample_with_temperature(output[0, -1], temperature)
        output_char = index_to_char(char_idx, model)  # Pass model to index_to_char
        print(output_char, end='', flush=True)
        char_indices.append(char_idx)
        input_tensor = torch.tensor([char_indices])

    print()  # to move to a new line after the generated sequence

print("Chat ended.")


#!/usr/bin/env python
# coding: utf-8

import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import Model, layers
from einops import rearrange, repeat
from einops.layers.tensorflow import Rearrange, Reduce

class MLPBlock(layers.Layer):
    def __init__(self, mixing, d_ff):
        super(MLPBlock, self).__init__()
        if mixing != 'token' and mixing != 'channel':
            raise ValueError("undefiend mixing")
        self.mixing = mixing
        self.d_ff = d_ff

    def build(self, input_shape):
        d_out = input_shape[-1] if self.mixing=='channel' else input_shape[-2]
        self.dense_1 = layers.Dense(self.d_ff, use_bias=False)
        self.dense_2 = layers.Dense(d_out, use_bias=False)
        self.norm = layers.LayerNormalization(epsilon=1e-6)

    def call(self, inputs):
        '''
        inputsは(batches, tokens, channels)
        Denseは最後の軸に作用
        '''
        x = self.norm(inputs)
        x = x if self.mixing!='token' else tf.transpose(x, perm=[0, 2, 1])
        x =self.dense_1(x)
        x = tf.keras.activations.gelu(x, approximate=True)
        x = self.dense_2(x)
        x = x if self.mixing!='token' else tf.transpose(x, perm=[0, 2, 1])
        return x + inputs

class MixerBlock(layers.Layer):
    def __init__(self, d_s, d_c):
        super(MixerBlock, self).__init__()
        self.mlp_1 = MLPBlock(mixing='token', d_ff=d_s)
        self.mlp_2 = MLPBlock(mixing='channel', d_ff=d_c)

    def call(self, inputs):
        x = self.mlp_1(inputs)
        x = self.mlp_2(x)
        return x

#StemがDenseModel
def MLP_Mixer1_build(**config):
    shapes = (config['image_size'], config['image_size'], config['channels'])
    hidden_dim  = config['hidden_dim']
    patch_size = config['patch_size']
    num_blocks = config['depth']
    tokens_mlp_dim = config['tokens_mlp_dim']
    channels_mlp_dim = config['channels_mlp_dim']
    num_classes = config['num_classes']
    size = (config['image_size']//config['patch_size'])  #4

    inputs = layers.Input(shape=shapes)
    x = Rearrange('b (h x) (w y) c -> b (h w) (x y c)', x=size, y=size)(inputs) #(B, 64, 48)
    x = layers.Dense(hidden_dim, use_bias=False)(x)

    for i in range(num_blocks):
      x = MixerBlock(tokens_mlp_dim, channels_mlp_dim)(x)

    x = layers.GlobalMaxPooling1D()(x)
    x = layers.LayerNormalization(epsilon=1e-6)(x)
    outputs = layers.Dense(num_classes, activation="softmax")(x)

    return Model(inputs, outputs)

#StemがConvModel
def MLP_Mixer2_build(**config):
    shapes = (config['image_size'], config['image_size'], config['channels'])
    hidden_dim  = config['hidden_dim']
    patch_size = config['patch_size']
    num_blocks = config['depth']
    tokens_mlp_dim = config['tokens_mlp_dim']
    channels_mlp_dim = config['channels_mlp_dim']
    num_classes = config['num_classes']
    size = config['image_size']//config['patch_size']

    inputs = layers.Input(shape=shapes)
    x = layers.Conv2D(hidden_dim, (size, size), strides=(size, size))(inputs)
    x = Rearrange('n h w c -> n (h w) c')(x)

    for _ in range(num_blocks):
      x = MixerBlock(tokens_mlp_dim, channels_mlp_dim)(x)

    x = layers.GlobalMaxPooling1D()(x)
    x = layers.LayerNormalization(epsilon=1e-6)(x)
    outputs = layers.Dense(num_classes, activation="softmax")(x)

    return Model(inputs, outputs)


from typing import List

import torch

from roar.core.classes import Loss, typecheck
from roar.core.neural_types import LossType, NeuralType

__all__ = ["AggregatorLoss"]

class AggregatorLoss(Loss):
    """
    Sums several losses into one.

    Args:
        num_inputs: number of input losses
        weights: a list of coefficient for merging losses
    """

    @property
    def input_types(self):
        """Returns definitions of module input ports."""
        input_types = {}
        for i in range(self._num_losses):
            input_types["loss_" + str(i + 1)] = NeuralType(elements_type=LossType())

        return input_types

    @property
    def output_types(self):
        """Returns definitions of module output ports."""
        return {"loss": NeuralType(elements_type=LossType())}

    def __init__(self, num_inputs: int = 2, weights: List[float] = None):
        super().__init__()
        self._num_losses = num_inputs
        if weights is not None and len(weights) != num_inputs:
            raise ValueError(
                "Length of weights should be equal to the number of inputs (num_inputs)"
            )

        self._weights = weights

    @typecheck()
    def forward(self, **kwargs):
        values = [kwargs[x] for x in sorted(kwargs.keys())]
        loss = torch.zeros_like(values[0])
        for loss_idx, loss_value in enumerate(values):
            if self._weights is not None:
                loss = loss.add(loss_value, alpha=self._weights[loss_idx])
            else:
                loss = loss.add(loss_value)
        return loss


import llm_toys  # For initlializing the hf modules; Look at __init__.py for more details.

import argparse
import random

from peft import get_peft_model, LoraConfig, prepare_model_for_kbit_training
from torch.utils.data import Dataset
from transformers import (
    AutoTokenizer,
    AutoModelForCausalLM,
    DataCollatorForLanguageModeling,
    Trainer,
    TrainingArguments,
)

from llm_toys.config import DEFAULT_3B_MODEL, get_bnb_config, TASK_TYPES, TaskType
from llm_toys.utils import dialogue_summary_topic_training_data, paraphrase_tone_training_data

class CustomDataset(Dataset):
    """Custom dataset for the trainer. Tokenizes and keeps the
    entire dataset in memory hence it's not suited for large datasets."""

    def __init__(self, tokenizer: AutoTokenizer, text: str, max_length: int = 1024):
        self.text = text
        self.tokenized = tokenizer(
            self.text,
            padding=True,
            truncation=True,
            max_length=max_length,
            return_tensors="pt",
        )

    def __getitem__(self, ix):
        return self.tokenized["input_ids"][ix]

    def __len__(self):
        return len(self.text)

class PeftQLoraTrainer:
    """Contains helpful methods for loading+transforming the data and training the model."""

    def __init__(self, args: dict):
        self.args = args
        for task_type in args["task_type"].split(","):
            assert (
                task_type.strip().lower() in TASK_TYPES
            ), f"Invalid task type {task_type}; Valid task types are {TASK_TYPES}"
        self.experiment_name = ""
        for task_type in args["task_type"].split(","):
            self.experiment_name += "-".join(task_type.strip().lower().split("_")) + "-"
        self.experiment_name = self.experiment_name[:-1]
        self._models_initialized = False

        if self.args["use_aim"]:
            from aim import Run

            self.run = Run(experiment=self.experiment_name)

    def _set_tokenizer(self) -> None:
        self.tokenizer = AutoTokenizer.from_pretrained(self.args["model_name"])
        self.tokenizer.pad_token = self.tokenizer.eos_token

    def _set_model(self) -> None:
        self.model = AutoModelForCausalLM.from_pretrained(
            self.args["model_name"],
            device_map={"": 0},
            trust_remote_code=True,
            quantization_config=get_bnb_config(),
        )
        self.model.gradient_checkpointing_enable()
        self.model = prepare_model_for_kbit_training(self.model)
        config = LoraConfig(
            r=self.args["lora_r"],
            lora_alpha=self.args["lora_alpha"],
            target_modules=["q_proj", "v_proj"] if "llama-2" in self.args["model_name"].lower() else ["query_key_value"],
            lora_dropout=self.args["lora_dropout"],
            bias="none",
            task_type="CAUSAL_LM",
        )
        self.model = get_peft_model(self.model, config)

    def load_data(self) -> list[str]:
        data = []
        for task_type in self.args["task_type"].split(","):
            task_type = task_type.strip().lower()
            if task_type == TaskType.PARAPHRASE_TONE:
                data += paraphrase_tone_training_data()
            elif task_type == TaskType.DIALOGUE_SUMMARY_TOPIC:
                data += dialogue_summary_topic_training_data()
            else:
                raise ValueError(
                    f"Probably an invalid task type {self.args['task_type']}; Valid task types are {TASK_TYPES}"
                )
        return data

    @staticmethod
    def get_train_evaluation_texts(eval_ratio: float, data: list[str]) -> tuple[list[str], list[str]]:
        val_ixs = set()
        if eval_ratio:
            n_val = int(eval_ratio * len(data))
            val_ixs = set(random.sample(range(len(data)), k=n_val))
        train_ixs = (i for i in range(len(data)) if i not in val_ixs)
        train_texts = [data[i] for i in train_ixs]
        val_texts = [data[i] for i in val_ixs]
        return train_texts, val_texts

    def set_trainer(self, train_dataset: Dataset, eval_dataset: Dataset) -> None:
        training_args = TrainingArguments(
            auto_find_batch_size=True if self.args["batch_size"] else False,
            per_device_train_batch_size=self.args["batch_size"],
            per_device_eval_batch_size=self.args["batch_size"],
            gradient_accumulation_steps=self.args["gradient_accumulation_steps"],
            num_train_epochs=self.args["num_train_epochs"],
            learning_rate=self.args["learning_rate"],
            fp16=True,
            do_eval=self.args["eval_ratio"],
            logging_strategy="steps",
            logging_steps=self.args["logging_steps"],
            evaluation_strategy="steps",
            eval_steps=self.args["eval_steps"],
            output_dir=f"models/{self.args['model_name']}",
            overwrite_output_dir=True,
            save_strategy="epoch",
            optim="paged_adamw_8bit",
            lr_scheduler_type="cosine",
            warmup_ratio=0.05,
        )
        self.trainer = Trainer(
            model=self.model,
            args=training_args,
            data_collator=DataCollatorForLanguageModeling(tokenizer=self.tokenizer, mlm=False),
            train_dataset=train_dataset,
            eval_dataset=eval_dataset,
        )

    def log_params(self):
        """Logs all the metadata and enables a UI to observe/compare them using Aim."""
        hparams = self.args
        self.run["hparams"] = hparams

        for log in self.trainer.state.log_history:
            if "eval_loss" in log:
                self.run.track(log["eval_loss"], name="loss", step=log["step"], context={"subset": "eval"})
            elif "loss" in log:
                self.run.track(log["loss"], name="loss", step=log["step"], context={"subset": "train"})

    def prep_data_train_log_and_save_model(self):
        """Loads the data, preps the train and eval datasets, trains+logs+saves the model."""
        if not self._models_initialized:
            self._set_tokenizer()
            self._set_model()
            self._models_initialized = True

        data = self.load_data()
        print(f"Loaded training data; {len(data)} records")

        random.seed(self.args["seed"])
        random.shuffle(data)
        train_texts, eval_texts = self.get_train_evaluation_texts(self.args["eval_ratio"], data)
        print(f"{'# Train':<15} : {len(train_texts)}")
        if self.args["eval_ratio"]:
            print(f"{'# Eval':<15} : {len(eval_texts)}")

        train_dataset = CustomDataset(self.tokenizer, train_texts, max_length=self.args["max_length"])
        print(f"{'# Train dataset':<15} : {len(train_dataset)}")
        if self.args["eval_ratio"]:
            eval_dataset = CustomDataset(self.tokenizer, eval_texts, max_length=args.max_length)
            print(f"{'# Eval dataset':<15} : {len(eval_dataset)}")

        self.set_trainer(train_dataset, eval_dataset if self.args["eval_ratio"] else None)
        self.model.config.use_cache = False
        self.trainer.train()

        self.model.save_pretrained(f"models/{self.args['model_name']}-{self.experiment_name}")

        if self.args["use_aim"]:
            self.log_params()

    def push_to_hub(self):
        self.model.push_to_hub(f"{self.args['model_name']}-{self.experiment_name}", private=True, use_auth_token=True)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--task_type", type=str, help=f"Valid task types are {TASK_TYPES}.")
    parser.add_argument("--model_name", type=str, default=DEFAULT_3B_MODEL, help="Model name.")
    parser.add_argument("--seed", type=int, default=10, help="Random seed.")
    parser.add_argument("--eval_ratio", type=float, default=0.1, help="Ratio of evaluation data.")
    parser.add_argument("--max_length", type=int, default=512, help="Maximum length of input sequence.")
    parser.add_argument("--lora_r", type=int, default=16, help="Lora r.")
    parser.add_argument("--lora_alpha", type=int, default=32, help="Lora alpha.")
    parser.add_argument("--lora_dropout", type=float, default=0.05, help="Lora dropout.")
    parser.add_argument("--batch_size", type=int, default=1, help="Batch size.")
    parser.add_argument("--gradient_accumulation_steps", type=int, default=4, help="Gradient accumulation steps.")
    parser.add_argument("--num_train_epochs", type=int, default=5, help="# Epochs.")
    parser.add_argument("--learning_rate", type=float, default=2e-4, help="Learning rate.")
    parser.add_argument("--logging_steps", type=int, default=100, help="Logging steps.")
    parser.add_argument("--eval_steps", type=int, default=100, help="Eval steps.")
    parser.add_argument("--use_aim", action=argparse.BooleanOptionalAction, help="To use AIM for logging or not.")
    args = parser.parse_args()

    peft_qlora_trainer = PeftQLoraTrainer(vars(args))
    peft_qlora_trainer.prep_data_train_log_and_save_model()


import numpy as np
import scipy.signal as scipysig
import torch
import torch.nn.functional as F
from scipy.stats import chi2
from typing import Tuple, List, NamedTuple
from momentchi2 import hbe

class CollectedData(NamedTuple):
    X: np.ndarray
    U: np.ndarray
    W: np.ndarray
    std_u: float
    std_w: float

class TestStatistics(NamedTuple):
    statistics: float
    pvalue: float

def collect_data(steps: int, std_u: float, std_w: float, sys: scipysig.StateSpace) -> Tuple[np.ndarray, np.ndarray]:
    """ Collect data from a system """
    dim_x, dim_u = sys.B.shape
    U = np.zeros((dim_u, steps))
    X = np.zeros((dim_x, steps + 1))
    W = np.zeros((dim_x, steps))
    X[:, 0] = np.random.normal(size=(dim_x))

    for i in range(steps):
        U[:, i] = std_u * np.random.normal(size=(dim_u))
        W[:, i] = std_w * np.random.normal(size=(dim_x))
        X[:, i+1] = sys.A @ X[:, i] +  np.squeeze(sys.B * U[:, i]) + W[:, i]

    return CollectedData(X, U, W, std_u, std_w)

def correlate(x: np.ndarray, num_lags: int) -> np.ndarray:
    """Correlate a signal"""
    n, T = x.shape
    R = np.zeros((num_lags,n,n))

    for m in range(num_lags):
        for i in range(T-m):
            R[m] += x[:,i:i+1] @ x[:, i+m:i+m+1].T

    return R/T

def torch_correlate(x: torch.Tensor, num_lags: int) -> List[torch.Tensor]:
    """Correlate a signal x using torch"""
    n, T = x.shape
    R = []

    for m in range(num_lags):
        R.append(F.conv1d(x.unsqueeze(1), x[:, m:].unsqueeze(1))[:,:,0]/T)

    return R

def whiteness_test(residuals: np.ndarray, num_lags: int) -> float:
    dim_x, T = residuals.shape
    R = correlate(residuals, num_lags)
    df = (T-dim_x-1) * (dim_x ** 2)
    R0Inv = np.linalg.inv(R[0])
    statistics = T * np.sum([np.trace(R[x].T @ R0Inv @ R[x] @ R0Inv) for x in range(1, num_lags)])
    q = chi2.cdf(statistics, df)
    return statistics

def pvalue_whiteness_test(R: np.ndarray, num_lags: int, T: int) -> TestStatistics:
    """Run whiteness test on the residuals

    Args:
        R (np.ndarray): correlation matrices, should be of shape (L,n,n), where L is the number of lags
        num_lags (int): number of lags to use to run the test. Should be lower than L
        T (int): Horizon length

    Returns:
        TestStatistics: Return the statistics of the test
    """
    R_lags, dim_x, _ = R.shape
    assert R_lags > num_lags, f"R has been computed only for {R_lags} lags, not for {num_lags} lags."
    df = (T-dim_x-1) * (dim_x ** 2)
    R0Inv = np.linalg.inv(R[0])
    statistics = T * np.sum([np.trace(R[x].T @ R0Inv @ R[x] @ R0Inv) for x in range(1, num_lags + 1)])
    p = chi2.cdf(statistics, df)
    return TestStatistics(statistics, p)

def pvalue_residuals_variance_test(residuals: np.ndarray, dim_u: int, sigma_eigs: List[float]) -> TestStatistics:
    """
    Compute the pvalue under H0 that the residuals follow a linear combination of chi_squared distribution

    :param residuals: a (dim_x, T) matrix with the residuals
    :param dim_u: dimension of the input
    :param sigma_eigs: list of eigenvalues of the noise covariance matrix
    :return pval: pvalue
    """
    assert isinstance(sigma_eigs, list), "Sigma_eigs needs to be a list"
    dim_x, T = residuals.shape
    assert len(sigma_eigs) == dim_x, "Sigma_eigs needs to have dim_x values"
    eigs = sigma_eigs * (T-dim_x-dim_u)
    fro_norm_squared  = np.linalg.norm(residuals, 'fro' ) ** 2
    p = hbe(coeff=eigs, x=fro_norm_squared)
    return TestStatistics(fro_norm_squared, 2*min(p, 1-p))


# -*- coding: UTF-8 -*-
from app import config
import os, sys, stat
import ipaddress
import hashlib

class Utils(object):

    @staticmethod
    def get_network_settings(ipaddr):
        addr4 = ipaddress.ip_address(ipaddr)
        gw = None
        nm = None
        found = False

        for subnet in config.local_networks:
            if addr4 in ipaddress.ip_network(subnet):
                gw = config.local_networks[subnet]['gateway']
                nm = config.local_networks[subnet]['netmask']
                pr = config.local_networks[subnet]['prefix']
                found = True

        if not found:
            raise Exception("NotFound", "Не найдены настройки сети для указанного IP-адреса {}".format(str(addr4)))

        return [gw, nm, pr]

    @staticmethod
    def get_dhcp_addr(ipaddr):
        addr4 = ipaddress.ip_address(ipaddr) + 1
        return str(addr4)

    @staticmethod
    def get_token():
        return hashlib.md5(os.urandom(32)).hexdigest()

    @staticmethod
    def create_preseed_conf(adman_id, params, token):
        preseed_template = '{preseeddir}/{os}{osver}_{diskpart}.seed'.format(
            preseeddir=config.utils['preseed-directory'],
            os=params['os'].lower(),
            osver=params['osver'].lower(),
            diskpart=params['diskpart'].lower()
        )

        preseed_config = '{confdir}/s{srv}/s{srv}.seed'.format(
            confdir=config.utils['config-directory'],
            srv=adman_id
        )

        mkconfig_cmd = '/bin/cp -f {src} {dst}'.format(
            src=preseed_template,
            dst=preseed_config
        )

        stream = os.popen(mkconfig_cmd)
        output = stream.read()

        # -- Fill template

        gateway, netmask, _ = Utils.get_network_settings(params['ipaddr'])

        # Read in the file
        with open(preseed_config, 'r') as file:
            filedata = file.read()

        # Replace the target string
        filedata = filedata.replace('__ipaddr__', str(params['ipaddr']))
        filedata = filedata.replace('__netmask__', netmask)
        filedata = filedata.replace('__gateway__', gateway)
        filedata = filedata.replace('__srv_name__', str(adman_id))
        filedata = filedata.replace('__passwdhash__', str(params['passwdhash']))
        filedata = filedata.replace('__token__', str(token))
        filedata = filedata.replace('__boot_host__', str(config.utils['boot_host']))

        # Write the file out again
        with open(preseed_config, 'w') as file:
            file.write(filedata)

        os.chmod(preseed_config, 436)

    @staticmethod
    def create_server_dir(server_id):
        create_cmd = "/bin/mkdir -p {dir}/s{server_id}".format(
            dir=config.utils['config-directory'],
            server_id=server_id
        )

        stream = os.popen(create_cmd)
        output = stream.read()
        return output

    @staticmethod
    def remove_symbol_link(link):
        rm_cmd = "/bin/rm -rf {dir}/{link}".format(
            dir = config.utils['config-directory'],
            link=link
        )

        stream = os.popen(rm_cmd)
        output = stream.read()
        return output

    @staticmethod
    def create_symbol_link(server_id, link):
        os.chdir(config.utils['config-directory'])
        create_cmd = "/bin/ln -s s{server_id} {link}".format(
            server_id=server_id,
            link=link
        )

        stream = os.popen(create_cmd)
        output = stream.read()
        return output

    @staticmethod
    def create_config(args, adman_id, token=''):

        diskpart = int(args['diskpart'])
        conf_dir = str(config.utils['config-directory'])
        url = ""

        if diskpart == 2:
            url = str(config.utils['iscsi_url'])
        else:
            url = str(config.utils['boot_url'])

        ipxe_config = '{confdir}/s{srv}/boot.ipxe'.format(
            confdir=conf_dir,
            srv=adman_id
        )

        tpl_key = "{os}_{osver}".format(os = args['os'].lower(), osver = args['osver'].lower())
        if diskpart == 2 and args['os'].lower() == 'local':
            tpl_key = "{os}_{osver}".format(os=args['os'].lower(), osver=2)

        tpl = config.templates[tpl_key]

        with open(ipxe_config, 'a') as file:
            file.truncate(0)
            file.writelines(tpl)

        # Read in the file
        with open(ipxe_config, 'r') as file:
            filedata = file.read()

        # Replace the target string
        filedata = filedata.replace('__srv_name__', str(adman_id))
        filedata = filedata.replace('__boot_url__', url)
        filedata = filedata.replace('__token__', str(token))

        # Write the file out again
        with open(ipxe_config, 'w') as file:
            file.write(filedata)

        os.chmod(ipxe_config, 436)

    @staticmethod
    def create_dhcp_config(adman_id, mac_addr, dhcp_addr, router=True):

        conf_dir = "/srv/hosts"
        dhcp_config = '{confdir}/s{srv}.conf'.format(
            confdir=conf_dir,
            srv=adman_id
        )

        with open(dhcp_config, 'a') as file:
            file.truncate(0)
            port_number = 0
            tpl_list = []
            for mac in mac_addr:
                tpl_list.append('#Autogenerated DHCP configuration\n')
                tpl_list.append('  host s{srv_name}p{port_number} {{\n'.format(srv_name=adman_id, port_number=port_number))
                tpl_list.append('    hardware ethernet {mac_addr};\n'.format(mac_addr=mac.mac_addr))
                tpl_list.append('    fixed-address {dhcp_addr};\n'.format(dhcp_addr=dhcp_addr))
                if router:
                    tpl_list.append('    option routers 10.0.224.4;\n')
                tpl_list.append('  }\n\n')
                port_number += 1

            file.writelines(tpl_list)

        os.chmod(dhcp_config, 436)

    @staticmethod
    def create_install_bat(args, adman_id):
        bat_config = '{confdir}/s{srv}/install.bat'.format(
            confdir=config.utils['config-directory'],
            srv=adman_id
        )

        osver = args['osver'].lower()

        with open(bat_config, 'a') as file:
            file.truncate(0)
            file.write('wpeinit\n')
            file.write('ping 10.0.222.1\n')
            file.write('ping 10.0.222.1\n')
            file.write('ping 10.0.222.1\n')
            file.write('net use j: \\\\__boot_host__\\images\\__os_ver__\\amd64 /user:user pass\n')
            file.write('net use k: \\\\__boot_host__\\configfile /user:user pass\n')
            file.write('j:\\setup.exe /unattend:k:\\s__srv_name__\\s__srv_name__.seed\n')

        # Read in the file
        with open(bat_config, 'r') as file:
            filedata = file.read()

        # Replace the target string
        filedata = filedata.replace('__srv_name__', str(adman_id))
        filedata = filedata.replace('__os_ver__', str(osver))
        filedata = filedata.replace('__boot_host__', str(config.utils['boot_host']))

        # Write the file out again
        with open(bat_config, 'w') as file:
            file.write(filedata)

        os.chmod(bat_config, 436)

    @staticmethod
    def create_win_set_ip_ps(ipaddr, token, adman_id):
        ps_config = '{confdir}/s{srv}/set-ip.ps1'.format(
            confdir=config.utils['config-directory'],
            srv=adman_id
        )

        with open(ps_config, 'a') as file:
            file.truncate(0)
            file.write('$ifaceindex = get-wmiobject win32_networkadapter -filter "netconnectionstatus = 2" | select -expand InterfaceIndex; ')
            file.write('netsh interface ipv4 set address name=$ifaceindex source=static addr=__ip_addr__  mask=__netmask__ gateway=__gateway__ gwmetric=1; ')
            file.write('netsh interface ipv4 add dns $ifaceindex 8.8.8.8 index=1; ')
            file.write('Timeout /T 15; ')
            file.write('ping 8.8.8.8; ')
            file.write('[System.Net.WebRequest]::Create("http://install.adman.cloud/api/v1.0/install/complete/__srv_name__?token=__token__").GetResponse(); ')

        gateway, netmask, _ = Utils.get_network_settings(ipaddr)
        # Read in the file
        with open(ps_config, 'r') as file:
            filedata = file.read()

        # Replace the target string
        filedata = filedata.replace('__ip_addr__', str(ipaddr))
        filedata = filedata.replace('__netmask__', netmask)
        filedata = filedata.replace('__gateway__', gateway)
        filedata = filedata.replace('__srv_name__', str(adman_id))
        filedata = filedata.replace('__token__', token)

        # Write the file out again
        with open(ps_config, 'w') as file:
            file.write(filedata)

        os.chmod(ps_config, 436)

    @staticmethod
    def remove_install_bat(adman_id):
        bat_config = '{confdir}/s{srv}/install.bat'.format(
            confdir=config.utils['config-directory'],
            srv=adman_id
        )

        if os.path.exists(bat_config):
            os.remove(bat_config)

    @staticmethod
    def remove_preseed_conf(adman_id):
        preseed_config = '{confdir}/s{srv}/s{srv}.seed'.format(
            confdir=config.utils['config-directory'],
            srv=adman_id
        )

        if os.path.exists(preseed_config):
            os.remove(preseed_config)

    @staticmethod
    def remove_win_set_ip_ps(adman_id):
        ps_config = '{confdir}/s{srv}/set-ip.ps1'.format(
            confdir=config.utils['config-directory'],
            srv=adman_id
        )

        if os.path.exists(ps_config):
            os.remove(ps_config)


import pickle
from subprocess import Popen

import matplotlib.pyplot as plt
import matplotlib

from stress_strain_evaluation import SS2506
from stress_strain_evaluation import experiments

matplotlib.style.use('classic')
plt.rc('text', usetex=True)
plt.rc('font', serif='Computer Modern Roman')
plt.rcParams.update({'font.size': 20})
plt.rcParams['text.latex.preamble'] = [r"\usepackage{amsmath}"]
plt.rc('font', **{'family': 'serif', 'serif': ['Computer Modern Roman'],
                  'monospace': ['Computer Modern Typewriter']})

class MaterialTest:
    def __init__(self, strain, hardness, austenite):
        self.strain = strain
        self.hardness = hardness
        self.austenite = austenite
        self.name = 'material_test_hv=' + str(hardness) + '_au=' + str(austenite).replace('.', '_')

    def _write_input_file(self):
        SS2506.write_material_input_file('SS2506material.inc')
        file_lines = ['**',
                      '**',
                      '** Autogenerated input file created by Case Hardening Simulation Toolbox, version 0.8.1',
                      '** Written by Niklas Melin and Erik Olsson',
                      '**',
                      '*Heading',
                      '\t Case Hardening Simulation Toolbox - Thermal - Niklas Melin 2012',
                      '*Preprint, echo=NO, model=NO, history=NO, contact=NO',
                      '**',
                      '** ----------------------------------------------------------------',
                      '** Load required include files',
                      '**',
                      '**   Create Geometry',
                      '*Node, nset=all_nodes',
                      '\t1, \t 0., 0., 0.',
                      '\t2, \t 1., 0., 0.',
                      '\t3, \t 1., 1., 0.',
                      '\t4, \t 0., 1., 0.',
                      '\t5, \t 0., 0., 1.',
                      '\t6, \t 1., 0., 1.',
                      '\t7, \t 1., 1., 1.',
                      '\t8, \t 0., 1., 1.',
                      '*Element, type=C3D8, elset=all_elements',
                      '\t1, 1, 2, 3, 4, 5, 6, 7, 8',
                      '**',
                      '** ----------------------------------------------------------------',
                      '**',
                      '**   Define material properties',
                      '**',
                      '*Solid Section, elset=all_elements, material=SS2506',
                      '*Material, name=SS2506',
                      '\t*Include, Input=SS2506material.inc',
                      '*INITIAL CONDITIONS, TYPE=FIELD, VAR=1',
                      '\tALL_NODES , ' + str(self.hardness),
                      '**',
                      '*INITIAL CONDITIONS, TYPE=FIELD, VAR=2',
                      '\tALL_NODES , ' + str(self.austenite),
                      '*Boundary',
                      '**',
                      '\t1, 1, 3',
                      '\t2, 2, 3',
                      '\t3, 3, 3',
                      '\t4, 1, 1',
                      '\t4, 3, 3',
                      '\t5, 1, 2',
                      '\t6, 2, 2',
                      '\t8, 1, 1',
                      '**',
                      '*STEP, NAME=material_test , INC=10000, Amplitude=Ramp',
                      '\t Material test of one element',
                      '\t*STATIC',
                      '\t\t0.01, 1.0, 1e-05, 1.0',
                      '\t*Boundary',
                      '\t\t5, 3, 3, ' + str(self.strain),
                      '\t\t6, 3, 3, ' + str(self.strain),
                      '\t\t7, 3, 3, ' + str(self.strain),
                      '\t\t8, 3, 3, ' + str(self.strain),
                      '\t*OUTPUT, FIELD, FREQ=1',
                      '\t\t*ELEMENT OUTPUT, directions=YES',
                      '\t\t\tS, E, FV',
                      '*END STEP']

        with open(self.name + '.inp', 'w') as inp_file:
            for line in file_lines:
                inp_file.write(line + '\n')
            inp_file.write('**EOF')

    @staticmethod
    def _write_env_file():
        file_lines = ['ask_delete = OFF']

        with open('abaqus_v6.env', 'w') as env_file:
            for line in file_lines:
                env_file.write(line + '\n')

    def run_material_test(self):
        self._write_input_file()
        self._write_env_file()
        abq = '/scratch/users/erik/SIMULIA/CAE/2018/linux_a64/code/bin/ABQLauncher'
        process = Popen(abq + ' j=' + self.name + ' interactive ', shell=True)
        process.wait()
        process = Popen(abq + ' python material_test_post_processing.py ' + self.name, shell=True)
        process.wait()

        with open('data_' + self.name + '.pkl', 'r') as pickle_handle:
            data = pickle.load(pickle_handle)
        return data

if __name__ == '__main__':
    for experiment in experiments:
        plt.figure(0)
        experiment.plot()

        test = MaterialTest(experiment.strain[-1], experiment.hardness, experiment.ra/100)
        stress_data = test.run_material_test()
        plt.plot(stress_data[:, 3], stress_data[:, 9], '--x' + experiment.color, lw=2)
    plt.show()


import pandas
import numpy as np
from sklearn.cluster import KMeans
from gestion.gestion_des_fichiers.afficheur import Afficheur
from gestion.elements_fichiers.data_base import Data_Base
from gestion.elements_fichiers.pays import Pays

class Resume(Afficheur):
	"""La classe Resume permet de renvoyer des data frames"""

	def table_criteres(self, liste_pays_a_afficher):
		"""Cette méthode permet de créer un data frame sur les pays choisis au préalable
		par la méthode critère_usuels à l'aide de la bibliothèque pandas.
		Valeurs_pays est un tableau qui prend toutes les informations chiffrées des différents pays. """

		noms_pays = [self.simplification(pays.get_name()) for pays in liste_pays_a_afficher]

		criteres = ['Superficie', 'Population', 'Croissance démographique', 'Inflation', 'Dette', 'Taux de chômage', 'Taux de dépenses en santé', 'Taux de dépenses en éducation', 'Taux de dépenses militaires']

		valeurs_pays = [[self.simplification(pays.get_superficie()) for pays in liste_pays_a_afficher],
						[self.simplification(pays.get_pop()) for pays in liste_pays_a_afficher],
						[self.simplification(pays.get_croissance_demo()) for pays in liste_pays_a_afficher],
						[self.simplification(pays.get_inflation()) for pays in liste_pays_a_afficher],
						[self.simplification(pays.get_dette()) for pays in liste_pays_a_afficher],
						[self.simplification(pays.get_chomage()) for pays in liste_pays_a_afficher],
						[self.simplification(pays.get_depenses_sante()) for pays in liste_pays_a_afficher],
						[self.simplification(pays.get_depenses_education()) for pays in liste_pays_a_afficher],
						[self.simplification(pays.get_depenses_militaires()) for pays in liste_pays_a_afficher]]

		return pandas.DataFrame(valeurs_pays, index = criteres, columns = noms_pays)

	def top_and_flop(self, critere, taille_classement):
		"""Cette méthode renvoie deux data frames.
		Le premier correspond aux 'taille_classement'(entier) premiers pays selon un certain critère.
		Le second data frame correspond aux 'taille_classement'(entier) dernier pays selon un certain critère."""

		donnees = Data_Base().donnees

		liste_triee_selon_critere = self.liste_triee_selon_critere(donnees, critere)

		top = []
		flop = []
		rang_top = range(1,taille_classement+1)
		rang_flop = [len(liste_triee_selon_critere)-i for i in range(taille_classement)]
		col = ['PAYS', critere.upper()]
		for i in range(taille_classement):
				top.append([liste_triee_selon_critere[-i-1][1].get_name(), self.simplification(liste_triee_selon_critere[-i-1][1].get_critere(critere))])
				flop.append([liste_triee_selon_critere[i][1].get_name(), self.simplification(liste_triee_selon_critere[i][1].get_critere(critere))])

		return (pandas.DataFrame(top, index = rang_top, columns = col), pandas.DataFrame(flop, index = rang_flop, columns = col))

	def sup_inf_seuil(self, critere, seuil):
		"""Cette méthode crée deux tableaux. Elle prend en argument un critère et un seuil.
		La premier tableau correspond aux pays dépassant le seuil déterminé par l'utilisateur.
		Le second correspond aux pays sous ce seuil. Elle renvoie 4 informations :
		le nombre de pays dépassant le seuil avec le tableau des pays correspondant avec les valeurs de ce critère,
		et le nombre de pays ne dépassant pas le seuil avec le tableau des pays correspondant avec les valeurs de ce critère"""

		donnees = Data_Base().donnees

		liste_triee_selon_critere = self.liste_triee_selon_critere(donnees, critere)
		liste_triee_selon_critere.reverse()
		pays_sup_seuil = []
		pays_inf_seuil = []
		col = ['PAYS', critere.upper()]

		for elm in liste_triee_selon_critere:
			if elm[0] >= seuil:
				pays_sup_seuil.append([elm[1].get_name(), self.simplification(elm[1].get_critere(critere))])
			else :
				pays_inf_seuil.append([elm[1].get_name(), self.simplification(elm[1].get_critere(critere))])

		tableau_pays_sup = pandas.DataFrame(pays_sup_seuil, index=[i+1 for i in range(len(pays_sup_seuil))], columns=col)
		tableau_pays_inf = pandas.DataFrame(pays_inf_seuil, index=[i+1 for i in range(len(pays_inf_seuil))], columns=col)

		return (len(pays_sup_seuil), tableau_pays_sup, len(pays_inf_seuil), tableau_pays_inf)

	def tableau_classes_age(self, liste_pays_a_afficher):
		"""Cette méthode renvoie un data frame concernant les différentes classes d'âges
		en prenant en compte les pays choisis au préalable."""

		noms_pays = [self.simplification(pays.get_name()) for pays in liste_pays_a_afficher]

		criteres = ["0-14", "15-24", "25-54", "55-64", ">= 65"]

		valeurs_classes_age = [[self.simplification(pays.get_classe_age1()) for pays in liste_pays_a_afficher],
							[self.simplification(pays.get_classe_age2()) for pays in liste_pays_a_afficher],
							[self.simplification(pays.get_classe_age3()) for pays in liste_pays_a_afficher],
							[self.simplification(pays.get_classe_age4()) for pays in liste_pays_a_afficher],
							[self.simplification(pays.get_classe_age5()) for pays in liste_pays_a_afficher]]

		return pandas.DataFrame(valeurs_classes_age, index = criteres, columns = noms_pays)

	def somme(self):
		"""Cette méthode renvoie un data frame d'une colonne. Elle fait la somme
		des différents critères sur les pays."""

		donnees = Data_Base().donnees
		superficie_tot = sum([elm[0] for elm in self.liste_triee_selon_critere(donnees, 'superficie')])
		population_tot = sum([elm[0] for elm in self.liste_triee_selon_critere(donnees, 'population')])
		dette_tot = sum([elm[0] for elm in self.liste_triee_selon_critere(donnees, 'dette')])

		criteres = ["La somme des superficies de tous les pays", "La somme des populations de tous les pays", "La somme des dettes de tous les pays"]
		index = criteres
		valeurs = [[superficie_tot],[population_tot],[dette_tot]]
		return pandas.DataFrame(valeurs, index = criteres, columns = [''])

	def summary(self):
		"""La méthode summary renvoie un data frame donnant des statistiques sur les différents critères (nombres de valeurs, moyenne, écart-type, ...)
		mais uniquement sur les colonnes numériques. Pour cela, la méthode crée un dictionnaire où chaque clé correspond à un critère
		que l'on associe à un data frame. Enfin, on utilise la méthode describe de pandas pour avoir notre résumé statistique."""

		donnees = Data_Base().donnees

		superficie = pandas.Series([elm[0] for elm in self.liste_triee_selon_critere(donnees, 'superficie')])
		population = pandas.Series([elm[0] for elm in self.liste_triee_selon_critere(donnees, 'population')])
		croissance_demographique = pandas.Series([elm[0] for elm in self.liste_triee_selon_critere(donnees, 'croissance démographique')])
		inflation = pandas.Series([elm[0] for elm in self.liste_triee_selon_critere(donnees, 'inflation')])
		dette = pandas.Series([elm[0] for elm in self.liste_triee_selon_critere(donnees, 'dette')])
		chomage = pandas.Series([elm[0] for elm in self.liste_triee_selon_critere(donnees, 'chômage')])
		depenses_sante = pandas.Series([elm[0] for elm in self.liste_triee_selon_critere(donnees, 'dépenses santé')])
		depenses_éducation = pandas.Series([elm[0] for elm in self.liste_triee_selon_critere(donnees, 'dépenses éducation')])
		depenses_militaires = pandas.Series([elm[0] for elm in self.liste_triee_selon_critere(donnees, 'dépenses militaires')])

		dic = {'Superficie' : superficie,
			'Population' : population,
			'Croissance démo' : croissance_demographique,
			'Inflation' : inflation,
			'Dette' : dette,
			'Chômage' : chomage,
			'Dépenses santé' : depenses_sante,
			'Dépenses éducation' : depenses_éducation,
			'Dépenses militaires' : depenses_militaires}

		return pandas.DataFrame(dic).describe()

	def clustering(self, nb_clusters):
		"""Cette méthode utilise la méthode des Kmeans pour permettre à la fonction de créer des classes entre les pays.
		Elle prend en argument le nombre de classes que l'utilisateur veut former.
		Cette méthode renvoie un data frame où chaque colonne correspond à chaque classe
		avec le nom des pays dans celles ci."""

		donnees = Data_Base().donnees

		criteres = ['superficie', 'population', 'croissance démographique', 'inflation', 'dette', 'chômage', 'dépenses santé', 'dépenses éducation', 'dépenses militaires']

		valeurs_pays = []
		liste_des_pays_comptabilises = []

		for num_pays in range(len(donnees)):
			pays = Pays(num_pays, donnees)
			if pays.get_name():
				if True not in [self.numerisation_critere(pays, critere) == 'NA' for critere in criteres]:
					liste_des_pays_comptabilises.append(self.simplification(pays.get_name()))
					valeurs_pays.append([self.numerisation_critere(pays, critere) for critere in criteres])

		clf = KMeans(n_clusters=nb_clusters)
		clf.fit(np.array(valeurs_pays))
		labels = clf.labels_

		pays_dans_clusters = [[] for i in range(nb_clusters)]

		for i in range(len(labels)):
			for cl in range(nb_clusters):
				if labels[i] == cl:
					pays_dans_clusters[cl].append(liste_des_pays_comptabilises[i])

		for cluster in pays_dans_clusters:
			while len(cluster) < max([len(cl) for cl in pays_dans_clusters]):
				cluster.append('')

		liste_des_clusters = ['Cluster {}'.format(i+1) for i in range(nb_clusters)]

		return pandas.DataFrame(pays_dans_clusters, index = liste_des_clusters).T


import os
import numpy as np
from qcircha.circuit_selector import _select_circ
from qiskit import QuantumCircuit
import qmatchatea as qmt
from qmatchatea.qk_utils import qk_transpilation_params
import qtealeaves.observables as obs
from qtealeaves.emulator import MPS
from tqdm import tqdm

depth = 14
dir_name = f"data/all_random"
num_qubits = 13
num_reps = 1
num_trials = 1
max_bond_dim = 128

if not os.path.isdir(dir_name):
    os.makedirs(dir_name)

observables = obs.TNObservables()
observables += obs.TNObsBondEntropy()
observables += obs.TNState2File('mps_state.txt', 'F')
conv_params = qmt.QCConvergenceParameters(int(max_bond_dim), singval_mode="C")
backend = qmt.QCBackend()
trans_params = qk_transpilation_params(linearize=True, tensor_compiler=False)

feature_map = _select_circ(num_qubits, "zzfeaturemap")
ansatz = _select_circ(num_qubits, "twolocal")
num_avg = 1000

for idx, data in tqdm(enumerate(range(num_avg))):
    entanglement_total = []
    singvals_cut_total = []
    states_total = []
    initial_state = "Vacuum"
    for num_reps in range(depth):
        print(f"\n__Reps {num_reps}/{depth}")
        random_params = np.pi * np.random.rand(len(ansatz.parameters))
        random_params_data = np.pi * np.random.rand(len(feature_map.parameters))

        binded_ansatz = ansatz.bind_parameters(random_params)
        binded_fmap = feature_map.bind_parameters(random_params_data)

        # Pick a PQC (modify the function)
        qc = QuantumCircuit(num_qubits)
        qc = qc.compose(binded_fmap)
        qc = qc.compose(binded_ansatz)

        io_info = qmt.QCIO(initial_state=initial_state)
        results = qmt.run_simulation(qc, convergence_parameters=conv_params,
                                transpilation_parameters=trans_params, backend=backend,
                                observables=observables, io_info=io_info)

        entanglement = np.array(list(results.entanglement.values()))
        initial_state = results.mps

        entanglement_total.append(entanglement)
        states_total.append(results.singular_values_cut)
        initial_state = MPS.from_tensor_list(initial_state)

        np.save( os.path.join(dir_name, f"entanglement_{idx+num_avg}.npy"), entanglement_total, allow_pickle=True)


import pymem
import pymem.process
import keyboard

dwEntityList = (0x4DFFFB4)
m_bSpotted = (0x93D)

def main():
    pm = pymem.Pymem("csgo.exe")
    client = pymem.process.module_from_name(pm.process_handle, "client.dll").lpBaseOfDll

    while True:
        if keyboard.is_pressed("end"):
            exit(0)

        for i in range(1,32):
            entity = pm.read_int(client + dwEntityList + i * 0x10)
            if entity:
                pm.write_uchar(entity + m_bSpotted, 1)

if __name__ == '__main__':
    print("cheat injected")
    main()


from django.urls import path

from users.views import *

app_name = 'users'
urlpatterns = [
    path('register/', RegisterView.as_view(), name='register'),  # 注册
    path('active/<slug:active_code>', ActiveView.as_view(), name='active'),  # 激活
    path('login/', LoginView.as_view(), name='login'),  # 登陆
    path('logout/', LogoutView.as_view(), name='logout'),  # 登出
    path('forget_pwd/', ForgetPwdView.as_view(), name='forget_pwd'),  # 忘记密码
    path('find/<slug:find_code>', ResetView.as_view(), name='find'),  # 从邮箱点击到找回密码页面
    path('new_pwd/', NewPwdView.as_view(), name='new_pwd'),  # 修改密码
    path('userinfo/', UserInfoView.as_view(), name='userinfo'),  # 个人中心
    path('userinfo/update', UpdateUserInfoView.as_view(), name='update_userinfo'),  # 修改个人信息
    path('userinfo/update_pwd', UpdateUserPwdView.as_view(), name='update_user_pwd'),  # 修改个人信息
]


from flask import Flask , render_template , request
import pickle

app =Flask(__name__ , template_folder="templates")

model = pickle.load(open("model/car-price-prediction-model.pkl", 'rb'))

@app.route("/home" , methods=["GET" , "POST"])
def home():
    output = 0
    if request.method == "POST":

        present_price = float(request.form["present_price"])
        kms_driven = float(request.form["kms_driven"])
        year = int(request.form["year"])
        fuel_type = request.form["fuel_type"]
        seller_type = request.form["seller_type"]

        if fuel_type == "Petrol":
            fuel_type = 171
        elif fuel_type == "Diesel":
            fuel_type = 33
        else:
            fuel_type = 1

        if seller_type == "Dealer":
            seller_type = 128
        else:
            seller_type = 77
        # print(present_price , kms_driven , year , fuel_type , seller_type)

        prediction = model.predict([[year , present_price , kms_driven , fuel_type , seller_type]])
        output = round(prediction[0] , 2)

    return render_template("templates.html" , output = output)

if __name__=="__main__":
    app.run(debug=True)


import requests
import os

class StockGetter:

    def __init__(self, ticker: str):
        self.yesterday_price: float
        self.day_before_yesterday_price: float
        self.ticker = ticker
        self.key = os.getenv('ALPHA_VANTAGE_KEY')

    def get_stock_price(self):
        params = {
            "function": "TIME_SERIES_DAILY",
            "symbol": self.ticker,
            "outputsize": "compact",
            "apikey": ENV_VAR,
        }

        response = requests.get(
            "https://www.alphavantage.co/query", params=params)
        response.raise_for_status()
        data = response.json()['Time Series (Daily)']
        data_list = [v for (k, v) in data.items()]

        self.yesterday_price = float(data_list[0]['4. close'])
        self.day_before_yesterday_price = float(data_list[1]['4. close'])

    def price_change(self) -> float:
        return float(
            (self.yesterday_price - self.day_before_yesterday_price) / self.day_before_yesterday_price) * 100


import pandas as pd
import numpy as np
from catboost import CatBoostClassifier, Pool
import os
import pickle

print("READ DATA...")
data = pd.read_csv("/data/X_train.csv")

print("LOAD STATISTICS...")
with open('models/global_stat.pickle', 'rb') as handle:
    global_stat = pickle.load(handle)

def get_cat_feature(df, col_name, stat):
    return df[col_name].map(lambda x: stat[col_name].loc[x].result if x in stat[col_name].index else stat[col_name].result.mean()).values

def generate_features(df, stat):
    features = []
    for col_name in ["email","ip","cardToken","paymentSystem",
                     "providerId","bankCountry","partyId","shopId",
                     "currency","bin_hash","ms_pan_hash"]:
        features.append(get_cat_feature(df, col_name, stat))

    features.append(df.amount * df.currency.map({"RUB":1, "USD": 63, "EUR": 63}))

    return np.array(features).T

print("GENERATE FEATURES")
features = generate_features(data, global_stat)

results = []
for model_name in os.listdir("models"):
    print("PREDICT MODEL:", model_name)
    if "model.cbt" not in model_name: continue
    cbc = CatBoostClassifier()
    cbc.load_model(os.path.join("models", model_name))
    res = cbc.predict_proba(features)[:, 1]
    results.append(res)
results = (np.array(results).mean(axis=0) > 0.15)

print("Сохраняем результат...")
data["command_result"] = results
data.to_csv("/data/OUTPUT_CONS.csv", index=False)


import sublime
import sublime_plugin
import xml.etree.ElementTree as ET
import xml.sax.saxutils as SAX

settings = sublime.load_settings("FileMaker.sublime-settings")

def get_selection(view, select_all=False):
    """ Returns regions of view to operate on
    """
    if select_all and not view.has_non_empty_selection_region():
        selection = [sublime.Region(0, view.size())]
    else:
        selection = view.sel()
    return selection

def syntax_to_filemaker(self):
    """Changes syntax to FileMaker if its in plain text
    """
    if "Plain text" in self.view.settings().get('syntax'):
        self.view.set_syntax_file("Packages/FileMaker/FileMaker.tmLanguage")

def syntax_to_clipboard(self):
    """Changes syntax to FileMaker Clipboard
    """
    self.view.set_syntax_file("Packages/FileMaker/FileMaker Clipboard.tmLanguage")

def quote(text):
    text = text.replace('\\', '\\\\')
    text = text.replace('"', '\\"')
    text = text.replace('¶', '\\¶')
    return '"' + text + '"'

def quote_and_append(text):
    textNew = ""
    for line in text.splitlines():
        textNew += quote(line) + ' & ¶ &' + "\n"
    return textNew + "\n" + '""'

def extract_table_field_names(text):
    root = ET.fromstring(text)
    textNew = ""
    for elem in root.iterfind('Field'):
      textNew += elem.get('name') + "\n"
    return textNew

def convert_to_script_comments(text):
    template = '<Step enable="True" id="" name="Comment"><Text>{0}</Text></Step>'
    textNew = '<fmxmlsnippet type="FMObjectList">' + "\n"
    for line in text.splitlines():
        lineNew = SAX.escape(line)
        textNew += template.format(' ' + lineNew + "\n")
    textNew += '</fmxmlsnippet>'
    return textNew

class FilemakerQuoteCommand(sublime_plugin.TextCommand):
    """ Quotes calculation as FileMaker string, escaping as necessary
    """
    def run(self, edit):
        view = self.view
        select_all = settings.get("use_entire_file_if_no_selection", True)
        selection = get_selection(view, select_all)
        for sel in selection:
            view.replace(edit, sel, quote(view.substr(sel)))

        if select_all:
            self.syntax_to_filemaker()

class FilemakerQuoteAndAppendCommand(sublime_plugin.TextCommand):
    """ Quotes each line in calculation as FileMaker string, appending each line of text with carriage return
    """
    def run(self, edit):
        view = self.view
        select_all = settings.get("use_entire_file_if_no_selection", True)
        selection = get_selection(view, select_all)
        for sel in selection:
            view.replace(edit, sel, quote_and_append(view.substr(sel)))

        if select_all:
            self.syntax_to_filemaker()

class FilemakerExtractTableFieldNamesCommand(sublime_plugin.TextCommand):
    """ Extracts list of field names from FileMaker clipboard object
    """
    def run(self, edit):
        view = self.view
        select_all = settings.get("use_entire_file_if_no_selection", True)
        selection = get_selection(view, select_all)
        for sel in selection:
            view.replace(edit, sel, extract_table_field_names(view.substr(sel)))

        if select_all:
            extract_table_field_names(view.substr(selection))

class FilemakerConvertToScriptComments(sublime_plugin.TextCommand):
    """ Converts each line of text to a script comment to be pasted into FileMaker
    """
    def run(self, edit):
        view = self.view
        select_all = settings.get("use_entire_file_if_no_selection", True)
        selection = get_selection(view, select_all)
        for sel in selection:
            view.replace(edit, sel, convert_to_script_comments(view.substr(sel)))

        if select_all:
            self.syntax_to_clipboard()


"""A unit test meant to catch accidental introduction of non-optional dependencies."""
from pathlib import Path

import toml

HERE = Path(__file__).parent

PYPROJECT_TOML = HERE / "../../pyproject.toml"

def test_required_dependencies() -> None:
    """A test that checks if a new non-optional dependency is being introduced.

    If this test is triggered, it means that a contributor is trying to introduce a new
    required dependency. This should be avoided in most situations.
    """
    with open(PYPROJECT_TOML) as f:
        pyproject = toml.load(f)

    # Get the dependencies from the [tool.poetry.dependencies] section
    dependencies = pyproject["tool"]["poetry"]["dependencies"]

    required_dependencies = [
        package_name
        for package_name, requirements in dependencies.items()
        if isinstance(requirements, str) or not requirements.get("optional", False)
    ]

    assert sorted(required_dependencies) == [
        "PyYAML",
        "SQLAlchemy",
        "aiohttp",
        "async-timeout",
        "dataclasses-json",
        "numexpr",
        "numpy",
        "openapi-schema-pydantic",
        "pydantic",
        "python",
        "requests",
        "tenacity",
    ]


from comet_ml import experiment
from data_loader.uts_classification_data_loader import UtsClassificationDataLoader
from models.uts_classification_model import UtsClassificationModel
from trainers.uts_classification_trainer import UtsClassificationTrainer
from evaluater.uts_classification_evaluater import UtsClassificationEvaluater
from utils.config import process_config_UtsClassification
from utils.dirs import create_dirs
from utils.utils import get_args
from comet_ml import Optimizer
import os
import time
from utils.config import get_config_from_json

def process_config_UtsClassification_bayes_optimization(json_file,learning_rate, num_epochs=50, batch_size=16,model_name='fcn'):
    config, _ = get_config_from_json(json_file)

    config.model.name = model_name
    config.model.learning_rate = learning_rate
    config.trainer.num_epochs = num_epochs
    config.trainer.batch_size = batch_size

    config.callbacks.tensorboard_log_dir = os.path.join("experiments",time.strftime("%Y-%m-%d/", time.localtime()),
                                                        config.exp.name, config.dataset.name,
                                                        config.model.name, "tensorboard_logs",
                                                        "lr=%s,epoch=%s,batch=%s" % (
                                                            config.model.learning_rate, config.trainer.num_epochs,
                                                            config.trainer.batch_size)
                                                       )
    config.callbacks.checkpoint_dir = os.path.join("experiments", time.strftime("%Y-%m-%d/", time.localtime()),
                                                   config.exp.name, config.dataset.name,
                                                   config.model.name, "%s-%s-%s" % (
                                                   config.model.learning_rate, config.trainer.num_epochs,
                                                   config.trainer.batch_size),
                                                   "checkpoints/")
    config.log_dir = os.path.join("experiments", time.strftime("%Y-%m-%d/", time.localtime()),
                                            config.exp.name, config.dataset.name,
                                            config.model.name, "%s-%s-%s" % (
                                            config.model.learning_rate, config.trainer.num_epochs,
                                            config.trainer.batch_size),
                                            "training_logs/")
    config.result_dir = os.path.join("experiments", time.strftime("%Y-%m-%d/", time.localtime()),
                                               config.exp.name, config.dataset.name,
                                               config.model.name, "%s-%s-%s" % (
                                                   config.model.learning_rate, config.trainer.num_epochs,
                                                   config.trainer.batch_size),
                                               "result/")
    return config

def main():

    os.environ['CUDA_VISIBLE_DEVICES'] = '2'

    # capture the config path from the run arguments
    # then process the json configuration file
    # try:
    args = get_args()
    config, _ = get_config_from_json(args.config)

    bayes_config = {
        "algorithm": "bayes",

        "parameters": {
            # "model": {"type": "categorical", "values": ['cnn','mlp']},
            "learning_rate": {"type": "float", "min": 0.001, "max": 0.01},
            # "batch_size": {"type": "integer", "min": 16, "max": 32},
            # "num_epochs": {"type": "integer", "min": 5, "max": 10},
        },
        "spec": {
            "maxCombo": 10,
            "objective": "minimize",
            "metric": "test_f1",
            "minSampleSize": 100,
            "retryAssignLimit": 0,

        },
        "trials": 1,
        "name": "Bayes",
    }
    opt = Optimizer(bayes_config, api_key=config.comet_api_key, project_name=config.exp_name)
    for exp in opt.get_experiments():
        args = get_args()
        # config = process_config_UtsClassification_bayes_optimization(args.config, exp.get_parameter('model'),exp.get_parameter('learning_rate'),
        #                                                              exp.get_parameter('batch_size'), exp.get_parameter('num_epochs'))
        config = process_config_UtsClassification_bayes_optimization(args.config, exp.get_parameter('learning_rate'))
        # except:
        #     print("missing or invalid arguments")
        #     exit(0)

        # create the experiments dirs

        print('Create the data generator.')
        data_loader = UtsClassificationDataLoader(config)

        print('Create the model.')

        model = UtsClassificationModel(config, data_loader.get_inputshape(), data_loader.get_nbclasses())

        print('Create the trainer')
        trainer = UtsClassificationTrainer(model.model, data_loader.get_train_data(), config)

        print('Start training the model.')
        trainer.train()

        # print('Create the evaluater.')
        # evaluater = UtsClassificationEvaluater(trainer.best_model, data_loader.get_test_data(), data_loader.get_nbclasses(),
        #                                        config)
        #
        # print('Start evaluating the model.')
        # evaluater.evluate()

        exp.log_metric("test_f1", trainer.best_model_val_loss)

        print('done')

if __name__ == '__main__':
    main()


from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import collections
from datetime import datetime, timedelta

from backtrader import AbstractDataBase, TimeFrame

class DataFiller(AbstractDataBase):
    '''This class will fill gaps in the source data using the following
    information bits from the underlying data source

      - timeframe and compression to dimension the output bars

      - sessionstart and sessionend

    If a data feed has missing bars in between 10:31 and 10:34 and the
    timeframe is minutes, the output will be filled with bars for minutes
    10:32 and 10:33 using the closing price of the last bar (10:31)

    Bars can be missinga amongst other things because

    Params:
      - ``fill_price`` (def: None): if None (or evaluates to False),the
        closing price will be used, else the passed value (which can be
        for example 'NaN' to have a missing bar in terms of evaluation but
        present in terms of time

      - ``fill_vol`` (def: NaN): used to fill the volume of missing bars

      - ``fill_oi`` (def: NaN): used to fill the openinterest of missing bars
    '''

    params = (
        ('fill_price', None),
        ('fill_vol', float('NaN')),
        ('fill_oi', float('NaN')),
        )

    def start(self):
        super(DataFiller, self).start()
        self._fillbars = collections.deque()
        self._dbar = False

    def preload(self):
        if len(self.p.dataname) == self.p.dataname.buflen():
            # if data is not preloaded .... do it
            self.p.dataname.start()
            self.p.dataname.preload()
            self.p.dataname.home()

        # Copy timeframe from data after start (some sources do autodetection)
        self.p.timeframe = self._timeframe = self.p.dataname._timeframe
        self.p.compression = self._compression = self.p.dataname._compression

        super(DataFiller, self).preload()

    def _copyfromdata(self):
        # Data is allowed - Copy size which is "number of lines"
        for i in range(self.p.dataname.size()):
            self.lines[i][0] = self.p.dataname.lines[i][0]

        self._dbar = False  # invalidate flag for read bar

        return True

    def _frombars(self):
        dtime, price = self._fillbars.popleft()

        price = self.p.fill_price or price

        self.lines.datetime[0] = self.p.dataname.date2num(dtime)
        self.lines.open[0] = price
        self.lines.high[0] = price
        self.lines.low[0] = price
        self.lines.close[0] = price
        self.lines.volume[0] = self.p.fill_vol
        self.lines.openinterest[0] = self.p.fill_oi

        return True

    # Minimum delta unit in between bars
    _tdeltas = {
        TimeFrame.Minutes: timedelta(seconds=60),
        TimeFrame.Seconds: timedelta(seconds=1),
        TimeFrame.MicroSeconds: timedelta(microseconds=1),
    }

    def _load(self):
        if not len(self.p.dataname):
            self.p.dataname.start()  # start data if not done somewhere else

            # Copy from underlying data
            self._timeframe = self.p.dataname._timeframe
            self._compression = self.p.dataname._compression

            self.p.timeframe = self._timeframe
            self.p.compression = self._compression

            # Calculate and save timedelta for timeframe
            self._tdunit = self._tdeltas[self._timeframe]
            self._tdunit *= self._compression

        if self._fillbars:
            return self._frombars()

        # use existing bar or fetch a bar
        self._dbar = self._dbar or self.p.dataname.next()
        if not self._dbar:
            return False  # no more data

        if len(self) == 1:
            # Cannot yet look backwards - deliver data as is
            return self._copyfromdata()

        # previous (delivered) close
        pclose = self.lines.close[-1]
        # Get time of previous (already delivered) bar
        dtime_prev = self.lines.datetime.datetime(-1)
        # Get time of current (from data source) bar
        dtime_cur = self.p.dataname.datetime.datetime(0)

        # Calculate session end for previous bar
        send = datetime.combine(dtime_prev.date(), self.p.dataname.sessionend)

        if dtime_cur > send:  # if jumped boundary
            # 1. check for missing bars until boundary (end)
            dtime_prev += self._tdunit
            while dtime_prev < send:
                self._fillbars.append((dtime_prev, pclose))
                dtime_prev += self._tdunit

            # Calculate session start for new bar
            sstart = datetime.combine(
                dtime_cur.date(), self.p.dataname.sessionstart)

            # 2. check for missing bars from new boundary (start)
            # check gap from new sessionstart
            while sstart < dtime_cur:
                self._fillbars.append((sstart, pclose))
                sstart += self._tdunit
        else:
            # no boundary jumped - check gap until current time
            dtime_prev += self._tdunit
            while dtime_prev < dtime_cur:
                self._fillbars.append((dtime_prev, pclose))
                dtime_prev += self._tdunit

        if self._fillbars:
            self._dbar = True  # flag a pending data bar is available

            # return an accumulated bar in current cycle
            return self._frombars()

        return self._copyfromdata()


import importlib
import sys
import os
import numpy as np
sys.path.append("../../")
import datetime
import optuna
import torch.distributed as dist
import pandas as pd
import torch
import os
from ml_graph_timer.model.graphsage import LayoutGraphModel,GraphModelArugments
from ml_graph_timer.dataset.layout_dataset import NpzDataset,GraphCollator
from ml_graph_timer.losses.losses import CustomMAELoss
from allrank.models.losses import listMLE,listNet
import shutil
import time

def get_latest_running_trial(study):
    # Fetch all trials from the study
    trials = study.get_trials()

    # Filter out trials that are still running
    running_trials = [trial for trial in trials if trial.state == optuna.trial.TrialState.RUNNING]

    # Sort the running trials by their start datetime and return the latest
    sorted_running_trials = sorted(running_trials, key=lambda x: x.datetime_start, reverse=True)

    return sorted_running_trials[0] if sorted_running_trials else None

def get_trial_by_name(study, name):
    for trial in reversed(study.get_trials()):
        if trial.user_attrs.get("unique_name") == name:
            return trial
    return None

if __name__ == "__main__":
    module_name = sys.argv[2]
    tune_id = sys.argv[3]
    UID = sys.argv[4]

    module = importlib.import_module(f"configs.{module_name}")
    base_obj = module.Configs()

    if base_obj.DISTRIBUTED:
        dist.init_process_group(backend='nccl',timeout=datetime.timedelta(seconds=7200000))

    from trainer.graphsage_trainer import Trainer
    if base_obj.DISTRIBUTED:
        rank = dist.get_rank()
    else:
        rank=0
    print("RANK, TUNEID, MODULENAME : ",rank,tune_id,module_name)

    study = optuna.create_study(direction="maximize", study_name='hptune_convtrans',storage="sqlite:///study_xla_default.db",load_if_exists=True)
    if rank==0:
        trail = study.ask()
        trail.set_user_attr("unique_name", UID+tune_id)
        max_configs = trail.suggest_categorical("max_config",[4,8,16,32,64])
        losstype= trail.suggest_categorical("losstype",["listnet","listmle"])
    dist.barrier()
    if rank!=0:
        time.sleep(5)
        trail = get_trial_by_name(study,  UID+tune_id)
        max_configs = trail.params["max_config"]
        losstype = trail.params["losstype"]

    base_obj = module.Configs()
    base_obj.train_dataset.max_configs = max_configs
    base_obj.criterion = listMLE if losstype=="listmle" else listNet

    trainer = Trainer(base_obj)

    try:
        value = trainer.train(prune_epochs=10)
    except Exception as e:
        print("ERROR OCCURED: ",str(e))
        if rank==0:
            trail.set_user_attr("error_name", str(e))
            value = trainer.evaluation_callback.opa

    if rank==0:
        if os.path.exists(base_obj.OUTPUTDIR):
            shutil.rmtree(base_obj.OUTPUTDIR)
        study.tell(trial=trail,values=value)
    dist.barrier()
    print(f"Completed Tuning for {module_name}!")