import os
import pandas as pd
import numpy as np
from os.path import join
import matplotlib.pyplot as plt

def search_best_id(root_path, index_list):
    """ This function, given the parameters, looks for the best realization
    of the experiment, among different learning rates.
    :param root_path: str of the folder containing results
    :param index_list: list of indices for the experiments with same parameters
    """
    loss_val = np.array([pd.read_csv(join(root_path,
                                          'train_%i/history.csv' % index))['val_loss'].values[-1]
                         for index in index_list])  # validation loss at last iteration
    loss_val[np.isnan(loss_val)] = np.inf
    best_id = index_list[np.argmin(loss_val)]       # best index
    return best_id

def flatten_train_json(df):
    """ We assume here that the train.json file has always the same keys.
    :param df: pandas dataframe
    :return: flatten df, each row corresponds to an experiment.
    """
    df = df.T
    dataset_keys = ['dataset_name',
                    'scenario',
                    'additional_dims',
                    'n_training',
                    'redundancy_amount']

    hyper_keys = ['learning_rate',
                  'architecture',
                  'epochs',
                  'batch_size',
                  'loss',
                  'optimizer',
                  'lr_at_plateau',
                  'reduction_factor',
                  'validation_check']

    upper_keys = ['id',
                  'output_path',
                  'train_completed']

    columns_name = dataset_keys + hyper_keys + upper_keys

    list_all_samples = []

    for i_ in range(df.shape[0]):
        list_per_sample = []
        for d_k_ in dataset_keys:
            list_per_sample.append(df['dataset'][i_][d_k_])
        for h_k_ in hyper_keys:
            list_per_sample.append(df['hyper'][i_][h_k_])
        for u_p_ in upper_keys:
            list_per_sample.append(df[u_p_][i_])
        list_all_samples.append(list_per_sample)

    return pd.DataFrame(list_all_samples, columns=columns_name)

def generate_bm(df, experiment_keys):
    """ Given the flatten DataFrame, containing all the experiment
    here we extract the experiments correspondent to the dictionary experiment_keys.
    :param df: pandas DataFrame containing the flatten df
    :param experiment_keys: dictionary with all the keys.
    :returns df_copy: the reduced dictionary.
    """
    df_copy = df.copy()
    for (k_, v_) in experiment_keys.items():
        df_copy = df_copy[df_copy[k_] == v_]
    return df_copy

def plot_validation_curves(df,
                           dataset_name,
                           root_path,
                           result_path,
                           scenario=1,
                           additional_dims=2,
                           architecture='FC',
                           epochs=500,
                           loss='cross_entropy'):
    """ Plot for same scenario and data dimensions.
     Each curve represents the model trained with different amount of data.
     :param df: pandas dataframe from the json, already flatten
     :param dataset_name: string with dataset name
     :param root_path: path where to retrieve results
     :param result_path: path where to store the plots
     :param scenario: int, representing the scenario
     :param additional_dims: int, dimension of the mnist - dataset
     :param architecture: str, architecture type
     :param epochs: int, number of epochs
     :param loss: str, square loss or cross entropy
     """

    experiment_keys = {'scenario': scenario,
                       'dataset_name': dataset_name,
                       'additional_dims': additional_dims,
                       'architecture': architecture,
                       'epochs': epochs,
                       'loss': loss}

    n_training_list = np.unique(df['n_training'])
    output_path = join(result_path,
                       dataset_name,
                       architecture,
                       loss,
                       'validation')
    os.makedirs(output_path,
                exist_ok=True)

    fig, ax = plt.subplots(figsize=(15, 5), nrows=1, ncols=2)
    plt.rc('xtick', labelsize=15)
    plt.rc('ytick', labelsize=15)
    for n_ in sorted(n_training_list):
        experiment_keys['n_training'] = n_

        # across all batch size and learning rate we pick the best performance
        index_list = generate_bm(df, experiment_keys=experiment_keys)['id'].values
        best_id = search_best_id(root_path, index_list)

        df_hist = pd.read_csv(join(root_path, 'train_%i/history.csv' % best_id))  # we retrieve id

        bs_ = df.iloc[best_id]['batch_size']  # we plot the convergence time
        n_epochs, _ = df_hist.shape
        convergence_time = (n_epochs * (n_ / bs_)) * np.linspace(0, 1, n_epochs)

        ax[0].loglog(convergence_time, df_hist['loss'], label='t: %i' % n_)
        ax[0].loglog(convergence_time, df_hist['val_loss'], '--', label='v')  # validation loss
        ax[0].set_xlabel('# convergence time', fontsize='xx-large')
        ax[0].set_ylabel('loss', fontsize='xx-large')
        ax[0].legend(fontsize='x-large')
        # ax[0].set_ylim([0, 2])
        ax[1].loglog(convergence_time, df_hist['accuracy'], label='t: %i' %n_)
        ax[1].loglog(convergence_time, df_hist['val_accuracy'], '--', label='v')  # validation accuracy
        ax[1].set_xlabel('# convergence time', fontsize='xx-large')
        ax[1].set_ylabel('accuracy', fontsize='xx-large')
        ax[1].legend(fontsize='x-large')
    fig.suptitle('Validation results', fontsize='xx-large')
    fig.tight_layout(rect=[0, 0.03, 1, 0.95])
    plt.savefig(join(output_path, 'scenario_%i_dim_%i.pdf' % (scenario,
                                                              additional_dims)))
    plt.close()

def plot_test_curves(df,
                     dataset_name,
                     root_path,
                     result_path,
                     scenario=1,
                     architecture='FC',
                     loss='cross_entropy'):
    """ Plot for same scenario. We change data dimensions and amount of data.
    Here we report the test loss and the test accuracy.

    This plot is for all the additional_dims values

    :param df: flattened dataframe
    :param dataset_name: str
    :param root_path: path where to retrieve results
    :param result_path: path where to save results
    :param scenario: int, denoting the scenario
    :param architecture: str, architecture name
    :param loss: str, name of the loss (cross_entropy or square_loss)
    """

    experiment_keys = {'scenario': scenario,
                       'dataset_name': dataset_name,
                       'architecture': architecture,
                       'loss': loss}

    n_training_list = np.unique(df['n_training'])
    dataset_dim_list = np.unique(df['additional_dims'])

    output_path = join(result_path,
                       dataset_name,
                       architecture,
                       loss,
                       'test')
    os.makedirs(output_path,
                exist_ok=True)

    ls_test = np.zeros((len(dataset_dim_list), len(n_training_list)))
    ac_test = np.zeros((len(dataset_dim_list), len(n_training_list)))
    plt.rc('xtick', labelsize=15)
    plt.rc('ytick', labelsize=15)
    fig, ax = plt.subplots(figsize=(15, 5), nrows=1, ncols=2)

    for i_, dim_ in enumerate(dataset_dim_list):
        experiment_keys['additional_dims'] = dim_

        for j_, n_ in enumerate(n_training_list):
            experiment_keys['n_training'] = n_
            index_list = generate_bm(df, experiment_keys=experiment_keys)['id'].values
            # across all batches and learning rate values
            best_id = search_best_id(root_path, index_list)

            ls, ac = np.load(join(root_path, 'train_%i/test.npy' % best_id))
            ls_test[i_, j_] = ls
            ac_test[i_, j_] = ac

        ax[0].loglog(n_training_list, ls_test[i_], 'o-', label='dim %i' % dim_)
        ax[0].set_xlabel('# training samples per class', fontsize='xx-large')
        ax[0].set_ylabel('cross entropy', fontsize='xx-large')

        ax[1].loglog(n_training_list, ac_test[i_], 'o-', label='dim %i' % dim_)
        ax[1].set_xlabel('# training samples per class', fontsize='xx-large')
        ax[1].set_ylabel('accuracy', fontsize='xx-large')

    ax[0].legend(fontsize='x-large')
    ax[1].legend(fontsize='x-large')

    fig.suptitle('Test results', fontsize='xx-large')
    fig.tight_layout(rect=[0, 0.03, 1, 0.95])
    plt.savefig(join(output_path, 'scenario_%i.pdf' % scenario))
    plt.close()

def plot_epochs_curves(df,
                       dataset_name,
                       root_path,
                       results_path,
                       scenario=1,
                       architecture='FC',
                       epochs=500,
                       loss='cross_entropy'):
    """Here we plot the curves related to the number of epochs before we stop training.

     :param df: flatten pandas DataFrame from the json, already flatten
     :param dataset_name: string with dataset name
     :param root_path: path where to retrieve results
     :param results_path: path where to store the plots
     :param scenario: int, representing the scenario
     :param additional_dims: int, dimension of the mnist - dataset
     :param architecture: str, architecture type
     :param epochs: int, number of epochs
     :param loss: str, square loss or cross entropy

     :returns None: plots
     """
    experiment_keys = {'scenario': scenario,
                       'dataset_name': dataset_name,
                       'architecture': architecture,
                       'epochs': epochs,
                       'loss': loss}

    n_training_array = np.unique(df['n_training'])
    additional_dims_array = np.unique(df['additional_dims'])

    output_path = join(results_path,
                       dataset_name,
                       architecture,
                       loss,
                       'time')
    os.makedirs(output_path,
                exist_ok=True)

    fig, ax = plt.subplots(figsize=(20, 5), nrows=1, ncols=3)
    plt.rc('xtick', labelsize=15)
    plt.rc('ytick', labelsize=15)

    for id_d_, d_ in enumerate(additional_dims_array):
        n_epochs_array = np.zeros(n_training_array.size, dtype=np.int)
        convergence_array = np.zeros(n_training_array.size)
        training_loss_array = np.zeros(n_training_array.size)
        experiment_keys['additional_dims'] = d_

        for id_n_, n_ in enumerate(n_training_array):
            experiment_keys['n_training'] = n_

            # across all batch size and learning rate we pick the best performance
            index_list = generate_bm(df, experiment_keys=experiment_keys)['id'].values
            best_id = search_best_id(root_path, index_list)

            df_hist = pd.read_csv(join(root_path, 'train_%i/history.csv' % best_id))  # we retrieve id

            bs_ = df.iloc[best_id]['batch_size']  # we plot the convergence time
            n_epochs, _ = df_hist.shape
            n_epochs_array[id_n_] = n_epochs
            convergence_array[id_n_] = n_epochs * (n_ / bs_)
            training_loss_array[id_n_] = df_hist['loss'].values[-1]

        ax[0].loglog(n_training_array, n_epochs_array, label='dim %i' %d_)
        ax[1].loglog(n_training_array, convergence_array, label='dim %i' %d_)
        ax[2].loglog(n_training_array, training_loss_array, label='dim %i' %d_)

    ax[0].set_xlabel('# training examples', fontsize='xx-large')
    ax[0].set_ylabel('# epochs', fontsize='xx-large')
    ax[0].legend(fontsize='x-large')

    ax[1].set_xlabel('# training examples', fontsize='xx-large')
    ax[1].set_ylabel('time ', fontsize='xx-large')
    ax[1].legend(fontsize='x-large')

    ax[2].set_xlabel("# training examples ")
    ax[2].set_ylabel("loss at last iteration")
    ax[2].legend(fontsize='x-large')

    fig.suptitle('Convergence time and loss last iteration', fontsize='xx-large')
    fig.tight_layout(rect=[0, 0.03, 1, 0.95])
    plt.savefig(join(output_path, 'scenario_%i.pdf' % scenario))
    plt.close()

def plot_test_across_repetitions(root_path_lst,
                                 results_path,
                                 scenario,
                                 architecture,
                                 dataset='standardized_MNIST_dataset',
                                 epochs=500):
    """
    Here we report the test results for the average of multiple repetitions.

    :param root_path_lst: list of str containing all the paths, for each experiment
    :param results_path: str for path where to store the plot
    :param scenario: value correspondent to the scenario
    :param architecture: architecture type
    :param dataset: dataset name, str
    :param epochs: int, number of epochs

    :return None: save plots
    """
    experiment_keys = {'scenario': scenario,
                       'dataset_name': dataset,
                       'architecture': architecture,
                       'epochs': epochs}
    output_path = join(results_path, dataset, architecture, 'test')
    os.makedirs(output_path, exist_ok=True)
    flag_find_dim = True
    for id_rep, root_path_ in enumerate(root_path_lst):
        df = flatten_train_json(pd.read_json(join(root_path_,
                                                  'train.json')))
        if flag_find_dim:  # first iteration over all repetitions
            n_training_list = sorted(list(set(df['n_training'])))
            dataset_dim_list = sorted(list(set(df['dataset_dimensions'])))
            ls_test = np.zeros((len(root_path_lst), len(dataset_dim_list), len(n_training_list)))
            ac_test = np.zeros((len(root_path_lst), len(dataset_dim_list), len(n_training_list)))
            flag_find_dim = False
        for id_d, dim_ in enumerate(dataset_dim_list):
            experiment_keys['dataset_dimensions'] = dim_
            for id_n, n_ in enumerate(n_training_list):
                experiment_keys['n_training'] = n_
                if epochs == 50:
                    experiment_keys['batch_size'] = 10 if n_ < 10 else 32
                index_list = generate_bm(df, experiment_keys=experiment_keys)['id'].values
                best_id = search_best_id(root_path_, index_list)
                ls, ac = np.load(join(root_path_, 'train_%i/test.npy' % best_id))
                ls_test[id_rep, id_d, id_n] = ls
                ac_test[id_rep, id_d, id_n] = ac
    plt.rc('xtick', labelsize=15)
    plt.rc('ytick', labelsize=15)
    fig, ax = plt.subplots(figsize=(15, 5), nrows=1, ncols=2)
    ax[0].set_xscale("log")
    ax[0].set_yscale("log")
    ax[1].set_xscale("log")
    mn_ls = np.mean(ls_test, axis=0)
    st_ls = np.std(ls_test, axis=0)
    mn_ac = np.mean(ac_test, axis=0)
    st_ac = np.std(ac_test, axis=0)

    for id_d, (dim_, ls_m, ls_s, ac_m, ac_s) in enumerate(zip(dataset_dim_list, mn_ls, st_ls,
                                                mn_ac, st_ac)):
        ax[0].errorbar(n_training_list, ls_m, ls_s, label='dim %i' % dim_)
        ax[1].errorbar(n_training_list, ac_m, ac_s, label='dim %i' % dim_)

    ax[1].set_xlabel('# training samples per class', fontsize='xx-large')
    ax[1].set_ylabel('accuracy', fontsize='xx-large')
    ax[0].set_xlabel('# training samples per class',
                     fontsize='xx-large')
    ax[0].set_ylabel('cross entropy', fontsize='xx-large')
    ax[0].legend(fontsize='x-large')
    ax[1].legend(fontsize='x-large')
    fig.suptitle('Test results', fontsize='xx-large')
    fig.tight_layout(rect=[0, 0.03, 1, 0.95])
    plt.savefig(join(output_path, 'scenario_%i.pdf' % scenario))
    plt.show()
    plt.close()

def plot_validation_across_repetitions(root_path_lst,
                                       results_path,
                                       scenario,
                                       architecture,
                                       dataset='standardized_MNIST_dataset',
                                       epochs=500):
    """ Here we report the test results for the average of multiple repetitions.
    :param root_path_lst: list of str containing all the paths, for each experiment
    :param results_path: str for path where to store the plot
    :param scenario: value correspondent to the scenario
    :param architecture: architecture type
    :param dataset: dataset name, str
    :param epochs: int, number of epochs
    :return None: save plots
    """
    experiment_keys = {'scenario': scenario,
                       'dataset_name': dataset,
                       'architecture': architecture,
                       'epochs': epochs}
    output_path = join(results_path, dataset, architecture, 'train')
    os.makedirs(output_path, exist_ok=True)

    df = flatten_train_json(pd.read_json(join(root_path_lst[0], 'train.json')))
    # these two array must be equivalent to all the repetitions
    # we use the previous scripts to generate clones for the same experiments
    n_training_list = sorted(list(set(df['n_training'])))
    dataset_dim_list = sorted(list(set(df['dataset_dimensions'])))
    epochs = sorted(list(set(df['epochs'])))[-1]  # the max epoch to realize the plots

    for id_d, dim_ in enumerate(dataset_dim_list):
        experiment_keys['dataset_dimensions'] = dim_

        history_rep = np.zeros((len(root_path_lst), len(n_training_list), epochs, 4))
        history_rep[:, :, :, :] = np.nan

        fig, ax = plt.subplots(figsize=(15, 5), nrows=1, ncols=2)
        plt.rc('xtick', labelsize=15)
        plt.rc('ytick', labelsize=15)

        for id_n, n_tr_ in enumerate(n_training_list):
            experiment_keys['n_training'] = n_tr_
            if epochs == 50:
                experiment_keys['batch_size'] = 10 if n_tr_ < 10 else 32

            bs_lst = []
            for id_rep, root_path_ in enumerate(root_path_lst):
                df = flatten_train_json(pd.read_json(join(root_path_,
                                                          'train.json')))
                index_list = generate_bm(df, experiment_keys=experiment_keys)['id'].values
                best_id = search_best_id(root_path_, index_list)
                bs_ = df.iloc[best_id]['batch_size']
                bs_lst.append(bs_)
                df_hist = pd.read_csv(join(root_path_, 'train_%i/history.csv' % best_id))
                max_size, _ = df_hist.shape
                # (tr_loss, tr_acc, vl_loss, vl_acc)
                # first and last index contain (df.index and learning rate)
                history_rep[id_rep, id_n, :max_size, :] = df_hist.values[:, 1:-1]

            bs_ = np.max(bs_lst)
            ax[0].set_yscale("log", nonposy='clip')
            convergence_time = np.linspace(0, 1, epochs) * epochs * (n_tr_ / bs_)
            ax[0].errorbar(convergence_time,
                           np.nanmean(history_rep, axis=0)[id_n, :, 0],
                           np.nanstd(history_rep, axis=0)[id_n, :, 0],
                           label='t: %i' % n_tr_)
            ax[0].errorbar(convergence_time,
                           np.nanmean(history_rep, axis=0)[id_n, :, 2],
                           np.nanstd(history_rep, axis=0)[id_n, :, 2],
                           label='v', fmt='o')
            ax[0].set_xlabel('# convergence time', fontsize='xx-large')
            ax[0].set_ylabel('cross entropy', fontsize='xx-large')
            ax[0].legend(fontsize='x-large')

            ax[1].errorbar(convergence_time,
                           np.nanmean(history_rep, axis=0)[id_n, :, 1],
                           np.nanstd(history_rep, axis=0)[id_n, :, 1],
                           label='t: %i' % n_tr_)
            ax[1].errorbar(convergence_time,
                           np.nanmean(history_rep, axis=0)[id_n, :, 3],
                           np.nanstd(history_rep, axis=0)[id_n, :, 3],
                           label='v', fmt='o')
            ax[1].set_xlabel('# convergence time', fontsize='xx-large')
            ax[1].set_ylabel('accuracy', fontsize='xx-large')
            ax[1].legend(fontsize='x-large')
        fig.suptitle('Learning results', fontsize='xx-large')
        fig.tight_layout(rect=[0, 0.03, 1, 0.95])
        plt.savefig(join(output_path, 'scenario_%i_dim_%i.pdf' % (scenario,
                                                                  dim_)))
        plt.show()
        plt.close()

def plot_time_across_repetitions(root_path_lst,
                                 result_path,
                                 dataset='standardized_MNIST_dataset',
                                 scenario=1,
                                 architecture='FC',
                                 epochs=500):
    """ For each n_tr we report the averaged convergence time
    :param root_path_lst: path where to retrieve results
    :param result_path: path where to store the plots
    :param dataset: string with dataset name
    :param scenario: int, representing the scenario
    :param architecture: str, architecture type
    :param epochs: int, max number of epochs
    """
    experiment_keys = {'scenario': scenario,
                       'dataset_name': dataset,
                       'architecture': architecture,
                       'epochs': epochs}
    output_path = join(result_path, dataset, architecture, 'time')
    os.makedirs(output_path, exist_ok=True)

    df = flatten_train_json(pd.read_json(join(root_path_lst[0], 'train.json')))
    n_training_array = np.unique(df['n_training'])
    dataset_dim_array = np.unique(df['dataset_dimensions'])

    convergence_time = np.zeros((dataset_dim_array.size,
                                 n_training_array.size,
                                 len(root_path_lst)))
    n_epochs_across = np.zeros((dataset_dim_array.size,
                                n_training_array.size,
                                len(root_path_lst)),
                               dtype=int)

    for id_r_, root_path_ in enumerate(root_path_lst):
        df = flatten_train_json(pd.read_json(join(root_path_, 'train.json')))

        for id_n_, n_ in enumerate(n_training_array):
            experiment_keys['n_training'] = n_
            if epochs == 50:
                experiment_keys['batch_size'] = 10 if n_ < 10 else 32

            for id_d_, d_ in enumerate(dataset_dim_array):
                experiment_keys['dataset_dimensions'] = d_

                index_list = generate_bm(df, experiment_keys=experiment_keys)['id'].values
                best_id = search_best_id(root_path_, index_list)
                bs_ = df.iloc[best_id]['batch_size']
                df_hist = pd.read_csv(join(root_path_, 'train_%i/history.csv' % best_id))  # we retrieve id
                n_epochs, _ = df_hist.shape
                convergence_time[id_d_, id_n_, id_r_] = n_epochs * (n_ / bs_)
                n_epochs_across[id_d_, id_n_, id_r_] = n_epochs

    fig, ax = plt.subplots()
    ax.set_xscale("log")
    ax.set_yscale("log")
    plt.rc('xtick', labelsize=15)
    plt.rc('ytick', labelsize=15)
    for (d_, ct_per_dim) in zip(dataset_dim_array, convergence_time):
        ct_mean = np.mean(ct_per_dim, axis=-1)
        ct_std = np.std(ct_per_dim, axis=-1)
        ax.errorbar(n_training_array, ct_mean, ct_std, label='d %i' % d_)
    ax.set_xlabel('#n per class', fontsize='xx-large')
    ax.set_ylabel('convergence time', fontsize='xx-large')
    ax.legend(fontsize='x-large')
    plt.tight_layout()
    plt.savefig(join(output_path, 'time_scenario_%i.pdf' % scenario))
    plt.close()

    fig, ax = plt.subplots()
    ax.set_xscale("log")
    plt.rc('xtick', labelsize=15)
    plt.rc('ytick', labelsize=15)
    for (d_, ep_per_dim) in zip(dataset_dim_array, n_epochs_across):
        ep_mean = np.mean(ep_per_dim, axis=-1)
        ep_std = np.std(ep_per_dim, axis=-1)
        ax.errorbar(n_training_array, ep_mean, ep_std, label='d %i' % d_)
    ax.set_xlabel('#n per class', fontsize='xx-large')
    ax.set_ylabel('# epochs', fontsize='xx-large')
    ax.legend(fontsize='x-large')
    plt.tight_layout()
    plt.savefig(join(output_path, 'n_epochs_scenario_%i.pdf' % scenario))
    plt.close()

def plot_pseudoinverse(df,
                       dataset_name,
                       root_path,
                       result_path,
                       scenario=1,
                       architecture='pseudoinverse',
                       loss='square_loss'):
    """ Here we plot the results over the test set for
    the pseudo-inverse solution.
    :param df: the flatten json file
    :param dataset_name: str, name of the dataset
    :param root_path: str, root path of the json to retrieve experiments
    :param result_path: str, where to save the plots
    :param scenario: int, 1,2,4
    :param architecture: str, architecture name
    :param loss: str, name of the loss
    """

    output_path = join(result_path,
                       dataset_name,
                       architecture,
                       'test')
    os.makedirs(output_path,
                exist_ok=True)

    exp_keys = {'dataset_name': dataset_name,
                'scenario': scenario,
                'architecture': architecture,
                'loss': loss}

    # we have different curves for each additional dimensions
    df_pinv = generate_bm(df, exp_keys)
    add_dims_array = np.unique(df_pinv['additional_dims'])
    n_train_array = np.unique(df_pinv['n_training'])

    fig, ax = plt.subplots(figsize=(15, 5), nrows=1, ncols=2)
    plt.rc('xtick', labelsize=15)
    plt.rc('ytick', labelsize=15)

    for add_d_ in add_dims_array:
        exp_keys['additional_dims'] = add_d_
        tmp_performance = np.zeros((2, n_train_array.size))

        for id_n_, n_tr_ in enumerate(n_train_array):
            print('index for the exp', id_n_, n_tr_)
            exp_keys['n_training'] = n_tr_
            tmp_output_path = generate_bm(df_pinv, exp_keys)['output_path'].values[0]

            print(join(tmp_output_path, 'test.npy'))

            tmp_performance[:, id_n_] = np.load(join(tmp_output_path, 'test.npy'))

        ax[0].plot(n_train_array, tmp_performance[0], 'o-', label='d:%i' % add_d_)
        ax[1].plot(n_train_array, tmp_performance[1], 'o-', label='d:%i' % add_d_)

    ax[0].set_xlabel("# training examples per class")
    ax[0].set_ylabel("MSE loss")
    ax[0].legend(fontsize='x-large')

    ax[1].set_xlabel("# training examples per class")
    ax[1].set_ylabel("accuracy")
    ax[1].legend(fontsize='x-large')

    fig.suptitle('Test results', fontsize='xx-large')
    fig.tight_layout(rect=[0, 0.03, 1, 0.95])
    plt.savefig(join(output_path, 'scenario_%i.pdf' % scenario))

def main():

    repetitions = 1
    dataset_name = 'dataset_2'

    # dataset_name = dataset_1 then repetition_0
    #              = dataset_2 then repetition_1

    if repetitions == 1:
        path_json = '/om/user/vanessad/synthetic_framework/results/repetition_1/train.json'
        path_plots = '/om/user/vanessad/synthetic_framework/plots'
    else:
        lst_path = ['repetition_%i' %i for i in range(repetitions)]
        lst_root_path = [join('/om2/user/vanessad/MNIST_framework/repetitions_500_epochs', res_folder_)
                         for res_folder_ in lst_path]
        path_avg_plot = '/om2/user/vanessad/MNIST_framework/viz_500_epochs_results_avg'

    if repetitions == 1:
        df_ = pd.read_json(path_json)
        df = df_.copy()
        df_flatten = flatten_train_json(df)

        scenario_list = np.unique(df_flatten['scenario'])
        additional_dims = np.unique(df_flatten['additional_dims'])
        architectures = np.unique(df_flatten['architecture'])
        loss_list = np.unique(df_flatten['loss'])

        print('scenarios', scenario_list)
        print('additional dimensions', additional_dims)
        print('architectures', architectures)
        print('loss list', loss_list)

        dct_loss = {'pseudoinverse': ['square_loss'],
                    'linear': ['square_loss', 'cross_entropy'],
                    'FC': ['cross_entropy']}

        for s_ in scenario_list:
            for arch_ in architectures:
                for loss_ in dct_loss[arch_]:
                    if arch_ != 'pseudoinverse':
                        """plot_test_curves(df_flatten,
                                         dataset_name='dataset_2',
                                         root_path=os.path.dirname(path_json),
                                         result_path=path_plots,
                                         scenario=s_,
                                         architecture=arch_,
                                         loss=loss_)"""

                        plot_epochs_curves(df_flatten,
                                           dataset_name=dataset_name,
                                           root_path=os.path.dirname(path_json),
                                           results_path=path_plots,
                                           scenario=s_,
                                           architecture=arch_,
                                           loss=loss_)
                        """for d_ in additional_dims:
                            plot_validation_curves(df_flatten,
                                                   dataset_name='dataset_2',
                                                   root_path=os.path.dirname(path_json),
                                                   result_path=path_plots,
                                                   additional_dims=d_,
                                                   scenario=s_,
                                                   architecture=arch_,
                                                   loss=loss_)"""

                    """else:
                        print('plot for the pseudo-inverse missing!')
                        plot_pseudoinverse(df_flatten,
                                           dataset_name='dataset_2',
                                           root_path=os.path.dirname(path_json),
                                           result_path=path_plots,
                                           scenario=s_,
                                           architecture='pseudoinverse',
                                           loss='square_loss')"""

    """ else:
        for s_ in scenario_list:
            for a_ in architecture_list:
                plot_validation_across_repetitions(lst_root_path, path_avg_plot, s_, a_)
                plot_test_across_repetitions(lst_root_path, path_avg_plot, s_, a_)
                plot_time_across_repetitions(lst_root_path,
                                             path_avg_plot,
                                             scenario=s_,
                                             architecture=a_)
    """

if __name__ == "__main__":
    main()


import typer
from dotenv import load_dotenv
from halo import Halo
from langchain.text_splitter import TokenTextSplitter

from chatflock.backing_stores import InMemoryChatDataBackingStore
from chatflock.base import Chat
from chatflock.code import LocalCodeExecutor
from chatflock.code.langchain import CodeExecutionTool
from chatflock.conductors.round_robin import RoundRobinChatConductor
from chatflock.participants.langchain import LangChainBasedAIChatParticipant
from chatflock.participants.user import UserChatParticipant
from chatflock.renderers.terminal import TerminalChatRenderer
from chatflock.web_research import WebSearch
from chatflock.web_research.page_analyzer import OpenAIChatPageQueryAnalyzer
from chatflock.web_research.page_retrievers.selenium_retriever import SeleniumPageRetriever
from chatflock.web_research.search import GoogleSerperSearchResultsProvider
from chatflock.web_research.web_research import WebResearchTool
from examples.common import create_chat_model, get_max_context_size

def chatgpt_clone_with_additional_tools(
    model: str = "gpt-4-1106-preview",
    model_for_page_analysis: str = "gpt-3.5-turbo-1106",
    temperature: float = 0.0,
    temperature_for_page_analysis: float = 0.0,
) -> None:
    chat_model = create_chat_model(model=model, temperature=temperature)
    chat_model_for_page_analysis = create_chat_model(
        model=model_for_page_analysis, temperature=temperature_for_page_analysis
    )

    max_context_size_for_page_analysis = get_max_context_size(chat_model_for_page_analysis) or 12_000

    page_retriever = SeleniumPageRetriever()
    web_search = WebSearch(
        chat_model=chat_model,
        search_results_provider=GoogleSerperSearchResultsProvider(),
        page_query_analyzer=OpenAIChatPageQueryAnalyzer(
            chat_model=chat_model_for_page_analysis,
            # Should `pip install selenium webdriver_manager` to use this
            page_retriever=page_retriever,
            text_splitter=TokenTextSplitter(
                chunk_size=max_context_size_for_page_analysis, chunk_overlap=max_context_size_for_page_analysis // 5
            ),
            use_first_split_only=True,
        ),
    )

    spinner = Halo(spinner="dots")
    ai = LangChainBasedAIChatParticipant(
        name="Assistant",
        chat_model=chat_model,
        tools=[
            CodeExecutionTool(executor=LocalCodeExecutor(spinner=spinner), spinner=spinner),
            WebResearchTool(web_search=web_search, n_results=3, spinner=spinner),
        ],
        spinner=spinner,
    )

    user = UserChatParticipant(name="User")
    participants = [user, ai]

    chat = Chat(
        backing_store=InMemoryChatDataBackingStore(), renderer=TerminalChatRenderer(), initial_participants=participants
    )

    chat_conductor = RoundRobinChatConductor()
    chat_conductor.initiate_dialog(chat=chat)

    page_retriever.close()

if __name__ == "__main__":
    load_dotenv()

    typer.run(chatgpt_clone_with_additional_tools)


import os
import shutil
import datetime

found_num = 0
keys = []
is_copyfile = True
copy_path = ''

def dfs(path):
    global found_num
    try:
        files = os.listdir(path)
    except FileNotFoundError:
        return
    for file in files:
        if os.path.isfile(path + '\\' + file):
            for key in keys:
                if key in file:
                    found_num += 1
                    print(found_num, file)
                    if is_copyfile:
                        shutil.copyfile(path + '\\' + file, copy_path + '\\' + file)
                    break
        else:
            try:
                if os.path.isdir(path + '\\' + file):
                    dfs(path + '\\' + file)
            except PermissionError:
                pass

def search_by_extension_name():
    global keys
    global is_copyfile
    global copy_path
    print('Do not add \\ to the end of all paths!')
    path = input('Please input the start path:')
    keys = input('Please input the file\'s extension name(Use , separate):')
    keys = keys.split(',')
    is_copy = input('If you need to copy this found files to a path(y/n):')
    if is_copy == 'y' or is_copy == 'Y':
        is_copyfile = True
        copy_path = input('Please input the copy path:')
        while not os.path.isdir(copy_path):
            print('Wrong path! please input again.')
            copy_path = input('Please input the copy path:')
    starttime = datetime.datetime.now()
    dfs(path)
    endtime = datetime.datetime.now()
    print('>>>Finish! It spent', (endtime - starttime).seconds, 'sec.', found_num, 'files were found.')

if __name__ == '__main__':
    search_by_extension_name()


import numpy as np
from scipy.spatial.distance import pdist
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
sns.set(color_codes=True)

# Implement required formulae.
def corners(n):
    return 2**n

def diag(n):
    # Diagonal from [0, ..., 0] to [1, ..., 1]
    # with length sqrt(1^2 + ... + 1^2).
    return np.sqrt(n)

def volume_ball(n, radius):
    # Recursive formula from
    # https://en.wikipedia.org/wiki/Volume_of_an_n-ball#Recursions
    if n == 1: return 2 * radius
    if n == 2: return np.pi * radius**2
    return 2*np.pi/n * radius**2 * volume_ball(n-2, radius)

def volume_skin(n, alpha):
    return 1 - (1 - 2*alpha)**n

def pairwise_distances(n, k):
    points = np.random.uniform(0, 1, (k, n))
    return pdist(points).flatten()

# Generate required statistics.
data = [
    (n, corners(n), diag(n), volume_ball(n, 0.5), volume_skin(n, 0.01))
    for n in range(1, 1+100)
]
df = pd.DataFrame(data=data, columns=["n", "Corners", "Diag", "VolumeB", "VolumeS"]).set_index("n")
with open("dimensional_stats.md", "w") as f:
    f.write(df.to_markdown())

# Generate required plot.
fig, ax = plt.subplots()
for n in range(1, 1+10):
    distances = pairwise_distances(2**n, 1000)
    sns.distplot(distances, bins=100, label=str(2**n), ax=ax)
ax.set_xlabel("Distance")
ax.set_ylabel("Kernel density estimate")
plt.title("Pairwise distances of random points in the n-dimensional unit hypercube.")
plt.tight_layout()
plt.legend(loc="upper right", title="n")
plt.savefig("pairwise_dist.pdf")
plt.show()


# STEPS FOR AMAZON HELP PAGE

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from behave import *
import time

# Scenario: User can search for solutions of Canceling an order on Amazon from Help page
@then('Locate "Search Help Library" field and search for "Cancel order"')
def locate_search_help_field(context):
    search_cancel_order = context.driver.find_element(By.ID,'helpsearch')
    search_cancel_order.clear()
    search_cancel_order.send_keys('Cancel order')
    search_cancel_order.send_keys(Keys.RETURN)

@then('Verify that "Cancel Items or Orders" text is displayed')
def verify_cancel_items_or_orders(context):
    expected_result = 'Cancel Items or Orders'
    actual_result = context.driver.find_element(By.XPATH,"//h1").text
    assert expected_result == actual_result, f'Test case failed! Actual text {actual_result} does not match expected {expected_result}'

# Scenario: Verify that Amazon Help page UI elements are present on the page
@then('Verify that Some things you can do here text result is displayed')
def verify_some_things(context):
    expected_result = 'Some things you can do here'
    actual_result = context.driver.find_element(By.CSS_SELECTOR,"div.a-section h2").text
    assert expected_result == actual_result, f'Test case failed! Actual text {actual_result} does not match expected {expected_result}'

@then('Verify that "Search the help library" text is displayed')
def search_help_library(context):
    expected_result = 'Search the help library'
    actual_result = context.driver.find_element(By.XPATH, "//label[@id='help-search-label']//span[contains(text(), 'Search the help library')]").text
    assert expected_result == actual_result, f'Test case failed! Actual text {actual_result} does not match expected {expected_result}'

@then('Verify that "Type something like, question about a charge" text result is displayed')
def type_smth_like(context):
    expected_result = 'Type something like, "question about a charge"'
    actual_result = context.driver.find_element(By.XPATH, "//label[@id='help-search-label']//span[contains(text(), 'Type something like')]").text
    assert expected_result == actual_result, f'Test case failed! Actual text {actual_result} does not match expected {expected_result}'

@then('Scroll down to "Browse Help Topics"')
def scroll_to_browse_help(context):
    element = context.driver.find_element(By.XPATH, "//*[contains(@class, 'help-content')]//*[contains(text(), 'Browse Help')]")
    context.driver.execute_script("arguments[0].scrollIntoView();", element)

@then('Verify that Browse Help Topics text is displayed')
def browse_help_topics(context):
    expected_result = 'Browse Help Topics'
    actual_result = context.driver.find_element(By.XPATH, "//*[contains(@class,'help-content')]//*[contains(text(),'Browse Help')]").text
    assert expected_result == actual_result, f'Test case failed! Actual text {actual_result} does not match expected {expected_result}'

@then('Verify that Where is My Stuff text is properly displayed')
def where_is_my_stuff(context):
    expected_result = "Where's My Stuff?"
    actual_result = context.driver.find_element(By.XPATH, "//*[contains(@rel, 'help-gateway-category') and contains(text(),'My Stuff?')]").text
    assert expected_result == actual_result, f'Test case failed! Actual text {actual_result} does not match expected {expected_result}'

@then('Verify that {find_text} text is displayed')
def find_text_1(context, find_text):
    expected_result = find_text
    actual_result = context.driver.find_element(By.XPATH, f"//*[contains(@class , 'a-column')]//*[ contains(text(), '{find_text}')]").text
    assert expected_result == actual_result, f'Test case failed! Actual text {actual_result} does not match expected {expected_result}'

@then('Verifying that "{find_text2}" text is displayed')
def find_text_2(context, find_text2):
    expected_result = find_text2
    actual_result = context.driver.find_element(By.XPATH, f"//*[contains(@rel, 'help-gateway-category') and contains(text(),'{find_text2}')]").text
    assert expected_result == actual_result, f'Test case failed! Actual text {actual_result} does not match expected {expected_result}'

@then('Verify that "Wrenches icon" is displayed')
def wrenches_icon(context):
    wrenches_pic = context.driver.find_element(By.CSS_SELECTOR, "#help-gateway-category-0 img.csg-hb-promo")


import os
import sys
from dataclasses import dataclass
sys.path.append(os.path.realpath('.'))

import pandas as pd

from sklearn.ensemble import (
    AdaBoostRegressor,
    GradientBoostingRegressor,
    RandomForestRegressor
)
from sklearn.linear_model import LinearRegression
from sklearn.metrics import r2_score
from sklearn.neighbors import KNeighborsRegressor
from sklearn.tree import DecisionTreeRegressor
from xgboost  import XGBRegressor

from src.exception import CustomException
from src.logger import logging
from flaml import AutoML
from src.utils import save_object, evaluate_models

@dataclass
class ModelTrainerConfig:
    trained_model_file_path  = os.path.join("artifact", "_{}_model.pkl")

class ModelTrainer:
    def __init__(self):
        self.model_trainer_config = ModelTrainerConfig()
        self.cities = ['Mariposa', 'Ramona','Redding',
                          'Sanel','Santa-Barbara','Union-City']

    def initiate_model_trainer(self):
        try:
            scores = []
            for city in self.cities:

                logging.info(f"{city}:splitting training and val input data")

                train_df = pd.read_csv(f"artifact/{city}_train.csv")
                X_train = train_df.drop(columns = ['time','FWI']).values
                y_train = train_df['FWI'].values

                test_df = pd.read_csv(f"artifact/{city}_test.csv")
                X_test = test_df.drop(columns = ['time','FWI']).values
                y_test = test_df['FWI'].values
                automl = AutoML()
                # Specify automl goal and constraint
                automl_settings = {
                    "time_budget": 120,  # in seconds
                    "metric": 'mae',
                    "task": 'regression',
                }
                # Train with labeled input data
                automl.fit(X_train=X_train, y_train=y_train,
                        **automl_settings)

                # models = {
                #     "Random Forest": RandomForestRegressor(),
                #     "Decision Tree": DecisionTreeRegressor(),
                #     "Gradient Boosting": GradientBoostingRegressor(),
                #     "Linear Regression": LinearRegression(),
                #     "XGBRegressor": XGBRegressor(),
                #     "AdaBoost Regressor": AdaBoostRegressor(),
                # }

                # params={
                #     "Decision Tree": {
                #         'criterion':['squared_error', 'friedman_mse', 'absolute_error', 'poisson'],
                #         # 'splitter':['best','random'],
                #         # 'max_features':['sqrt','log2'],
                #     },
                #     "Random Forest":{
                #         # 'criterion':['squared_error', 'friedman_mse', 'absolute_error', 'poisson'],

                #         # 'max_features':['sqrt','log2',None],
                #         'n_estimators': [8,16,32,64,128,256]
                #     },
                #     "Gradient Boosting":{
                #         # 'loss':['squared_error', 'huber', 'absolute_error', 'quantile'],
                #         'learning_rate':[.1,.01,.05,.001],
                #         'subsample':[0.6,0.7,0.75,0.8,0.85,0.9],
                #         # 'criterion':['squared_error', 'friedman_mse'],
                #         # 'max_features':['auto','sqrt','log2'],
                #         'n_estimators': [8,16,32,64,128,256]
                #     },
                #     "Linear Regression":{},
                #     "XGBRegressor":{
                #         'learning_rate':[.1,.01,.05,.001],
                #         'n_estimators': [8,16,32,64,128,256]
                #     },
                #     "AdaBoost Regressor":{
                #         'learning_rate':[.1,.01,0.5,.001],
                #         # 'loss':['linear','square','exponential'],
                #         'n_estimators': [8,16,32,64,128,256]
                #     }

                # }

                # logging.info(f"{city}:Evaluating models")

                # model_report:dict = evaluate_models(X_train = X_train,
                #                                 y_train = y_train,
                #                                 X_test = X_test,
                #                                 y_test = y_test,
                #                                 models = models,
                #                                 params = params)
                # best_score = 0
                # best_model_name = ''
                # # getting best model from dictionary
                # for model in model_report:
                #     score = model_report[model]
                #     if score > best_score:
                #         best_model_name = model
                #         best_score = score

                best_model = automl.model#models[best_model_name]
                # if best_score < 0.6:
                #     raise CustomException("no best model found", sys)
                # logging.info(f"best model found {best_model_name}")

                save_object(
                    file_path=self.model_trainer_config.trained_model_file_path.format(city),
                    obj=best_model
                )

                predicted = best_model.predict(X_test)
                score = r2_score(y_test, predicted)
                scores.append(score)
            return score
        except Exception as e:
            raise CustomException(e,sys)

if __name__ == '__main__':
    di = ModelTrainer()
    di.initiate_model_trainer()


from django.shortcuts import render
from django.http import HttpResponse, JsonResponse
from .forms import PdfUploadForm
from .models import Category
import PyPDF2

# Create your views here.

# FBV
def pdf_extract(request):
    if request.method == 'POST':
        form = PdfUploadForm(request.POST, request.FILES)
        if form.is_valid():
            page_num = int(request.POST.get("page", 0))
            page_index = page_num - 1
            f = request.FILES['file']

            with open('original.pdf',"wb+") as pdfFileObj:
                for chunk in f.chunks():
                    pdfFileObj.write(chunk)
                pdfReader = PyPDF2.PdfFileReader(pdfFileObj)

    else:
        form = PdfUploadForm()
        return render(request, 'pdf/pdf_upload.html', {'form': form})

from rest_framework.response import Response

def category(request):
    if request.method == "GET":
        # root_category = Category.objects.get(name='Root Category')
        # child_categories = root_category.children.all()

        # 根据name获取他的所有父节点
        for item in Category.objects.filter(name="steverocket1.1.1").get_ancestors(include_self=True):
            print(item.name)

    elif request.method == "POST":
        root_category = Category.objects.create(name='Root Category')
        child_category1 = Category.objects.create(name='Child Category 1', parent=root_category)
        child_category2 = Category.objects.create(name='Child Category 2', parent=root_category)
    # return Response(data={})
    # return JsonResponse({"category": root_category})
    return JsonResponse({})

def whitenoise_server(request):
    # whitenoise 不支持template
    # from django.conf import settings
    # settings.STATIC_ROOT
    # return render(request, "index.html")
    pass

    # return render(request, "README.md")
    # https://gitee.com/SteveRocket/staticfiles/blob/master/staticfiles/index.html
    # return render(request, "https://gitee.com/SteveRocket/staticfiles/blob/master/staticfiles/index.html")


# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy
from scrapy.loader.processors import MapCompose, TakeFirst, Join
from scrapy.loader import ItemLoader #拿到takefirst
import re
import datetime

# class ArticalsoiderItem(scrapy.Item):
#     # define the fields for your item here like:
#     # name = scrapy.Field()
#     title = scrapy.Field()
#     create_time = scrapy.Field()
#     front_image_url = scrapy.Field()
#     front_image_path = scrapy.Field()
#     praise_number = scrapy.Field()
#     colloct_number = scrapy.Field()
#     conmment_number = scrapy.Field()
#     content = scrapy.Field()
#     tag = scrapy.Field()
#     url_hashcode = scrapy.Field() #得到URL生成的哈希码

class Item_Takefirst(ItemLoader):#自定义拿到takefirst
    default_output_processor = TakeFirst()

def time_convert(value):
    #value = (re.match(".*(\d+)/(\d+)/(\d+).*?", value).group(1))
    value = value.replace("\r", "").replace("\n", "").replace("·", "").strip()
    try:
        value = datetime.datetime.strptime(value, "%Y/%m/%d").date()
    except Exception as e:
        value = datetime.datetime.now().date()
        #print(e)
    return value

def praise_convert(value):
    try:
        value = int(value)
    except:
        value = 0
    return value

def colloct_convert(value):
    try:
        value = int(re.match(".*(\d+).*?", value).group(1))
    except:
        value = 0
    return value

def comment_convert(value):
    try:
        value = (re.match(".*(\d+).*?", value).group(1))
    except:
        value = 0
    return value

def tag_convert(value):
    try:
        value = [element for element in value if not element.strip().endswith("评论")]  # 去掉评论这个标签 输出结果['职场', ' 7 评论 ', '程序员', '职场']
        # value = dict.fromkeys(value,1)
        # value = list(value.keys())
        # value = ",".join(value)
    except:
        value = "没有标签"
    return value

def return_value(value):#去掉takefirst效果
    return value

class ArticalsoiderItem(scrapy.Item):#重置item用于配置item_loder
    # define the fields for your item here like:
    # name = scrapy.Field()
    title = scrapy.Field()  #output_processor = TakeFirst()
    create_time = scrapy.Field(
        input_processor=MapCompose(time_convert)
    )
    front_image_url = scrapy.Field(
        out_processor=MapCompose(return_value)#image_url需要是一个list，所以要去掉str属性
    )
    front_image_path = scrapy.Field()
    praise_number = scrapy.Field(
        input_processor=MapCompose(praise_convert)
    )
    colloct_number = scrapy.Field(
        input_processor=MapCompose(colloct_convert)
    )
    conmment_number = scrapy.Field(
        input_processor=MapCompose(comment_convert)
    )
    content = scrapy.Field()
    tag = scrapy.Field(
        out_processor=MapCompose(tag_convert)#和前面的不一样，因为tag本身就是一个list所以在经过takefirst进行str以后就会出问题，所以现在要对tag进行还原成list，即重写out_processor
    )
    url_hashcode = scrapy.Field() #得到URL生成的哈希码


import json
import logging

import requests

LOGGER = logging.getLogger(__name__)

class RundeckAPI:
    def __init__(
        self,
        api_version,
        auth_token,
        base_url,
        max_executions_for_request=1000,
    ):
        self.api_version = api_version
        self.auth_token = auth_token
        self.base_url = base_url
        self.max_executions_for_request = max_executions_for_request
        self.url = f'{self.base_url}/{self.api_version}/'

        self.headers = {
            'Content-Type': 'application/json',
            'Accept': 'application/json',
            'X-Rundeck-Auth-Token': self.auth_token,
        }

    def _get(self, url_path, params=None):
        return requests.get(
            f'{self.url}{url_path}', headers=self.headers, params=params,
        )

    def _delete(self, url_path):
        return requests.delete(f'{self.url}{url_path}', headers=self.headers)

    def _post(self, url_path, data):
        return requests.post(
            f'{self.url}{url_path}', data=data, headers=self.headers,
        )

    def get_project_names(self):
        response = self._get(url_path='projects')
        if response.status_code == 200:
            return [project['name'] for project in response.json()]

        LOGGER.error('No project created')
        return []

    def get_project(self, name):
        response = self._get(url_path=f'project/{name}/')
        if response.status_code == 200:
            return response.json()

        LOGGER.error('%s project does not exist', name)
        return ''

    def get_job_ids_for_project(self, project_name):
        params = {'project': project_name}
        response = self._get(url_path='jobs', params=params)

        if response.status_code == 200:
            return [job['id'] for job in response.json()]

        LOGGER.error('No jobs for the %s project found', project_name)
        return []

    def get_executions_for_job(self, job_id):
        response = self._get(url_path=f'job/{job_id}/executions/')

        if response.status_code == 200:
            executions = response.json()['executions']
            return [execution['id'] for execution in executions]

        LOGGER.error('No executions for the %s job found', job_id)
        return []

    def get_executions_for_project(self, project_name, filters=None):
        if filters:
            params = filters

        response = self._get(
            url_path=f'project/{project_name}/executions/', params=params,
        )
        if response.status_code == 200:
            executions = response.json()['executions']
            return [execution['id'] for execution in executions]

        LOGGER.error(
            'No executions for the %s project with the parameters %s found',
            project_name,
            filters,
        )
        return []

    def delete_execution(self, execution_id):
        response = self._delete(url_path=f'execution/{execution_id}')

        return response.status_code

    def bulk_delete_executions(self, execution_ids):
        response = self._post(
            url_path='executions/delete/',
            data=json.dumps({'ids': execution_ids}),
        )

        return response.status_code


from time import sleep

from bs4 import BeautifulSoup

from Base.SmartCSV import smart_csv
from Base.grab import abstract_grab

AOC_MONITOR_HOST = 'http://b2b.aocmonitor.com.cn'

def get_item_info(index):
    uri = '%s/product/%s.html' % (AOC_MONITOR_HOST, index)

    html = abstract_grab(uri, phone_agent=False)

    soup = BeautifulSoup(html, 'html.parser')

    images = soup.find(id='J_Focus').find(class_='carousel-inner').find_all('img')
    image_list = []
    for image in images:
        image_list.append(image['src'])

    table = soup.find(class_='cpBox').find('table')
    items = table.find_all('tr', class_='proParaRow')
    result = dict()
    result['产品ID'] = index

    for item in items:
        key = item.find(class_='proParaName').text
        value = item.find(class_='proParaValue').text
        result[key] = value

    result['图片列表'] = image_list

    return result

def main():
    result_list = []
    for i in range(57):
        try_time = 3
        item_info = None
        while try_time:
            try:
                item_info = get_item_info(i + 1)
                break
            except Exception:
                sleep(3)
                try_time -= 1
                if try_time:
                    print(i + 1, '出错重爬')
                else:
                    print(i + 1, '停止爬取')
        if item_info:
            result_list.append(item_info)
    smart_csv('b2b.aocmonitor.csv', result_list)

main()


import os

from django.conf import settings

from domain.Process import exonstodomain as exd
from domain.Process import process_data as pr
import pandas as pd
import numpy as np
import networkx as nx
from django.urls import reverse

PPI=exd.load_obj("PPI")
DDI=exd.load_obj("DDI")
g2d=exd.load_obj("g2d")

data = pd.read_csv( "domain/data/all_Proteins.csv")

# --- Create folder
# Global jobs table path
jobs_table_path = os.path.join(settings.MEDIA_ROOT, 'jobs', 'tables')
if not os.path.exists(jobs_table_path):
    os.makedirs(jobs_table_path)

# Global jobs networks path
jobs_networks_path = os.path.join(settings.MEDIA_ROOT, 'jobs', 'networks')
if not os.path.exists(jobs_networks_path):
    os.makedirs(jobs_networks_path)

def Construct_network(proteins_id, missing,job_ID):

      E=[]
      N=[]
      affected_nodes=[]
      DDI_nodes=[]
      p1=[]
      p2=[]
      p1_name=[]
      p2_name=[]
      DDIs=[]
      missing_DDIs=[]

      #DDIS with liknks for web display only:
      DDIs2=[]
      missing_DDIs2=[]

      #Score = (retained DDI/All interactions)
      score=[]
      source=[]
      home=reverse('home')
      # Get the subgraph:
      H = PPI.subgraph(proteins_id)
      #print('interactions:',H)
      if len(H.edges())==0: return 0
      for e in H.edges():
          if e[0] not in N: N.append(e[0])
          if e[1] not in N: N.append(e[1])

          gene1=entrez_to_ensembl(e[0])
          gene2=entrez_to_ensembl(e[1])

          #check if gene is in missing dictionary
          #missing{} have all proteins with domain

          #Draw The PPI edge for every case

          #one of the gene have a domain
          if (gene1 in missing) and (gene2 in missing):
              inter=False

              if e[0] in g2d:
                    domains1=g2d[e[0]]
              else:
                  domains1=[]
              if e[1] in g2d:
                  domains2=g2d[e[1]]
              else:
                  domains2=[]

              DDIs_tmp=[]
              missing_DDIs_tmp=[]

              #DDIS with liknks for web display only:

              DDIs_tmp2=[]
              missing_DDIs_tmp2=[]

              for d1 in domains1:
                  for d2 in domains2:
                      if DDI.has_edge(d1,d2):
                           inter=True

                           #Interacted domain is missing
                           if (d1 in missing[gene1]) or (d2 in missing[gene2]):
                                  E.append("{from: '"+e[0]+"', to: '"+e[1]+"', dashes:  true,title:' Lost "+d1+'-'+d2+"', color: 'red'},")
                                  #print('yeaaaaah')
                                  #print(d1,d2)
                                  #print(gene1,gene2)
                                  affected_nodes.append(e[0])
                                  affected_nodes.append(e[1])

                                  missing_DDIs_tmp.append(d1+'-'+d2)
                                  missing_DDIs_tmp2.append(link(d1)+'-'+link(d2))

                           else:
                                  #Interaction retained
                                  E.append("{from: '"+e[0]+"', to: '"+e[1]+"',title:'"+d1+'-'+d2+"',  color: 'red'},")
                                  DDI_nodes.append(e[0])
                                  DDI_nodes.append(e[1])

                                  DDIs_tmp.append(d1+'-'+d2)
                                  DDIs_tmp2.append(link(d1)+'-'+link(d2))

              if inter:
                   # inter=True: a Domain-Domain Interaction was found:
                   #for the table
                   p1.append(e[0])
                   p1_name.append(pr.entrez_to_name(e[0]))
                   p2.append(e[1])
                   p2_name.append(pr.entrez_to_name(e[1]))

                   if DDIs_tmp!=[]:
                      DDIs.append(' ; '.join(DDIs_tmp))
                      DDIs2.append(' ; '.join(DDIs_tmp2))

                   else:
                      DDIs.append('-')
                      DDIs2.append('-')

                   if missing_DDIs_tmp!=[]:
                      missing_DDIs.append(' ; '.join(missing_DDIs_tmp))
                      missing_DDIs2.append(' ; '.join(missing_DDIs_tmp2))

                   else:
                        missing_DDIs.append('-')
                        missing_DDIs2.append('-')

                   score.append(str(float("{0:.2f}".format(len(DDIs_tmp)/(len(DDIs_tmp)+len(missing_DDIs_tmp))))))
                   source.append('PPI-DDI')

              if not inter:
                   E.append("{from: '"+e[0]+"', to: '"+e[1]+"', title:'PPI',  color: CHOOSEN,       smooth: {type: 'continuous'}},")
                   p1.append(e[0])
                   p1_name.append(pr.entrez_to_name(e[0]))
                   p2.append(e[1])
                   p2_name.append(pr.entrez_to_name(e[1]))

                   DDIs.append('-')
                   DDIs2.append('-')

                   missing_DDIs.append('-')
                   missing_DDIs2.append('-')

                   score.append('1.0')
                   source.append('PPI')
          else : #Both genes have no domains
                   E.append("{from: '"+e[0]+"', to: '"+e[1]+"', title:'PPI evidence',  color: CHOOSEN,       smooth: {type: 'continuous'}},")
                   E.append("{from: '"+e[0]+"', to: '"+e[1]+"', title:'PPI',  color: CHOOSEN,       smooth: {type: 'continuous'}},")
                   p1.append(e[0])
                   p1_name.append(pr.entrez_to_name(e[0]))
                   p2.append(e[1])
                   p2_name.append(pr.entrez_to_name(e[1]))

                   DDIs.append('-')
                   DDIs2.append('-')
                   missing_DDIs.append('-')
                   missing_DDIs2.append('-')

                   score.append('1.0')
                   source.append('PPI')
      nodes=[]
      for n in N:
          ensembl=entrez_to_ensembl(n)
          if (n in affected_nodes) and (missing[ensembl]!=[]):
              nodes.append("{id: \""+n+"\",url:  '"+home+"ID/gene/"+ensembl+"' , color: CHOOSEN2, label:  '"+pr.entrez_to_name(n)+"'}, ")

          elif n in DDI_nodes:
              nodes.append("{id: \""+n+"\", url:  '"+home+"ID/gene/"+ensembl+"' , color: CHOOSEN3, label:  '"+pr.entrez_to_name(n)+"'},")

          else:
              nodes.append("{id: \""+n+"\", url:  '"+home+"ID/gene/"+ensembl+"' , label:  '"+pr.entrez_to_name(n)+"'},")

      # Table of interactions
      pd_interaction=pd.DataFrame( data=[p1,p1_name,p2,p2_name,DDIs2,  missing_DDIs2,DDIs,missing_DDIs, score,source ], index=['Protein 1', 'Protein 1 name','Protein 2', 'Protein 2 name', "Retained domain-domain interactions","Missing domain-domain interactions",'Retained DDIs', 'Lost DDIs','Score','Source of the interaction'] )
      pd_interaction=pd_interaction.transpose()
      pd.set_option('display.max_colwidth',1000)

      pd_interaction.drop(columns=["Retained domain-domain interactions","Missing domain-domain interactions"]).to_csv(f'{jobs_table_path}/{job_ID}.csv', index=False)

      pd_html=pd_interaction.drop(columns=['Protein 1', 'Protein 2','Retained DDIs', 'Lost DDIs'])

      pd_html.sort_values(by=['Source of the interaction'], ascending=[True])

      pd_html=pd_html.rename(columns={

          "Score": "Score*",

          })

      # Convert Table to HTML
      pd_html=pd_html.to_html(table_id='results', **settings.TO_HTML_RESPONSIVE_PARAMETERS)

      #Generate the network file (only edges and scores)
      pd_interaction=pd_interaction.rename(columns={
          "Score": "weight",
          })

      pd_interaction['weight']=pd_interaction['weight'].astype(float)
      edges =pd_interaction.drop(columns=['Protein 1 name', 'Protein 2 name','Retained DDIs','Lost DDIs','Source of the interaction',"Retained domain-domain interactions","Missing domain-domain interactions"])
      edges.to_csv(f'{jobs_networks_path}/{job_ID}.sif', index=False,sep='\t')

      edges=edges.rename(columns={"Protein 1": "source",
       "Protein 2": "target",

       })

      weighted_G=nx.from_pandas_edgelist(edges, edge_attr=True)

      #print(weighted_G.edges())
      nx.write_graphml(weighted_G, f'{jobs_networks_path}/{job_ID}.graphml')

      return nodes,E,pd_interaction,pd_html

def analysis_input_isoforms(Inputs):

        filtred=filter_proteins_list(Inputs)
        print('-----------filtred--------------')

        n=len(filtred)
        #Inputs are so many
        if n>2000:
              return False

        else:
                print('-----------yeaaaaaaah--------------')
                gene_domains={}
                #all genes and missing domains
                missing={}
                # all entrex ID of the genes
                proteins_id=[]

                for tr in filtred:
                          #print(tr)
                          gene=tr_to_gene(tr)
                          domains=tr_to_domain(tr)

                          if len(domains)==0:
                              #print(tr,domains)
                              gene_domains[gene]=[]

                          if len(domains)>0:
                              #Check for every domain
                              for domain in domains:

                                      if gene not in gene_domains:

                                          gene_domains[gene]=[domain]

                                      else :
                                          gene_domains[gene].append(domain)
                                          gene_domains[gene]=Remove(gene_domains[gene])

                for tr in filtred:
                    ensembl=tr_to_gene(tr)
                    entrez=str(ensembl_to_entrez(ensembl))
                    proteins_id.append(entrez)
                    if entrez in g2d:
                        domains=g2d[entrez]

                        #all domains are lost for this gene
                        if ensembl not in gene_domains :
                          missing[ensembl]=domains

                        #check if all domains are there
                        else:
                          missing[ensembl]=[]
                          for d in domains:
                              if d not in gene_domains[ensembl]:
                                  missing[ensembl].append(d)

                proteins_id=Remove(proteins_id)

                return proteins_id, missing,n

def analysis_input_genes(Inputs):

    protein_id=[]
    missing={}
    for i in Inputs:
        f=i.replace(" ", "")
        f=f.split(".")[0]

        if len(f)==15 and f[0:4]=='ENSG' :

              #check if gene is coding:
              if  len(data[data['Gene stable ID'].isin([f])])!=0:

                  #print(f)
                  #check PPI status:

                  entrez_id=str(ensembl_to_entrez(f))
                  #print(entrez_id)
                  if PPI.has_node(entrez_id):
                      print('yeaaaaaah')
                      protein_id.append(entrez_id)
                      missing[f]=[]
    print(len(protein_id))
    if len(protein_id)>2000 or len(protein_id)<1:
            return False
    else : return protein_id,missing,0

def pr_to_tr(pr):
    df_filter = data['Protein stable ID'].isin([pr])
    try: return data[df_filter]['Transcript stable ID'].unique()[0]
    except IndexError:
        return False

def tr_to_gene(tr):
    df_filter = data['Transcript stable ID'].isin([tr])
    try: return data[df_filter]['Gene stable ID'].unique()[0]
    except IndexError:
        return False

def ensembl_to_entrez(gene):
    df_filter = data['Gene stable ID'].isin([gene])
    try: return data[df_filter]['NCBI gene ID'].unique()[0]
    except IndexError:
        return False

def entrez_to_ensembl(gene):
    df_filter = data['NCBI gene ID'].isin([gene])
    try: return data[df_filter]['Gene stable ID'].unique()[0]
    except IndexError:
        return False

def tr_to_domain(tr):
    df_filter = data['Transcript stable ID'].isin([tr])
    tdata=data[df_filter]
    tdata=tdata[tdata["Pfam ID"].notna()].drop_duplicates()
    try: return tdata["Pfam ID"].unique()
    except IndexError:
        return False

#add to view
def check_PPI_status(tr):
    df_filter = data['Transcript stable ID'].isin([tr])
    return PPI.has_node(data[df_filter]['NCBI gene ID'].astype('str').unique()[0])

def tr_is_coding(tr):
    return  len(data[data['Transcript stable ID'].isin([tr])])!=0

def filter_proteins_list(List):
    filtred_list=[]
    for tr in List:
        #Correct spaces, dotsof version....
        ftr=tr=tr.replace(" ", "")
        ftr=ftr.split(".")[0]
        ftr=ftr.split("'\'")[0]
        ftr=ftr.split("+")[0]
        #print(ftr)
        #make sure Correct ID
        ftr=ftr[0:15]
        if ftr[0:3]=='ENS':

            #Check if protein coding

            #input is a protein and coverted successfully
            if ftr[3]=='P' :

                tmp= pr_to_tr(ftr)

                # The protein is converted to a transcript
                if tmp!=False:
                        # Check PPI status
                        if check_PPI_status(tmp):
                            filtred_list.append(tmp)

            #check if transcript is a coding protein
            elif ftr[3]=='T'and tr_is_coding(ftr) and check_PPI_status(ftr):
               filtred_list.append(ftr)

    return filtred_list

def Remove(duplicate):
    final_list = []
    for num in duplicate:
        if num not in final_list:
            final_list.append(num)
    return final_list

def link(id):
  return '<a href="http://pfam.xfam.org/family/'+id+'" target="_blank">'+id+' </a>'


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

###
# © 2018 The Board of Trustees of the Leland Stanford Junior University
# Nathaniel Watson
# nathankw@stanford.edu
###

"""
Backports biosamples from the ENCODE Portal. Biosample identifiers on the ENCODE Portal can be
provided in one of three ways:

    1. An input file with one ID per line.
    2. On the command-line.
    3. Via an ENCODE Portal search URL.
"""

import argparse
import pdb

import encode_utils.connection as euc
import pulsarpy_to_encodedcc.backport_from_encode_portal.backport as backport

ENC_CONN = euc.Connection("prod")
# Note that ex_url below has a double '%', where the second is used to escape the original.
ex_url = "https://www.encodeproject.org/search/?type=Biosample&lab.title=Michael+Snyder%%2C+Stanford&award.rfa=ENCODE4&biosample_type=tissue"

def get_parser():
    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawTextHelpFormatter)
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument("-u", "--url", help="""
    A search URL (wrapped in quotation marks to avoid shell expansion, to indicate which biosamples to import, i.e. {}.""".format(ex_url))
    group.add_argument("-i", "--infile", help="File containing biosample record IDs, one per line.")
    group.add_argument("-r", "--rec-ids", nargs="+", help="One or more biosample record identifiers separated by a space.")
    return parser

def main():
    parser = get_parser()
    args = parser.parse_args()
    url = args.url
    infile = args.infile
    rec_ids = args.rec_ids
    inputs = []
    if infile:
        with open(infile) as fh:
            for line in fh:
                line = line.strip()
                if not line or line.startswith("#"):
                    continue
                inputs.append(line)
    elif rec_ids:
        inputs = rec_ids
    else:
        res = ENC_CONN.search(url=url)
        for record in res: # i is a JSON object.
            # GET the object. Only part of the object is given in the search results. For example,
            # the property 'genetic_modifications' isn't present.
            inputs.append(record["@id"])

    for i in inputs:
        print("Backporting Biosample {}".format(i))
        backport.biosample(rec_id=i)

if __name__ == "__main__":
    main()


import unittest
import shutil
import os
import tempfile
import pandas as pd
import numpy as np
import numpy.testing as npt
import pandas.testing as pdt
import bio_test_artifacts.prebuilt as test_prebuilt
from inferelator.workflow import inferelator_workflow
from inferelator.utils import loader

class TestExpressionLoader(unittest.TestCase):

    def setUp(self):

        self.worker = inferelator_workflow()

    def test_tsv(self):

        file, data = test_prebuilt.counts_yeast_single_cell_chr01(filetype='tsv')
        data.to_csv(file, sep="\t")

        self.worker.set_expression_file(tsv=file)
        self.worker.read_expression()

        npt.assert_array_almost_equal(data.values, self.worker.data.expression_data)

    def test_h5ad(self):
        file, data = test_prebuilt.counts_yeast_single_cell_chr01(filetype='h5ad')

        self.worker.set_expression_file(h5ad=file)
        self.worker.read_expression()

        npt.assert_array_almost_equal(data.values, self.worker.data.expression_data)

    def test_hdf5(self):
        file, data = test_prebuilt.counts_yeast_single_cell_chr01(filetype='hdf5')

        self.worker.set_expression_file(hdf5=file)
        self.worker.read_expression()

        npt.assert_array_almost_equal(data.values, self.worker.data.expression_data)

    def test_mtx(self):
        (file1, file2, file3), data = test_prebuilt.counts_yeast_single_cell_chr01(filetype='mtx')

        self.worker.set_expression_file(mtx=file1, mtx_feature=file2, mtx_barcode=file3)
        self.worker.read_expression()

        npt.assert_array_almost_equal(data.values, self.worker.data.expression_data.A)

    def test_10x(self):
        (file1, file2, file3), data = test_prebuilt.counts_yeast_single_cell_chr01(filetype='mtx')

        with tempfile.TemporaryDirectory() as txdir:
            shutil.copy(file1, os.path.join(txdir, "matrix.mtx"))
            shutil.copy(file2, os.path.join(txdir, "genes,tsv"))
            shutil.copy(file3, os.path.join(txdir, "barcodes.tsv"))

            self.worker.set_expression_file(tenx_path=txdir)
            self.worker.read_expression()

            npt.assert_array_almost_equal(data.values, self.worker.data.expression_data.A)

    def test_10x_ranger3(self):
        (file1, file2, file3), data = test_prebuilt.counts_yeast_single_cell_chr01(filetype='mtx', gzip=True)

        with tempfile.TemporaryDirectory() as txdir:
            shutil.copy(file1, os.path.join(txdir, "matrix.mtx.gz"))
            shutil.copy(file2, os.path.join(txdir, "features.tsv.gz"))
            shutil.copy(file3, os.path.join(txdir, "barcodes.tsv.gz"))

            self.worker.set_expression_file(tenx_path=txdir)
            self.worker.read_expression()

            npt.assert_array_almost_equal(data.values, self.worker.data.expression_data.A)

    def test_df_decode(self):
        idx = pd.Index(['str1', b'str2', b'str3', 'str4', 5, 17.4, np.inf, ('str1',)])
        correct = pd.Index(['str1', 'str2', 'str3', 'str4', 5, 17.4, np.inf, ('str1',)])

        df1 = pd.DataFrame(idx.tolist(), index=idx)
        df1_c = pd.DataFrame(correct.tolist(), index=correct)

        loader._safe_dataframe_decoder(df1)
        pdt.assert_frame_equal(df1, df1_c)

        vals = np.random.rand(len(idx), len(idx))
        df2 = pd.DataFrame(vals, index=idx, columns=idx)
        df2_c = pd.DataFrame(vals, index=correct, columns=correct)

        loader._safe_dataframe_decoder(df2)

        pdt.assert_frame_equal(df2, df2_c)

        df3 = pd.DataFrame(pd.Categorical(idx.tolist()), index=idx)
        df3_c = pd.DataFrame(pd.Categorical(correct.tolist()), index=correct)

        loader._safe_dataframe_decoder(df3)

        pdt.assert_frame_equal(df3, df3_c)


import ast
import random
import time
import pika
import requests
from flask import Flask
from flask_restful import Api

app = Flask(__name__)
api = Api(app)
app.config['DEBUG'] = True

connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
channel = connection.channel()
channel.queue_declare(queue='wotnot', durable=True)

def callback(ch, method, properties, body):
    data = body.decode("utf-8")
    data = ast.literal_eval(data)
    if data['random'] % 10 == 0:
        r = random.randint(0, 60)
        time.sleep(4)
        if r % 10 == 0:
            data['category'] = "Failed"
        else:
            data['category'] = "Retried"
    payload = {
        "data": [
            {
                "id": data['id'],
                "message": data['message'],
                "count": data['count'],
                "category": data['category']
            }
        ]
    }
    response = requests.post("http://127.0.0.1:3000/tracker/insert", json=payload)
    print(data)
    print(response.json())
    print(response.status_code)

    ch.basic_ack(delivery_tag=method.delivery_tag)

channel.basic_consume(queue='wotnot', on_message_callback=callback)
channel.start_consuming()


from collections import deque
import sys

input = sys.stdin.readline

N = int(input())
M = int(input())
costs = [int(1e9)] * (N + 1)

buses = [[] for _ in range(N + 1)]
for _ in range(M):
    a, b, c = map(int, input().split())
    buses[a].append([b, c])

start, end = map(int, input().split())

def dijkstra(start):
    queue = deque()
    queue.append((0, start))
    costs[start] = 0
    while queue:
        cost, now = queue.pop()
        if costs[now] < cost:
            continue
        for bus in buses[now]:
            temp = cost + bus[1]
            if temp < costs[bus[0]]:
                costs[bus[0]] = temp
                queue.append((temp, bus[0]))

dijkstra(start)
print(costs[end])

백준/Gold/1916. 최소비용 구하기/최소비용 구하기.py


import numpy as np
import matplotlib.pyplot as plt
from scipy.integrate import odeint
from scipy.optimize import curve_fit
from scipy.signal import find_peaks
from harmonic_analysis import amplitude_sweep, fourier_func

'''
Solving the differential equations describing a Gaussian perturbation to a
two-level system. By varying the perturbation frequency and measuring the state at time
'''

#%%
'''
This cell scans for various standard deviations, amplitudes or means to find
the largest peak for each. Scans one at a time, depending on what is and isn't
commented.
'''
if __name__ == "__main__":
    plt.close()

    omega_fi = 0.2           # frequency between states
    std = 2.5
    t0 = 12.8
    num_points = 10000      # number of time array points
    num_measurements = 1024
    frequency = omega_fi    # resonance
    initial_conditions = [1,0,0,0]

    # arrays
    N = 1000
    amp_array = np.linspace(0.1, 5, N)
    t_array = np.linspace(0, 4*t0, num_points)
    state_array = np.zeros(N)

    probability_averages = amplitude_sweep(omega_fi, std, t0, frequency, t_array, amp_array, initial_conditions)

    ### vectorised?
    # rand_nums = np.random.rand(N)
    # state_array = np.where(rand_nums >= probability_averages, 1, 0)
    # state = np.sum(state_array)

    for i, probability in enumerate(probability_averages):
        state = 0
        for n in range(num_measurements):
            rand_num = np.random.rand()
            if rand_num <= probability:
                state += 1
            else:
                pass
        state = state/num_measurements
        state_array[i] = state
    state_array -= np.mean(state_array)

    # fourier transform
    power, freq = fourier_func(state_array, np.max(t_array))
    sine_freq = freq[np.argmax(power)]
    print("Signal frequency is "+str(sine_freq))

    linestyles = ["#6edbfa","#82a6bd","#957080","#a93b42","#bc0505"]
    labels = []

    # figure for plotting the signal strength
    fig_signal = plt.figure(figsize=(8,6))
    ax_signal = fig_signal.gca()
    ax_signal.set_xlabel("Amplitude", fontsize=16)
    ax_signal.set_ylabel("Signal Strength", fontsize=16)
    ax_signal.set_title("Average Signal Strength over "+str(num_measurements)+" measurements", fontsize=20)
    # ax_signal.set_ylim(0, 1)

    ax_signal.plot(amp_array, state_array, "b")
    # ax_signal.plot(amp_array, np.sin(amp_array + 0.1))

    fig_fourier = plt.figure(figsize=(8,6))
    ax_fourier = fig_fourier.gca()
    ax_fourier.set_xlabel("Signal Frequency $t$", fontsize=16)
    ax_fourier.set_ylabel("Power", fontsize=16)
    ax_fourier.set_title("Frequency Spectrum of Signal", fontsize=20)
    ax_fourier.plot(freq, power, "ko")
#


from posts.models import Comment, Follow, Group, Post, User
from rest_framework import serializers
from rest_framework.relations import SlugRelatedField

class GroupSerializer(serializers.ModelSerializer):
    class Meta:
        model = Group
        fields = '__all__'

class PostSerializer(serializers.ModelSerializer):
    author = SlugRelatedField(slug_field='username', read_only=True)

    class Meta:
        fields = '__all__'
        model = Post

class CommentSerializer(serializers.ModelSerializer):
    author = serializers.SlugRelatedField(
        read_only=True, slug_field='username'
    )

    class Meta:
        fields = '__all__'
        model = Comment
        read_only_fields = ('post',)

class FollowSerializer(serializers.ModelSerializer):
    user = serializers.SlugRelatedField(
        read_only=True,
        slug_field='username',
        default=serializers.CurrentUserDefault()
    )
    following = serializers.SlugRelatedField(
        queryset=User.objects.all(),
        slug_field='username'
    )

    def validate(self, data):
        if self.context['request'].user == data['following']:
            raise serializers.ValidationError(
                'Вы пытаетесь подписаться на себя')
        return data

    class Meta:
        fields = ('user', 'following')
        model = Follow


"""season calendar

Revision ID: 14883a5dd702
Revises: e0ab6f1a73be
Create Date: 2019-09-25 15:18:04.337300

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '14883a5dd702'
down_revision = 'e0ab6f1a73be'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    with op.batch_alter_table('season', schema=None) as batch_op:
        batch_op.add_column(sa.Column('calendar_link', sa.String(length=512), nullable=True))

    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    with op.batch_alter_table('season', schema=None) as batch_op:
        batch_op.drop_column('calendar_link')

    # ### end Alembic commands ###


from tkinter import *
from tkinter import ttk
import requests
from dotenv import dotenv_values
import datetime

root = Tk()
root.title('Football result app by Nassim Bouziane')
root.geometry('500x500')

year = datetime.datetime.today().year - 1
lst_Years = list(range(year, year - 13, -1)) # Getting a list of year from todays year to 13years ago

config = dotenv_values('.env') # getting api key from .env

frm = ttk.Frame(root, padding=0)
frm.pack()

# Define Leagues Tuple
Leagues= ('Premier League','Liga','Ligue 1','Serie A','Liga Nos')

# Function to print the index of selected option in Combobox
def callback(*arg):
   label['text'] = 'Result of '+var.get()

   #https://v3.football.api-sports.io/leagues to get leagues
   league = 0
   if var.get() == 'Premier League':
    league = 39
   elif var.get() == 'Liga':
        league = 140
   elif var.get() == 'Ligue 1':
        league = 61
   elif var.get() == 'Liga Nos':
        league = 94
   elif var.get() == 'Serie A':
        league = 135

   if league == 94:
    if(int(varYears.get()) >= 2014):
        len = 18
    elif int(varYears.get()) < 2014:
        len = 16
   elif league != 94:
    len = 20

   headers = {'x-rapidapi-key': config['API_KEY'] }
   r = requests.get('https://v3.football.api-sports.io/standings?season='+varYears.get()+'&league='+str(league),headers=headers)
   standing = r.json()['response'][0]['league']['standings'][0] # from standing retrieve rank, team name, ponts, goaldiff form, description played win draw lose
   list = []

   for i in range(len):
      list.append(str(standing[i]['rank'])+' '+ standing[i]['team']['name'] +' ' + str(standing[i]['points']) + ' ' + str(standing[i]['goalsDiff'])
      + ' ' + standing[i]['form']+ ' ' + str(standing[i]['description'])+ ' ' + str(standing[i]['all']['played'])+ ' ' + str(standing[i]['all']['win'])
      + ' ' + str(standing[i]['all']['draw'])+ ' ' + str(standing[i]['all']['lose']))

   labelStanding['text'] = "\n.".join(list)
   labelStanding['font'] = ('Times'),13
   labelStanding['borderwidth'] = '2'
   labelStanding['relief'] = 'solid'
   labelStanding['justify'] ='left'

#Canvas :
canvas = Canvas(root)
scrollbar = ttk.Scrollbar(root, orient="vertical", command=canvas.yview)
scrollable_frame = ttk.Frame(canvas)
scrollable_frame.bind(
    "<Configure>",
    lambda e: canvas.configure(
        scrollregion=canvas.bbox("all")
    )
)
canvas.create_window((0, 0), window=scrollable_frame, anchor="nw")
canvas.configure(yscrollcommand=scrollbar.set,borderwidth=0,highlightthickness=0)
canvas.pack(side="left", fill="both", expand=True)
scrollbar.pack(side="right", fill="y")

# Create a combobox widget
var = StringVar() # the var we are tracing
var.set('Premier League')

varYears = StringVar()
varYears.set(datetime.datetime.today().year - 1)

cb= ttk.Combobox(scrollable_frame, textvariable= var)
cb['values']= Leagues
cb['state']= 'readonly'
cb.pack(fill='x',padx= 5, pady=5)

label = Label(scrollable_frame, text='Result of '+var.get())
label.pack()

cbYear = ttk.Combobox(scrollable_frame,textvariable=varYears)
cbYear['values'] = lst_Years
cbYear['state'] = 'readonly'
cbYear.pack(fill='x', padx='10',pady='5')

labelStanding = Label(scrollable_frame)
labelStanding.pack()

# Set the tracing for the given variable
var.trace('w', callback)
varYears.trace('w', callback)

root.mainloop()


from django.core.exceptions import ObjectDoesNotExist
from django.db.utils import DatabaseError
from django.test import TestCase
from unittest.mock import patch

from base.tests.factories.learning_unit_year import LearningUnitYearFactory
from base.tests.factories.group_element_year import GroupElementYearFactory
from osis_common.models.auditable_model import AuditableModel

class AuditableModelTest(TestCase):
    def setUp(self):
        pass

    def test_auditable_model_delete(self):
        luy = LearningUnitYearFactory()
        geys = [GroupElementYearFactory(child_leaf=luy) for i in range(20)]

        # First delete only a group
        result = geys[0].delete()
        self.assertTrue(result)

        self.assertEqual(result, [geys[0]])
        self.is_not_existing_for_orm(geys[0])

        # Now delete all the data with the learning unit
        luy.delete()
        self.is_not_existing_for_orm(luy)
        for gey in geys:
            self.is_not_existing_for_orm(gey)

    def test_auditable_model_delete_with_database_error(self):
        luy = LearningUnitYearFactory()
        geys = [GroupElementYearFactory(child_leaf=luy) for i in range(20)]

        result = None
        with patch.object(AuditableModel, 'save') as mock_method:
            mock_method.side_effect = DatabaseError("test error")

            try:
                luy.delete()
            except DatabaseError as e:
                result = e

        self.assertIsInstance(result, DatabaseError)
        self.is_existing_for_orm(luy)
        for gey in geys:
            self.is_existing_for_orm(gey)

    def is_existing_for_orm(self, obj):
        self.assertEqual(obj.__class__.objects.get(id=obj.id), obj)

    def is_not_existing_for_orm(self, obj):
        with self.assertRaises(ObjectDoesNotExist):
            obj.__class__.objects.get(id=obj.id)