'''
v6
Uses datagen.flow_from_dataframe instead of datagen.flow_from_directory
v5
This version provides support to Resnet, and training some backend model layers
From:
https://www.apriorit.com/dev-blog/647-ai-applying-deep-learning-to-classify-skin-cancer-types
It does not use .h5 but the new format to save models.
v4
Code that saves all training history in a pickle file.
Saves AUC, ROC, etc as part of the history, in this pickle file.
'''
#tutorial in part from https://wandb.ai/sayakpaul/efficientnet-tl/reports/Transfer-Learning-With-the-EfficientNet-Family-of-Models--Vmlldzo4OTg1Nw

from tensorflow.keras.models import Sequential, load_model
from tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Input, Dropout, GlobalAveragePooling2D
import tensorflow as tf
from tensorflow.keras.optimizers import Adam
import tensorflow_hub as hub
from tensorflow.keras.preprocessing.image import ImageDataGenerator
import matplotlib.pyplot as plt
import numpy as np
import sklearn.metrics
from sklearn.metrics import confusion_matrix, roc_curve, auc, recall_score, f1_score, precision_score, precision_recall_curve
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
import seaborn as sn
import os
import sys
import shutil
import pickle
import argparse
import pandas as pd
from tensorflow.keras.applications.resnet import ResNet152, preprocess_input
from tensorflow.keras.callbacks import TensorBoard

#To avoid the warning in
#https://github.com/tensorflow/tensorflow/issues/47554
from absl import logging
logging.set_verbosity(logging.ERROR)

def get_training_model_xception(url, trainable=False):
    #https://www.analyticsvidhya.com/blog/2022/04/binary-classification-on-skin-cancer-dataset-using-dl/
    model = Sequential()
    base = Xception(include_top=False, weights="imagenet", input_shape=(200,200,3), pooling='avg')
    for lay in base.layers: lay.trainable = True # false for transfer learning
    model.add(base)
    model.add(Dropout(0.5))
    model.add(Flatten())
    model.add(Dense(10,'relu'))
    model.add(Dropout(0.5))
    model.add(Dense(1,'sigmoid'))
    return model

def get_training_model_resnet(trainable=False):
    # from https://www.apriorit.com/dev-blog/647-ai-applying-deep-learning-to-classify-skin-cancer-types
    #Download data from https://storage.googleapis.com/tensorflow/keras-applications/resnet/resnet152_weights_tf_dim_ordering_tf_kernels_notop.h5
    base_model = ResNet152(weights='imagenet', include_top=False)
    x = base_model.output
    x = GlobalAveragePooling2D()(x)
    #x = Dense(1000, activation='relu')(x)
    x = Dense(100, activation='relu')(x)
    x = Dropout(0.25)(x)
    predictions = Dense(1, activation='sigmoid')(x)

    for i, layer in enumerate(base_model.layers):
        print(i, layer.name)

    if True:
        # freeze all layers
        for layer in base_model.layers:
            layer.trainable = False
    else:
        # freeze up to given layer
        layer_num = 483
        for layer in base_model.layers[:layer_num]:
            layer.trainable = False
        for layer in base_model.layers[layer_num:]:
            layer.trainable = True

    model = Model(inputs=base_model.input, outputs=predictions)

    return model

def get_training_model_effnet(url, trainable=False):
    # Load the respective EfficientNet model but exclude the classification layers
    extractor = hub.KerasLayer(url, input_shape=(img_size, img_size, 3), trainable=trainable)

    # Construct the head of the model that will be placed on top of the
    # the base model
    model = tf.keras.models.Sequential([
        extractor,
        tf.keras.layers.Dropout(0.8),
        tf.keras.layers.Dense(200, activation="relu"),
        tf.keras.layers.BatchNormalization(),
        #tf.keras.layers.Dense(80, activation="relu"),
        #tf.keras.layers.BatchNormalization(),
        #tf.keras.layers.Dropout(0.5),
        #tf.keras.layers.Dense(60, activation="relu"),
        #tf.keras.layers.BatchNormalization(),
        #tf.keras.layers.Dropout(0.5),
        tf.keras.layers.Dense(1, activation="sigmoid")
    ])

    #layers = model.weights
    #for i in range( len(layers) ):
    #    print(i, layers[i].name)

    return model

def get_training_model_fixed():

    img_width = 240
    img_height = 240
    # Define the input shape of the images
    input_shape = (img_width, img_height, 3)

    num_output_neurons = 1

    # Define the CNN model
    model = Sequential()
    if True:
        model.add(Conv2D(60, (8,8), input_shape=input_shape, activation='swish'))
        model.add(MaxPooling2D((2,2)))
        model.add(Dropout(0.4))
        model.add(Conv2D(53, (5,5), activation='swish'))
        model.add(MaxPooling2D((2,2)))
        model.add(Dropout(0.4))
        model.add(Flatten())
        model.add(Dense(1, activation='sigmoid'))

    return model

def decrease_num_negatives(df, desired_num_negative_examples):
    '''
    Create dataframe with desired_num_rows rows from df
    '''
    shuffled_df = df.sample(frac=1).reset_index(drop=True)
    neg_examples = shuffled_df[shuffled_df['target'] == '0'].copy()
    neg_examples = neg_examples.head( round(desired_num_negative_examples) ).copy()

    pos_examples = shuffled_df[shuffled_df['target'] == '1'].copy()
    newdf = pd.concat([neg_examples, pos_examples], ignore_index=True)
    newdf = newdf.sample(frac=1).reset_index(drop=True) #shuffle again
    return newdf

def get_balanced_dataframe(df, desired_num_negative_examples, desired_num_positive_examples):
    '''
    Create dataframe with desired_num_rows rows from df
    '''
    shuffled_df = df.sample(frac=1).reset_index(drop=True)
    neg_examples = shuffled_df[shuffled_df['target'] == '0'].copy()
    neg_examples = neg_examples.head( round(desired_num_negative_examples) ).copy()

    pos_examples = shuffled_df[shuffled_df['target'] == '1'].copy()
    pos_examples = pos_examples.head( round(desired_num_positive_examples) ).copy()

    newdf = pd.concat([neg_examples, pos_examples], ignore_index=True)
    newdf = newdf.sample(frac=1).reset_index(drop=True) #shuffle again
    return newdf

if __name__ == '__main__':
    print("=====================================")
    print("Train NN classifier")

    parser = argparse.ArgumentParser()
    #required arguments
    parser.add_argument('--num_desired_train_examples',type=int,help='Desired number of training examples (for both classes 0 and 1, assuming there are 324 examples of 1)',required=True)

    args = parser.parse_args()
    num_desired_train_examples = args.num_desired_train_examples

    plt.close("all")

    #%% CHECK ALL VARIABLES AND REPLACE VALUES AS NEEDED
    simulation_ID = 22 # model from Optuna, but using AUC as loss instead of accuracy
    #--------------------------------------------------------------------------
    #num_desired_train_examples = 40
    img_size = 240
    #model_url = 'https://tfhub.dev/google/efficientnet/b1/feature-vector/1' #non-trainable
    #model_url = 'https://tfhub.dev/google/efficientnet/b1/classification/1'

    #From https://colab.research.google.com/github/tensorflow/hub/blob/master/examples/colab/tf2_image_retraining.ipynb?hl=vi#scrollTo=FlsEcKVeuCnf
    model_url = 'https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet1k_b1/feature_vector/2'
    use_augment = False
    type_augment = "Albumentations"
    test_id = str(simulation_ID) #test number
    data_id = "unbal" #ratio
    effnet_model = str(1)
    #--------------------------------------------------------------------------

    # Define the folders for train, validation, and test data
    #Laptop:
    #train_folder = 'D:/temp/test_50percent_teste4/train_50percent_teste4/'
    #validation_folder = 'D:/temp/test_50percent_teste4/val_50percent_teste4/'
    #test_folder = 'D:/temp/test_50percent_teste4/test_50percent_teste4'

    # Define the folders for train, validation, and test data
    #PC:
    train_folder = 'D:/sofia/ufpa/tcc/dataset_updated/train'
    validation_folder = 'D:/sofia/ufpa/tcc/dataset_updated/val'
    test_folder = 'D:/sofia/ufpa/tcc/dataset_updated/test'

    train_csv = 'D:/sofia/ufpa/tcc/train_data.csv'
    test_csv = 'D:/sofia/ufpa/tcc/test_data.csv'
    validation_csv = 'D:/sofia/ufpa/tcc/val_data.csv'

    if False:
        print(traindf.info())
        print(traindf.head())
        print(traindf.describe())
        print(traindf.value_counts())

    if True:
        #do not remove header
        traindf=pd.read_csv(train_csv,dtype=str)
        testdf=pd.read_csv(test_csv,dtype=str)
        validationdf=pd.read_csv(validation_csv,dtype=str)
    else:
        #remove header
        traindf=pd.read_csv(train_csv,dtype=str, header=None)
        testdf=pd.read_csv(test_csv,dtype=str, header=None)
        validationdf=pd.read_csv(validation_csv,dtype=str, header=None)

    '''
    D:/sofia/ufpa/tcc/data_intofolders
    Train NN classifier
    train:
    target
    1    324
    0    18227 = 18551-324
    Name: count, dtype: int64
    test:
    target
    0    9754
    1     184
    Name: count, dtype: int64
    validation:
    target
    0    4561
    1      76
    Name: count, dtype: int64
    '''

    if num_desired_train_examples >= 324:
        #the 324 positive examples will always be part of the training set
        num_desired_negative_train_examples = num_desired_train_examples - 324
        traindf = decrease_num_negatives(traindf, num_desired_negative_train_examples)
    else:
        num_desired_negative_train_examples = np.floor(num_desired_train_examples / 2.0)
        desired_num_positive_examples = num_desired_train_examples - num_desired_negative_train_examples
        traindf = get_balanced_dataframe(traindf, num_desired_negative_train_examples, desired_num_positive_examples)

    testdf = decrease_num_negatives(testdf, 184)
    validationdf = decrease_num_negatives(validationdf, 76)

    print('train:')
    print(traindf['target'].value_counts())
    print('test:')
    print(testdf['target'].value_counts())
    print('validation:')
    print(validationdf['target'].value_counts())

    # Define the image size and other parameters
    image_size = (img_size, img_size)
    batch_size = 12
    epochs = 100
    num_classes = 2

    if use_augment:
        title_id = 'Augment - ' + type_augment + ' - test: ' + test_id + ' -' + ' ratio: ' + data_id
        base_name = 'effnetb' + effnet_model + '_augment_testeC' + test_id + '_' + data_id + '_tr_ex_' + str(num_desired_train_examples)
    else:
        title_id = 'No augment - ' + 'test: ' + test_id + ' -' + ' ratio: ' + data_id
        base_name = 'effnetb' + effnet_model + '_no_augment_testeC' + test_id + '_' + data_id + '_tr_ex_' + str(num_desired_train_examples)

    #Important: output folder
    output_dir = os.path.join('../outputs/unbalanced/id_' + str(simulation_ID), base_name)

    if not os.path.exists(output_dir):
        os.makedirs(output_dir)
        print("Created folder ", output_dir)

    #%%
    #copy script
    copied_script = os.path.join(output_dir, os.path.basename(sys.argv[0]))
    shutil.copy2(sys.argv[0], copied_script)
    print("Just copied current script as file", copied_script)

    #model = get_training_model_effnet(model_url, trainable=False)
    # model = get_training_model_resnet(trainable=False)
    model = get_training_model_fixed()
    print("just got the model")

    model.compile(loss='binary_crossentropy',
                  optimizer=Adam(learning_rate=0.005),
                  metrics=["accuracy", tf.keras.metrics.AUC()])
                  #metrics=["accuracy"])

    model.summary()

    train_datagen = ImageDataGenerator(rescale=1./255)

    train_generator = train_datagen.flow_from_dataframe(
            dataframe = traindf,
            directory = train_folder,
            x_col="image_name",
            y_col="target",
            target_size=image_size,
            batch_size=batch_size,
            class_mode='binary',
            shuffle=True
        )

    # Loading and preprocessing the training, validation, and test data
    validation_datagen = ImageDataGenerator(rescale=1./255)
    test_datagen = ImageDataGenerator(rescale=1./255)

    validation_generator = validation_datagen.flow_from_dataframe(
            dataframe = validationdf,
            directory = validation_folder,
            x_col="image_name",
            y_col="target",
            target_size=image_size,
            batch_size=batch_size,
            class_mode='binary',
            shuffle=True
    )

    test_generator = test_datagen.flow_from_dataframe(
            dataframe = testdf,
            directory = test_folder,
            x_col="image_name",
            y_col="target",
            target_size=image_size,
            batch_size=batch_size,
            class_mode='binary',
            shuffle=True
    )

    # Count effective number of examples, to make sure
    #print(train_generator.classes)  # numpy array with all labels
    #actual_num_desired_train_examples = train_generator.classes.shape[0]  #flow from directory
    actual_num_desired_train_examples = train_generator.n #flow from dataframe
    print("actual_num_desired_train_examples=", actual_num_desired_train_examples)
    if actual_num_desired_train_examples != num_desired_train_examples:
        print("Updating num_desired_train_examples to", actual_num_desired_train_examples)
        num_desired_train_examples = actual_num_desired_train_examples
        # update base_name
        base_name = 'effnetb' + effnet_model + '_no_augment_testeC' + test_id + '_' + data_id + '_tr_ex_' + str(num_desired_train_examples)

    # Define the EarlyStopping callback
    metric_to_monitor = 'val_auc' #'val_accuracy'
    metric_mode = 'max'
    early_stopping = EarlyStopping(monitor=metric_to_monitor, patience=10, mode=metric_mode)
    #early_stopping = EarlyStopping(monitor='val_auc', patience=5)

    #look at https://www.tensorflow.org/guide/keras/serialization_and_saving
    #do not use HDF5 (.h5 extension)
    #best_model_name = 'best_model_' + base_name + '.h5'
    best_model_name = 'best_model_' + base_name
    best_model_name = os.path.join(output_dir, best_model_name)
    mcp_save = ModelCheckpoint(best_model_name, save_best_only=True, monitor=metric_to_monitor, mode=metric_mode)
    reduce_lr_loss = ReduceLROnPlateau(monitor=metric_to_monitor, factor=0.5, patience=3, verbose=1, min_delta=1e-5, mode=metric_mode)
    # Define Tensorboard as a Keras callback
    tensorboard = TensorBoard(
    log_dir= os.path.join('.\logsunb', base_name),
    #log_dir= '.\logs',
    histogram_freq=1,
    write_images=True
    )

    # Training the model
    history = model.fit(
        train_generator,
        steps_per_epoch=train_generator.samples // batch_size,
        epochs=epochs,
        validation_data=validation_generator,
        callbacks=[early_stopping, mcp_save, reduce_lr_loss, tensorboard]
    )

    # Save the last model
    #look at https://www.tensorflow.org/guide/keras/serialization_and_saving
    #do not use HDF5 (.h5 extension)
    #last_model_name = 'last_model_' + base_name + '.h5'
    last_model_name = 'last_model_' + base_name
    last_model_name = os.path.join(output_dir, last_model_name)
    print("Saving ", last_model_name, "...")
    model.save(last_model_name)

    # Read the best model
    # add custom_objects according to https://stackoverflow.com/questions/61814614/unknown-layer-keraslayer-when-i-try-to-load-model
    model = load_model(best_model_name, custom_objects={'KerasLayer':hub.KerasLayer})

    # Evaluating the best model on the test set
    test_loss, test_accuracy, test_auc = model.evaluate(test_generator)
    print('Test loss:', test_loss)
    print('Test accuracy:', test_accuracy)
    print('Test AUC:', test_auc)

    # Generate predictions --> labels: predicted and true
    predictions = model.predict(test_generator)

    # defining the metrics
    train_loss = history.history['loss']
    val_loss = history.history['val_loss']
    train_acc = history.history['accuracy']
    val_acc = history.history['val_accuracy']

    # Plot the metrics over epochs
    plt.plot(train_loss, label='Training Loss')
    plt.plot(val_loss, label='Validation Loss')
    plt.plot(train_acc, label='Training Accuracy')
    plt.plot(val_acc, label='Validation Accuracy')
    plt.xlabel('Epoch')
    plt.ylabel('Metric')
    plt.legend()
    #plt.show()

    metrics_name = 'metrics_' + base_name + '.png'
    plt.title('Metrics - ' + title_id)
    plt.savefig(os.path.join(output_dir, metrics_name))

    # Convert predictions into values 0 or 1
    N = len(predictions)
    pred_labels = np.zeros( (N,) )
    my_threshold = 0.5 # assume a threshold given predictions are in range [0, 1]
    for i in range(N):
        if predictions[i] > my_threshold:
            pred_labels[i] = 1
        else:
            pred_labels[i] = 0
    true_labels = test_generator.classes

    true_labels = test_generator.classes

    fpr, tpr, thresholds = roc_curve(true_labels, predictions, pos_label=1)
    auc = auc(fpr, tpr)
    print('AUC:', auc)

    recall = recall_score(true_labels, pred_labels)
    f1 = f1_score(true_labels, pred_labels)
    precis = precision_score(true_labels, pred_labels)

    pr_precision, pr_recall, pr_thresholds = precision_recall_curve(true_labels, predictions, pos_label=1)

    print('Recall: ', recall)
    print('Precision: ', precis)
    print('F1-score: ', f1)

    # Compute confusion matrix
    cm = confusion_matrix(true_labels, pred_labels)

    # Plot confusion matrix with Seaborn
    plt.figure()
    sn.heatmap(cm, annot=True, fmt='d')
    plt.title('Confusion matrix - ' + title_id)
    plt.xlabel('Predicted label')
    plt.ylabel('True label')
    #plt.show()

    #save the cm
    cm_filename = 'cm_' + base_name + '.png'
    plt.savefig(os.path.join(output_dir, cm_filename))

    # add to history
    history.history['num_desired_train_examples'] = num_desired_train_examples
    history.history['test_accuracy'] = test_accuracy
    history.history['test_loss'] = test_loss
    history.history['test_confusion_matrix'] = cm
    history.history['recall'] = recall
    history.history['f1'] = f1
    history.history['precis'] = precis
    history.history['auc'] = auc
    history.history['roc_fpr'] = fpr
    history.history['roc_tpr'] = tpr
    history.history['roc_thresholds'] = thresholds
    history.history['pr_precision'] = pr_precision
    history.history['pr_recall'] = pr_recall
    history.history['pr_thresholds'] = pr_thresholds

    # https://stackoverflow.com/questions/41061457/keras-how-to-save-the-training-history-attribute-of-the-history-object
    pickle_file_path = os.path.join(output_dir, 'trainHistoryDict.pickle')
    with open(pickle_file_path, 'wb') as file_pi:
        pickle.dump(history.history, file_pi)

    # Write accuracies and losses to a text file
    file_name = 'classification_output_' + base_name + '.txt'
    with open(os.path.join(output_dir, file_name), 'w') as f:
        f.write('validation_acc\n')
        f.write(str(val_acc))
        f.write('\ntrain_acc\n')
        f.write(str(train_acc))
        f.write('\nvalidation_loss\n')
        f.write(str(val_loss))
        f.write('\ntrain_loss\n')
        f.write(str(train_loss))
        f.write('\ntest_acc\n')
        f.write(str(test_accuracy))
        f.write('\nnum_desired_train_examples\n')
        f.write(str(num_desired_train_examples))


from typing import Any
from PyQt5.QtGui import QIcon, QPixmap
from PyQt5.QtCore import Qt, QRect
from PyQt5.QtWidgets import QApplication, QMainWindow, QWidget, QGridLayout, QVBoxLayout, QFrame, QGraphicsScene, QGraphicsView, QScrollArea, QHBoxLayout, QPushButton, QSpacerItem, QSizePolicy, QLabel, QLineEdit, QComboBox, QSpinBox, QSlider
import rospkg
import os
from FlatlandBodyEditor import *
from ArenaScenario import *
from QtExtensions import *

class PedsimAgentEditor(QWidget):
    def __init__(self, pedsimAgentWidget, **kwargs):
        super().__init__(**kwargs)
        self.pedsimAgent = pedsimAgentWidget.pedsimAgent
        self.pedsimAgentWidget = pedsimAgentWidget
        self.tempFlatlandModel = FlatlandModel()
        self.setup_ui()
        self.updateValuesFromPedsimAgent()
        self.updateWidgetsFromSelectedType()

    def setup_ui(self):
        self.setWindowTitle("Pedsim Agent Editor")
        self.setLayout(QtWidgets.QGridLayout())
        self.setWindowModality(QtCore.Qt.WindowModality.ApplicationModal)
        self.resize(600, 600)

        # scrollarea
        self.scrollArea = QScrollArea(self)
        self.scrollArea.setWidgetResizable(True)
        self.scrollArea.setMinimumWidth(400)
        self.layout().addWidget(self.scrollArea, 0, 0, 1, -1)
        # frame
        self.scrollAreaFrame = QFrame()
        self.scrollAreaFrame.setLayout(QGridLayout())
        self.scrollAreaFrame.setSizePolicy(QSizePolicy.Policy.Maximum, QSizePolicy.Policy.Maximum)
        self.scrollAreaFrame.setMinimumWidth(400)
        self.scrollArea.setWidget(self.scrollAreaFrame)

        # heading "general"
        general_label = QLabel("#### General")
        general_label.setTextFormat(QtCore.Qt.TextFormat.MarkdownText)
        self.scrollAreaFrame.layout().addWidget(general_label, 0, 0, QtCore.Qt.AlignmentFlag.AlignLeft)
        line = Line()
        self.scrollAreaFrame.layout().addWidget(line, 0, 1, QtCore.Qt.AlignmentFlag.AlignRight)

        # name
        ## label
        name_label = QLabel("Name")
        name_label.setTextFormat(QtCore.Qt.TextFormat.MarkdownText)
        self.scrollAreaFrame.layout().addWidget(name_label, 1, 0, QtCore.Qt.AlignmentFlag.AlignLeft)
        ## editbox
        self.name_edit = QLineEdit(self.pedsimAgentWidget.name_label.text())
        self.name_edit.setFixedSize(200, 30)
        self.scrollAreaFrame.layout().addWidget(self.name_edit, 1, 1, QtCore.Qt.AlignmentFlag.AlignRight)

        # model file
        ## label
        name_label = QLabel("Flatland Model")
        name_label.setTextFormat(QtCore.Qt.TextFormat.MarkdownText)
        self.scrollAreaFrame.layout().addWidget(name_label, 2, 0, QtCore.Qt.AlignmentFlag.AlignLeft)
        ## choose button
        self.modelButton = QPushButton("Choose...")
        self.modelButton.setFixedSize(200, 30)
        self.modelButton.clicked.connect(self.onModelButtonClicked)
        self.scrollAreaFrame.layout().addWidget(self.modelButton, 2, 1, QtCore.Qt.AlignmentFlag.AlignRight)

        # type
        ## label
        type_label = QLabel("Type")
        type_label.setTextFormat(QtCore.Qt.TextFormat.MarkdownText)
        self.scrollAreaFrame.layout().addWidget(type_label, 3, 0, QtCore.Qt.AlignmentFlag.AlignLeft)
        ## dropdown
        self.type_dropdown = QComboBox()
        for agent_type in PedsimAgentType:
            self.type_dropdown.insertItem(agent_type.value, agent_type.name.lower())
        self.type_dropdown.setFixedSize(200, 30)
        self.type_dropdown.currentIndexChanged.connect(self.updateWidgetsFromSelectedType)
        self.scrollAreaFrame.layout().addWidget(self.type_dropdown, 3, 1, QtCore.Qt.AlignmentFlag.AlignRight)

        # amount
        ## label
        amount_label = QLabel("Amount")
        amount_label.setTextFormat(QtCore.Qt.TextFormat.MarkdownText)
        self.scrollAreaFrame.layout().addWidget(amount_label, 4, 0, QtCore.Qt.AlignmentFlag.AlignLeft)
        ## spin box
        amount_spin_box = QSpinBox()
        amount_spin_box.setValue(1)
        amount_spin_box.setMinimum(1)
        amount_spin_box.setFixedSize(200, 30)
        self.scrollAreaFrame.layout().addWidget(amount_spin_box, 4, 1, QtCore.Qt.AlignmentFlag.AlignRight)

        # waypoint mode
        ## label
        label = QLabel("Waypoint Mode")
        label.setTextFormat(QtCore.Qt.TextFormat.MarkdownText)
        self.scrollAreaFrame.layout().addWidget(label, 5, 0, QtCore.Qt.AlignmentFlag.AlignLeft)
        ## dropdown
        self.waypoint_mode_dropdown = QComboBox()
        for mode in PedsimWaypointMode:
            self.waypoint_mode_dropdown.insertItem(mode.value, mode.name.lower())
        self.waypoint_mode_dropdown.setFixedSize(200, 30)
        self.scrollAreaFrame.layout().addWidget(self.waypoint_mode_dropdown, 5, 1, QtCore.Qt.AlignmentFlag.AlignRight)

        # vmax
        ## label
        self.vmax_label = QLabel("Velocitymax")
        self.vmax_label.setTextFormat(QtCore.Qt.TextFormat.MarkdownText)
        self.scrollAreaFrame.layout().addWidget(self.vmax_label, 6, 0, QtCore.Qt.AlignmentFlag.AlignLeft)
        ## slider
        self.vmax_slider = ArenaSliderWidget(0, 20, 0.1, "m/s")
        self.scrollAreaFrame.layout().addWidget(self.vmax_slider, 6, 1, QtCore.Qt.AlignmentFlag.AlignRight)

        # max talking distance
        ### label
        self.maxTalkingDistanceLabel = QLabel("Max Talking Distance")
        self.maxTalkingDistanceLabel.setTextFormat(QtCore.Qt.TextFormat.MarkdownText)
        self.scrollAreaFrame.layout().addWidget(self.maxTalkingDistanceLabel, 7, 0, QtCore.Qt.AlignmentFlag.AlignLeft)
        ### slider
        self.maxTalkingDistanceSlider = ArenaSliderWidget(0, 10, 1, "m")
        self.scrollAreaFrame.layout().addWidget(self.maxTalkingDistanceSlider, 7, 1, QtCore.Qt.AlignmentFlag.AlignRight)

        # forces
        ## heading
        label = QLabel("#### Force Factors")
        label.setTextFormat(QtCore.Qt.TextFormat.MarkdownText)
        self.scrollAreaFrame.layout().addWidget(label, 8, 0, QtCore.Qt.AlignmentFlag.AlignLeft)
        line = Line()
        self.scrollAreaFrame.layout().addWidget(line, 8, 1, QtCore.Qt.AlignmentFlag.AlignRight)
        ## Desired
        ### label
        label = QLabel("Desired")
        label.setTextFormat(QtCore.Qt.TextFormat.MarkdownText)
        self.scrollAreaFrame.layout().addWidget(label, 9, 0, QtCore.Qt.AlignmentFlag.AlignLeft)
        ### slider
        slider = ArenaSliderWidget(0, 20, 1, "")
        slider.slider.setValue(1)
        self.scrollAreaFrame.layout().addWidget(slider, 9, 1, QtCore.Qt.AlignmentFlag.AlignRight)
        ## Obstacle
        ### label
        label = QLabel("Obstacle")
        label.setTextFormat(QtCore.Qt.TextFormat.MarkdownText)
        self.scrollAreaFrame.layout().addWidget(label, 10, 0, QtCore.Qt.AlignmentFlag.AlignLeft)
        ### slider
        slider = ArenaSliderWidget(0, 20, 1, "")
        slider.slider.setValue(1)
        self.scrollAreaFrame.layout().addWidget(slider, 10, 1, QtCore.Qt.AlignmentFlag.AlignRight)
        ## Social
        ### label
        label = QLabel("Social")
        label.setTextFormat(QtCore.Qt.TextFormat.MarkdownText)
        self.scrollAreaFrame.layout().addWidget(label, 11, 0, QtCore.Qt.AlignmentFlag.AlignLeft)
        ### slider
        slider = ArenaSliderWidget(0, 20, 1, "")
        slider.slider.setValue(1)
        self.scrollAreaFrame.layout().addWidget(slider, 11, 1, QtCore.Qt.AlignmentFlag.AlignRight)
        ## Robot
        ### label
        label = QLabel("Robot")
        label.setTextFormat(QtCore.Qt.TextFormat.MarkdownText)
        self.scrollAreaFrame.layout().addWidget(label, 12, 0, QtCore.Qt.AlignmentFlag.AlignLeft)
        ### slider
        slider = ArenaSliderWidget(0, 20, 1, "")
        slider.slider.setValue(1)
        self.scrollAreaFrame.layout().addWidget(slider, 12, 1, QtCore.Qt.AlignmentFlag.AlignRight)

        # individual talking
        ## heading
        self.individualTalkingLabel = QLabel("#### Individual Talking")
        self.individualTalkingLabel.setTextFormat(QtCore.Qt.TextFormat.MarkdownText)
        self.scrollAreaFrame.layout().addWidget(self.individualTalkingLabel, 13, 0, QtCore.Qt.AlignmentFlag.AlignLeft)
        self.individualTalkingLine = Line()
        self.scrollAreaFrame.layout().addWidget(self.individualTalkingLine, 13, 1, QtCore.Qt.AlignmentFlag.AlignRight)
        ## probability
        ### label
        self.individualTalkingProbabilityLabel = QLabel("Probability")
        self.individualTalkingProbabilityLabel.setTextFormat(QtCore.Qt.TextFormat.MarkdownText)
        self.scrollAreaFrame.layout().addWidget(self.individualTalkingProbabilityLabel, 14, 0, QtCore.Qt.AlignmentFlag.AlignLeft)
        ### slider
        self.individualTalkingProbabilitySlider = ArenaProbabilitySliderWidget()
        self.scrollAreaFrame.layout().addWidget(self.individualTalkingProbabilitySlider, 14, 1, QtCore.Qt.AlignmentFlag.AlignRight)
        ## base time
        ### label
        self.individualTalkingBaseTimeLabel = QLabel("Base Time")
        self.individualTalkingBaseTimeLabel.setTextFormat(QtCore.Qt.TextFormat.MarkdownText)
        self.scrollAreaFrame.layout().addWidget(self.individualTalkingBaseTimeLabel, 15, 0, QtCore.Qt.AlignmentFlag.AlignLeft)
        ### slider
        self.individualTalkingBaseTimeSlider = ArenaSliderWidget(0, 100, 1, "s")
        self.scrollAreaFrame.layout().addWidget(self.individualTalkingBaseTimeSlider, 15, 1, QtCore.Qt.AlignmentFlag.AlignRight)

        # group talk
        ## heading
        self.groupTalkLabel = QLabel("#### Group Talk")
        self.groupTalkLabel.setTextFormat(QtCore.Qt.TextFormat.MarkdownText)
        self.scrollAreaFrame.layout().addWidget(self.groupTalkLabel, 16, 0, QtCore.Qt.AlignmentFlag.AlignLeft)
        self.groupTalkLine = Line()
        self.scrollAreaFrame.layout().addWidget(self.groupTalkLine, 16, 1, QtCore.Qt.AlignmentFlag.AlignRight)
        ## probability
        ### label
        self.groupTalkProbabilityLabel = QLabel("Probability")
        self.groupTalkProbabilityLabel.setTextFormat(QtCore.Qt.TextFormat.MarkdownText)
        self.scrollAreaFrame.layout().addWidget(self.groupTalkProbabilityLabel, 17, 0, QtCore.Qt.AlignmentFlag.AlignLeft)
        ### slider
        self.groupTalkProbabilitySlider = ArenaProbabilitySliderWidget()
        self.scrollAreaFrame.layout().addWidget(self.groupTalkProbabilitySlider, 17, 1, QtCore.Qt.AlignmentFlag.AlignRight)
        ## base time
        ### label
        self.groupTalkBaseTimeLabel = QLabel("Base Time")
        self.groupTalkBaseTimeLabel.setTextFormat(QtCore.Qt.TextFormat.MarkdownText)
        self.scrollAreaFrame.layout().addWidget(self.groupTalkBaseTimeLabel, 18, 0, QtCore.Qt.AlignmentFlag.AlignLeft)
        ### slider
        self.groupTalkBaseTimeSlider = ArenaSliderWidget(0, 100, 1, "s")
        self.scrollAreaFrame.layout().addWidget(self.groupTalkBaseTimeSlider, 18, 1, QtCore.Qt.AlignmentFlag.AlignRight)

        # talk and walk
        ## heading
        self.talkWalkLabel = QLabel("#### Talk & Walk")
        self.talkWalkLabel.setTextFormat(QtCore.Qt.TextFormat.MarkdownText)
        self.scrollAreaFrame.layout().addWidget(self.talkWalkLabel, 19, 0, QtCore.Qt.AlignmentFlag.AlignLeft)
        self.talkWalkLine = Line()
        self.scrollAreaFrame.layout().addWidget(self.talkWalkLine, 19, 1, QtCore.Qt.AlignmentFlag.AlignRight)
        ## probability
        ### label
        self.talkWalkProbabilityLabel = QLabel("Probability")
        self.talkWalkProbabilityLabel.setTextFormat(QtCore.Qt.TextFormat.MarkdownText)
        self.scrollAreaFrame.layout().addWidget(self.talkWalkProbabilityLabel, 20, 0, QtCore.Qt.AlignmentFlag.AlignLeft)
        ### slider
        self.talkWalkProbabilitySlider = ArenaProbabilitySliderWidget()
        self.scrollAreaFrame.layout().addWidget(self.talkWalkProbabilitySlider, 20, 1, QtCore.Qt.AlignmentFlag.AlignRight)
        ## base time
        ### label
        self.talkWalkBaseTimeLabel = QLabel("Base Time")
        self.talkWalkBaseTimeLabel.setTextFormat(QtCore.Qt.TextFormat.MarkdownText)
        self.scrollAreaFrame.layout().addWidget(self.talkWalkBaseTimeLabel, 21, 0, QtCore.Qt.AlignmentFlag.AlignLeft)
        ### slider
        self.talkWalkBaseTimeSlider = ArenaSliderWidget(0, 100, 1, "s")
        self.scrollAreaFrame.layout().addWidget(self.talkWalkBaseTimeSlider, 21, 1, QtCore.Qt.AlignmentFlag.AlignRight)

        # requesting service
        ## heading
        self.requestingServiceLabel = QLabel("#### Requesting Service")
        self.requestingServiceLabel.setTextFormat(QtCore.Qt.TextFormat.MarkdownText)
        self.scrollAreaFrame.layout().addWidget(self.requestingServiceLabel, 22, 0, QtCore.Qt.AlignmentFlag.AlignLeft)
        self.requestingServiceLine = Line()
        self.scrollAreaFrame.layout().addWidget(self.requestingServiceLine, 22, 1, QtCore.Qt.AlignmentFlag.AlignRight)
        ## probability
        ### label
        self.requestingServiceProbabilityLabel = QLabel("Probability")
        self.requestingServiceProbabilityLabel.setTextFormat(QtCore.Qt.TextFormat.MarkdownText)
        self.scrollAreaFrame.layout().addWidget(self.requestingServiceProbabilityLabel, 23, 0, QtCore.Qt.AlignmentFlag.AlignLeft)
        ### slider
        self.requestingServiceProbabilitySlider = ArenaProbabilitySliderWidget()
        self.scrollAreaFrame.layout().addWidget(self.requestingServiceProbabilitySlider, 23, 1, QtCore.Qt.AlignmentFlag.AlignRight)
        ## max service distance
        ### label
        self.maxServiceDistanceLabel = QLabel("Max Service Distance")
        self.maxServiceDistanceLabel.setTextFormat(QtCore.Qt.TextFormat.MarkdownText)
        self.scrollAreaFrame.layout().addWidget(self.maxServiceDistanceLabel, 24, 0, QtCore.Qt.AlignmentFlag.AlignLeft)
        ### slider
        self.maxServiceDistanceSlider = ArenaSliderWidget(0, 10, 1, "m")
        self.scrollAreaFrame.layout().addWidget(self.maxServiceDistanceSlider, 24, 1, QtCore.Qt.AlignmentFlag.AlignRight)
        ## base time requesting
        ### label
        self.requestingServiceRequestingBaseTimeLabel = QLabel("Base Time (req.)")
        self.requestingServiceRequestingBaseTimeLabel.setTextFormat(QtCore.Qt.TextFormat.MarkdownText)
        self.scrollAreaFrame.layout().addWidget(self.requestingServiceRequestingBaseTimeLabel, 25, 0, QtCore.Qt.AlignmentFlag.AlignLeft)
        ### slider
        self.requestingServiceRequestingBaseTimeSlider = ArenaSliderWidget(0, 100, 1, "s")
        self.scrollAreaFrame.layout().addWidget(self.requestingServiceRequestingBaseTimeSlider, 25, 1, QtCore.Qt.AlignmentFlag.AlignRight)
        ## base time receiving
        ### label
        self.requestingServiceReceivingBaseTimeLabel = QLabel("Base Time (recv.)")
        self.requestingServiceReceivingBaseTimeLabel.setTextFormat(QtCore.Qt.TextFormat.MarkdownText)
        self.scrollAreaFrame.layout().addWidget(self.requestingServiceReceivingBaseTimeLabel, 26, 0, QtCore.Qt.AlignmentFlag.AlignLeft)
        ### slider
        self.requestingServiceReceivingBaseTimeSlider = ArenaSliderWidget(0, 100, 1, "s")
        self.scrollAreaFrame.layout().addWidget(self.requestingServiceReceivingBaseTimeSlider, 26, 1, QtCore.Qt.AlignmentFlag.AlignRight)

        # requesting guide
        ## heading
        self.requestingGuideLabel = QLabel("#### Requesting Guide")
        self.requestingGuideLabel.setTextFormat(QtCore.Qt.TextFormat.MarkdownText)
        self.scrollAreaFrame.layout().addWidget(self.requestingGuideLabel, 27, 0, QtCore.Qt.AlignmentFlag.AlignLeft)
        self.requestingGuideLine = Line()
        self.scrollAreaFrame.layout().addWidget(self.requestingGuideLine, 27, 1, QtCore.Qt.AlignmentFlag.AlignRight)
        ## probability
        ### label
        self.requestingGuideProbabilityLabel = QLabel("Probability")
        self.requestingGuideProbabilityLabel.setTextFormat(QtCore.Qt.TextFormat.MarkdownText)
        self.scrollAreaFrame.layout().addWidget(self.requestingGuideProbabilityLabel, 28, 0, QtCore.Qt.AlignmentFlag.AlignLeft)
        ### slider
        self.requestingGuideProbabilitySlider = ArenaProbabilitySliderWidget()
        self.scrollAreaFrame.layout().addWidget(self.requestingGuideProbabilitySlider, 28, 1, QtCore.Qt.AlignmentFlag.AlignRight)

        # requesting follower
        ## heading
        self.requestingFollowerLabel = QLabel("#### Requesting Follower")
        self.requestingFollowerLabel.setTextFormat(QtCore.Qt.TextFormat.MarkdownText)
        self.scrollAreaFrame.layout().addWidget(self.requestingFollowerLabel, 29, 0, QtCore.Qt.AlignmentFlag.AlignLeft)
        self.requestingFollowerLine = Line()
        self.scrollAreaFrame.layout().addWidget(self.requestingFollowerLine, 29, 1, QtCore.Qt.AlignmentFlag.AlignRight)
        ## probability
        ### label
        self.requestingFollowerProbabilityLabel = QLabel("Probability")
        self.requestingFollowerProbabilityLabel.setTextFormat(QtCore.Qt.TextFormat.MarkdownText)
        self.scrollAreaFrame.layout().addWidget(self.requestingFollowerProbabilityLabel, 30, 0, QtCore.Qt.AlignmentFlag.AlignLeft)
        ### slider
        self.requestingFollowerProbabilitySlider = ArenaProbabilitySliderWidget()
        self.scrollAreaFrame.layout().addWidget(self.requestingFollowerProbabilitySlider, 30, 1, QtCore.Qt.AlignmentFlag.AlignRight)

        # save button
        self.save_button = QtWidgets.QPushButton("Save and Close")
        self.save_button.clicked.connect(self.onSaveClicked)
        self.layout().addWidget(self.save_button, 1, 0, -1, -1)

    def updateWidgetsFromSelectedType(self):
        agent_type = PedsimAgentType(self.type_dropdown.currentIndex())
        if agent_type in [PedsimAgentType.ADULT, PedsimAgentType.CHILD, PedsimAgentType.ELDER]:
            # max talking distance
            self.maxTalkingDistanceLabel.show()
            self.maxTalkingDistanceSlider.show()
            # individual talking
            self.individualTalkingLabel.show()
            self.individualTalkingLine.show()
            self.individualTalkingProbabilityLabel.show()
            self.individualTalkingProbabilitySlider.show()
            self.individualTalkingBaseTimeLabel.show()
            self.individualTalkingBaseTimeSlider.show()
            # group talk
            self.groupTalkLabel.show()
            self.groupTalkLine.show()
            self.groupTalkProbabilityLabel.show()
            self.groupTalkProbabilitySlider.show()
            self.groupTalkBaseTimeLabel.show()
            self.groupTalkBaseTimeSlider.show()
            # talk and walk
            self.talkWalkLabel.show()
            self.talkWalkLine.show()
            self.talkWalkProbabilityLabel.show()
            self.talkWalkProbabilitySlider.show()
            self.talkWalkBaseTimeLabel.show()
            self.talkWalkBaseTimeSlider.show()
            # requesting service
            self.requestingServiceLabel.show()
            self.requestingServiceLine.show()
            self.requestingServiceProbabilityLabel.show()
            self.requestingServiceProbabilitySlider.show()
            self.maxServiceDistanceLabel.hide()
            self.maxServiceDistanceSlider.hide()
            self.requestingServiceRequestingBaseTimeLabel.show()
            self.requestingServiceRequestingBaseTimeSlider.show()
            self.requestingServiceReceivingBaseTimeLabel.show()
            self.requestingServiceReceivingBaseTimeSlider.show()
            # requesting guide
            self.requestingGuideLabel.show()
            self.requestingGuideLine.show()
            self.requestingGuideProbabilityLabel.show()
            self.requestingGuideProbabilitySlider.show()
            # requesting follower
            self.requestingFollowerLabel.show()
            self.requestingFollowerLine.show()
            self.requestingFollowerProbabilityLabel.show()
            self.requestingFollowerProbabilitySlider.show()

        elif agent_type == PedsimAgentType.VEHICLE:
            # max talking distance
            self.maxTalkingDistanceLabel.hide()
            self.maxTalkingDistanceSlider.hide()
            # individual talking
            self.individualTalkingLabel.hide()
            self.individualTalkingLine.hide()
            self.individualTalkingProbabilityLabel.hide()
            self.individualTalkingProbabilitySlider.hide()
            self.individualTalkingBaseTimeLabel.hide()
            self.individualTalkingBaseTimeSlider.hide()
            # group talk
            self.groupTalkLabel.hide()
            self.groupTalkLine.hide()
            self.groupTalkProbabilityLabel.hide()
            self.groupTalkProbabilitySlider.hide()
            self.groupTalkBaseTimeLabel.hide()
            self.groupTalkBaseTimeSlider.hide()
            # talk and walk
            self.talkWalkLabel.hide()
            self.talkWalkLine.hide()
            self.talkWalkProbabilityLabel.hide()
            self.talkWalkProbabilitySlider.hide()
            self.talkWalkBaseTimeLabel.hide()
            self.talkWalkBaseTimeSlider.hide()
            # requesting service
            self.requestingServiceLabel.hide()
            self.requestingServiceLine.hide()
            self.requestingServiceProbabilityLabel.hide()
            self.requestingServiceProbabilitySlider.hide()
            self.maxServiceDistanceLabel.hide()
            self.maxServiceDistanceSlider.hide()
            self.requestingServiceRequestingBaseTimeLabel.hide()
            self.requestingServiceRequestingBaseTimeSlider.hide()
            self.requestingServiceReceivingBaseTimeLabel.hide()
            self.requestingServiceReceivingBaseTimeSlider.hide()
            # requesting guide
            self.requestingGuideLabel.hide()
            self.requestingGuideLine.hide()
            self.requestingGuideProbabilityLabel.hide()
            self.requestingGuideProbabilitySlider.hide()
            # requesting follower
            self.requestingFollowerLabel.hide()
            self.requestingFollowerLine.hide()
            self.requestingFollowerProbabilityLabel.hide()
            self.requestingFollowerProbabilitySlider.hide()

        elif agent_type == PedsimAgentType.SERVICEROBOT:
            # max talking distance
            self.maxTalkingDistanceLabel.hide()
            self.maxTalkingDistanceSlider.hide()
            # individual talking
            self.individualTalkingLabel.hide()
            self.individualTalkingLine.hide()
            self.individualTalkingProbabilityLabel.hide()
            self.individualTalkingProbabilitySlider.hide()
            self.individualTalkingBaseTimeLabel.hide()
            self.individualTalkingBaseTimeSlider.hide()
            # group talk
            self.groupTalkLabel.hide()
            self.groupTalkLine.hide()
            self.groupTalkProbabilityLabel.hide()
            self.groupTalkProbabilitySlider.hide()
            self.groupTalkBaseTimeLabel.hide()
            self.groupTalkBaseTimeSlider.hide()
            # talk and walk
            self.talkWalkLabel.hide()
            self.talkWalkLine.hide()
            self.talkWalkProbabilityLabel.hide()
            self.talkWalkProbabilitySlider.hide()
            self.talkWalkBaseTimeLabel.hide()
            self.talkWalkBaseTimeSlider.hide()
            # requesting service
            self.requestingServiceLabel.show()
            self.requestingServiceLine.show()
            self.requestingServiceProbabilityLabel.hide()
            self.requestingServiceProbabilitySlider.hide()
            self.maxServiceDistanceLabel.show()
            self.maxServiceDistanceSlider.show()
            self.requestingServiceRequestingBaseTimeLabel.hide()
            self.requestingServiceRequestingBaseTimeSlider.hide()
            self.requestingServiceReceivingBaseTimeLabel.hide()
            self.requestingServiceReceivingBaseTimeSlider.hide()
            # requesting guide
            self.requestingGuideLabel.hide()
            self.requestingGuideLine.hide()
            self.requestingGuideProbabilityLabel.hide()
            self.requestingGuideProbabilitySlider.hide()
            # requesting follower
            self.requestingFollowerLabel.hide()
            self.requestingFollowerLine.hide()
            self.requestingFollowerProbabilityLabel.hide()
            self.requestingFollowerProbabilitySlider.hide()

    def updateValuesFromPedsimAgent(self):
        self.tempFlatlandModel.load(self.pedsimAgent.yaml_file)
        # TODO update all other values

    def onModelButtonClicked(self):
        rospack = rospkg.RosPack()
        initial_folder = os.path.join(rospack.get_path("simulator_setup"), "dynamic_obstacles")
        res = QtWidgets.QFileDialog.getOpenFileName(parent=self, directory=initial_folder)
        path = res[0]
        if path != "":
            self.setModelPath(path)

    def setModelPath(self, path: str):
        self.tempFlatlandModel.load(path)
        self.modelButton.setText(path.split("/")[-1])

    def onSaveClicked(self):
        self.save()
        self.pedsimAgentWidget.update()
        self.hide()

    def save(self):
        self.pedsimAgent.name = self.name_edit.text()
        self.pedsimAgent.yaml_file = self.tempFlatlandModel.path
        self.pedsimAgent.flatlandModel = self.tempFlatlandModel
        # TODO save data in widgets into pedsim agent


from setuptools import setup, find_packages
from codecs import open
from os import path

here = path.abspath(path.dirname(__file__))

# Get the long description from the README file
with open(path.join(here, 'README.rst'), encoding='utf-8') as f:
    long_description = f.read()

setup(
    name='poloniex_aio',
    use_scm_version=True,
    setup_requires=['setuptools_scm'],
    url='https://github.com/OlivierCazade/poloniex_aio',
    author='Olivier Cazade',
    author_email='olivier.cazade@gmail.com',
    license='MIT',
    packages=find_packages(),
    description='An async library for Poloniex api',
    long_description=long_description,
    python_requires='>=3.5'
)


from django.core.management.base import BaseCommand

from elasticsearch_dsl import connections

from catalog.constants import ES_INDEX

class Command(BaseCommand):
    help = 'Creates an Elasticsearch index.'

    def handle(self, *args, **kwargs):
        self.stdout.write(f'Creating index "{ES_INDEX}"...')
        connection =  connections.get_connection()
        if connection.indices.exists(index=ES_INDEX):
            self.stdout.write(f'Index "{ES_INDEX}" already exists')
        else:
            connection.indices.create(index=ES_INDEX, body={
                'settings': {
                    'number_of_shards': 1,
                    'number_of_replicas': 0,
                }
            })
            self.stdout.write(f'Index "{ES_INDEX}" created successfully')


from datetime import timedelta
from typing import Optional

from mockseries.transition.lambda_transition import LambdaTransition

class LinearTransition(LambdaTransition):
    """Linear transition.

    The transition from the base value to the switch value is linear.
    If a stop window is provided, the transition from the switch value
    to the base value is also linear.
    """

    def __init__(
        self,
        transition_window: timedelta,
        stop_window: Optional[timedelta] = None,
    ):
        super().__init__(transition_window, lambda x: x, stop_window)


import requests
from bs4 import BeautifulSoup
import pandas as pd
from sqlalchemy import create_engine

def get_html(url):
    r = requests.get(url)
    if r.status_code == 200:
        return r.text
        print(r.text)
    if r.status_code == 404:
        print('Страница не существует!')

def get_date(html):
    soup = BeautifulSoup(html, 'lxml')
    try:
        data_date = [p.text.strip()
                     for p in soup.find_all('td')if p.text is not None]
    except Exception.DoesNotExist:
        data_date = ''
    return data_date

def find_archive(page_archive):
    soup = BeautifulSoup(page_archive, 'lxml')
    try:
        data_arc = [p.text.replace('↓', '\n').strip()
                    for p in soup.find_all('td') if p.text is not None]
    except Exception.DoesNotExist:
        data_arc = ''
    return data_arc

def find_right(page_right):
    soup = BeautifulSoup(page_right, 'lxml')
    try:
        data_right = [p.text.replace('↓', '\n').strip()
                      for p in soup.find_all('table', {'class': 'items'})
                      if p.text is not None]
    except Exception.DoesNotExist:
        data_right = ''
    return data_right

def parse_table(table):
    # парсинг со страницы "реестр членов СРО" из 2 по 6 столбца информации
    res = pd.DataFrame()
    name_org = ''
    inn = ''
    ogrn = ''
    status = ''
    type = ''
    reg_num = ''

    data_tr = table.find('tr', {'class': 'sro-link'})
    name_org = data_tr.find_all('td')[1].text.strip()
    inn = data_tr.find_all('td')[2].text.strip()
    ogrn = data_tr.find_all('td')[3].text.strip()
    status = data_tr.find_all('td')[4].text.strip()
    type = data_tr.find_all('td')[5].text.strip()
    reg_num = data_tr.find_all('td')[6].text.strip()

    res = res.append(pd.DataFrame([[
        name_org, inn, ogrn, status, type,
        reg_num]],
        columns=[
        'name_org', 'inn', 'ogrn', 'status',
        'type', 'reg_num']))
    return res

def parse_page_2(page_2):
    # парсинг со страницы "Сведения о члене СРО" из 7 по 8 столбца информации
    res = pd.DataFrame()
    date_reg = ''
    date_close = ''

    date_reg = page_2[6]
    date_close_td = page_2[7]
    if date_close_td is not None:
        date_close = date_close_td

    res = res.append(pd.DataFrame(
        [[date_reg, date_close]],
        columns=['date_reg', 'date_close']), ignore_index=True)
    return res

def parse_page_arch(page_arch):
    # парсинг со страницы "Архив"
    res = pd.DataFrame()
    num_sv = ''
    date_open = ''
    basis = ''
    status_sv = ''

    num_sv = page_arch[7]
    date_open_td = page_arch[8]
    basis = page_arch[9]
    status_sv = page_arch[11]
    if date_open_td is not None:
        date_open = date_open_td

    res = res.append(pd.DataFrame([[
        num_sv, date_open, basis, status_sv]],
        columns=[
            'num_sv', 'date_open', 'basis', 'status_sv']), ignore_index=True)
    return res

def parse_page_right(page_right):
    # парсинг со страницы "Сведения о наличии права"
    res = pd.DataFrame()
    inform_of_the_right = ''
    inform_right = page_right[0]
    if inform_right is not None:
        inform_of_the_right = inform_right
    res = res.append(
        pd.DataFrame([[inform_of_the_right]], columns=['inform_of_the_right']))
    return res

def main():
    for i in num_inn:
        url_inn = base_url+(rel_inn)+i
        soup = BeautifulSoup(get_html(url_inn), 'lxml')
        tables = soup.find_all('table', {'class': 'items'})
        rel_cl = [tag.get('rel')for tag in soup.find_all('tr')
                  if not tag.get('rel') is None]
        num_clienta = rel_cl[0]
        url_client = base_url + num_clienta
        html = get_html(url_client)
        url_arch = url_client + '/certificates'
        url_of_the_right = url_client + '/rights'
        page_get_right = get_html(url_of_the_right)
        page_archive = get_html(url_arch)
        for item in tables:
            res = parse_table(item)
        data_date = parse_page_2(get_date(html))
        archive = parse_page_arch(find_archive(page_archive))
        the_right = parse_page_right(find_right(page_get_right))
        result = pd.DataFrame()
        df = pd.concat([res, data_date, archive, the_right], axis=1)
        result = result.append(df, ignore_index=True)
        engine = create_engine('sqlite:///pars_data.db')
        result.to_sql('users', con=engine, if_exists='append')

if __name__ == '__main__':
    base_url = 'http://reestr.nostroy.ru'
    rel_inn = '/reestr?m.fulldescription=&m.shortdescription=&m.inn='
    print("\n---------\n")
    num_inn = input("Введите ваш номер ИНН или список через пробел: ").split()
    # 0278128540 770965012811
    main()
    print("\n---------\n")


"""Fake FLI USB camera module"""

import numpy as np
import astropy.io.fits as pyfits
import time
import os
import threading

class FliError(Exception):
    """Exception for FLI camera"""
    pass

def numberOfCamera():
    """Return number of available FLI cameras"""
    return numCams

def getLibVersion():
    """Get the current library version"""
    return "Software Development Library for Linux 1.999.1"

class Camera:
    """FLI usb camera"""

    def __init__(self, id, devsn, imgPath=None):
        """(id) : index of the camera device"""
        if id < 0 or id >= numCams:
            raise FliError("Camera[%d] not available" % id)
        self.id = id
        self.status = CLOSED
        self.exposureID = 0
        self.agcid = -1
        self.abort = 0
        self.temp = None
        self.devname = "MicroLine ML4720"
        self.devsn = devsn
        self.hwRevision = 256
        self.fwRevision = 512
        self.mode = 0

        # read simulated image, contains single or 6 image extensions
        if imgPath is not None:
            hdulist = pyfits.open(imgPath)
            if len(hdulist) > 1:
                if hdulist[id+1].data is None:
                    self.rawdata = np.zeros((1033, 1072), dtype=np.uint16)
                else:
                    self.rawdata = hdulist[id+1].data.astype(np.uint16)
            else:
                self.rawdata = hdulist[0].data.astype(np.uint16)
        else:
            self.rawdata = np.zeros((1033, 1072), dtype=np.uint16)
        self.lock = threading.Lock()

    def getStatusStr(self):
        with self.lock:
            status = self.status
        return Status[status]

    def isClosed(self):
        with self.lock:
            status = self.status
        return status == CLOSED

    def isReady(self):
        with self.lock:
            status = self.status
        return status == READY

    def isExposing(self):
        with self.lock:
            status = self.status
        return status == EXPOSING

    def isSetmode(self):
        with self.lock:
            status = self.status
        return status == SETMODE

    def open(self):
        """Open the camera device"""
        if dev[self.id] != FLI_INVALID_DEVICE:
            raise FliError("Device already opened")
        dev[self.id] = FLIDEVICE_CAMERA

        # set default parameters
        self.setTemperature(CCD_TEMP)
        self.setHBin(1)
        self.setVBin(1)
        self.setExpTime(0)
        self.setFrame(0, 0, 1072, 1033)
        with self.lock:
            self.status = READY
            self.visibleExpArea = (24, 9, 1048, 1033)
            self.defaultExpArea = (0, 0, 1072, 1033)
            self.expArea = (0, 0, 1072, 1033)
            self.regions = ((0, 0, 0), (0, 0, 0))

    def close(self):
        """Close the camera device"""
        if dev[self.id] == FLI_INVALID_DEVICE:
            raise FliError("Device already closed or not initialized")
        dev[self.id] = FLI_INVALID_DEVICE
        with self.lock:
            self.status = CLOSED

    def setExpTime(self, exptime):
        """Set the exposure time in ms"""
        with self.lock:
            self.exptime = exptime

    def setHBin(self, hbin):
        """Set the horizontal binning"""
        with self.lock:
            self.hbin = hbin

    def setVBin(self, vbin):
        """Set the vertical binning"""
        with self.lock:
            self.vbin = vbin

    def setFrame(self, x1, y1, width, height):
        """Set the image area"""
        with self.lock:
            self.xsize = width
            self.ysize = height
            self.expArea = (x1, y1, x1+width, y1+height)

    def resetFrame(self):
        """Reset the image area"""
        with self.lock:
            hbin = self.hbin
            vbin = self.vbin
        if hbin != 1:
            self.setHBin(1)
        if vbin != 1:
            self.setVBin(1)
        with self.lock:
            self.expArea = self.defaultExpArea
            x1, y1, x2, y2 = self.expArea
            self.xsize = x2 - x1
            self.ysize = y2 - y1

    def setTemperature(self, temp):
        """Set the CCD temperature"""
        with self.lock:
            self.temp = temp

    def getTemperature(self):
        """Get the CCD temperature"""
        with self.lock:
            temp = self.temp
        return temp

    def getCoolerPower(self):
        """Get the cooler power in percentage"""
        return 90.0

    def getPixelSize(self):
        """Get the pixel sizes in micron"""
        return (0.000013, 0.000013)

    def wfits(self, filename=None):
        """Write the image to a FITS file"""
        with self.lock:
            dark = self.dark
        if not filename:
            if dark != 0:
                filename = self.getNextFilename("dark")
            else:
                filename = self.getNextFilename("object")
        with self.lock:
            if(self.data.size == 0):
                raise FliError("No image available")
            hdu = pyfits.PrimaryHDU(self.data)
        hdr = hdu.header
        with self.lock:
            hdr.set('DATE', self.timestamp, 'exposure begin date')
            hdr.set('INSTRUME', self.devname, 'this instrument')
            hdr.set('SERIAL', self.devsn, 'serial number')
            hdr.set('EXPTIME', self.exptime, 'exposure time (ms)')
            hdr.set('VBIN', self.vbin, 'vertical binning')
            hdr.set('HBIN', self.hbin, 'horizontal binning')
            hdr.set('CCD-TEMP', self.temp, 'CCD temperature')
            if dark != 0:
                hdr.set('SHUTTER', 'CLOSE', 'shutter status')
            else:
                hdr.set('SHUTTER', 'OPEN', 'shutter status')
            hdr.set('CCDAREA', '[%d:%d,%d:%d]' % self.expArea, 'image area')
        hdu.writeto(filename, overwrite=True, checksum=True)
        with self.lock:
            self.filename = filename

    def getNextFilename(self, expType):
        """Fetch the next image filename"""
        with self.lock:
            self.exposureID += 1
            exposureID = self.exposureID
        path = os.path.join("$ICS_MHS_DATA_ROOT", 'agcc')
        path = os.path.expandvars(os.path.expanduser(path))
        if not os.path.isdir(path):
            os.makedirs(path, 0o755)
        with self.lock:
            timestamp = self.timestamp
        return os.path.join(path, 'AGC%d_%s_%06d_%s.fits' % \
               (self.agcid + 1, expType, exposureID, timestamp))

    def cancelExposure(self):
        """Cancel current exposure"""
        with self.lock:
            status = self.status
        if status == EXPOSING:
            with self.lock:
                self.abort = 1

    def expose(self, dark=False, blocking=True):
        """Do exposure and return the image"""
        with self.lock:
            status = self.status
        if status != READY:
            raise FliError("Camera not ready, abort expose command")
        with self.lock:
            self.dark = dark
            self.tstart = time.time()
            self.timestamp = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(self.tstart))
            self.status = EXPOSING

        thr = threading.Thread(target=self.exposeHandler)
        thr.start()
        if blocking:
            thr.join()

    def exposeHandler(self):
        # Check if the exposure is done and write the image
        tstart = time.time();
        with self.lock:
            # add 350ms readout time
            exptime = (self.exptime + 350.0) / 1000.0
        while (time.time() - tstart < exptime):
            time.sleep(POLL_TIME)
            with self.lock:
                abort = self.abort
            if abort != 0:
                break

        with self.lock:
            if self.abort != 0:
                # Exposure aborted
                self.abort = 0
                self.tend = 0
            else:
                xsize = self.xsize
                ysize = self.ysize
                self.data = self.rawdata[self.expArea[1]:self.expArea[3], self.expArea[0]:self.expArea[2]]
                self.tend = time.time()
            self.status = READY

    def expose_test(self):
        """Return the test image"""
        with self.lock:
            self.dark = 1
            self.tstart = time.time()
            self.timestamp = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(self.tstart))
            imagesize = (self.expArea[3] - self.expArea[1],
                         self.expArea[2] - self.expArea[0])
            self.data = np.ones(shape=imagesize, dtype=np.uint16)
            self.tend = time.time()

    def getModeString(self, mode):
        """Get the camera mode string"""
        if mode == 0:
            return "4 MHz"
        elif mode == 1:
            return "500 KHz"
        else:
            raise FliError("FLIGetCameraModeString failed")

    def getMode(self):
        """Get the camera mode string"""
        with self.lock:
            mode = self.mode
        return mode

    def setMode(self, mode):
        """Get the camera mode string"""
        if mode == 0 or mode == 1:
            with self.lock:
                self.mode = mode
        else:
            raise FliError("FLISetCameraMode failed")

    def getTotalTime(self):
        """ get the total readout + exposure time in second """
        with self.lock:
            if self.tend == 0:
                total = -1
            else:
                total = self.tend - self.tstart
        return total

# module initialization
CLOSED, READY, EXPOSING, SETMODE = range(4)
Status = {CLOSED:"CLOSED", READY:"READY", EXPOSING:"EXPOSING", SETMODE:"SETMODE"}
POLL_TIME = 0.02
CCD_TEMP = -30
FLI_INVALID_DEVICE, FLIDEVICE_CAMERA = 0, 1

numCams = 6
dev = np.zeros(numCams, int)


# encoding: UTF-8

'''
实现读取wind行情历史数据.mat格式
'''

import scipy.io as sio
import pandas as pd
import datetime, re, logging
from module_mylog import gLogger

class LoadMatFile(object):

    def __init__(self, matFile):
        self.matFile = matFile
        self.data = sio.loadmat(self.matFile)
        gLogger.info("load mat file completed")
        self.dfData = self.convert2df()

    def convert2df(self):
        try:
            colNames = ['vtSymbol','symbol','date','time','lastPrice','lastVolume','lastTurnover','matchItems','openInterest','tradeFlag','bsFlag','volume','turnover'
                        ,'highPrice','lowPrice','openPrice','preClosePrice','settlementPrice','position','curDelta','preSettlementPrice','prePosition','askPrice1'
                        ,'askVolume1','bidPrice1','bidVolume1','askAvPrice','bidAvPrice','totalAskVolume','totalBidVolume','index','stocks','ups','downs','holdLines']

            seriesNames = locals()
            for k,v in enumerate(self.data['temp'][0][0].tolist()):
                if colNames[k] in ['askPrice1','askVolume1','bidPrice1','bidVolume1']:
                    v = v[:,0]
                if v.ndim  == 1:
                    seriesNames['SN_%s' % k] = pd.Series(v, name = colNames[k])
                elif v.ndim > 1:
                    v = v.ravel()
                    seriesNames['SN_%s' % k] = pd.Series(v, name=colNames[k])
                else:
                    print ("index = %d, dim = %d" %(k, v.ndim))

            df_data = pd.concat([seriesNames[a] for a in seriesNames if 'SN_' in a], axis = 1)
            df_data = df_data.dropna(axis=0,how='any')
            dfData = self.normalizeData(df_data)
            return dfData
        except Exception as e:
            gLogger.exception("Exception when convert to df: %s" %e)

    def normalizeData(self, df):
        try:
            gLogger.info("start normalize df Data")
            f = lambda x:str(int(x))
            df["date"] = df["date"].map(f)
            df["time"] = df["time"].map(f)
            df["time"] = df["time"].map(lambda x: x.zfill(9))
            vtSymbol = self.matFile.split('\\')[-1].split('_')[0]
            symbol = "".join([a for a in vtSymbol if a.isalpha()]).lower()
            date = self.matFile.split('\\')[-1].split('_')[-1][:-4]
            df["vtSymbol"] = vtSymbol
            df["symbol"] = symbol
            df["DT"] = df["date"] + ' ' + df["time"]
            df["datetime"] = df["DT"].map(self.convert2datetime)
            del df["DT"]
            df["date"] = date
            return df
        except Exception as e:
            gLogger.exception("Exception when convert to df: %s" %e)

    def convert2datetime(self, target):
        try:
            return datetime.datetime.strptime(target, "%Y%m%d %H%M%S%f")
        except:
            gLogger.error("Error when convert to df: %s" % target)
            return None


#coding:utf-8

"""
ID:          issue-5406
ISSUE:       5406
TITLE:       Expression index may not be used by the optimizer if created and used in different connection charsets
DESCRIPTION:
JIRA:        CORE-5122
FBTEST:      bugs.core_5122
"""

import pytest
from firebird.qa import *

db = db_factory()

act = python_act('db')

expected_stdout = """
    PLAN (TEST INDEX (TEST_CALC_S))
"""

@pytest.mark.version('>=3')
def test_1(act: Action):
    with act.db.connect(charset='iso8859_1') as con1:
        cur1 = con1.cursor()
        cur1.execute("recreate table test(s varchar(10))")
        cur1.execute("create index test_calc_s on test computed by ('zxc' || s)")
        con1.commit()
    #
    act.expected_stdout = expected_stdout
    act.isql(switches=['-n'],
               input="set planonly; select * from test where 'zxc' || s starting with 'qwe';")
    assert act.clean_stdout == act.clean_expected_stdout


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from conans import ConanFile, CMake, tools
import os

class OpenALConan(ConanFile):
    name = "openal"
    version = "1.18.2"
    url = "https://github.com/kcat/openal-soft.git"
    description = "A software implementation of the OpenAL 3D audio API."
    license = "https://github.com/kcat/openal-soft/blob/master/COPYING"
    exports_sources = ["CMakeLists.txt", "LICENSE"]
    settings = "os", "arch", "compiler", "build_type"
    options = {"shared": [True, False]}
    default_options = "shared=False"
    generators = "cmake"

    def source(self):
        source_url = "https://github.com/kcat/openal-soft/archive"
        tools.get("{}/openal-soft-{}.tar.gz".format(source_url, self.version))
        os.system("ls")
        extracted_dir = "openal-soft-openal-soft-" + self.version
        os.rename(extracted_dir, "sources")

    def build(self):
        cmake = CMake(self)
        cmake.configure()
        cmake.build()

    def package(self):
        self.copy(pattern="*.h", dst="include", src="sources/include")
        self.copy(pattern="*.hh", dst="include", src="sources/include")
        self.copy(pattern="*.hpp", dst="include", src="sources/include")
        with tools.chdir("sources"):
            self.copy(pattern="LICENSE")
            self.copy(pattern="*.dll", dst="bin", keep_path=False)
            self.copy(pattern="*.lib", dst="lib", keep_path=False)
            self.copy(pattern="*.a", dst="lib", keep_path=False)
            self.copy(pattern="*.so*", dst="lib", keep_path=False)
            self.copy(pattern="*.dylib", dst="lib", keep_path=False)

    def package_info(self):
        self.cpp_info.libs = tools.collect_libs(self)
        if not self.options.shared:
            self.cpp_info.defines = ["FLAC__NO_DLL"]


import joblib
import numpy as np
from flask import Flask, jsonify, request
import joblib
import logging
from datetime import date
import random
import os
import pickle
import pandas as pd

# Get the current directory of the Python file
base_dir = os.path.dirname(os.path.relpath(__file__))

model_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'model', 'logreg_model.joblib')
column_name_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'model', 'col_names.pk1')

app = Flask(__name__)
app.logger.setLevel(logging.DEBUG)

# Load the KNN model
logreg = joblib.load(model_path)
with open(column_name_path, 'rb') as f:
    col_names = pickle.load(f)

@app.route('/predict', methods=['POST'])
def predict():

    today = date.today()  # get today's date
    jan_1 = date(today.year, 1, 1)  # create a date object for January 1st of this year
    delta = today - jan_1  # calculate the time difference between January 1st and today
    days_since_jan_1 = delta.days + 1  # add 1 to include January 1st

    odd_or_even = days_since_jan_1%2

    if odd_or_even == 0:
        # Return the predicted activity as a JSON response
        activity = ['BubblePopActivity', 'YogaActivity', 'ColoringActivity', 'RainbowActivity', 'SelfHugActivity',
         'WaterDrinkingActivity']
        print("Inside If", days_since_jan_1)
        return jsonify({'activity': random.choice(activity)})

    else:
        # Get the emotion and feedback values from the request
        print("Inside else", days_since_jan_1)

        emotion = request.json['emotion']
        feedback = request.json['feedback']

        # Create a new DataFrame with a single row containing the encoded input values
        input_data = pd.DataFrame({'emotion_' + emotion: [1], 'feedback_' + feedback: [1]})

        # Add any missing columns to the input_data DataFrame
        missing_cols = set(col_names) - set(input_data.columns)
        for c in missing_cols:
            input_data[c] = 0

        # Reorder the columns in the input_data DataFrame to match the order of the columns in X
        input_data = input_data[col_names]

        prediction = logreg.predict(input_data)

        # Return the predicted activity as a JSON response
        return jsonify({'activity': prediction[0]})

# Test api for outside access tested.
@app.route('/predict/test', methods=['GET'])
def predicthi():
    return "Predict Till Test"

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=8080)
    # app.run(host='127.0.0.1', port=5000)


# -*- coding: utf-8 -*-

import requests
from bs4 import BeautifulSoup
import json
import pandas as pd
import threading
import time
import os
import re

with open('academy_links.json', 'r') as f:
    academy = f.read()

academy = json.loads(academy)

# print(academy)

jianjie_dict = {
    '马克思主义学院': 'http://mkszyxy.zuel.edu.cn/1703/list.htm',
    '哲学院': 'http://zxy.zuel.edu.cn/2654/list.htm',
    '经济学院': 'http://jjxy.zuel.edu.cn/2747/list.htm',
    '财政税务学院': 'http://csxy.zuel.edu.cn/4929/list.htm',
    '金融学院': 'http://finance.zuel.edu.cn/1103/list.htm',
    '法学院': 'http://law.zuel.edu.cn/3709/list.htm',
    '刑事司法学院': 'http://cjs.zuel.edu.cn/2013/0730/c3111a65935/page.htm',
    '外国语学院': 'http://wgyxy.zuel.edu.cn/830/list.htm',
    '新闻与文化传播学院': 'http://xwcb.zuel.edu.cn/49/list.htm',
    '工商管理学院': 'http://gsxy.zuel.edu.cn/4111/list.htm',
    '会计学院(会硕中心)': 'http://kjxy.zuel.edu.cn/3916/list.htm',
    '公共管理学院(MPA中心)': 'http://ggglxy.zuel.edu.cn/959/list.htm',
    '统计与数学学院': 'http://tsxy.zuel.edu.cn/4806/list.htm',
    '信息与安全工程学院': 'http://xagx.zuel.edu.cn/2096/list.htm',
    '文澜学院': 'http://wls.zuel.edu.cn/3771/list.htm',
    '知识产权学院': 'http://www.iprcn.com/IL_Zxjs.aspx?News_PI=1',
    'MBA学院': 'http://mba.zuel.edu.cn/2504/list.htm',
    '继续教育学院(网络教育学院)': 'http://sce.zuel.edu.cn/2918/list.htm',
    '中韩新媒体学院':'http://zhxmt.zuel.edu.cn/2659/list.htm'
}

class myThread(threading.Thread):
    def __init__(self, name, url, headers):
        threading.Thread.__init__(self)
        self.name = name
        self.url = url
        self.headers = headers

    def run(self):
        print("开始线程：" + self.name)
        #threadLock.acquire()
        self.soup = crawl(self.url, self.headers)
        try:
            self.pattern = re.compile(r'.+(学院简介.+)',re.S)
            self.jianjie = self.pattern.match(self.soup.text).group(1)
        except:
            self.jianjie = self.soup.text
        filename = '{}.txt'.format(self.name)
        if os.path.isfile(filename):
            print('File {0} already exists. Deleting file...'.format(filename))
            os.remove(filename)
        f=open(filename, mode='x',encoding='utf-8')
        f.write(self.name+'\n')
        f.write(self.jianjie)
        f.close()
        #threadLock.release()
        print("退出线程：" + self.name)

#threadLock = threading.Lock()

def crawl(url, headers):
    res = requests.get(url, headers=headers)
    res.encoding = 'utf-8'
    soup = BeautifulSoup(res.text, 'lxml')

    return soup

names = list(jianjie_dict.keys())
# 创建新线程
threads = []
for each_name in names:
    each_url = jianjie_dict[each_name]
    each_h = academy[each_name]
    each_headers = {
        'Host': '%s' % each_h[7:-1],
        're-Requests': '1',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0'
    }
    thread = myThread(each_name, each_url, each_headers)
    threads.append(thread)

for each_thread in threads:
    each_thread.start()

for each_thread in threads:
    each_thread.join()

# .replace('\xa0','')
#pattern = re.compile(r'.+(学院简介.+)%s'%threads[0].name,re.S)
#
#m = pattern.match(soup.text)


# A linked list of length n is given such that each node contains an additional random pointer, which could point to any node in the list, or null.
# Construct a deep copy of the list. The deep copy should consist of exactly n brand new nodes, where each new node has its value set to the value of its corresponding original node. Both the next and random pointer of the new nodes should point to new nodes in the copied list such that the pointers in the original list and copied list represent the same list state. None of the pointers in the new list should point to nodes in the original list.
# For example, if there are two nodes X and Y in the original list, where X.random --> Y, then for the corresponding two nodes x and y in the copied list, x.random --> y.
# Return the head of the copied linked list.
# The linked list is represented in the input/output as a list of n nodes. Each node is represented as a pair of [val, random_index] where:
    # val: an integer representing Node.val
    # random_index: the index of the node (range from 0 to n-1) that the random pointer points to, or null if it does not point to any node.
# Your code will only be given the head of the original linked list.

# Definition for a Node.
from typing import Optional
from collections import defaultdict

class Node:
    def __init__(self, x: int, next: 'Node' = None, random: 'Node' = None):
        self.val = int(x)
        self.next = next
        self.random = random

class Solution:
    def copyRandomList(self, head: Optional[Node]) -> Optional[Node]:
        if not head: return head
        head2 = Node(head.val)
        conn = defaultdict(list)

        runner1, runner2 = head, head2
        while runner1:
            if runner1.random == None:
                runner2.random == None
            else:
                conn[runner1.random].append(runner2)
            if not runner1.next: break
            runner1 = runner1.next
            runner2.next = Node(runner1.val)
            runner2 = runner2.next

        runner1, runner2 = head, head2
        while runner1 and conn:
            if runner1 in conn:
                for i in conn[runner1]:
                    i.random = runner2
                conn.pop(runner1, None)
            runner1 = runner1.next
            runner2 = runner2.next
        return head2

node5 = Node(1)
node4 = Node(10, node5)
node3 = Node(11, node4)
node2 = Node(13, node3)
node1 = Node(7, node2)

node2.random = node1
node3.random = node5
node4.random = node3
node2.random = node1

s = Solution()
print(s.copyRandomList(node1))


import numpy as np
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit

# Parameters
b_n = [0.1, 1, 10]
d_n = [0.2, 2, 5]
dt = 0.01
max_it = 100000
td_list = []
tb_list = []

fig, axs = plt.subplots(nrows=len(b_n), ncols=2, figsize=(10, 6))
for j, (b, d) in enumerate(zip(b_n, d_n)):
    td_data = np.zeros(max_it)
    tb_data = np.zeros(max_it)
    for i in range(max_it):
        t = 0
        td = 0
        tb = 0
        while tb == 0 or td == 0:
            t += dt
            r = np.random.rand()
            if tb == 0 and r < b*dt:
                tb = t
                tb_data[i] = tb
            if td == 0 and r < d*dt:
                td = t
                td_data[i] = td

    tb_data = tb_data[tb_data != 0]
    td_data = td_data[td_data != 0]
    tb_list.append(tb_data)
    td_list.append(td_data)

    # Plot distribution of tb
    ax = axs[j, 0]
    tb_hist, tb_edges = np.histogram(tb_list[j], bins=100, density=True)
    # tb_centers = (tb_edges[:-1] + tb_edges[1:]) / 2
    ax.hist(tb_edges[:-1], tb_edges, weights=tb_hist, density=True, color='#91413f', edgecolor='k')
    ax.plot(tb_list[j], np.exp(-b_n[j] * np.array(tb_list[j])), 'b-')
    ax.set_yscale('log')
    ax.set_xlabel(r'$t_b$')
    ax.set_ylabel(r'log($P(t_b)$)')
    ax.set_title(fr'$b_n$={b}')

    # Plot distribution of td
    ax = axs[j, 1]
    td_hist, td_edges = np.histogram(td_list[j], bins=100, density=True)
    # td_centers = (td_edges[:-1] + td_edges[1:]) / 2
    ax.hist(td_edges[:-1], td_edges, weights=td_hist, density=True, color='#21d18d', edgecolor='k')
    ax.plot(td_list[j], np.exp(-d_n[j] * np.array(td_list[j])), 'b-')
    ax.set_yscale('log')
    ax.set_xlabel(r'$t_d$')
    ax.set_ylabel(r'log($P(t_d)$)')
    ax.set_title(fr'$d_n$={d}')

fig.tight_layout()
plt.show()

def exp_fit(x, y):
    def f(x, A, lambda_):
        return A * np.exp(-lambda_ * x)

    popt, pcov = curve_fit(f, x, y)

    return popt[1], pcov[1, 1]

# Fit exponential function to infection data
for j, (b, d) in enumerate(zip(b_n, d_n)):
    tb_data_j = tb_list[j][tb_list[j] > 0]
    hist, bin_edges = np.histogram(tb_data_j, bins=100, density=True)
    bin_centers = (bin_edges[:-1] + bin_edges[1:]) / 2
    lambda_i, cov_i = exp_fit(bin_centers, hist)
    print(f'For b = {b}, lambda_inf = {lambda_i:.3f}')

# Fit exponential function to recovery data
for j, (b, d) in enumerate(zip(b_n, d_n)):
    td_data_j = td_list[j][td_list[j] > 0]
    hist, bin_edges = np.histogram(td_data_j, bins=100, density=True)
    bin_centers = (bin_edges[:-1] + bin_edges[1:]) / 2
    lambda_r, cov_r = exp_fit(bin_centers, hist)
    print(f'd = {d}, lambda_rec = {lambda_r:.3f}')

for i in range(len(tb_list)):
    np.save(f'td_arr{i}', td_list[i])
    np.save(f'tb_arr{i}', tb_list[i])

for i in range(len(tb_list)):
    print('b = {}, <t_b> = {}'.format(b_n[i], np.mean(tb_list[i])))
    print('d = {}, <t_d> = {}'.format(d_n[i], np.mean(td_list[i])))