from SVM_Experiment1 import *
import pandas as pd
import numpy as np
import librosa
import librosa.display
import matplotlib.pyplot as plt
import matplotlib
import pywt
import pywt.data

# Notification
import time

# File IO
import h5py
import os

# Deep learning
# Keras-related imports
import keras
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten, LSTM
from keras.layers import Convolution1D, MaxPooling2D, Convolution2D
from keras import backend as K
# K.set_image_dim_ordering('th')
from keras.callbacks import ModelCheckpoint
from keras.optimizers import SGD
from keras.callbacks import RemoteMonitor
from keras.callbacks import EarlyStopping
from keras.models import load_model
from keras.layers import Conv2D, Conv1D, MaxPooling1D, GlobalAveragePooling1D
from keras.layers.normalization import BatchNormalization
from keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau, CSVLogger

# Result processing
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from sklearn.utils.multiclass import unique_labels
from sklearn.metrics import f1_score
import skimage.io
import pylab
from skimage.feature import local_binary_pattern
import cv2
from scipy import ndimage

# For plotting headlessly
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas

np. set_printoptions(threshold=np. inf)

#return indexes for num_from_each_cat TN, FN, FP, TP classified data so we can see what they look like (1 means noise, 0 means mosquito
def get_classified_samples (num_from_each_cat, y_test, y_predicted):
    tn_idxes = [idx for idx, elt in enumerate(y_test) if elt == 0 and y_predicted[idx] == 0]
    fn_idxes = [idx for idx, elt in enumerate(y_test) if elt == 1 and y_predicted[idx] == 0]
    fp_idxes = [idx for idx, elt in enumerate(y_test) if elt == 0 and y_predicted[idx] == 1]
    tp_idxes = [idx for idx, elt in enumerate(y_test) if elt == 1 and y_predicted[idx] == 1]

    return tn_idxes[0:num_from_each_cat], fn_idxes[0:num_from_each_cat], fp_idxes[0:num_from_each_cat], tp_idxes[0:num_from_each_cat]

#return some TN, FN, FP, TP  feature maps
def get_feature_maps(tn_idxes, fn_idxes, fp_idxes, tp_idxes, X_test, model, new_output_layer_idx):
    intermediate_layer_model = keras.Model(inputs=model.input, outputs=model.layers[new_output_layer_idx].output)
    print("Model Layers:", model.layers)
    tn_list, fn_list, fp_list, tp_list = [], [], [], []
    print("\n\nlen tn_idx:", len(tn_idxes))
    for i in range (len(tn_idxes)):
        tn_list.append(intermediate_layer_model.predict(X_test[tn_idxes[i]].reshape(1,1,X_test.shape[-1])))
        fn_list.append(intermediate_layer_model.predict(X_test[fn_idxes[i]].reshape(1,1,X_test.shape[-1])))
        fp_list.append(intermediate_layer_model.predict(X_test[fp_idxes[i]].reshape(1,1,X_test.shape[-1])))
        tp_list.append(intermediate_layer_model.predict(X_test[tp_idxes[i]].reshape(1,1,X_test.shape[-1])))

    return np.array(tn_list), np.array(fn_list), np.array(fp_list), np.array(tp_list)

#another way of computing feature map using a Keras function, in [x, 0], 0 means output in test mode, 1 means in train mode
#    get_3rd_layer_output = K.function([model.layers[0].input, K.learning_phase()],
#                                  [model.layers[3].output])
#
#    # output in test mode = 0
#    layer_output = get_3rd_layer_output([x, 0])[0]
#
#    # output in train mode = 1
#    layer_output = get_3rd_layer_output([x, 1])[0]

def get_data(target_names):

        # Read about 20K of recs from every species
        # Note: All wav files must be the same sampling frequency and number of datapoints!

        X = []                    # holds all data raw
        Z = []                    # holds all frequency domain data
        y = []                    # holds all class labels

        filenames = []            # holds all the file names
        target_count = []         # holds the counts in a class

        feature_type = 'dwt'  # Select from {'log-mel', 'mfcc', 'dwt'}

        start_time = time.time()
        for i, target in enumerate(target_names):
            target_count.append(0)  # initialize target count
            path='/media/hermann/Tonpi/tonpi/Collegecourses/PSU/Graduate-School/Thesis/Dataset/Dataset #3 (Mosquito Wingbeats Recording (Kaggle))/archive/Wingbeats/' + target + '/'    # assemble path string

            for [root, dirs, files] in os.walk(path, topdown=False):
                for filename in files:
                    name,ext = os.path.splitext(filename)
                    if ext=='.wav':
                        name=os.path.join(root, filename)
                        data, fs = librosa.load(name, sr=None) #data, fs = sf.read(name)
#                        stft = librosa.stft(data)
                        mel_spec = librosa.feature.melspectrogram(y=data, sr=fs)
#                        chroma_cqt = librosa.feature.chroma_cqt(y=data, sr=fs, fmin=30)
#                        D = librosa.stft(y=data)
#                        S_db = librosa.amplitude_to_db(np.abs(D), ref=np.max)
                        Z.append(mel_spec)
#                        cA, cD = pywt.dwt(data, 'db1')
#                        X.append(data)
#                        Z.append(cA)
                        y.append(i)
                        filenames.append(name)
                        target_count[i]+=1
                        if target_count[i]>20000:
                                break
                        if target_count[i]%1000 == 0:
                            time_used = time.time()-start_time

                            print('Iteration', str(i), "%i"%(time_used))

            print (target,'#recs = ', target_count[i])

#        X = np.vstack(X)
        Z = np.array(Z)
        y = np.hstack(y)

#        X = X.astype("float32")
        Z = Z.astype("float32")
        print ("")
        print ("Total dataset size:")
#        print ('# of classes: %d' % len(np.unique(y)))
        print ('total dataset size: %d' % Z.shape[0])
        print ('Sampling frequency = %d Hz' % fs)
        print ("n_samples: %d" % Z.shape[1])
        print ("duration (sec): %f" % (Z.shape[1]/fs))
        print ("\nshape of dataset", Z.shape)

        return X, Z, y

## Save files
def save_files(raw_data, freq_data, labels):

#        hf = h5py.File('/media/hermann/Tonpi/tonpi/Collegecourses/PSU/Graduate-School/Thesis/Dataset/Dataset #3 (Mosquito Wingbeats Recording (Kaggle))/archive/Extracted_Features/raw/raw_data.h5', 'w')
#        hf.create_dataset('raw_data', data=raw_data)
#        hf.close()

#        hf = h5py.File('/media/hermann/Tonpi/tonpi/Collegecourses/PSU/Graduate-School/Thesis/Dataset/Dataset #3 (Mosquito Wingbeats Recording (Kaggle))/archive/Extracted_Features/stft/stft_data.h5', 'w')
#        hf.create_dataset('stft_data', data=freq_data)
#        hf.close()
#
#        hf = h5py.File('/media/hermann/Tonpi/tonpi/Collegecourses/PSU/Graduate-School/Thesis/Dataset/Dataset #3 (Mosquito Wingbeats Recording (Kaggle))/archive/Extracted_Features/stft/labels.h5', 'w')
#        hf.create_dataset('labels', data=labels)
#        hf.close()

        hf = h5py.File('/media/hermann/Tonpi/tonpi/Collegecourses/PSU/Graduate-School/Thesis/Dataset/Dataset #3 (Mosquito Wingbeats Recording (Kaggle))/archive/Extracted_Features/mel_spec/mel_log_data.h5', 'w')
        hf.create_dataset('mel_log_data', data=freq_data)
        hf.close()

# Load files
def load_files():
#    hf = h5py.File('/media/hermann/Tonpi/tonpi/Collegecourses/PSU/Graduate-School/Thesis/Dataset/Dataset #3 (Mosquito Wingbeats Recording (Kaggle))/archive/Extracted_Features/raw/raw_data.h5', 'r')
#    spec_matrix_read = np.array(hf.get('raw_data'))
#    hf.close()
#
#    hf = h5py.File('/media/hermann/Tonpi/tonpi/Collegecourses/PSU/Graduate-School/Thesis/Dataset/Dataset #3 (Mosquito Wingbeats Recording (Kaggle))/archive/Extracted_Features/raw/labels.h5', 'r')
#    y = np.array(hf.get('labels'))
#    hf.close()

#    hf = h5py.File('/media/hermann/Tonpi/tonpi/Collegecourses/PSU/Graduate-School/Thesis/Dataset/Dataset #3 (Mosquito Wingbeats Recording (Kaggle))/archive/Extracted_Features/db1/db1_data.h5', 'r')
#    spec_matrix_read = np.array(hf.get('db1_data_cD'))
#    hf.close()

#    hf = h5py.File('/media/hermann/Tonpi/tonpi/Collegecourses/PSU/Graduate-School/Thesis/Dataset/Dataset #3 (Mosquito Wingbeats Recording (Kaggle))/archive/Extracted_Features/db1/db1_data_cA.h5', 'r')
#
#    spec_matrix_read = np.concatenate((spec_matrix_read, np.array(hf.get('db1_data_cA'))), axis = 1)
#    #spec_matrix_read_2 = np.array(hf.get('db1_data_cA'))
#    hf.close()

#    hf = h5py.File('/media/hermann/Tonpi/tonpi/Collegecourses/PSU/Graduate-School/Thesis/Dataset/Dataset #3 (Mosquito Wingbeats Recording (Kaggle))/archive/Extracted_Features/db1/labels.h5', 'r')
#    y = np.array(hf.get('labels'))
#    hf.close()

#    hf = h5py.File('/media/hermann/Tonpi/tonpi/Collegecourses/PSU/Graduate-School/Thesis/Dataset/Dataset #3 (Mosquito Wingbeats Recording (Kaggle))/archive/Extracted_Features/mel_spec/mel_log_data.h5', 'r')
#    spec_matrix_read = np.array(hf.get('mel_log_data'))
#    hf.close()

    hf = h5py.File('/media/hermann/Tonpi/tonpi/Collegecourses/PSU/Graduate-School/Thesis/Dataset/Dataset #3 (Mosquito Wingbeats Recording (Kaggle))/archive/Extracted_Features/mel_spec/mel_lbp_data.h5', 'r')
    spec_matrix_read = np.array(hf.get('mel_lbp_data')) #np.concatenate((spec_matrix_read, np.array(hf.get('mel_lbp_data'))), axis=1)
    hf.close()

    hf = h5py.File('/media/hermann/Tonpi/tonpi/Collegecourses/PSU/Graduate-School/Thesis/Dataset/Dataset #3 (Mosquito Wingbeats Recording (Kaggle))/archive/Extracted_Features/mel_spec/labels.h5', 'r')
    y = np.array(hf.get('labels'))
    hf.close()

#    hf = h5py.File('/media/hermann/Tonpi/tonpi/Collegecourses/PSU/Graduate-School/Thesis/Dataset/Dataset #3 (Mosquito Wingbeats Recording (Kaggle))/archive/Extracted_Features/chroma/chroma_data.h5', 'r')
#    spec_matrix_read = np.array(hf.get('chroma_data'))
#    hf.close()
#
#    hf = h5py.File('/media/hermann/Tonpi/tonpi/Collegecourses/PSU/Graduate-School/Thesis/Dataset/Dataset #3 (Mosquito Wingbeats Recording (Kaggle))/archive/Extracted_Features/chroma/labels.h5', 'r')
#    y = np.array(hf.get('labels'))
#    hf.close()

#    hf = h5py.File('/media/hermann/Tonpi/tonpi/Collegecourses/PSU/Graduate-School/Thesis/Dataset/Dataset #3 (Mosquito Wingbeats Recording (Kaggle))/archive/Extracted_Features/stft/stft_data.h5', 'r')
#    spec_matrix_read = np.array(hf.get('stft_data'))
#    hf.close()
#
#    hf = h5py.File('/media/hermann/Tonpi/tonpi/Collegecourses/PSU/Graduate-School/Thesis/Dataset/Dataset #3 (Mosquito Wingbeats Recording (Kaggle))/archive/Extracted_Features/stft/labels.h5', 'r')
#    y = np.array(hf.get('labels'))
#    hf.close()

    #spec_matrix_read = db1_data
    #y = labels

    # Convert to dB

    spec_matrix_db = spec_matrix_read #np.zeros_like(spec_matrix_read)

#    for i, spec in enumerate(spec_matrix_read):
#        spec_matrix_db[i] = librosa.power_to_db(spec,ref=np.max)

#    for i, spec in enumerate(spec_matrix_read):
#        spec_matrix_db[i] = librosa.amplitude_to_db(np.abs(spec), ref=np.max)

#    pred_list = []
#    y_test_list = []

    print("Spec_matrix_db min and max:", np.amin(spec_matrix_db), np.amax(spec_matrix_db))
    print("Spec_matrix_db shape:", spec_matrix_db.shape)
    return spec_matrix_db, y

def scale_minmax(X, min=0.0, max=1.0):
    X_std = (X - X.min()) / (X.max() - X.min())
    X_scaled = X_std * (max - min) + min
    return X_scaled

def spectrogram_image(mels, out):
    # use log-melspectrogram
#    mels = librosa.feature.melspectrogram(y=y, sr=sr, n_mels=n_mels,
#                                            n_fft=hop_length*2, hop_length=hop_length)
#    mels = numpy.log(mels + 1e-9) # add small number to avoid log(0)

    # min-max scale to fit inside 8-bit range
    img = scale_minmax(mels, 0, 255).astype(np.uint8)
    img = np.flip(img, axis=0) # put low frequencies at the bottom in image
    img = 255-img # invert. make black==more energy

    # save as PNG
    skimage.io.imsave(out, img)

#plot some samples
def plot_time_series(spec_matrix_db, y):
    fig, ax = plt.subplots(nrows=6, sharex=True, sharey=True) #nrows = 6 because we want 6 plots
    for i in range(0, 6):
        index = np.nonzero(y == i)[0][0] #return index of first occurence of number i in y
        librosa.display.waveplot(spec_matrix_db[index], sr=8000, ax=ax[i]) #8000 comes from printing sr (sample rate) in the get_data method
        ax[i].set(title='class ' + str(i))
        ax[i].label_outer()
    plt.show()

def plot_mel_spec(spec_matrix_db):
    output = spec_matrix_db
    for i in range(0, 1):#spec_matrix_db.shape[0]):
        fig = plt.Figure()
        canvas = FigureCanvas(fig)
        ax = fig.add_subplot(111)
        img = librosa.display.specshow(spec_matrix_db[i], ax=ax)
#        output[i] = img.get_array().reshape(img._meshHeight, img._meshWidth)
#        output[i] = scale_minmax(output[i], 0, 255)
#        plt.specgram(spec_matrix_db[i], Fs=8000)
        plt.show()
        fig.savefig('spec.png', bbox_inches='tight', pad_inches=0)
    return output

def MEL_LBP(spec_matrix_db):
    mel_lbp_hists = []
    for i in range(0, spec_matrix_db.shape[0]):
        fig = plt.Figure()
        canvas = FigureCanvas(fig)
        ax = fig.add_subplot(111)
        img = librosa.display.specshow(spec_matrix_db[i], ax=ax)
        fig.savefig('spec_2.png', bbox_inches='tight', pad_inches=0)
#        lbp_hist = LBP('spec.png', 512)
        lbp_hist = LOG('spec_2.png', 512)
#        print(lbp_hist)
        mel_lbp_hists.append(lbp_hist)
    return np.array(mel_lbp_hists)

def LBP(path, n_bins):
    radius = 3
    n_points = 8 * radius
    METHOD = 'uniform'
    eps = 1e-7
    img_bgr = cv2.imread(path, 1)
    img_gray = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2GRAY)
    img_lbp = local_binary_pattern(img_gray, n_points, radius, METHOD)
    (hist, _) = np.histogram(img_lbp.ravel(), bins=n_bins)# range=(0, numPoints + 2))
    # normalize the histogram
    hist = hist.astype("float")
    hist /= (hist.sum() + eps)

    return hist

def LOG(path, n_bins):
    eps = 1e-7
    img_bgr = cv2.imread(path, 1)
    img_gray = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2GRAY)
    img_log = ndimage.gaussian_laplace(img_gray, sigma=3)
    (hist, _) = np.histogram(img_log.ravel(), bins=n_bins)# range=(0, numPoints + 2))
    # normalize the histogram
    hist = hist.astype("float")
    hist /= (hist.sum() + eps)

    return hist

def train_classify(spec_matrix_db, y):
    for random_state in [2021]: #[10,20,30,40,50,60,70,80,90,100]:

        X_train, X_test, y_train, y_test = train_test_split(spec_matrix_db, np.array(y), test_size=0.20,
                                                            random_state=random_state)
#        # Normalise by statistics of training data
        X_train= (X_train - np.amin(X_train))/(np.amax(X_train) - np.amin(X_train))
#        X_train= (X_train - np.mean(X_train))/np.std(X_train)
        X_test= (X_test - np.amin(X_test))/(np.amax(X_test) - np.amin(X_test))
#        X_test= (X_test - np.mean(X_train))/np.std(X_train)
        print("X_train Sha[e:", X_train.shape)
#        X_train = X_train.reshape(X_train.shape[0], 1, X_train.shape[1])#X_train.shape[1], X_train.shape[2])
#        X_test= X_test.reshape(X_test.shape[0], 1, X_test.shape[1])#X_test.shape[1], X_test.shape[2])

        # Convert label to onehot
        y_train = keras.utils.to_categorical(y_train, num_classes=6)
        y_test = keras.utils.to_categorical(y_test, num_classes=6)

        #KNN and SVM
#        X_train = X_train.reshape(X_train.shape[0], X_train.shape[1] * X_train.shape[2]) # 1, X_train.shape[1], X_train.shape[2])
#        X_test= X_test.reshape(X_test.shape[0], X_test.shape[1] * X_test.shape[2]) #1, X_test.shape[1], X_test.shape[2])

#        #CNN 1D Mel_Spec
#        X_train = X_train.reshape(X_train.shape[0], X_train.shape[1] * X_train.shape[2], 1) # 1, X_train.shape[1], X_train.shape[2])
#        X_test= X_test.reshape(X_test.shape[0], X_test.shape[1] * X_test.shape[2], 1) #1, X_test.shape[1], X_test.shape[2])

        #CNN 1D Wavelet
        X_train = X_train.reshape(X_train.shape[0], X_train.shape[1], 1)
        X_test= X_test.reshape(X_test.shape[0], X_test.shape[1], 1)

        #CNN 2D
#        X_train = X_train.reshape(X_train.shape[0], X_train.shape[1], X_train.shape[2])
#        X_test= X_test.reshape(X_test.shape[0], X_test.shape[1], X_test.shape[2])

#        (Acc, Time) = KNNClassification(X_train, y_train, X_test, y_test)
#        (Acc, Time) = SVM(X_train, y_train, X_test, y_test)
#        (Acc, Time) = RF(X_train, y_train, X_test, y_test)
#        (Acc, Time) = MLP(X_train, y_train, X_test, y_test)

#        ################################ CONVOLUTIONAL NEURAL NETWORK ################################
#        ## NN parameters
#        class_weight = {0: 1.,
#                        1: 1.,
#                        2: 1.,
#                        3: 1.,
#                        4: 1.,
#                        5: 1.,
#                        }
##        input_shape = (X_train.shape[1], X_train.shape[0],  X_train.shape[-1]) #1D
#        input_shape = (1, X_train.shape[2], X_train.shape[-1]) #2D
        input_shape = (X_train.shape[1], X_train.shape[-1]) #1D Kaggle Mel
        # Build the Neural Network
        model = Sequential()

        model.add(Conv1D(16, 3, activation='relu', input_shape=input_shape))
        model.add(Conv1D(16, 3, activation='relu'))
        model.add(BatchNormalization())

        model.add(Conv1D(32, 3, activation='relu'))
        model.add(Conv1D(32, 3, activation='relu'))
        model.add(BatchNormalization())

        model.add(MaxPooling1D(2))
        model.add(Conv1D(64, 3, activation='relu'))
        model.add(Conv1D(64, 3, activation='relu'))
        model.add(BatchNormalization())

        model.add(MaxPooling1D(2))
        model.add(Conv1D(128, 3, activation='relu'))
        model.add(Conv1D(128, 3, activation='relu'))
        model.add(BatchNormalization())

        model.add(MaxPooling1D(2))
        model.add(Conv1D(256, 3, activation='relu'))
        model.add(Conv1D(256, 3, activation='relu'))
        model.add(BatchNormalization())
        model.add(GlobalAveragePooling1D())

        model.add(Dropout(0.5))
        model.add(Dense(6, activation='softmax'))

        model.compile(loss='categorical_crossentropy',
                      optimizer='adam',
                      metrics=['accuracy'])

        # Plot model
        # from keras.utils import plot_model
        # plot_model(model, to_file='model.png')

        model_name = 'deep_1'
        top_weights_path = 'model_' + str(model_name) + '.h5'

        callbacks_list = [ModelCheckpoint(top_weights_path, monitor = 'val_acc', verbose = 1, save_best_only = True, save_weights_only = True),
            EarlyStopping(monitor = 'val_acc', patience = 6, verbose = 0),
            ReduceLROnPlateau(monitor = 'val_acc', factor = 0.1, patience = 3, verbose = 1),
            CSVLogger('model_' + str(model_name) + '.log')]

        model.fit(X_train, y_train, batch_size=128, epochs=50, validation_data = [X_test, y_test], callbacks = callbacks_list)

        model.load_weights(top_weights_path)
        loss, acc = model.evaluate(X_test, y_test, batch_size=16)
        pred = model.predict(X_test)

        #print('loss', loss)
        print('Test accuracy:', acc)

        pred = (pred > 0.5)
        c_f = confusion_matrix(y_test.argmax(axis=1), pred.argmax(axis=1))

        print(c_f)

#
#        model = Sequential()
#        n_dense = 128
#        nb_classes = 6
#        # number of convolutional filters
#        nb_conv_filters = 32
#        # num_hidden = 236
#        nb_conv_filters_2 = 64
#        convout1 = Activation('relu')
#        convout2 = Activation('relu')
#
#        model.add(Conv2D(nb_conv_filters, kernel_size = (3,3),
#             activation = 'relu', padding = 'valid', strides = 1,
#             input_shape = input_shape, data_format='channels_first'))
#        model.add(MaxPooling2D(pool_size= (2, 2)))
#
##        model.add(Conv1D(nb_conv_filters, kernel_size = (5),
##             activation = 'relu', padding = 'valid', strides =1, data_format='channels_first'))
##        model.add(MaxPooling1D(pool_size=2, data_format='channels_first'))
##
##
##        model.add(Dropout(0.6))
##        model.add(Conv1D(nb_conv_filters_2, kernel_size = (2),
##             activation = 'relu', padding = 'valid', strides =1, data_format='channels_first'))
##        model.add(MaxPooling1D(pool_size=2, data_format='channels_first'))
#
#        model.add(Dropout(0.2))
#        model.add(Conv2D(nb_conv_filters_2, kernel_size = (5,5),
#              activation = 'relu', padding = 'valid'))
#        model.add(MaxPooling2D(pool_size=(2, 2)))
#
#        model.add(Dropout(0.4))
#        model.add(Flatten())
#        # Shared between MLP and CNN:
#        model.add(Dense(n_dense, activation='relu'))
#        model.add(Dense(nb_classes, activation='softmax'))
#        model.compile(loss='categorical_crossentropy',
#                        optimizer='adam',
#                        metrics=['accuracy'])
#
#        model.fit(x=X_train, y=y_train, batch_size=None, epochs=20, verbose=1, callbacks=None, validation_split=0.1,
#                  shuffle=True, sample_weight=None, initial_epoch=0,
#                  steps_per_epoch=None)
#
#
#        loss, acc = model.evaluate(x=X_test, y=y_test, batch_size=None, verbose=0, sample_weight=None, steps=None)
#        pred = model.predict(X_test)
#        plt.hist(pred[:,1]) # Optional: visualise histogram of labels
#    #     plt.show()
#        print(random_state, loss, acc)
#
#    #    pred_list.append(pred)  # Collect y_test to report classification performance
#    #    y_test_list.append(y_test)  # Collect y_test to report classification performance
#
#        pred = (pred > 0.5)
#        c_f = confusion_matrix(y_test.argmax(axis=1), pred.argmax(axis=1))
#
#        print(c_f)

    #    tn_idxes, fn_idxes, fp_idxes, tp_idxes = get_classified_samples (5, y_test, np.rint(pred))
    #    tn_list, fn_list, fp_list, tp_list = get_feature_maps(tn_idxes, fn_idxes, fp_idxes, tp_idxes, X_test, model, 3) #index 3 = second maxpooling layer
    #    print(c_f, "\n\n TN samples: ", tn_list, "\n\n FN samples: ", fn_list, "\n\n FP samples: ", fp_list, "\n\n TP samples: ", tp_list)
    #    print(c_f, "\n\n TN samples: ", tn_list.shape, "\n\n FN samples: ", fn_list.shape, "\n\n FP samples: ", fp_list.shape, "\n\n TP samples: ", tp_list.shape)

    #print(wav.shape, fs)
    #print(len(cA))
    #print(np.array(y))

def main():
#    target_names = ['Ae. aegypti', 'Ae. albopictus', 'An. gambiae', 'An. arabiensis', 'C. pipiens', 'C. quinquefasciatus']
#    raw_data, freq_data, labels = get_data(target_names)
#    save_files (raw_data, freq_data, labels)
    spec_matrix_db, y = load_files()
#    plot_time_series(spec_matrix_db, y)
#    mel_lbp_hists = MEL_LBP(spec_matrix_db)
#    save_files ([], mel_lbp_hists, y)
#    spectrogram_image(spec_matrix_db[5], "out.png")
    train_classify(spec_matrix_db, y)
##
if __name__ == '__main__':
    main()

Python Programming/Machine Learning/Audio Signal Processing/Wavelet_Transform_and_Classification_Wingbeats.py


from django.shortcuts import render
from django.utils import timezone

class Person():
    def __init__(self, name):
        self.name = name

    def say_hello(self):
        return 'hello'

def test(request):
    people = ['Amy', 'Josh', 'Tobey', 'Josh']
    person = Person('Amy')
    person_list = []
    now = timezone.now()
    past_dt = timezone.datetime(1971,8,22,0,0)
    criteria_dt = timezone.datetime(2001,3,19,0,0)
    future_dt = timezone.datetime(2037,1,1,0,0)

    value ='''
        Miracles happen to only those who believe in them.
        Think like a man of action and act like man of thought.
        Courage is very important. Like a muscle, it is strengthened by use.
        Life is the art of drawing sufficient conclusions from insufficient premises.
        By doubting we come at the truth.
        A man that has no virtue in himself, ever envies virtue in others.
        When money speaks, the truth keeps silent.
        Better the last smile than the first laughter.
    '''

    value1 = 'Joel is a slug'
    value2 = '
Joel is a slug
'
    value3 = "https://www.example.org/foo?a=b&c=d"
    value4 = "Check out www.djangoproject.com"
    value5 = "Send questions to foo@example.com"
    value6 = 'Joel \n is a slug'

    return render(request, 'mytest/test.html', {'people':people, 'person':person,
                                                'person_list':person_list,
                                                'datetime_obj':now, 'past_dt':past_dt,
                                                'criteria_dt':criteria_dt, 'future_dt':future_dt,
                                                'value':value, 'value1':value1,
                                                'value2':value2, 'value3':value3,
                                                'value4':value4, 'value5':value5, 'value6':value6})


import matplotlib.pyplot as plt
import numpy as np
import logging

from scipy.signal import find_peaks
from skimage import filters

def calculate_emittance(image, scale, slit_sep, drift):
    '''
    calculate emittance using processed beam image

    Arguments
    ---------
    image : ndarray, size (N,M)
        Post-processed screen image with screen removed and noise present

    scale : float
        Pixel scaling factor in meters per pixel

    slit_sep : float
        Slit seperation in meters

    drift : float
        Longitudinal drift distance between screen and slits

    threshold : float in range [0,1), optional
        Fractional threshold to set proj to zero as a fraction of global maximum,
        default 0.0 (no change)
    '''

    logger = logging.getLogger(__name__)
    logging.getLogger('matplotlib').setLevel(logging.WARNING)

    #get projection
    orig_proj = np.sum(image, axis = 1)
    orig_proj = orig_proj / np.max(orig_proj)

    #apply threshold
    #proj = np.where(proj > threshold * np.max(proj), proj, 0.0)
    try:
        triangle_threshold = filters.threshold_triangle(orig_proj)
    except ValueError:
        fig,ax = plt.subplots(1,2)
        ax[0].plot(orig_proj)
        ax[1].imshow(image)
        print(image)
    logger.debug(f'triangle_threshold: {triangle_threshold}')
    proj = np.where(orig_proj > triangle_threshold + 0.05, orig_proj, 0)
    #proj = orig_proj

    #plot proj if in debugging
    if 0:
        fig,ax = plt.subplots(1,3)
        ax[0].imshow(image)
        ax[1].plot(orig_proj)
        ax[2].plot(proj)

        plt.show()

    #we assume that the beam is divergent, as a result the peaks should be at least
    #2 mm apart
    peaks,_ = find_peaks(proj, distance = 0.5e-3 / scale)

    if len(peaks) < 5:
        logger.warning(f'detected only {len(peaks)} peaks '
                       '-- emittance might be underestimated')

    logger.debug(f'peak finding found {len(peaks)} peaks')
    logger.debug(f'found peaks at {peaks} px')
    logger.debug(f'mean seperation {np.mean(peaks[1:] - peaks[:-1])*scale:.2e}')
    logger.debug(f'rms seperation {np.std(peaks[1:] - peaks[:-1])*scale:.2e}')

    #calculate mid points and number of blobs
    mid_pts = (peaks[1:] + peaks[:-1])/2
    mid_pts = mid_pts.astype(np.int)
    mid_pts = np.array([0, *mid_pts, len(proj)])

    x_screen_px = np.arange(len(proj))
    n_blobs = len(mid_pts) - 1

    #calculate gaussian fit stats
    a = np.empty(n_blobs) # relative intensity
    b = np.empty(n_blobs) #'y' peak position
    c = np.empty(n_blobs) #'y' sigma

    for n in range(n_blobs):
        sub_proj = proj[mid_pts[n] : mid_pts[n+1]]
        sub_x = x_screen_px[mid_pts[n] : mid_pts[n+1]]

        a[n] = np.sum(sub_proj)
        b[n] = np.average(sub_x, weights = sub_proj)
        c[n] = np.sqrt(np.average((sub_x - b[n])**2, weights=sub_proj))

    #sort peaks by central position
    sorted_idx = np.argsort(b)
    a,b,c = a[sorted_idx],b[sorted_idx],c[sorted_idx]

    logger.debug(f'mean beamlet with_px: {np.mean(c)}')
    logger.debug(a)
    logger.debug(b)
    logger.debug(c)
    #convert pixel lengths to meters
    b = b * scale
    c = c * scale

    #define slit locations
    x_slit_m = np.linspace(-(n_blobs-1)/2*slit_sep, (n_blobs-1)/2*slit_sep, n_blobs)
    logger.debug(x_slit_m)
    #calculate beam centroid at slits
    ixi = np.sum(a * x_slit_m) / np.sum(a)

    #center b coords on beam center on screen
    b = b - np.average(b, weights = a)
    logger.debug(b)

    #calculate rms spread at slits
    ixxi = np.sum(a * (x_slit_m - ixi)**2) / np.sum(a)

    #calc mean divergence of each beamlet
    xp = (b - x_slit_m) / drift

    #calc mean divergence of beam at slits
    ixpi = np.sum(a * xp) / np.sum(a)

    #calc spread in divergence for each beamlet
    sp = c / drift

    #calc rms divergence at slits
    ixpxpi = np.sum(a * sp**2) / np.sum(a)
    #ixpxpi = np.sum(a * sp**2 + a * (xp - ixpi)**2) / np.sum(a)

    #calc correlation term at slits
    ixxpi = (np.sum(a * x_slit_m * xp) - np.sum(a) * ixi *ixpi) / np.sum(a)

    logger.debug('emittance calculation results')
    logger.debug(f'ixi:{ixi}')
    logger.debug(f'ixpi:{ixpi}')
    logger.debug(f'ixxi:{ixxi}')
    logger.debug(f'ixpxpi:{ixpxpi}')
    logger.debug(f'ixxpi:{ixxpi}')

    #calculate emittance
    #emittance = np.sqrt(ixxi * ixpxpi - ixxpi**2)
    emittance = np.sqrt(ixxi * ixpxpi)
    logger.info(f'calculated emittance: {emittance:.2e}, n_peaks:{len(peaks)}')

    return emittance, ixxi, ixpxpi, ixxpi

if __name__ == '__main__':
    logging.basicConfig(level = logging.DEBUG)
    #testing
    minscreen2mm = np.genfromtxt('test_images/onenc400200.txt',names=True)
    img, _, _ = np.histogram2d(minscreen2mm['x'], minscreen2mm['y'],
                               range = np.array(((-25.0,25.0),
                                                 (-25.0,25.0)))*1e-3,
                               bins=(700,700))

    scale = 50e-3 / 700
    drift = 1.5748
    slit_sep = 0.002

    calculate_emittance(img, scale, slit_sep, drift, threshold = 0.0)


"""
Tests for view - dashboard
"""
import pytest

from datetime import date, datetime, timedelta

from pytest_django.asserts import assertContains

from django.http import HttpResponse
from django.urls import reverse

from accessibility_monitoring_platform.apps.common.models import ChangeToPlatform

from ...cases.models import (
    Case,
    STATEMENT_COMPLIANCE_STATE_COMPLIANT,
    WEBSITE_COMPLIANCE_STATE_COMPLIANT,
    CASE_COMPLETED_NO_SEND,
    REPORT_APPROVED_STATUS_APPROVED,
    CASE_COMPLETED_SEND,
    ENFORCEMENT_BODY_PURSUING_YES_COMPLETED,
)
from ...cases.utils import create_case_and_compliance
from ...common.models import BOOLEAN_TRUE
from ...reminders.models import Reminder

def test_dashboard_loads_correctly_when_user_logged_in(admin_client):
    """Tests if a logged in user can access the Dashboard"""
    response: HttpResponse = admin_client.get(reverse("dashboard:home"))

    assert response.status_code == 200
    assertContains(response, "Home |")

def test_dashboard_redirects_to_login_when_user_not_logged_in(client):
    """Tests if a unauthenticated user returns a 302 response"""
    url: str = reverse("dashboard:home")
    response: HttpResponse = client.get(url)

    assert response.status_code == 302
    assert response.url == f"/account/login/?next={url}"  # type: ignore

@pytest.mark.parametrize(
    "dashboard_view",
    ["View+your+cases", "View+all+cases"],
)
def test_dashboard_shows_qa_auditors(dashboard_view, admin_client):
    """Tests if dashboard views are showing the expected QA auditors column"""
    response: HttpResponse = admin_client.get(
        f'{reverse("dashboard:home")}?view={dashboard_view}'
    )

    assert response.status_code == 200
    assertContains(response, "On call")

def test_dashboard_shows_oldest_unassigned_cases_first(admin_client):
    """Tests dashboard shows unassigned cases in order of oldest first"""
    case_1: Case = Case.objects.create(organisation_name="Organisation One")
    case_2: Case = Case.objects.create(organisation_name="Organisation Two")
    case_3: Case = Case.objects.create(organisation_name="Organisation Three")

    response: HttpResponse = admin_client.get(reverse("dashboard:home"))

    assert response.status_code == 200

    content: str = str(response.content)
    case_1_position: int = content.index(case_1.organisation_name)
    case_2_position: int = content.index(case_2.organisation_name)
    case_3_position: int = content.index(case_3.organisation_name)

    assert case_1_position < case_2_position
    assert case_2_position < case_3_position

def test_dashboard_shows_link_to_closed_and_sent_cases(admin_client, admin_user):
    """Check dashboard contains link to find closed and sent to equalities body cases"""
    create_case_and_compliance(
        home_page_url="https://www.website.com",
        organisation_name="org name",
        auditor=admin_user,
        website_compliance_state_initial=WEBSITE_COMPLIANCE_STATE_COMPLIANT,
        statement_compliance_state_initial=STATEMENT_COMPLIANCE_STATE_COMPLIANT,
        report_review_status=BOOLEAN_TRUE,
        report_approved_status=REPORT_APPROVED_STATUS_APPROVED,
        report_sent_date=datetime.now(),
        report_acknowledged_date=datetime.now(),
        twelve_week_update_requested_date=datetime.now(),
        twelve_week_correspondence_acknowledged_date=datetime.now(),
        case_completed=CASE_COMPLETED_SEND,
        sent_to_enforcement_body_sent_date=datetime.now(),
    )

    response: HttpResponse = admin_client.get(reverse("dashboard:home"))

    assert response.status_code == 200

    assertContains(
        response,
        f"""

                View all your cases with status "Case closed and sent to equalities body"

""",
        html=True,
    )

def test_dashboard_shows_link_to_completed_cases(admin_client, admin_user):
    """Check dashboard contains link to find completed cases"""
    create_case_and_compliance(
        home_page_url="https://www.website.com",
        organisation_name="org name",
        auditor=admin_user,
        website_compliance_state_initial=WEBSITE_COMPLIANCE_STATE_COMPLIANT,
        statement_compliance_state_initial=STATEMENT_COMPLIANCE_STATE_COMPLIANT,
        case_completed=CASE_COMPLETED_NO_SEND,
    )

    response: HttpResponse = admin_client.get(reverse("dashboard:home"))

    assert response.status_code == 200

    assertContains(
        response,
        f"""

                View all your cases with status "Complete"

""",
        html=True,
    )

def test_dashboard_shows_warning_of_recent_changes_to_platform(admin_client):
    """Check dashboard contains link to find completed cases"""
    ChangeToPlatform.objects.create(name="Recent change")

    response: HttpResponse = admin_client.get(reverse("dashboard:home"))

    assert response.status_code == 200

    assertContains(
        response,
        f"""

            !

                Warning
                An update has been made to the platform. View the update in

                    Settings > Platform version history

""",
        html=True,
    )

def test_dashboard_shows_correct_number_of_active_cases(admin_client, admin_user):
    """Check dashboard shows correct number of active cases"""
    # Creates unassigned case
    Case.objects.create()

    # Creates test in progress case
    Case.objects.create(
        home_page_url="https://www.website.com",
        organisation_name="org name",
        auditor=admin_user,
    )

    # Creates deactivated case
    Case.objects.create(
        home_page_url="https://www.website.com",
        organisation_name="org name",
        is_deactivated=True,
    )

    # Creates completed case
    create_case_and_compliance(
        home_page_url="https://www.website.com",
        organisation_name="org name",
        auditor=admin_user,
        statement_compliance_state_initial=STATEMENT_COMPLIANCE_STATE_COMPLIANT,
        website_compliance_state_initial=WEBSITE_COMPLIANCE_STATE_COMPLIANT,
        report_review_status=BOOLEAN_TRUE,
        report_approved_status=REPORT_APPROVED_STATUS_APPROVED,
        report_sent_date=datetime.now(),
        report_acknowledged_date=datetime.now(),
        twelve_week_update_requested_date=datetime.now(),
        twelve_week_correspondence_acknowledged_date=datetime.now(),
        case_completed=CASE_COMPLETED_SEND,
        sent_to_enforcement_body_sent_date=datetime.now(),
        enforcement_body_pursuing=ENFORCEMENT_BODY_PURSUING_YES_COMPLETED,
    )

    # Creates closed sent to equalities-body case
    create_case_and_compliance(
        home_page_url="https://www.website.com",
        organisation_name="org name",
        auditor=admin_user,
        statement_compliance_state_initial=STATEMENT_COMPLIANCE_STATE_COMPLIANT,
        website_compliance_state_initial=WEBSITE_COMPLIANCE_STATE_COMPLIANT,
        report_review_status=BOOLEAN_TRUE,
        report_approved_status=REPORT_APPROVED_STATUS_APPROVED,
        report_sent_date=datetime.now(),
        report_acknowledged_date=datetime.now(),
        twelve_week_update_requested_date=datetime.now(),
        twelve_week_correspondence_acknowledged_date=datetime.now(),
        case_completed=CASE_COMPLETED_SEND,
        sent_to_enforcement_body_sent_date=datetime.now(),
    )

    response: HttpResponse = admin_client.get(reverse("dashboard:home"))
    assert Case.objects.all().count() == 5
    assert response.status_code == 200
    expected_number_of_active_cases: str = """

 Total active cases

 2

    """
    assertContains(
        response,
        expected_number_of_active_cases,
        html=True,
    )

def test_dashboard_shows_correct_number_of_your_active_cases(admin_client, admin_user):
    """Check dashboard shows correct number of your active cases"""

    # Creates unassigned case
    Case.objects.create()
    Case.objects.create()

    # Case assigned to you
    Case.objects.create(auditor=admin_user)

    response: HttpResponse = admin_client.get(reverse("dashboard:home"))
    assert Case.objects.all().count() == 3
    assert response.status_code == 200
    expected_number_of_your_active_cases: str = """

 Your active cases

 1

    """
    assertContains(
        response,
        expected_number_of_your_active_cases,
        html=True,
    )

    expected_number_of_unnassigned_cases: str = """

 Unassigned cases

 2

    """
    assertContains(
        response,
        expected_number_of_unnassigned_cases,
        html=True,
    )

def test_dashboard_shows_link_to_reminder(admin_client, admin_user):
    """
    Tests dashboard shows link to reminder for cases of status
    reviewing changes
    """
    case: Case = Case.objects.create(
        home_page_url="https://www.website.com",
        organisation_name="org name",
        auditor=admin_user,
        report_review_status=BOOLEAN_TRUE,
        report_approved_status=REPORT_APPROVED_STATUS_APPROVED,
        report_sent_date=datetime.now(),
        report_acknowledged_date=datetime.now(),
        twelve_week_update_requested_date=datetime.now(),
        twelve_week_correspondence_acknowledged_date=datetime.now(),
    )
    case.compliance.statement_compliance_state_initial = (
        STATEMENT_COMPLIANCE_STATE_COMPLIANT
    )
    case.compliance.website_compliance_state_initial = (
        WEBSITE_COMPLIANCE_STATE_COMPLIANT
    )
    case.compliance.save()
    case.save()
    reminder: Reminder = Reminder.objects.create(
        case=case, due_date=date.today() + timedelta(days=10)
    )

    response: HttpResponse = admin_client.get(reverse("dashboard:home"))

    assert response.status_code == 200

    assertContains(
        response, reverse("reminders:edit-reminder", kwargs={"pk": reminder.id})
    )


#!usr/bin/env python
# coding:utf-8

import zhconv
import codecs
import os
import re
import jieba

all_tang_song_verse = "../corpus/corpus-master/hans_全宋词_全唐五代词.txt"
tang_300_peom = "../corpus/corpus-master/hans_唐诗三百首.txt"
song_300_verse = "../corpus/corpus-master/hans_宋词三百首.txt"
shi_jing = "../corpus/corpus-master/hans_诗经.txt"
chu_ci = "../corpus/corpus-master/hans_楚辞.txt"
lun_yu = "../corpus/corpus-master/hans_论语.txt"
# 设置分句的标志符号；可以根据实际需要进行修改
cutlist = "。！？；".decode('utf-8')

def convert_file_to_hans(target_file):
    with codecs.open(target_file, 'r', encoding='utf-8') as f:
        content = f.read()
        dir_name, base_name = os.path.split(target_file)
        hans_base_name = "hans_" + zhconv.convert(base_name.decode('utf-8'), 'zh-hans').encode('utf-8')
        hans = zhconv.convert(content, 'zh-hans')
        with codecs.open(dir_name + "/" + hans_base_name, 'w', encoding='utf-8') as res:
            res.write(hans)

def convert_hant2hans():
    input_dir = raw_input("directory to convert: ")
    for f in os.listdir(input_dir):
        if f.endswith('.txt') and not f.startswith("hans"):
            convert_file_to_hans(input_dir + "/" + f)
    print("convert finished")

# 检查某字符是否分句标志符号的函数；如果是，返回True，否则返回False
def FindToken(cutlist, char):
    if char in cutlist:
        return True
    else:
        return False

# 进行分句的核心函数
def Cut(cutlist, lines):  # 参数1：引用分句标志符；参数2：被分句的文本，为一行中文字符
    l = []  # 句子列表，用于存储单个分句成功后的整句内容，为函数的返回值
    line = []  # 临时列表，用于存储捕获到分句标志符之前的每个字符，一旦发现分句符号后，就会将其内容全部赋给l，然后就会被清空

    for i in lines:  # 对函数参数2中的每一字符逐个进行检查 （本函数中，如果将if和else对换一下位置，会更好懂）
        if FindToken(cutlist, i):  # 如果当前字符是分句符号
            line.append(i)  # 将此字符放入临时列表中
            l.append(''.join(line))  # 并把当前临时列表的内容加入到句子列表中
            line = []  # 将符号列表清空，以便下次分句使用
        else:  # 如果当前字符不是分句符号，则将该字符直接放入临时列表中
            line.append(i)
    return l

def create_corpus(resource, peom_prefix="", impurities=[]):
    sentences_list = []
    with open(resource, 'r') as text_file:
        for text_line in text_file:
            sentences = Cut(list(cutlist), list(text_line.decode('utf-8')))
            for sentence in sentences:
                if sentence.strip() != "":
                    sentence = sentence.strip()
                    if len(peom_prefix.strip()) > 0:
                        sentence = sentence.replace(peom_prefix, u"\n")
                    for s in impurities:
                        sentence = re.sub(s, "", sentence)
                    sentences_list.append(sentence)
        print("Cut sentence finished")
    dir_name, base_name = os.path.split(resource)
    corpus = dir_name + '/corpus_' + base_name
    with open(corpus, 'w') as corpus:
        for sentence in sentences_list:
            s = sentence.encode('utf-8')
            corpus.write("%s\n" % s)
        print("corpus created")

# 全宋词_全唐五代词
def create_corpus_of_all_tang_song_verse():
    create_corpus(all_tang_song_verse, u"词文：", [u'【.*】?。?'])

# 300唐诗
def create_corpus_of_300_tang_peom():
    create_corpus(tang_300_peom, u'诗文:', [u"\(.*\)"])

# 300宋词
def create_corpus_of_300_song_verse():
    create_corpus(song_300_verse, u"词文:")

# 楚辞
def create_corpus_of_chu_ci():
    create_corpus(chu_ci)

# 诗经
def create_corpus_of_shi_jing():
    create_corpus(shi_jing)

# 论语
def create_corpus_of_lun_yu():
    create_corpus(lun_yu)

if __name__ == '__main__':

    # convert_hant2hans()

    # create_corpus_of_300_tang_peom()

    # create_corpus_of_300_song_verse()

    # create_corpus_of_chu_ci()

    # create_corpus_of_shi_jing()

    create_corpus_of_lun_yu()


import pytest ; pytest

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

# External imports
from flaky import flaky
from selenium.webdriver.common.keys import Keys

# Bokeh imports
from bokeh._testing.util.selenium import RECORD
from bokeh.layouts import row
from bokeh.models import (
    Circle,
    ColumnDataSource,
    CustomAction,
    CustomJS,
    MultiChoice,
    Plot,
    Range1d,
)

#-----------------------------------------------------------------------------
# Tests
#-----------------------------------------------------------------------------

pytest_plugins = (
    "bokeh._testing.plugins.project",
)

def modify_doc(doc):
    source = ColumnDataSource(dict(x=[1, 2], y=[1, 1], val=["a", "b"]))
    plot = Plot(plot_height=400, plot_width=400, x_range=Range1d(0, 1), y_range=Range1d(0, 1), min_border=0)
    plot.add_glyph(source, Circle(x='x', y='y', size=20))
    plot.add_tools(CustomAction(callback=CustomJS(args=dict(s=source), code=RECORD("data", "s.data"))))
    input_box = MultiChoice(css_classes=["foo"])
    input_box.title = "title"
    input_box.options = ["100001", "12344556", "12344557", "3194567289", "209374209374"]
    input_box.value = ["12344556", "12344557"]
    def cb(attr, old, new):
        source.data['val'] = [old, new]
    input_box.on_change('value', cb)
    doc.add_root(row(input_box, plot))

@pytest.mark.selenium
class Test_MultiChoice(object):

    def test_displays_multi_choice(self, bokeh_model_page) -> None:
        text_input = MultiChoice(css_classes=["foo"], options = ["100001", "12344556", "12344557", "3194567289", "209374209374"])

        page = bokeh_model_page(text_input)

        el = page.driver.find_element_by_css_selector('.foo input')
        assert el.get_attribute('type') == "text"

        assert page.has_no_console_errors()

    def test_displays_title(self, bokeh_model_page) -> None:
        text_input = MultiChoice(title="title", css_classes=["foo"], options = ["100001", "12344556", "12344557", "3194567289", "209374209374"])

        page = bokeh_model_page(text_input)

        el = page.driver.find_element_by_css_selector('.foo label')
        assert el.text == "title"
        el = page.driver.find_element_by_css_selector('.foo input')
        assert el.get_attribute('placeholder') == ""
        assert el.get_attribute('type') == "text"

        assert page.has_no_console_errors()

    @flaky(max_runs=10)
    def test_displays_menu(self, bokeh_model_page) -> None:
        text_input = MultiChoice(title="title", css_classes=["foo"], options = ["100001", "12344556", "12344557", "3194567289", "209374209374"])

        page = bokeh_model_page(text_input)

        el = page.driver.find_element_by_css_selector('.foo .choices__list--dropdown')
        assert 'is-active' not in el.get_attribute('class')

        # double click to highlight and overwrite old text
        inp = page.driver.find_element_by_css_selector('.foo input')
        inp.click()
        assert 'is-active' in el.get_attribute('class')

        inp.send_keys(Keys.ENTER)

        selected = page.driver.find_element_by_css_selector('.foo .choices__list--multiple')
        items = selected.find_elements_by_css_selector("div")
        assert len(items) == 1

        item = page.driver.find_element_by_css_selector('.foo .choices__list--multiple div.choices__item')
        assert '100001' == item.get_attribute('data-value')

        delete_button = page.driver.find_element_by_css_selector('.foo .choices__item button')
        assert "Remove item: '100001'" == delete_button.get_attribute('aria-label')

        delete_button.click()

        selected = page.driver.find_element_by_css_selector('.foo .choices__list--multiple')
        items = selected.find_elements_by_css_selector("div")
        assert len(items) == 0

        assert page.has_no_console_errors()

    @flaky(max_runs=10)
    def test_server_on_change_round_trip_on_enter(self, bokeh_server_page) -> None:
        page = bokeh_server_page(modify_doc)

        inp = page.driver.find_element_by_css_selector('.foo input')
        inp.click()

        inp.send_keys(Keys.ENTER)

        button = page.driver.find_element_by_class_name("bk-toolbar-button-custom-action")
        button.click()

        results = page.results
        assert results['data']['val'] == [['12344556', '12344557'], ['12344556', '12344557', '100001']]


import PyQt5
from PyQt5 import QtCore
from PyQt5.QtCore import *
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *
import sys
import operator
import json
from tkinter import *
import time
import os.path
import tkinter.ttk as ttk
from tkinter import messagebox
from tkinter.filedialog import *
from tkinter import scrolledtext
import tkinter.font as tkFont
from tkcalendar import *
import tkinter as tk
import datetime
from datetime import timedelta
from PIL import Image, ImageTk
import getpass as gp
from multiprocessing import Process, Queue
import threading
import pprint
import pandas as pd
import openpyxl
from openpyxl.styles import Font,Alignment,PatternFill,Color, alignment
import shutil
from openpyxl.styles.borders import Border, Side
import webbrowser
import winshell
import schedule
import pandas

today_file_path='data/today_result.json'
history_file_path='data/today_history.json'
setting_file_path='data/setting.json'
icon_path='checkbox/logo.ico'
excel_path='data_excel.xlsx'
history_excel_path='history_excel.xlsx'
images_path=['places/notebook.png','places/print.png','places/dvd.png']

FILE_CLOSE = True

def ctrlEvent(event):#crtl+c제외 키 블락
    if(12==event.state and event.keysym=='c' ):
        return
    else:
        return "break"

def date_range(start, end):
    start = datetime.datetime.strptime(start, "%Y-%m-%d")
    end = datetime.datetime.strptime(end, "%Y-%m-%d")
    dates = [date.strftime("%Y-%m-%d") for date in pd.date_range(start, periods=(end-start).days+1)]
    return dates

def today_date_str():
    return datetime.datetime.today().strftime("%Y-%m-%d")

def now_time_str():
    return today_date_str()+datetime.datetime.now().strftime(" %I:%M:%S %p")

def imageModifier(FILE_PATH,x,y):#아미지 사이즈 재설정 함수
    img1=Image.open(FILE_PATH)
    img1=img1.resize((x,y),Image.ANTIALIAS)
    resized_img1=ImageTk.PhotoImage(img1)
    return resized_img1

class Json_Data():
    def __init__(
        self,FilePath=today_file_path,
        HistoryPath=history_file_path,
        SettingPath=setting_file_path,
        iconpath=icon_path,
        whereList=["노트북",'프린트','관내열람'],
        genderList=['남성','여성'],
        imagelist=images_path,
        savetime='17:50',
        savedir=winshell.desktop()):#생성자

        if os.path.isfile(iconpath):pass#아이콘 체크
        else:iconpath=''

        for image in imagelist:#이미지 리스트 체크
            if os.path.isfile(image):pass
            else:imagelist.remove(image)

        self.SettingPath=SettingPath
        self.settings={}

        self.load_settings()#세팅 파일 로드
        self.check_setting(FilePath,HistoryPath,whereList,genderList,imagelist,savetime,savedir,iconpath)

        self.__Today_Data__=[]
        self.data={}
        self.history={}

        if self.check_save():
            with open(self.FilePath,"r") as f:
                self.data = json.load(f)
            self.check_date()
            if self.check_his():
                with open(self.HistoryPath,"r") as h:
                    self.history=json.load(h)
                    self.add_history_member(type='init')
            else:
                self.history['init']=[]
                self.history['push']=[]
                self.history['show']=[]
                self.history['built']=[]
                self.history['fail']=[]
                self.add_history_member(type='built')
                self.add_history_member(type='init')
        else:
            if self.check_his():
                with open(self.HistoryPath,"r") as h:
                    self.history=json.load(h)
                    self.add_history_member(type='built')
                    self.add_history_member(type='init')
            else:
                self.history['init']=[]
                self.history['push']=[]
                self.history['show']=[]
                self.history['built']=[]
                self.history['fail']=[]
                self.add_history_member(type='built')
                self.add_history_member(type='init')
            self.data['cumulative']=0#전체 누적 이용자 수
            self.data['typecode']="TR"#데이터 타입
            for where in self.__WhereName__:
                self.data[where]=[]
                k={
                    "date":today_date_str(),
                    "where":where
                }
                for gender in self.__Gender__:
                    k[gender]=0
                self.data[where].append(k)
            self.check_date()
            self.save_data()

    def save_data(self):#데이터 저장
        with open(self.FilePath,'w') as fk:
            json.dump(self.data, fk,indent=4)

    def load_settings(self):
        if os.path.isfile(self.SettingPath):
            with open(self.SettingPath,'r') as stpr:
                self.settings=json.load(stpr)

    def check_setting(self,FilePath,HistoryPath,whereList,genderList,imagelist,savetime,savedir,iconpath):
        '''
        세팅 파일에서 읽어온 데이터 체크
        ->만약 내용이 없다면 1번(새로 만들고 저장 후 자기함수호출)
        ->내용이 불완전하다 2번(세팅 읽어온거 다 지우고 자기함수호출)
        ->내용이 맞다면 3번(세이브 파일 요소로 등록 후 함수 나옴)
        '''
        if self.settings=={}:
            self.settings['FilePath']=[]
            self.settings['FilePath'].append({
                'path':FilePath,
                'type':'Data'
            })
            self.settings['FilePath'].append({
                'path':HistoryPath,
                'type':'History'
            })
            self.settings['NameList']=[]
            self.settings['NameList'].append({
                'list':whereList,
                'type':'Place'
            })
            self.settings['NameList'].append({
                'list':genderList,
                'type':'Gender'
            })
            self.settings['ImagePath']=imagelist
            self.settings['DailySave']={
                'time':savetime,
                'dir':os.path.abspath(savedir)
            }
            self.settings['IconPath']=iconpath
            self.save_settings()
            self.check_setting(FilePath,HistoryPath,whereList,genderList,imagelist,savetime,savedir,iconpath)
        else:
            if sorted(list(self.settings.keys()))!=sorted(['FilePath','NameList','ImagePath','DailySave','IconPath']):
                self.settings={}
                self.check_setting(FilePath,HistoryPath,whereList,genderList,imagelist,savetime,savedir,iconpath)
            else:
                self.settings_to_self()

    def settings_to_self(self):#세팅 dict로부터 요소들 읽어오기
        for d in self.settings['FilePath']:
            if d['type']=='Data':self.FilePath=d['path']
            elif d['type']=='History':self.HistoryPath=d['path']
        for f in self.settings['NameList']:
            if f['type']=='Place': self.__WhereName__=f['list']
            elif f['type']=='Gender':self.__Gender__=f['list']
        self.imagelist=self.settings['ImagePath']
        self.savetime=self.settings['DailySave']['time']
        self.savedir=self.settings['DailySave']['dir']
        self.icon_path=self.settings['IconPath']

    def change_settings(self,**kwargs):#요소를 바꾸고 세팅 dict에 넣은 후 저장, 바뀐 것이 있가면 True 리턴
        ischanged=False
        for k,v in kwargs.items():
            print (k)
            print(v)
            print('===========================')
            k=k.lower()
            if k=='savedir' and os.path.isdir(v):
                print('change savedir')
                self.savedir=v
                self.settings['DailySave']['dir']=self.savedir
                self.save_settings()
                ischanged=True
            if k=='iconpath' and os.path.isfile(v):
                print('change icon path')
                self.icon_path=v
                self.settings['IconPath']=self.icon_path
                self.save_settings()
                ischanged=True
            if k=='savetime' and v is str:
                try:
                    time.strftime(v,'%H:%M')
                    print('change savetime')
                    self.savetime=v
                    self.settings['DailySave']['time']=self.savetime
                    self.save_settings()
                    ischanged=True
                except ValueError:
                    pass
            if k=='imagelist' and type(v)==list:
                for vd in v:#이미지 리스트 체크
                    if os.path.isfile(vd):pass
                    else:v.remove(vd)
                print('change imagelist')
                self.imagelist=v
                self.settings['ImagePath']=self.imagelist
                self.save_settings()
                ischanged=True
            if k=='wherelist' and type(v)==list:
                for vd in v:
                    if type(vd) is not str:
                        v.remove(vd)
                print('change places name')
                self.__WhereName__=v
                for f in self.settings['NameList']:
                    if f['type']=='Place': f['list']=self.__WhereName__
                self.save_settings()
                ischanged=True
            if k=='genderlist' and type(v)==list:
                for vd in v:
                    if type(vd) is not str:
                        v.remove(vd)
                print('change Gender name')
                self.__Gender__=v
                for f in self.settings['NameList']:
                    if f['type']=='Gender': f['list']=self.__Gender__
                self.save_settings()
                ischanged=True
            if k=='filepath':
                print('Change FilePath')
                self.FilePath=v
                for d in self.settings['FilePath']:
                    if d['type']=='Data':d['path']=self.FilePath
                self.save_settings()
                ischanged=True
            if k=='historypath':
                print('Change HistoryPath')
                self.HistoryPath=v
                for d in self.settings['FilePath']:
                    if d['type']=='History':d['path']=self.HistoryPath
                self.save_settings()
                ischanged=True
        return ischanged

    def show_settings(self,*args):
        tmp={}
        for arg in args:
            k=None
            if arg.lower()=='filepath':k=self.FilePath
            elif arg.lower()=='historypath':k=self.HistoryPath
            elif arg.lower()=='wherelist':k=self.__WhereName__
            elif arg.lower()=='genderlist':k=self.__Gender__
            elif arg.lower()=='imagelist':k=self.imagelist
            elif arg.lower()=='savetime':k=self.savetime
            elif arg.lower()=='savedir':k=self.savedir
            elif arg.lower()=='iconpath':k=self.icon_path
            if k!=None:tmp[arg]=k
        return tmp
    def save_settings(self):
        with open(self.SettingPath,'w') as kosk:
            json.dump(self.settings,kosk, indent=4)

    def save_history(self):#내역 저장
        with open(self.HistoryPath,'w') as hk:
            json.dump(self.history,hk,indent=4)

    def add_history_member(self,type='push',count=1,gender="",place="",showdates=(today_date_str(),today_date_str())):
        tmp={}
        if type =='built'or type=='init':
            tmp['user']=gp.getuser()
            tmp['time']=now_time_str()
            tmp['type']=type
            tmp['isSucess']=TRUE
        elif type=='push' and gender in self.__Gender__ and place in self.__WhereName__:
            tmp['user']=gp.getuser()
            tmp['time']=now_time_str()
            tmp['type']=type
            tmp['isSucess']=TRUE
            tmp['gender']=gender
            tmp['place']=place
            tmp['count']=count
        elif type=='show' and len(date_range(showdates[0],showdates[1]))>0:
            tmp['user']=gp.getuser()
            tmp['time']=now_time_str()
            tmp['type']=type
            tmp['isSucess']=TRUE
            tmp['dates']=date_range(showdates[0],showdates[1])
        else:
            tmp['user']=gp.getuser()
            tmp['time']=now_time_str()
            tmp['type']=type
            tmp['isSucess']=FALSE
            tmp['count']=count
            if gender!="":tmp['gender']=gender
            if place!="":tmp['place']=place
            self.history['fail'].append(tmp)
            self.save_history()
            return FALSE
        self.history[tmp['type']].append(tmp)
        self.save_history()
        return TRUE

    def check_date(self,date_w=today_date_str()):#오늘 날짜 멤버 체크 및 생성
        today=date_w
        print('check_date')
        print(today)
        self.__Today_Data__=[]
        for where in self.__WhereName__:
            for item in self.data[where]:
                if today==item['date']:
                    self.__Today_Data__.append(item)
                    break
        print('sada')
        print(self.__Today_Data__)
        if len(self.__Today_Data__)<len(self.__WhereName__):
            hasMember=[]
            for item in self.__Today_Data__:
                hasMember.append(item['where'])
            tmp=list(set(self.__WhereName__)-set(hasMember))
            print('fdasfasd')
            print(tmp)
            for place in tmp:
                kal={
                    "date":today,
                    "where":place
                }
                for gender in self.__Gender__:
                    kal[gender]=0
                self.data[place].append(kal)
            self.check_date(date_w=today)
        elif len(self.__WhereName__)==len(self.__Today_Data__):
            self.save_data()
            return
        else: self.check_date(date_w=today)

    def show_data(self,start=today_date_str(),end=today_date_str(),mode=FALSE):#특정 기간의 데이터 검색
        end_date=datetime.datetime.strptime(end, '%Y-%m-%d')
        start_date=datetime.datetime.strptime(start, '%Y-%m-%d')
        result=[]
        delta=end_date - start_date
        if delta.days==0:
            for place in self.__WhereName__:
                for data in self.data[place]:
                    if datetime.datetime.strptime(data['date'],'%Y-%m-%d')==start_date:
                        result.append(data)
                        break
        elif delta.days>=1:
            for time in [start_date+timedelta(days=i) for i in range(delta.days+1)]:
                for place in self.__WhereName__:
                    for data in self.data[place]:
                        if datetime.datetime.strptime(data['date'],'%Y-%m-%d')==time:
                            result.append(data)
                            break
        if mode:self.add_history_member(type='show',showdates=(start,end))
        return result

    def push_data(self,date=today_date_str(),gender="",where="",count=1):#날짜/성별/장소 를 지정 count만큼 추가
        sucess=FALSE
        print('hey')
        if gender not in self.__Gender__:pass#Invalid Gender
        elif where not in self.__WhereName__:pass#Invalid Place
        elif [data for data in self.show_data(date,date) if data['where']==where]==[]:
            print(self.show_data(date,date))#No Member0
            print(where)
        else:
            target=[data for data in self.show_data(date,date) if data['where']==where][0]
            print('kola')
            print(target)
            for member in self.data[where]:
                if member==target:
                    member[gender]+=count
                    if member[gender]<0:
                        member[gender]=0
                    self.data['cumulative']+=count
                    if self.data['cumulative']<=0:
                        self.data['cumulative']=0
                    sucess=TRUE
                    break
        self.save_data()
        self.add_history_member(type='push',gender=gender,place=where,count=count)
        return sucess

    def modify_history_excel(self,excel_path=history_excel_path):
        """
        excel_path : if mode is TRUE excel file will save at excel_path. Defalt :'history_excel.xlsx'
        """
        columns_h={}
        for key in list(self.history.keys()):
            try:
                columns_h[key]=list(self.history[key][0].keys())
            except IndexError:
                columns_h[key]=[]

        wb=openpyxl.Workbook()
        for key in list(self.history.keys()):
            ws=wb.create_sheet(key)
            if len(columns_h[key])!=0:
                for cl in range(len(columns_h[key])):
                    ws.cell(row=1,column=cl+2).value=columns_h[key][cl]
                    ws.cell(row=1,column=cl+2).alignment=Alignment(horizontal='center',vertical='center')
                    ws.cell(row=1,column=cl+2).fill=PatternFill(patternType='solid',fgColor=Color('FFC000'))
                for index in range(len(self.history[key])):
                    ws.cell(row=index+2,column=1).value=index+1
                    ws.cell(row=index+2,column=1).alignment=Alignment(horizontal='center',vertical='center')
                    ws.cell(row=index+2,column=1).fill=PatternFill(patternType='solid',fgColor=Color('008000'))
                    ws.cell(row=index+2,column=1).font=Font(bold=TRUE,color=Color('ffffff'))
                    for cl in range(len(columns_h[key])):
                        if type(self.history[key][index][columns_h[key][cl]])!=list:ws.cell(row=index+2,column=cl+2).value=self.history[key][index][columns_h[key][cl]]
                        else:ws.cell(row=index+2,column=cl+2).value=str(self.history[key][index][columns_h[key][cl]])
                        ws.cell(row=index+2,column=cl+2).alignment=Alignment(horizontal='center',vertical='center')
                self.adjust_cell_width(ws)
            else:
                pass
        wb.remove(wb['Sheet'])
        try:
            wb.save(excel_path)
            os.startfile(excel_path)
        except PermissionError:
            messagebox.showerror("PermissionError","파일을 닫아주세요.")

    def adjust_cell_width(self, ws,value=1.2,cl=['A'],ri=[1]):
        """
        Adjust cell width/make (.1)&(A.) thick
        ws : worksheet handle from openpyxl
        cl : options to make other columns thick default : ['a']
        ri : options to make other rows thick default : [1]
        """
        thick_border=Border(left=Side(style='thick'),right=Side(style='thick'),top=Side(style='thick'),bottom=Side(style='thick'))
        for col in ws.columns:
            max_length =0
            column_col=col[0].column_letter
            for cell in col:
                cell.alignment=Alignment(horizontal='center',vertical='center')
                try:
                    if cell.column_letter in cl or cell.row in ri:cell.border=thick_border
                    if cell.value==0:cell.value='-'
                    if len(str(cell.value)) > max_length:
                        max_length = len(str(cell.value))
                except:
                    print('kr')
                    pass
            adjusted_width=(max_length+2)*value
            print(column_col)
            print(int(adjusted_width))
            ws.column_dimensions[str(column_col)].width=int(adjusted_width)

    def give_savedir(self):#만약 프로그램을 다른 컴퓨터에서 실행한다면 데스크톱 유저 디렉토리도 바뀌므로 그에 대비해서 만든 함수
        tmp=''
        if os.path.isdir(self.savedir):
            pass
        elif self.savedir.split('\\')[-1]=='Desktop':
            tmp=winshell.desktop()
        else:
            tmp='data'
        if tmp!='':self.change_settings(savedir=tmp)
        return self.savedir

    def modify_data_excelike(self,selected_dates=[],mode=TRUE,excel_path=excel_path):
        """
        mode : make Excel file default : True
        selected_dates : list of dates that required for showing info DeFault : show_everything
        excel_path : if mode is TRUE excel file will save at excel_path. Defalt :'data_excel.xlsx'
        """
        datesd=[]
        for place in self.__WhereName__:
            for data in self.data[place]:
                if data['date'] not in datesd:datesd.append(data['date'])
        dates=[]
        if selected_dates!=[]:
            for date in datesd:
                if date in selected_dates:
                    dates.append(date)
        else:dates=datesd
        DATE=sorted(dates)
        dates=DATE
        result={}
        columns_d=["Date"]
        columns_name=[]
        for where in self.__WhereName__:
            for gender in self.__Gender__:
                columns_name.append(where+" "+gender)
        columns_d+=columns_name
        columns_d+=['총합']
        for date in range(len(dates)):
            sd=self.show_data(start=dates[date],end=dates[date],mode=TRUE)
            tmp=[dates[date]]
            for place in self.__WhereName__:
                for gender in self.__Gender__:
                    tmp.append(0)
            tmp.append(0)
            for member in sd:
                for i in range(len(self.__WhereName__)):
                    if member['where']==self.__WhereName__[i]:
                        for j in range(len(self.__Gender__)):
                            tmp[i*2+j+1]+=member[self.__Gender__[j]]
                            tmp[-1]+=tmp[i*2+j+1]
            result[str(date+1)]=tmp
        pprint.pprint(result)
        if mode:
            df=pd.DataFrame.from_dict(result,orient="index",columns=columns_d)
            try:
                df.to_excel(excel_path)
                wb=openpyxl.load_workbook(excel_path,data_only=True)
                ws=wb['Sheet1']
                sum_row=[]
                for c in ws.columns:
                    column_ind=c[0].column_letter
                    c[0].alignment=Alignment(horizontal='center',vertical='center')
                    c[0].fill=PatternFill(patternType='solid',fgColor=Color('FFC000'))
                    if column_ind !='A' and column_ind !='B':
                        tmp_add=0
                        for cell in c:
                            cell.alignment=Alignment(horizontal='center',vertical='center')
                            if cell.row!=1:
                                tmp_add+=int(cell.value)
                        sum_row.append(tmp_add)
                    elif column_ind=='A':
                        F=Font(bold=TRUE,color=Color('ffffff'))
                        for cell in c:
                            cell.alignment=Alignment(horizontal='center',vertical='center')
                            cell.fill=PatternFill(patternType='solid',fgColor=Color('008000'))
                            cell.font=F
                    else:
                        F=Font(bold=TRUE,color=Color('ffffff'))
                        for cell in c:
                            cell.alignment=Alignment(horizontal='center',vertical='center')
                            cell.fill=PatternFill(patternType='solid',fgColor=Color('008000'))
                            cell.font=F
                ws.append(["총합"," "]+sum_row)
                kolase=[]
                for kas in range(len(self.__Gender__)):
                    j=0
                    for suro in range(len(sum_row)):
                        if suro%len(self.__Gender__)==kas:
                            j+=sum_row[suro]
                    kolase.append(j)
                kolase[0]-=sum_row[-1]
                kartli=[]
                for genko in range(len(self.__Gender__)):
                    kartli.append(self.__Gender__[genko]+ " 총합 :")
                    kartli.append(kolase[genko])
                place_total=[]
                GT=len(self.__Gender__)
                for plag in range(len(self.__WhereName__)):
                    loki=0
                    for geni in range(GT):
                        loki+=sum_row[plag*GT+geni]
                    place_total.append(self.__WhereName__[plag]+" 총합 :")
                    place_total.append(loki)
                ws.append(["",""]+kartli)
                ws.append(["",""]+place_total)
                self.adjust_cell_width(ws,value=2.4,cl=['A','B'])
                ws.column_dimensions['A'].width=5
                wb.save(excel_path)
                os.startfile(excel_path)
            except PermissionError:
                messagebox.showerror("PermissionError","파일을 닫아주세요.")
        return result

    def columns_name(self):
        columns_name=[]
        for where in self.__WhereName__:
            for gender in self.__Gender__:
                columns_name.append(where+" "+gender)
        return columns_name

    def button_event(self,event,where="",gender="",mode=TRUE):
        if where not in self.__WhereName__ or gender not in self.__Gender__:pass
        else:
            if mode==TRUE:self.push_data(gender=gender,where=where)
            else:self.push_data(gender=gender,where=where,count=-1)

    def check_save(self,path=""):
        isValid=FALSE
        if path=="":path=self.FilePath
        if os.path.isfile(path):
            check_tmp={}
            with open(path,'r') as check:
                check_tmp=json.load(check)
            if sorted(list(check_tmp.keys()))==sorted(self.__WhereName__+['cumulative','typecode']) and sorted(list(check_tmp[self.__WhereName__[0]][0].keys()))==sorted(list(self.__Gender__+['date','where'])):
                isValid=TRUE
                print("세이브 파일 무결성 확인")
        else:
            pass
        return isValid

    def check_his(self,path=""):
        isValid=FALSE
        if path=="":path=self.HistoryPath
        if os.path.isfile(path):
            check_tmp={}
            with open(path,'r') as check:
                check_tmp=json.load(check)
            if sorted(list(check_tmp.keys()))==sorted(['init','built','show','push','fail']):
                isValid=TRUE
                print("기록 파일 무결성 확인")
        else:
            pass
        return isValid

    def quit(self,mode=TRUE):
        global FILE_CLOSE
        FILE_CLOSE=mode
        print("자동저장 후 종료")
        self.save_data()
        self.save_history()
        self.save_settings()
        print('저장 완료')
        del self

class MyDelegate(QItemDelegate):
    def __init__(self,parent):
        super().__init__(parent)

    def paint(self,painter=QPainter,option=QStyleOptionViewItem,index=QModelIndex):
        super().paint(painter,option,index)
        painter.setPen(Qt.red)
        painter.drawRect( option.rect )
class MyApp(QWidget):
    def __init__(self,save_file=None,master=None):
        super().__init__()
        self.savefile=save_file
        self.master=master
        self.InitUi()
        self.SetUi()

    def InitUi(self):
        self.set_menu()
        self.set_statusbar()
        self.set_widgets()

    def set_menu(self):
        pass

    def set_statusbar(self):
        pass

    def set_widgets(self):
        self.table=QTableWidget(self)
        self.table.resize(1000,250)
        self.table.move(0,0)
        self.setTableWidgetData()

    def setTableWidgetData(self):
        dta=self.savefile.modify_data_excelike(mode=False)
        df=pandas.DataFrame.from_dict(dta,orient='index',columns=['Date(날짜)']+self.savefile.columns_name()+['Total(총합)'])
        self.table.setColumnCount(len(df.columns))
        self.table.setRowCount(len(df.index))
        self.table.setHorizontalHeaderLabels(df.columns.tolist())
        self.table.horizontalHeader().setSectionResizeMode(QHeaderView.Stretch)
        for i in range(len(df.index)):
            for j in range(len(df.columns)):
                oak=QTableWidgetItem(str(df.iloc[i, j]))
                oak.setTextAlignment(QtCore.Qt.AlignCenter)
                self.table.setItem(i,j,oak)
        self.table.setAutoScroll(True)
        self.table.resizeColumnsToContents()
        self.table.resizeRowsToContents()
        self.table.setAutoFillBackground(True)
        self.table.setStyleSheet('QTableView {border: 4px solid ;} QTableView::column{border: 4px solid ;}')
        self.table.repaint()
    def SetUi(self):
        self.setWindowTitle(today_date_str()+'일 기록')
        self.resize(1000,600)
        self.center()
        self.setWindowIcon(QIcon(self.savefile.icon_path))
        self.show()

    def center(self):#make window move to center
        qr = self.frameGeometry()
        cp = QDesktopWidget().availableGeometry().center()
        qr.moveCenter(cp)
        self.move(qr.topLeft())

    def closeEvent(self,event):
        event.ignore()
        print('Quit all')
        self.savefile.quit()
        event.accept()

if __name__ == "__main__":
    app=QApplication(sys.argv)
    print(app)
    jd=Json_Data()
    Mw=MyApp(save_file=jd,master=app)
    sys.exit(app.exec_())


import typing
from dataclasses import dataclass

from Cryptodome.Cipher import AES
from Cryptodome.Util import Counter

from bkcrypto import constants, types

from .. import configs, options
from . import base

@dataclass
class AESSymmetricRuntimeConfig(configs.BaseSM4SymmetricConfig, base.BaseSymmetricRuntimeConfig):

    mode_class: types.AESModeClass = None

    def __post_init__(self):
        super().__post_init__()

        if self.key_size not in AES.key_size:
            raise ValueError(f"Optional key sizes are {AES.key_size}, but got {self.key_size}")

        try:
            self.mode_class = {
                constants.SymmetricMode.CTR: AES.MODE_CTR,
                constants.SymmetricMode.CBC: AES.MODE_CBC,
                constants.SymmetricMode.GCM: AES.MODE_GCM,
                constants.SymmetricMode.CFB: AES.MODE_CFB,
            }[self.mode]

        except KeyError:
            raise ValueError(f"Unsupported mode: {self.mode}")

class AESSymmetricCipher(base.BaseSymmetricCipher):

    CIPHER_TYPE: str = constants.SymmetricCipherType.AES.value

    CONFIG_DATA_CLASS: typing.Type[AESSymmetricRuntimeConfig] = AESSymmetricRuntimeConfig

    OPTIONS_DATA_CLASS: typing.Type[options.AESSymmetricOptions] = options.AESSymmetricOptions

    config: AESSymmetricRuntimeConfig = None

    def get_block_size(self) -> int:
        return self.config.key_size

    def init_ctx(self, encryption_metadata: base.EncryptionMetadata):
        mode_init_args: typing.List[bytes] = []
        mode_init_kwargs: typing.Dict[str : typing.Any] = {}

        if self.config.enable_iv:
            if self.config.mode == constants.SymmetricMode.CTR:
                # Size of the counter block must match block size
                mode_init_kwargs["counter"] = Counter.new(
                    self.get_block_size() * 8, initial_value=int.from_bytes(encryption_metadata.iv, byteorder="big")
                )
            else:
                mode_init_args.append(encryption_metadata.iv)

        cipher_ctx = AES.new(self.config.key, self.config.mode_class, *mode_init_args, **mode_init_kwargs)
        if self.config.enable_aad:
            cipher_ctx.update(encryption_metadata.aad)

        return cipher_ctx

    def _encrypt(self, plaintext_bytes: bytes, encryption_metadata: base.EncryptionMetadata) -> bytes:

        cipher_ctx = self.init_ctx(encryption_metadata)

        if self.config.mode == constants.SymmetricMode.GCM:
            ciphertext_bytes, tag = cipher_ctx.encrypt_and_digest(plaintext_bytes)
            encryption_metadata.tag = tag
            return ciphertext_bytes
        else:
            return cipher_ctx.encrypt(plaintext_bytes)

    def _decrypt(self, ciphertext_bytes: bytes, encryption_metadata: base.EncryptionMetadata) -> bytes:

        cipher_ctx = self.init_ctx(encryption_metadata)

        if self.config.mode == constants.SymmetricMode.GCM:
            plaintext_bytes: bytes = cipher_ctx.decrypt_and_verify(ciphertext_bytes, encryption_metadata.tag)
            return plaintext_bytes
        else:
            return cipher_ctx.decrypt(ciphertext_bytes)


#!/usr/bin/python3
import sys, re, os
from collections import deque

# if len(sys.argv) < 2:
#     print("SCRIPT INPUT_FILE OUTPUT_FILE")

# print(sys.argv[1])
result_file = open(sys.argv[1], "w")

source_file = sys.stdin

add_list = []
final_data = []
count = 0
prev_val = 0
q = deque()

with sys.stdin as f:
    for line in f:
        if(len(line) == 0):
            continue
        x = re.search("0x[a-f0-9]*", line)
        if( x == None):

            continue
        # pbar.update(len(line))
        addr = x.group()
        # print(addr)
        if "PC" in line:
            pc = addr

            for addr in add_list:
                delta = str(hex(int(addr, 16) - prev_val))
                q.append((int(addr, 16), int(pc, 16)))

            # print("Entering")
            while len(q) != 1:

                addr = q[0][0]
                pc = q[0][1]
                delta = q[1][0] - addr
                result_file.write("{},{},{}\n".format(addr, pc, delta))
                q.popleft()
            add_list.clear()
        else:
            # if addr not in add_list:
            add_list.append(addr)

source_file.close()
result_file.close()


import time
import random
import numpy

import pyadvantg
import Logger
import datetime
import SpacePartition1D
import SpacePartition3D
import matplotlib
import matplotlib.pyplot as pyplot

__author__ = 'etnc6d'

class EvolAlg(object):

    def __init__(self):
        self.max_evals = 1E5
        self.timeout = 30*60  # Seconds
        self.pop_size = 15
        self.children = 5

        today = datetime.datetime.now()
        Logger.Logger.initialize("logfile_" + str(today.year) + "_" + str(today.month) + "_" + str(today.day) + "_" + str(today.hour) + "_" + str(today.minute) + ".log")

    def finish(self):
        Logger.Logger.finish()

    def begin(self):
        eval_hist = []
        terminated = False
        start_time = time.time()
        best_solution = None
        generation = 0
        generation_hist = []
        best_solution_hist = []
        pop_size_hist = []
        gen_best_solution = []

        xpop = EvolAlg.initialize(self.pop_size, 10)
        # for i in range(len(xpop)):
        #    EvolAlg.fixify_x(xpop[i])
        ypop = EvolAlg.initialize(self.pop_size, 10)
        zpop = EvolAlg.initialize(self.pop_size, 10)

        xpop[0].relBins = [0, .3, .7, 1]
        xpop[1].relBins = [0, .6, .9, 1]

        # Comment

        print(xpop[0])
        print(xpop[1])
        print(str(xpop[0].diff(xpop[1])) + "   " + str(xpop[0].diff(xpop[1], True)))

        print(xpop[0])
        print(xpop[2])
        print(str(xpop[0].diff(xpop[2])) + "   " + str(xpop[0].diff(xpop[2], True)))

        print(xpop[0])
        print(xpop[3])
        print(str(xpop[0].diff(xpop[3])) + "   " + str(xpop[0].diff(xpop[3], True)))

        teams = EvolAlg.make_teams(xpop, ypop, zpop, 5)
        solutions = []
        for team in teams:
            sol = SpacePartition3D.SpacePartition3D()
            sol.x = team[0]
            sol.y = team[1]
            sol.z = team[2]
            sol.fit = EvolAlg.evaluate_team(sol.x, sol.y, sol.z)
            solutions.append(sol)
            Logger.Logger.write_eval(team, sol.fit)

        maxfit = -1
        for sol in solutions:
            if sol.fit.fomavg > maxfit:
                maxfit = sol.fit.fomavg
                best_solution = sol

        generation_hist.append(0)
        best_solution_hist.append(best_solution)
        gen_best_solution.append(best_solution)
        pop_size_hist.append(self.pop_size)
        eval_hist.append(len(solutions))

        while not terminated:
            generation += 1

            xparents, yparents, zparents = EvolAlg.select_parents(xpop, ypop, zpop)
            xoff, yoff, zoff = EvolAlg.xover_coev(xparents, yparents, zparents, self.children)

            xoff = [EvolAlg.mutate(x) for x in xoff]

            # for i in range(len(xoff)):
            #    EvolAlg.fixify_x(xoff[i])

            yoff = [EvolAlg.mutate(y) for y in yoff]
            zoff = [EvolAlg.mutate(z) for z in zoff]

            for xx in xoff:
                if xx.get_bin_count() < 4:
                    print("Problem!")

            for xx in yoff:
                if xx.get_bin_count() < 4:
                    print("Problem!")

            for xx in zoff:
                if xx.get_bin_count() < 4:
                    print("Problem!")

            xpop.extend(xoff)
            ypop.extend(yoff)
            zpop.extend(zoff)

            # xpop, ypop, zpop = EvolAlg.tournament(xpop, ypop, zpop, self.pop_size, solutions)
            teams = EvolAlg.make_teams(xpop, ypop, zpop, 5)
            # solutions = []
            for team in teams:
                sol = SpacePartition3D.SpacePartition3D()
                sol.x = team[0]
                sol.y = team[1]
                sol.z = team[2]
                sol.fit = EvolAlg.evaluate_team(sol.x, sol.y, sol.z)
                solutions.append(sol)
                Logger.Logger.write_eval(team, sol.fit)

            xpop, ypop, zpop = EvolAlg.tournament(xpop, ypop, zpop, self.pop_size, solutions)

            for sol in solutions:
                if sol.fit.fomavg > best_solution.fit.fomavg:
                    best_solution = sol

            generation_hist.append(generation)
            best_solution_hist.append(best_solution)
            pop_size_hist.append(len(xpop))
            eval_hist.append(len(solutions))

            if len(solutions) > self.max_evals:
                terminated = True
                print('Max evaluations reached')

            if time.time() - start_time > self.timeout:
                terminated = True
                print('Timeout')

        print("Generations: " + str(generation))
        print("Evaluations: " + str(len(solutions)))
        print(best_solution)

        '''
        matplotlib.rcParams.update({'font.size': 22})
        matplotlib.rcParams.update({'figure.autolayout': True})

        pyplot.figure()
        pyplot.plot(generation_hist, [s.fit.fomavg for s in best_solution_hist])
        pyplot.title('Best Fitness')
        pyplot.xlabel('Generation')
        pyplot.ylabel('Fitness')
        # pyplot.title('Best Fitness', fontname="Times New Roman")
        # pyplot.xlabel('Generation', fontname="Times New Roman")
        # pyplot.ylabel('Fitness', fontname="Times New Roman")

        pyplot.figure()
        pyplot.loglog(generation_hist, [s.fit.fomavg for s in best_solution_hist])
        pyplot.title('Best Fitness (Log Log)')
        pyplot.xlabel('Generation')
        pyplot.ylabel('Fitness')

        pyplot.figure()
        pyplot.plot([s.fit.fomavg for s in solutions])
        pyplot.title('Fitness with Evaluations')
        pyplot.xlabel('Evaluation')
        pyplot.ylabel('Fitness')

        pyplot.figure()
        pyplot.plot([s.x.get_bin_count() for s in solutions])
        pyplot.title('X Bin Size')
        pyplot.xlabel('Evaluation')
        pyplot.ylabel('X Bins')

        pyplot.figure()
        pyplot.plot([s.y.get_bin_count() for s in solutions])
        pyplot.title('Y Bin Size')
        pyplot.xlabel('Evaluation')
        pyplot.ylabel('Y Bins')

        pyplot.figure()
        pyplot.plot([s.z.get_bin_count() for s in solutions])
        pyplot.title('Z Bin Size')
        pyplot.xlabel('Evaluation')
        pyplot.ylabel('Z Bins')

        pyplot.figure()
        pyplot.plot([s.fit.mcnp_t for s in solutions])
        pyplot.title('MCNP Runtime')
        pyplot.xlabel('Evaluation')
        pyplot.ylabel('Runtime')

        pyplot.figure()
        pyplot.plot([s.fit.adv_t for s in solutions])
        pyplot.title('ADVANTG Runtime')
        pyplot.xlabel('Evaluation')
        pyplot.ylabel('Runtime')

        pyplot.figure()
        totaltime = [s.fit.adv_t + s.fit.mcnp_t for s in solutions]
        cumsumtime = numpy.cumsum(totaltime).tolist()
        globalfit = [s.fit.fomavg / ct for s, ct in zip(solutions, cumsumtime)]
        pyplot.plot(globalfit)
        pyplot.title('Global Fitness')
        pyplot.xlabel('Evaluation')
        pyplot.ylabel('Fitness')

        pyplot.figure()
        totaltime = [s.fit.adv_t + s.fit.mcnp_t for s in solutions]
        cumsumtime = numpy.cumsum(totaltime).tolist()
        globalfit = [s.fit.fomavg / ct for s, ct in zip(best_solution_hist, cumsumtime)]
        pyplot.plot(globalfit)
        pyplot.title('Global Best Fitness')
        pyplot.xlabel('Evaluation')
        pyplot.ylabel('Fitness')

        pyplot.figure()
        pyplot.plot([s.fit.avg_unc() for s in solutions])
        pyplot.title('Average Uncertainty')
        pyplot.xlabel('Evaluation')
        pyplot.ylabel('Uncertainty')

        fig = pyplot.figure()
        fig.set_alpha(0.5)

        voxel_count = [s.x.get_bin_count()*s.y.get_bin_count()*s.z.get_bin_count() for s in solutions]
        time_per_voxel = [t/v for t, v in zip(totaltime, voxel_count)]
        # pyplot.scatter(voxel_count, time_per_voxel, c='k', alpha=.2)
        pyplot.scatter(voxel_count, totaltime, c='k', alpha=.2)
        # pyplot.ylim(0, 1.01*max(time_per_voxel))
        pyplot.ylim(0, 1.01*max(totaltime))
        pyplot.title('Time per voxel')
        pyplot.xlabel('Voxel Count')
        pyplot.ylabel('Evaluation Time / Voxel [min]')

        pyplot.figure()
        pyplot.plot(generation_hist, pop_size_hist)
        pyplot.title('Population Size')
        pyplot.xlabel('Generation')
        pyplot.ylabel('# Of Individuals per dimension')

        pyplot.show()
        '''

    @staticmethod
    def evaluate_team(xindiv, yindiv, zindiv):
        """Takes an individual from each coev pop and evaluates the
        """
        return pyadvantg.eval_co_fitness(xindiv, yindiv, zindiv)  # random.random()

    @staticmethod
    def make_teams(xpop, ypop, zpop, pair_count):
        """Pairs members of the coev pops together
        xpop - A list of x SpacePartition1D individuals
        ypop - A list of y SpacePartition1D individuals
        zpop - A list of z SpacePartition1D individuals
        pair_count - The number of pairings that will be made
        """

        tups = []
        for i in range(pair_count):
            x = random.choice(xpop)
            y = random.choice(ypop)
            z = random.choice(zpop)
            tups.append((x, y, z))

        return tups

    @staticmethod
    def select_parents(xpop, ypop, zpop):
        return xpop, ypop, zpop

    @staticmethod
    def xover1D(parent1, parent2):
        children = []

        xpt = random.random()

        child1 = SpacePartition1D.SpacePartition1D()
        child2 = SpacePartition1D.SpacePartition1D()

        # Construct child 1 from parent1 and parent2
        for gene in parent1.relBins:
            if gene < xpt:
                child1.relBins.append(gene)
        for gene in parent2.relBins:
            if gene >= xpt:
                child1.relBins.append(gene)
        while child1.get_bin_count() < 4:
            EvolAlg.mutate_add(child1)

        # Construct child 2 form parent2 and parent1
        for gene in parent2.relBins:
            if gene < xpt:
                child2.relBins.append(gene)
        for gene in parent1.relBins:
            if gene >= xpt:
                child2.relBins.append(gene)
        while child2.get_bin_count() < 4:
            EvolAlg.mutate_add(child2)

        children.append(child1)
        children.append(child2)

        return children

    @staticmethod
    def xover3D(x1, y1, z1, x2, y2, z2):
        xchildren = EvolAlg.xover1D(x1, x2)
        ychildren = EvolAlg.xover1D(y1, y2)
        zchildren = EvolAlg.xover1D(z1, z2)
        return xchildren, ychildren, zchildren

    @staticmethod
    def xover_coev(xpop, ypop, zpop, xover_count):
        xchildren = []
        ychildren = []
        zchildren = []

        for i in range(xover_count):
            p1 = random.randint(0, len(xpop)-1)
            p2 = random.randint(0, len(xpop)-1)
            xc, yc, zc = EvolAlg.xover3D(xpop[p1], ypop[p1], zpop[p1], xpop[p2], ypop[p2], zpop[p2])

            #for i in range(len(xc)):
            #    EvolAlg.fixify_x(xc[i])
            xchildren.extend(xc)
            ychildren.extend(yc)
            zchildren.extend(zc)

        return xchildren, ychildren, zchildren

    @staticmethod
    def mutate(indiv):
        if random.random() < 0.5:
            EvolAlg.mutate_add(indiv)
        else:
            EvolAlg.mutate_delete(indiv)

        # EvolAlg.fixify(indiv)

        return indiv

    # @staticmethod
    # def fixify_x(indiv):
    #    if 107.5/112.5 not in indiv.relBins:
    #        indiv.relBins.append(107.5/112.5)
    #        indiv.relBins = sorted(indiv.relBins)

    @staticmethod
    def mutate_add(indiv):
        indiv.relBins.append(random.random())
        indiv.relBins = sorted(indiv.relBins)

    @staticmethod
    def mutate_densify(indiv):
        p = 0.3

        t1 = random.random()
        t2 = random.random()
        pt1 = min(t1, t2)
        pt2 = max(t1, t2)

        tlist = []
        for i in range(0, indiv.get_bin_count()):
            if random.random() < p and (pt1 < indiv.relBins[i] < pt2 or pt1 < indiv.relBins[i+1] < pt2):
                r = random.random()
                r = (indiv.relBins[i] - indiv.relBins[i+1])*r + indiv.relBins[i]
                tlist.append(r)

        indiv.relBins.extend(tlist)
        indiv.relBins = sorted(indiv.relBins)

    @staticmethod
    def mutate_sparcify(indiv):
        p = 0.3

        t1 = random.random()
        t2 = random.random()
        pt1 = min(t1, t2)
        pt2 = max(t1, t2)

        for i in range(indiv.get_bin_count(), -1, -1):
            if random.random() < p and pt1 < indiv.relBins[i] < pt2:
                del self.relBins[i]

    @staticmethod
    def mutate_delete(indiv):
        if indiv.get_bin_count() > 4:
            remove_gene = random.randint(0, len(indiv.relBins)-1)
            del indiv.relBins[remove_gene]

    @staticmethod
    def tournament(x, y, z, survivor_count, solutions):

        newx = EvolAlg.tournament_single(x, survivor_count, solutions)
        newy = EvolAlg.tournament_single(y, survivor_count, solutions)
        newz = EvolAlg.tournament_single(z, survivor_count, solutions)

        return newx, newy, newz

    @staticmethod
    def tournament_single(x, survivor_count, solutions):

        if survivor_count >= len(x):
            return x

        xfit = []
        for indiv in x:
            ifit = 0
            icnt = 0
            for sol in solutions:
                if sol.x == indiv:
                    ifit += sol.fit.fomavg
                    icnt += 1
            if icnt > 0:
                ifit /= icnt
            xfit.append(ifit)

        mostfit = max(xfit)
        pressure = 2.0

        xfit = [q+(mostfit/pressure) for q in xfit]

        xfitsum = sum(xfit)

        if xfitsum == 0:
            return numpy.random.choice(x, survivor_count, False).tolist()

        xfit = [q/xfitsum for q in xfit]

        return numpy.random.choice(x, survivor_count, False, xfit).tolist()

    @staticmethod
    def initialize(indivCount, max_genes):
        init_pop = []

        for i in range(indivCount):
            new_indiv = SpacePartition1D.SpacePartition1D()
            EvolAlg.randomize(new_indiv, random.randint(5, max_genes))
            init_pop.append(new_indiv)

        return init_pop

    @staticmethod
    def randomize(indiv, genes):
        for i in range(genes):
            indiv.relBins.append(random.random())
            indiv.relBins = sorted(indiv.relBins)
        return indiv