import os
import numpy as np
import pandas as pd
from time import time
import cvxopt.solvers
import numpy.linalg as la
import matplotlib.cm as cm
import matplotlib.pyplot as plt
from prettytable import PrettyTable
from sklearn import metrics
from sklearn.metrics import precision_recall_curve
from sklearn.metrics import auc
os.chdir(os.path.dirname(__file__))

MIN_SUPPORT_VECTOR_MULTIPLIER = 1e-5

class Kernel(object):
    @staticmethod
    def polykernel(dimension, offset, gamma):
        return lambda x, y: ((offset + gamma*np.dot(x.T,y)) ** dimension)
    @staticmethod
    def linear():
        return lambda x,y:np.dot(x.T,y)

    @staticmethod
    def sigmoid(gamma, offset):
        return lambda x, y: np.tanh(gamma*np.dot(x.T,y) + offset)

    @staticmethod
    def radial_basis(gamma):
        return lambda x, y: np.exp(-gamma*la.norm(np.subtract(x, y)))

class SVMTrainer(object):

    def __init__(self, kernel, c):
        self.kernel = kernel
        self.c = c

    def train(self, X, y):
        lagrange_multipliers = self.compute_multipliers(X, y)
        return self.construct_predictor(X, y, lagrange_multipliers)

    def kernel_matrix(self, X, n_samples):
        K = np.zeros((n_samples, n_samples))
        # f = open("K.csv", "w+")
        # f.write('header \n')
        for i, x_i in enumerate(X):
            temp = ""
            for j, x_j in enumerate(X):
                K[i, j] = self.kernel(x_i, x_j)
                temp = temp + str(K[i,j]) + ','
            # f.write(temp + '\n')
        # f.close()
        return K

    def construct_predictor(self, X, y, lagrange_multipliers):
        support_vector_indices = lagrange_multipliers > MIN_SUPPORT_VECTOR_MULTIPLIER
        support_multipliers = lagrange_multipliers[support_vector_indices]
        support_vectors = X[support_vector_indices]
        support_vector_labels = y[support_vector_indices]

        bias = np.mean(
            [y_k - SVMPredictor(
                kernel=self.kernel,
                bias=0.0,
                weights=support_multipliers,
                support_vectors=support_vectors,
                support_vector_labels=support_vector_labels).predict(x_k)[0]
             for (y_k, x_k) in zip(support_vector_labels, support_vectors)])

        return SVMPredictor(
            kernel=self.kernel,
            bias=bias,
            weights=support_multipliers,
            support_vectors=support_vectors,
            support_vector_labels=support_vector_labels)

    def compute_multipliers(self, X, y):
        n_samples, n_features = X.shape
        # K = pd.read_csv('K.csv', header=0, dtype='d').values
        K = self.kernel_matrix(X,n_samples)
        P = cvxopt.matrix(np.outer(y, y) * K)
        q = cvxopt.matrix(-1 * np.ones(n_samples))
        G_std = cvxopt.matrix(np.diag(np.ones(n_samples) * -1))
        h_std = cvxopt.matrix(np.zeros(n_samples)) #[1, n] = all 0

        G_slack = cvxopt.matrix(np.diag(np.ones(n_samples)))
        h_slack = cvxopt.matrix(np.ones(n_samples) * self.c) #[1, n] = all c

        G = cvxopt.matrix(np.vstack((G_std, G_slack))) #7240x3620
        h = cvxopt.matrix(np.vstack((h_std, h_slack))) #[2, n]

        A = cvxopt.matrix(y, (1, n_samples) , 'd')
        b = cvxopt.matrix(0.0)
        solution = cvxopt.solvers.qp(P, q, G, h, A, b)
        return np.ravel(solution['x'])

class SVMPredictor(object):
    def __init__(self,
                 kernel,
                 bias,
                 weights,
                 support_vectors,
                 support_vector_labels):
        self._kernel = kernel
        self._bias = bias
        self._weights = weights
        self._support_vectors = support_vectors
        self._support_vector_labels = support_vector_labels
        assert len(support_vectors) == len(support_vector_labels)
        assert len(weights) == len(support_vector_labels)

    def predict(self, x):
        result = self._bias
        for z_i, x_i, y_i in zip(self._weights,
                                 self._support_vectors,
                                 self._support_vector_labels):
            # result += w * support_vector_labels * K
            result += z_i * y_i * self._kernel(x_i, x)
        return np.sign(result).item(), result

def calculate(true_positive,false_positive, false_negative, true_negative):
    result = {}
    result['precision'] = true_positive / (true_positive + false_positive)
    result['recall'] = true_positive / (true_positive + false_negative)
    return result

def confusion_matrix(true_positive,false_positive,false_negative, true_negative, y_pred):
    matrix = PrettyTable([' ', 'Ham' , 'Spam'])
    matrix.add_row(['Ham', true_positive , false_positive])
    matrix.add_row(['Spam', false_negative , true_negative])
    return matrix , y_pred, calculate(true_positive,false_positive,false_negative, true_negative)

def implementSVM(X_train,Y_train,X_test,Y_test,parameters,type, c, y_pred_prob, k):
    ham_spam = 0
    spam_spam = 0
    ham_ham = 0
    spam_ham = 0
    if(type == "polykernel"):
        dimension = parameters['dimension']
        offset = parameters['offset']
        gamma = parameters['gamma']
        trainer = SVMTrainer(Kernel.polykernel(dimension, offset, gamma),float(c))
        predictor = trainer.train(X_train,Y_train)
    elif(type=="linear"):
        trainer = SVMTrainer(Kernel.linear(),float(c))
        predictor = trainer.train(X_train,Y_train)
    elif(type == "sigmoid"):
        gamma = parameters['gamma']
        offset = parameters['offset']
        trainer = SVMTrainer(Kernel.sigmoid(gamma,offset),float(c))
        predictor = trainer.train(X_train,Y_train)
    elif(type == "rbf"):
        gamma = parameters['gamma']
        trainer = SVMTrainer(Kernel.radial_basis(gamma),float(c))
        predictor = trainer.train(X_train,Y_train)

    y_pred_prob[str(type)][k] = []
    for i in range(X_test.shape[0]):
        ans = predictor.predict(X_test[i])[0]
        if(ans==-1 and Y_test[i]==-1):
            spam_spam+=1
        elif(ans==1 and Y_test[i]==-1):
            spam_ham+=1
        elif(ans==1 and Y_test[i]==1):
            ham_ham+=1
        elif(ans==-1 and Y_test[i]==1):
            ham_spam+=1
        y_pred_prob[str(type)][k].append(predictor.predict(X_test[i])[1])
    if(type == "polykernel"):
        y_pred_prob[str(type)][k].append('b')
    elif(type == "linear"):
        y_pred_prob[str(type)][k].append('r')
    elif(type == "sigmoid"):
        y_pred_prob[str(type)][k].append('k')
    elif(type == "rbf"):
        y_pred_prob[str(type)][k].append('m')

    return confusion_matrix(ham_ham,ham_spam,spam_ham,spam_spam, y_pred_prob[str(type)][k])

def write_to_file(matrix, result, parameters, type, start_time, auc):
    F1_score = 2*(float(result['precision'])*float(result['recall']))/(float(result['precision']) + float(result['recall']))
    f = open("results.txt","a")
    if(type=="polykernel"):
        f.write("Polykernel model parameters")
        f.write("\n")
        f.write("Dimension : " + str(parameters['dimension']))
        f.write("\n")
        f.write("Offset : " + str(parameters['offset']))
        f.write("\n")
        f.write("Gamma : " + str(parameters['gamma']))
    elif(type=="linear"):
        f.write("Linear model")
    elif(type=="sigmoid"):
        f.write("sigmoid model")
    elif(type=="rbf"):
        f.write("rbf model")
        f.write("\n")
        f.write("Gamma : " + str(parameters['gamma']))

    f.write("\n")
    f.write(matrix.get_string())
    f.write("\n")
    f.write("Precision : " + str(round(result['precision'],4)))
    f.write("\n")
    f.write("Recall : " + str(round(result['recall'],4)))
    f.write("\n")
    f.write("AUC : " + str(round(auc,4)))
    f.write("\n")
    f.write("F1 score : " + str(round(F1_score,4)))
    f.write("\n")
    f.write("Time spent for model : " + str(round(time()-start_time,2)))
    f.write("\n")
    f.write("\n")
    f.write("\n")
    f.close()

def read_data(wordlist, frequency, percent):
    df1 = pd.read_csv(wordlist)
    df2 = pd.read_csv(frequency, header=0)
    input_output = df2.values
    X = input_output[:,:-1] # ignore final colummn
    Y = input_output[:,-1:] # output
    total = X.shape[0]
    train = int(X.shape[0] * int(percent) / 100)
    X_train = X[:train,:]
    Y_train = Y[:train,:]
    X_test = X[train:,:]
    Y_test = Y[train:,:]
    return X_train, Y_train, X_test, Y_test

def get_parameter_hernel(type):
    if type == 'polykernel':
        dimension = 4
        offset = 7
        gamma = [1, 2, 3]
        c = [0.1]
    elif type == 'linear':
        dimension = 3
        offset = 7
        gamma = [1]
        c = [0.1]
    elif type == 'sigmoid':
        dimension = 3
        offset = 7
        gamma = [1, 2]
        c = [0.1]
    elif type == 'rbf':
        dimension = 3
        offset = 1
        gamma = [1, 2, 3]
        c = [0.1]
    return dimension, offset, gamma, c

def run(X_train, Y_train, X_test, Y_test, type, dimension, offset, gamma, c, y_pred_prob):
    k=0
    t_type = type
    parameters = {}
    for i in range(2,int(dimension)):
        for j in range(0, int(offset), 2):
            for tc in c:
                for tgamma in gamma:
                    parameters['dimension'] = i
                    parameters['offset'] = j
                    parameters['gamma'] = tgamma
                    start_time = time()
                    matrix, y_pred, result = implementSVM(X_train, Y_train, X_test, Y_test, parameters, str(t_type), tc, y_pred_prob, k)
                    t_precision, t_recall, _  = precision_recall_curve(Y_test, y_pred[:-1])
                    t_auc = auc(t_recall, t_precision)
                    write_to_file(matrix, result, parameters, t_type, start_time, t_auc)
                    k +=1
                    print(str(type) + " done : " + str(k))

def model(model, global_start_time = None):
    if model == 'start':
        f = open("results.txt","w+")
        f.close()
    elif model == 'stop':
        f = open("results.txt","a")
        f.write("Time spent for entire code : " + str(round(time()-global_start_time,2)))
        f.close()
    else:
        pass

def plot_AUC(Y_test, y_pred_prob):

    for kernel, proba in y_pred_prob.items():
        arrange = {}
        best_Key_auc = 0
        best_value_auc = 0
        if len(proba) != 0:
            for i in range(len(proba)):
                t_precision, t_recall, _  = precision_recall_curve(Y_test, proba[i][:-1])
                t_auc = auc(t_recall, t_precision)
                if len(arrange) == 0:
                    arrange[i] = t_auc
                else:
                    arrange[i] = t_auc
            for key, value in arrange.items():
                if value > best_value_auc:
                    best_value_auc = value
                    best_Key_auc = key
                else:
                    pass
            plt.title('Receiver Operating Characteristic')
            plt.plot(t_recall, t_precision, str(y_pred_prob[kernel][best_Key_auc][-1]), label = 'AP' + kernel + '= %0.4f' % best_value_auc)
        else:
            print('Kernel ' + kernel + ' not found')
    plt.legend(loc = 'lower right')
    plt.plot([0, 1], [0, 1],'r--')
    plt.xlim([-0.03, 1.03])
    plt.ylim([-0.03, 1.03])
    plt.ylabel('True Positive Rate')
    plt.xlabel('False Positive Rate')
    plt.show()

if __name__ == '__main__':
    global_start_time = time()
    cvxopt.solvers.options['show_progress'] = False
    wordlistfile = 'wordslist.csv'
    frequenceyfile = 'frequency.csv'
    percent = 70
    y_pred_prob = {
                    'polykernel': {},
                    'linear': {},
                    'sigmoid': {},
                    'rbf': {}
                  }
    type = ['polykernel', 'linear', 'rbf']
    model('start')
    X_train, Y_train, X_test, Y_test = read_data(wordlistfile, frequenceyfile, percent)
    for i_type in type:
        dimension, offset, gamma, c = get_parameter_hernel(i_type)
        run(X_train, Y_train, X_test, Y_test, i_type, dimension, offset, gamma, c, y_pred_prob)
    model('stop', global_start_time)
    plot_AUC(Y_test, y_pred_prob)


import numpy as np
from flask import Flask, request, jsonify, render_template
from keras.models import model_from_json
from keras.optimizers import Adam
import keras
import numpy as np
from sklearn.preprocessing import LabelEncoder
import os
import wfdb
from wfdb import processing
import scipy.signal as sign
import matplotlib.pyplot as plt
import numpy as np
from ecgdetectors import Detectors
import pandas as pd
import time
from tensorflow.keras.layers import Input, Conv2D, MaxPooling2D, GlobalAveragePooling2D, BatchNormalization, Conv1D
import tflearn
from tflearn.layers.conv import conv_2d, max_pool_2d
from tflearn.layers.core import input_data, dropout, fully_connected
from tflearn.layers.estimator import regression
from tensorflow.keras import datasets, layers, models
class model:
    # def __init__(self):
    #     json_file = open('ResNetNaive.json', 'r')
    #     loaded_model_json = json_file.read()
    #     json_file.close()
    #     self.loaded_model = model_from_json(loaded_model_json)
    #     self.loaded_model.load_weights("ResNetNaive.h5")
    #     opt = Adam(lr=0.001)
    #     self.loaded_model.compile(optimizer=opt, loss=keras.losses.categorical_crossentropy, metrics=['accuracy'])
    #
    #     json_file1 = open('incML.json', 'r')
    #     loaded_model_json1 = json_file1.read()
    #     json_file1.close()
    #     self.loaded_model1 = model_from_json(loaded_model_json1)
    #     self.loaded_model1.load_weights("incMLs.h5")
    #     opt = Adam(lr=0.001)
    #     self.loaded_model1.compile(optimizer=opt, loss=keras.losses.categorical_crossentropy, metrics=['accuracy'])

    def de_noise(self,fs,sig):
        low_cut = (1) / (fs)
        high_cut = (40) / (fs)
        a, b = sign.butter(2, [low_cut, high_cut], 'pass')
        f = sign.filtfilt(a, b, sig, axis=0)
        return f

    def read(self,path):
        sig, fi = wfdb.rdsamp(path)

        return sig, fi

    def dynamic_segmentation(self,fs, sig):
        detectors = Detectors(fs)
        l_peaks = []
        l_pqrst = []
        for i in sig:
            r_peaks = detectors.pan_tompkins_detector(i)
            end = len(i) - 1
            l = []
            for i in range(len(r_peaks)):
                if i == 0:
                    continue
                if i == len(r_peaks) - 1:
                    continue
                rp = r_peaks[i] - r_peaks[i - 1]
                rn = r_peaks[i + 1] - r_peaks[i]
                before = (1 / 3) * max(rp, rn)
                after = (2 / 3) * max(rp, rn)
                rbefore = r_peaks[i] - before
                rafter = r_peaks[i] + after
                l.append(int(rbefore))
                l.append(int(rafter))
            l_peaks.append(r_peaks)
            l_pqrst.append(l)
        return l_peaks, l_pqrst

    def to_one_dict(self,dict):
        l = {}
        for key, value in dict.items():
            if key == 'comments':
                for i in value:
                    ls = i.split(":")
                    l[ls[0]] = ls[1]

            else:
                l[key] = value
        return l

    def build_header_file(self,p_fi):
        i = self.to_one_dict(p_fi)
        del i['sig_name']
        del i['units']
        df = pd.DataFrame([i])
        return df

    def cuts(self,f, pqrst, sign_name):
        list_peats = {}
        for l in range(len(f)):
            peats = []
            i = 0
            while i < len(pqrst[l]) - 1:
                si = f[l][pqrst[l][i]:pqrst[l][i + 1]]
                samp = sign.resample(si, 300)
                peats.append(samp)
                i += 2
            list_peats[sign_name[l]] = peats
        return list_peats

    def normalize(self,f):
        ft = f.T
        for i in range(len(ft)):
            ft[i] = (ft[i] - min(ft[1])) / (max(ft[1]) - min(ft[1]))
        return ft.T

    def get_name(self,folder, lis):
        folder = folder + "/"
        l = []
        l_out = []
        for i in lis:
            st = i[:-4]
            if (st not in l):
                l.append(st)
        for i in l:
            l_out.append(folder + i)
        return l_out

    def processing(self,path):
        sig, fi = wfdb.rdsamp(path)
        fs = fi['fs']
        f = self.de_noise(fs, sig)
        f = self.normalize(f)
        ft = f.T
        # df = pd.DataFrame(f, columns=fi['sig_name'])
        y, z = self.dynamic_segmentation(fs, ft)
        head = self.build_header_file(fi)
        peats = self.cuts(ft, z, fi['sig_name'])
        return peats, head, fi

    def col(self,peats):
        f = 0
        min = len(peats['i'])
        for key, value in peats.items():
            t = len(value)
            if t < min:
                min = t

        leads = ['i', 'ii', 'iii', 'avr', 'avl', 'avf', 'v1', 'v2', 'v3', 'v4', 'v5', 'v6', 'vx', 'vy', 'vz']
        for i in leads:

            arr = np.array(peats[i])
            arr=arr[:min]
            if f == 0:
                lis = arr.reshape(-1, 300, 1)
                f = 1
            else:
                lis2 = arr.reshape(-1, 300, 1)
                lis = np.concatenate((lis, lis2), axis=2)
        return lis
    loaded_model=None
    loaded_model1=None

    def predict(self,x):
        # model = Sequential()
        # '''model = models.Sequential()
        # model.add(layers.Conv2D(32, (5, 5), activation='relu', input_shape=(300, 15,1)))
        # model.add(layers.MaxPooling2D((5,5 )))
        # model.add(layers.Conv2D(64, (5, 5), activation='relu'))
        # model.add(layers.MaxPooling2D((5, 5)))
        # model.add(layers.Conv2D(128, (5, 5), activation='relu'))
        # model.add(layers.MaxPooling2D((5,5 )))
        # model.add(layers.Conv2D(256, (5, 5), activation='relu'))
        # model.add(layers.MaxPooling2D((5, 5)))
        # model.add(layers.Conv2D(128, (5, 5), activation='relu'))
        # model.add(layers.Conv2D(64, (5, 5), activation='relu'))
        # model.add(layers.MaxPooling2D((5, 5)))'''
        # tflearn.init_graph(num_cores=8, gpu_memory_fraction=0.5)
        conv_input = input_data(shape=[None, 300, 15, 1], name='input')
        conv0 = conv_2d(conv_input, 32, 5, activation='relu')
        pool0 = max_pool_2d(conv0, 5)

        conv1 = conv_2d(pool0, 64, 5, activation='relu')
        pool1 = max_pool_2d(conv1, 5)

        conv2 = conv_2d(pool1, 128, 5, activation='relu')
        pool2 = max_pool_2d(conv2, 5)

        conv3 = conv_2d(pool2, 256, 5, activation='relu')
        pool3 = max_pool_2d(conv3, 5)

        conv4 = conv_2d(pool3, 128, 5, activation='relu')
        pool4 = max_pool_2d(conv4, 5)

        conv5 = conv_2d(pool4, 64, 5, activation='relu')
        pool5 = max_pool_2d(conv5, 5)

        fully_layer = fully_connected(pool5, 1024, activation='relu')
        fully_layer = dropout(fully_layer, 0.5)

        cnn_layers = fully_connected(fully_layer, 15, activation='softmax')

        cnn_layers = regression(cnn_layers, optimizer='adam', learning_rate=0.001, loss='categorical_crossentropy',
                                name='targets')
        loaded_model = tflearn.DNN(cnn_layers, tensorboard_dir='log', tensorboard_verbose=3)

        loaded_model.load('cnn.h5')
        # peats, df, fi = pre.processing(x)
        # X_test = pre.col(peats)
        # print (X_test.shape)
        # X_test = np.array([i for i in X_test]).reshape(-1, 300, 15,1)
        # print (X_test.shape)
        # fi = pre.to_one_dict(fi)
        # print(fi)
        # diasease = fi['Reason for admission']
        # d = ['Healthy control', 'Bundle branch block', 'Cardiomyopathy', 'Dysrhythmia', 'Myocarditis', 'Palpitation',
        #      'Stable angina', 'Unstable angina', 'Heart failure (NYHA 2)', 'Heart failure (NYHA 3)',
        #      'Valvular heart disease',
        #      'Myocardial infraction', 'Heart failure (NYHA 4)', 'Hypertrophy']

        d = [['Healthy control', 0], ['Bundle branch block', 1], ['Cardiomyopathy', 2], ['Dysrhythmia', 3],
             ['Myocarditis', 4], ['Palpitation', 5], ['Stable angina', 6], ['Unstable angina', 7],
             ['Heart failure (NYHA 2)', 8], ['Heart failure (NYHA 3)', 9], ['Valvular heart disease', 10],
             ['Myocardial infraction', 11], ['Heart failure (NYHA 4)', 12], ['Hypertrophy', 13], ['normal', 14]]
        df_names = pd.DataFrame(d, columns=['class', 'Label'])

        # json_file = open('ResNetNaive.json', 'r')
        # loaded_model_json = json_file.read()
        # json_file.close()
        # loaded_model = model_from_json(loaded_model_json)
        # loaded_model.load_weights("ResNetNaive.h5")
        # opt = Adam(lr=0.001)
        # loaded_model.compile(optimizer=opt, loss=keras.losses.categorical_crossentropy, metrics=['accuracy'])
        # y_predict = model.predict(X_test)
        # y_predict = y_predict[0].argmax()
        # input = df_names.loc[df_names['Label'] == y_predict]['class'].values[0]

        # labels = LabelEncoder()
        # train_labels = labels.fit(d)
        # first = labels.inverse_transform(y_predict)[0]
        #print(self.first)
        # return input
        peats, df, fi = self.processing(x)
        X_test = self.col(peats)
        print(X_test.shape)
        X_test = X_test.reshape(-1, 300, 15, 1)
        print(X_test.shape)
        fi = self.to_one_dict(fi)
        print(fi)
        # diasease = fi['Reason for admission']
        # d = ['Healthy control', 'Bundle branch block', 'Cardiomyopathy', 'Dysrhythmia', 'Myocarditis', 'Palpitation',
        #      'Stable angina', 'Unstable angina', 'Heart failure (NYHA 2)', 'Heart failure (NYHA 3)',
        #      'Valvular heart disease',
        #      'Myocardial infraction', 'Heart failure (NYHA 4)', 'Hypertrophy','normal']
        # json_file = open('ResNetNaive.json', 'r')
        # loaded_model_json = json_file.read()
        # json_file.close()
        # loaded_model = model_from_json(loaded_model_json)
        # loaded_model.load_weights("ResNetNaive.h5")
        # opt = Adam(lr=0.001)
        # loaded_model.compile(optimizer=opt, loss=keras.losses.categorical_crossentropy, metrics=['accuracy'])
        # y_predict = loaded_model.predict(X_test)
        # y_predict = y_predict.argmax(axis=1)
        #loaded_model.compile(optimizer=opt, loss=keras.losses.categorical_crossentropy, metrics=['accuracy'])
        y_predict = loaded_model.predict(X_test)
        y_predict = y_predict[0].argmax()
        input = df_names.loc[df_names['Label'] == y_predict]['class'].values[0]
        # labels = LabelEncoder()
        # train_labels = labels.fit(d)
        # first =labels.inverse_transform(y_predict)[0]
        # print(self.first)
        return input

    def predictmyco(self,x):
        k = ['anterior', 'antero-lateral', 'antero-septal', 'antero-septo-lateral', 'inferior', 'infero-lateral',
             'infero-postero-lateral', 'lateral', 'posterior', 'postero-lateral', 'infero-posterior']
        peats, df, fi = self.processing(x)
        X_test = self.col(peats)
        X_test = X_test.reshape(-1, 300, 15)
        fi = self.to_one_dict(fi)
        print(fi)
        diasease = fi['Acute infarction (localization)']
        # json_file = open('vggML.json', 'r')
        # loaded_model_json = json_file.read()
        # json_file.close()
        # loaded_model = model_from_json(loaded_model_json)

        img_input = Input((300, 15))
        # Block 1
        x = layers.Conv1D(64, 3,
                          activation='relu',
                          padding='same',
                          name='block1_conv1')(img_input)
        x = layers.Conv1D(64, 3,
                          activation='relu',
                          padding='same',
                          name='block1_conv2')(x)
        x = layers.MaxPooling1D(2, strides=2, name='block1_pool', padding='same')(x)

        # Block 2
        x = layers.Conv1D(128, 3,
                          activation='relu',
                          padding='same',
                          name='block2_conv1')(x)
        x = layers.Conv1D(128, 3,
                          activation='relu',
                          padding='same',
                          name='block2_conv2')(x)
        x = layers.MaxPooling1D(2, strides=2, name='block2_pool', padding='same')(x)

        # Block 3
        x = layers.Conv1D(256, 3,
                          activation='relu',
                          padding='same',
                          name='block3_conv1')(x)
        x = layers.Conv1D(256, 3,
                          activation='relu',
                          padding='same',
                          name='block3_conv2')(x)
        x = layers.Conv1D(256, 3,
                          activation='relu',
                          padding='same',
                          name='block3_conv3')(x)
        x = layers.MaxPooling1D(2, strides=2, name='block3_pool', padding='same')(x)

        # Block 4
        x = layers.Conv1D(512, 3,
                          activation='relu',
                          padding='same',
                          name='block4_conv1')(x)
        x = layers.Conv1D(512, 3,
                          activation='relu',
                          padding='same',
                          name='block4_conv2')(x)
        x = layers.Conv1D(512, 3,
                          activation='relu',
                          padding='same',
                          name='block4_conv3')(x)
        x = layers.MaxPooling1D(2, strides=2, name='block4_pool', padding='same')(x)

        # Block 5
        x = layers.Conv1D(512, 3,
                          activation='relu',
                          padding='same',
                          name='block5_conv1')(x)
        x = layers.Conv1D(512, 3,
                          activation='relu',
                          padding='same',
                          name='block5_conv2')(x)
        x = layers.Conv1D(512, 3,
                          activation='relu',
                          padding='same',
                          name='block5_conv3')(x)
        x = layers.MaxPooling1D(2, strides=2, name='block5_pool', padding='same')(x)

        # Classification block
        x = layers.Flatten(name='flatten')(x)
        x = layers.Dense(128, activation='relu', name='fc1')(x)  # reduced dim for 1-d task
        x = layers.Dense(128, activation='relu', name='fc2')(x)
        x = layers.Dense(14, activation='softmax', name='predictions')(x)

        # Create model.
        loaded_model = models.Model(img_input, x, name='vgg16')

        loaded_model.load_weights("vggML.h5")
        opt = Adam(lr=0.001)
        loaded_model.compile(optimizer=opt, loss=keras.losses.categorical_crossentropy, metrics=['accuracy'])
        y_predict = loaded_model.predict(X_test)
        print(y_predict)
        y_predict =y_predict.argmax(axis=1)
        labels = LabelEncoder()
        train_labels = labels.fit(k)
        second = labels.inverse_transform(y_predict)[0]
        return second


from datetime import datetime

import graphene
import math

from django.contrib.auth.models import Group
from django.db import transaction

from authentication.models import User
from common.methods.create_notification import create_notification
from cs.models import Coupon
from order.models import OrderMaster, ZipCode
from order.models.order_detail import OrderDetail
from order.types.order_master_type import OrderMasterType
from order.types.ordered_product import OrderedProductInputType
from product.models import ProductMaster, Product, Draft, NewDraft

class PlaceOrder(graphene.Mutation):
    class Arguments:
        # ordering_user = OrderingUserInputType()
        user_id = graphene.Int()
        ordered_products = graphene.List(OrderedProductInputType)
        is_pick_up = graphene.Boolean(default_value=False)

    success = graphene.Boolean()
    order_master = graphene.Field(OrderMasterType)

    @classmethod
    @transaction.atomic()
    def mutate(cls, _, info, ordered_products, user_id=None, is_pick_up=False):

        delivery_price_normals = [0]
        delivery_price_divisions = [0]
        delivery_price_individuals = [0]
        if user_id is not None:
            user = User.objects.get(pk=user_id)
        else:
            user = info.context.user

        address = user.addresses.get(default=True)

        if not is_pick_up:
            # product_master가 같은 상품들끼리 묶기
            product_masters = {}
            for ordered_product in ordered_products:
                product_master_id = str(ordered_product.product_master_id)
                if product_master_id in product_masters:
                    product_masters[product_master_id]['quantity'] += ordered_product.quantity
                else:
                    product_masters[product_master_id] = {
                        'quantity': ordered_product.quantity
                    }

            # 도서산간 추가배송비
            extra_delivery = ZipCode.objects.filter(zip_code=address.zip_code).first()

            # 합친 딕셔너리로 배달비 구하기
            for key, value in product_masters.items():
                product_master = ProductMaster.objects.get(pk=int(key))
                delivery_type = product_master.delivery_type
                quantity = value['quantity']
                price_delivery = product_master.price_delivery
                max_quantity_per_box = product_master.max_quantity_per_box
                # 도서산간 지역의 zip code를 가지고 있으면 추가 배송비 지챈
                if extra_delivery:
                    extra_delivery_price = extra_delivery.additional_delivery_price
                    if delivery_type == '일반배송상품':
                        total_price_delivery = price_delivery + extra_delivery_price
                        delivery_price_normals.append(total_price_delivery)
                    elif delivery_type == '분할배송상품':
                        delivery_price_division = math.ceil(quantity / max_quantity_per_box)
                        total_price_delivery = (price_delivery + extra_delivery_price) * delivery_price_division
                        delivery_price_divisions.append(total_price_delivery)
                    else:
                        total_price_delivery = (price_delivery + extra_delivery_price) * quantity
                        delivery_price_individuals.append(total_price_delivery)
                else:
                    if delivery_type == '일반배송상품':
                        delivery_price_normals.append(price_delivery)
                    elif delivery_type == '분할배송상품':
                        delivery_price_division = math.ceil(quantity / max_quantity_per_box)
                        total_price_delivery = delivery_price_division * price_delivery
                        delivery_price_divisions.append(total_price_delivery)
                    else:
                        total_price_delivery = price_delivery * quantity
                        delivery_price_individuals.append(total_price_delivery)

        total_delivery_price = max(delivery_price_normals) + sum(delivery_price_divisions) + sum(delivery_price_individuals)

        # 주어진 파라미터 + 배달비 넣어주기(order master,order detail 생성)

        # order master 생성
        order_master = OrderMaster.objects.create(user=user, receiver=user.gym.name,
                                                  order_number='O{}{}'.format(user.phone[-4:], datetime.now().strftime('%y%m%d%H%M%S')),
                                                  phone=user.phone,
                                                  zip_code=address.zip_code,
                                                  address=address.address,
                                                  is_pick_up=is_pick_up,
                                                  detail_address=address.detail_address,
                                                  price_delivery=total_delivery_price,
                                                  )

        for index, ordered_product in enumerate(ordered_products):
            product = Product.objects.get(pk=ordered_product.product_id)
            print(product)

            draft = None
            price_total = product.product_master.price_gym + product.price_additional
            if ordered_product.draft_id:
                draft = NewDraft.objects.get(pk=ordered_product.draft_id)
                price_total += draft.price_work
                admin = Group.objects.get(name="관리자")
                admin_users = User.objects.filter(groups=admin)
                # for admin_user in admin_users:
                #     Notification.objects.create(user=admin_user, title="주문요청알림 - 작업실배정필요",
                #                                 contents="{}님께서 주문 요청 및 작업실 배정 요청을 하였습니다.. 작업실 배정해주세요"
                #                                 .format(order_master.order_name),
                #                                 notification_type="주문 및 작업실 배정 요청알림")
            else:
                admin = Group.objects.get(name="관리자")
                admin_users = User.objects.filter(groups=admin)
                # for admin_user in admin_users:
                #     Notification.objects.create(user=admin_user, title="주문요청알림",
                #                                 contents="{}님께서 주문 요청을 하였습니다. 주문관리에서 확인 부탁드립니다."
                #                                 .format(order_master.order_name),
                #                                 notification_type="부분입고완료")

            # order detail 생성
            product_master = product.product_master
            order_detail =OrderDetail.objects.create(order_master=order_master,
                                                     state="구매요청",
                                                     order_detail_number=order_master.order_number+str(index),
                                                     product_master=product.product_master,
                                                     product=product,
                                                     quantity=ordered_product.quantity,
                                                     user_request=ordered_product.user_request,
                                                     new_draft=draft,
                                                     price_option=product.price_additional * ordered_product.quantity,
                                                     price_total=price_total * ordered_product.quantity,
                                                     price_products=product.product_master.price_gym * ordered_product.quantity,
                                                     price_work=draft.price_work * ordered_product.quantity if draft is not None else 0,
                                                     price_work_labor=draft.price_work_labor * ordered_product.quantity if draft is not None else 0,
                                                     student_names=ordered_product.student_names,
                                                     price_consumer=product_master.price_consumer,
                                                     price_parent=product_master.price_parent,
                                                     price_gym=product_master.price_gym,
                                                     price_vendor=product_master.price_vendor
                                                     )

            gym = order_detail.order_master.user.gym
            gym.total_purchased_amount += order_detail.price_gym * order_detail.quantity + order_detail.price_work
            gym.save()
        return PlaceOrder(success=True, order_master=order_master)


from tkinter import *
from PIL import Image, ImageTk
from tkinter import messagebox

# Cores
white = '#FFFFFF'
gray = '#919191'
font = '#3e4146'

# Configurando janela
app = Tk()
app.title('Calculadora de Salário')
app.geometry('550x330')
app.config(bg=white)
app.resizable(False, False)

# Criando cabeçalho
header = Frame(app, width=551, height=30, bg=white,
               relief='raised', borderwidth=1)
header.place(x=0, y=0)

# Preparando imagens
iconHeader = Image.open('CalculadoraSalario/images/money.png')
iconHeader = iconHeader.resize((28, 28))
iconHeader = ImageTk.PhotoImage(iconHeader)

iconBig = Image.open('calculadoraSalario/images/money.png')
iconBig = iconBig.resize((100, 100))
iconBig = ImageTk.PhotoImage(iconBig)

circle = Image.open('CalculadoraSalario/images/circles.png')
circle = circle.resize((39, 100))
circle = ImageTk.PhotoImage(circle)

# Fazendo header
iconHead = Label(header, image=iconHeader, width=20, height=23,
                 compound=CENTER, anchor=CENTER, bg=white, relief=FLAT)
iconHead.place(x=4, y=0)

nameApp = Label(header, text='Calculadora de salário', width=20,
                height=1, anchor=CENTER, font=('Ivy 12 bold'), bg=white, fg=font)
nameApp.place(x=30, y=3)

# Criando labels e entrys
name = Label(app, text='Escreva seu nome completo: ', width=30,
             height=1, anchor='w', font=('Ivy 9 bold'), bg=white, fg=font)
name.place(x=5, y=50)

nameEntry = Entry(app, width=25, justify='left', relief='solid', borderwidth=1)
nameEntry.place(x=190, y=50)

hoursDay = Label(app, text='Quantas horas você trabalha por dia? ',
                 width=40, height=1, anchor='w', font=('Ivy 9 bold'), bg=white, fg=font)
hoursDay.place(x=5, y=85)

hoursEntry = Entry(app, width=15, justify='center',
                   relief='solid', borderwidth=1)
hoursEntry.place(x=246, y=85)

daysWeek = Label(app, text='Quantos dias você trabalha por semana? ',
                 width=40, height=1, anchor='w', font=('Ivy 9 bold'), bg=white, fg=font)
daysWeek.place(x=5, y=120)

daysEntry = Entry(app, width=15, justify='center',
                  relief='solid', borderwidth=1)
daysEntry.place(x=246, y=120)

yearSalary = Label(app, text='Qual é o seu salário anual ?', width=30,
                   height=1, anchor='w', font=('Ivy 9 bold'), bg=white, fg=font)
yearSalary.place(x=5, y=155)

yearEntry = Entry(app, width=15, justify='center',
                  relief='solid', borderwidth=1)
yearEntry.place(x=246, y=155)

# Criando função
def calc():
    try:
        # Coletando informações dos entrys
        nameGet = nameEntry.get()
        hoursGet = int(hoursEntry.get())
        daysGet = int(daysEntry.get())
        yearGet = float(yearEntry.get())
        hoursWeek = int(hoursGet * daysGet)

        if nameGet == '':
            messagebox.showinfo('Atenção!!!',
                                'Preencha o campo de nome.')
        else:
            nameShow = Label(app, text=nameGet, width=28, height=1, anchor='w', font=(
                'Ivy 10'), relief=RAISED, bg=gray, fg=font)
            nameShow.place(x=5, y=190)

            hoursShow = Label(app, text=f'Horas de trabalho por dia: {hoursGet}', width=28, height=1, anchor='w', font=(
                'Ivy 10'), relief=RAISED, bg=gray, fg=font)
            hoursShow.place(x=5, y=212)

            daysShow = Label(app, text=f'Dias de trabalho por semana: {daysGet}', width=28, height=1, anchor='w', font=(
                'Ivy 10'), relief=RAISED, bg=gray, fg=font)
            daysShow.place(x=5, y=234)

            yearShow = Label(app, text=f"R${yearGet:,.2f}", width=28, height=1, anchor='w', font=(
                'Ivy 10 bold'), relief=RAISED, bg=gray, fg=font)
            yearShow.place(x=5, y=256)

            # Mostrando o salário anual, mensal, diário e por hora
            iconCircles = Label(app, image=circle, width=39, height=100,
                                compound=CENTER, anchor=CENTER, bg=white, relief=FLAT)
            iconCircles.place(x=245, y=195)

            salHour = Label(app, text=f'Salário por hora: R$ {yearGet/(hoursWeek*52):,.2f}', width=25, height=1, anchor='w', font=(
                'Arial 10'), bg=white, fg=font)
            salHour.place(x=290, y=190)

            salDay = Label(app, text=f'Salário Diário: R$ {yearGet/(daysGet*52):,.2f}', width=25, height=1, anchor='w', font=(
                'Arial 10'), bg=white, fg=font)
            salDay.place(x=290, y=212)

            salWeek = Label(app, text=f'Salário Semanal: R$ {yearGet/52:,.2f}', width=25, height=1, anchor='w', font=(
                'Arial 10'), bg=white, fg=font)
            salWeek.place(x=290, y=234)

            salMonth = Label(app, text=f'Salário Mensal: R$ {yearGet/12:,.2f}', width=25, height=1, anchor='w', font=(
                'Arial 10'), bg=white, fg=font)
            salMonth.place(x=290, y=256)

            salYear = Label(app, text=f'Salário Anual: R$ {yearGet:,.2f}', width=25, height=1, anchor='w', font=(
                'Arial 10'), bg=white, fg=font)
            salYear.place(x=290, y=278)
    except:
        messagebox.showerror('Erro!!!',
                              'Insira números')

# Colocando imagem e botão de calcular
img = Label(app, image=iconBig, width=114, height=100,
            compound=CENTER, anchor=CENTER, bg=white, relief=FLAT)
img.place(x=380, y=40)

calc_button = Button(app, command=calc, text='Calcular', width=17, height=1, anchor='center', font=(
    'Ivy 9 bold'), relief='raised', overrelief='sunken', bg=gray, fg=font, borderwidth=2,)
calc_button.place(x=370, y=151)

app.mainloop()


## original code https://medium.com/towards-data-science/lstm-by-example-using-tensorflow-feb0c1968537
###the adapted code uses word characteristics
from __future__ import print_function

import numpy as np
import tensorflow as tf
from tensorflow.contrib import rnn
import random
import collections
import time
from tensorflow.python import debug as tf_debug

start_time = time.time()
def elapsed(sec):
    if sec<60:
        return str(sec) + " sec"
    elif sec<(60*60):
        return str(sec/60) + " min"
    else:
        return str(sec/(60*60)) + " hr"

#targets log path
logs_path = '/tmp/tensorflow/rnn_words'
writer = tf.summary.FileWriter(logs_path)

#text file containing words for training
training_file = '/home/sean/christina/git/lstm_rnn_word2vec/beiling_the_cat.txt'

def read_data(fname):
    with open(fname) as f:
        content = f.readlines()
    content = [x.strip() for x in content]
    content = [content[i].split() for i in range(len(content))]
    content = np.array(content)
    content = np.reshape(content,[-1,])
    return content

training_data = read_data(training_file)
print("Loaded training data...")

#might be better to do hashing? https://www.tensorflow.org/api_guides/python/string_ops#Conversion
def build_dataset(words):
    count = collections.Counter(words).most_common()
    dictionary = dict()
    for word, _ in count:
        dictionary[word] = len(dictionary)
    reverse_dictionary = dict(zip(dictionary.values(),dictionary.keys()))
    return dictionary, reverse_dictionary

dictionary, reverse_dictionary = build_dataset(training_data)
vocab_size = len(dictionary)

#Parameters
#learning_rate = 0.001
training_iters = 100000
display_step = 50
n_input = 3
characteristics_size = 4
input_size = n_input*characteristics_size

#number of units in RNN cell
n_hidden = 512

x = tf.placeholder("float", [input_size])
y = tf.placeholder("float", [characteristics_size])
y2 = tf.placeholder("int64", [])

weights = {
    'out': tf.Variable(tf.random_uniform([n_hidden, input_size])),
    'dict': tf.Variable(tf.random_uniform([vocab_size,characteristics_size]))
}

biases = {
    'out': tf.Variable(tf.random_uniform([input_size]))
}

def input_list(offset, n_input):
        input_list = tf.Variable([],dtype="float32")
        for i in range(offset,offset+n_input):
            toconcat = tf.slice(weights['dict'],[dictionary[str(training_data[i])],0],[1,characteristics_size])
            toconcat = tf.reshape(toconcat,[-1])
            input_list = tf.concat([input_list,toconcat],0)
        input_list = tf.reshape(input_list,[-1])
        return input_list

def RNN(x, weights, biases):

    #reshape to [1, n_input]
    x = tf.reshape(x, [-1, input_size])
    x = tf.split(x, n_input, 1)

    # print(x.eval())

    #2-layer LSTM, each layer has n_hidden units
    rnn_cell = rnn.MultiRNNCell([rnn.BasicLSTMCell(n_hidden),rnn.BasicLSTMCell(n_hidden)])

    #generate prediction
    outputs, states = rnn.static_rnn(rnn_cell, x, dtype=tf.float32)

    # we only want the last output (due to nature of LSTM)
    y_ = tf.matmul(outputs[-1], weights['out']) + biases['out']
    y_ = y_[:,-characteristics_size:]
    y_ = tf.reshape(y_,[-1])
    return y_

#find closest word with the predicted characteristics
def RNN2(y_):
    y_onehot = tf.subtract(weights['dict'],y_)
    y_onehot = tf.square(y_onehot)
    y_onehot = tf.reduce_sum(y_onehot, 1)
    y_onehot = tf.argmin(y_onehot)

    y_onehot = tf.cast(y_onehot, dtype="int64")

    return y_onehot

#find the characteristics of the predicted word
def pred_characteristics(y_onehot):
    index = tf.cast(y_onehot, dtype="int32")
    y_final = weights['dict'][index, :]
    return y_final

actual_pred = RNN(x, weights, biases)
onehot_pred = RNN2(actual_pred)
pred_characteristics = pred_characteristics(onehot_pred)

cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = pred_characteristics, labels = y))
optimizer = tf.train.AdamOptimizer(learning_rate = .01).minimize(cost)

#model evaluation compares the predicted characteristics and actual characteristics of correct word
# note, the model can predict the correct word and have an accuracy <100% because the predicted characteristics weren't correct
correct_pred = tf.square(tf.subtract(pred_characteristics, y))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))

#initialize the variables
init = tf.global_variables_initializer()

#launch the graph
with tf.Session() as session:
    session.run(init)
    #session = tf_debug.LocalCLIDebugWrapperSession(session)
    #session.add_tensor_filter("has_inf_or_nan", tf_debug.has_inf_or_nan)
    step = 0
    offset = random.randint(0, n_input+1)
    end_offset = n_input + 1
    acc_total = 0
    loss_total = 0

    writer.add_graph(session.graph)

    while step < training_iters:
        #generate a minibatch. add some randomness on selection process
        if offset > (len(training_data)-end_offset):
           offset = random.randint(0, n_input+1)

        symbols_in_keys = input_list(offset, n_input).eval()

        symbols_out = tf.slice(weights['dict'],[dictionary[str(training_data[offset+n_input])],0],[1,characteristics_size])
        symbols_out = tf.reshape(symbols_out,[-1])
        symbols_out = symbols_out.eval()

        y_hot = offset+n_input

        _, acc, loss, onehot_p, pred, actual_pred2= session.run([optimizer, accuracy, cost, onehot_pred, pred_characteristics, actual_pred], \
                                                feed_dict={x: symbols_in_keys, y: symbols_out, y2: y_hot})
        loss_total += loss
        acc_total += acc
        if (step+1) % display_step == 0:
            print("Iter= " + str(step+1) + ", Current Loss= " + \
                  "{:.6f}".format(loss) + ", Average Accuracy= " + \
                  "{:.2f}%".format(100*acc_total/display_step))
            acc_total = 0
            loss_total = 0
            symbols_in = [training_data[i] for i in range(offset, offset + n_input)]
            symbols_out = training_data[offset + n_input]
            symbols_out_pred = reverse_dictionary[onehot_p]
            print("%s - Actual: [%s] vs Pred: [%s]" % (symbols_in,symbols_out,symbols_out_pred))
            #the below will print the actual and predicted word characteristics
            #print("Actual:[%s] vs Pred:[%s]" % (pred,actual_pred2))
        step += 1
        offset += (n_input+1)
    print("Optimization Finished!")
    print("Elapsed time: ", elapsed(time.time() - start_time))
    print("Run on command line.")
    print("\ttensorboard --logdir=%s" % (logs_path))
    print("Point your web browser to: http://localhost:6006/")
    while True:
        prompt = "%s words: " % n_input
        sentence = input(prompt)
        sentence = sentence.strip()
        words = sentence.split(' ')
        if len(words) != n_input:
            continue
        try:
            symbols_in_keys = [dictionary[str(words[i])] for i in range(len(words))]
            for i in range(32):
                keys = np.reshape(np.array(symbols_in_keys), [-1, n_input, 1])
                onehot_p = session.run(pred, feed_dict={x: keys})
                onehot_pred_index = int(tf.argmax(onehot_p, 1).eval())
                sentence = "%s %s" % (sentence,reverse_dictionary[onehot_pred_index])
                symbols_in_keys = symbols_in_keys[1:]
                symbols_in_keys.append(onehot_pred_index)
            print(sentence)
        except:
            print("Word not in dictionary")


from rest_framework.filters import BaseFilterBackend

class LessonParentFilter(BaseFilterBackend):
    def filter_queryset(self, request, queryset, view):
        parent_only = request.query_params.get('parent_only')

        if parent_only and parent_only == '1':
            queryset = queryset.filter(**{
                'parent__isnull': True,
            })

        return queryset


from main import app
from fastapi.testclient import TestClient

client = TestClient(app)

url = "/properties"

def test_get_properties_success():
    """
    Test if get properties endpoint return success status code
    """
    response = client.get(url)
    assert response.status_code == 200

def test_get_properties_list():
    """
    Test if get properties endpoint return a list
    """
    response = client.get(url)
    assert response.status_code == 200
    assert type(response.json()) == list

def test_get_properties_year_param():
    """
    Test if get properties endpoint is successful
    setting the year param
    """
    year_param = {
        "year": 2000
    }
    response = client.get(url, params=year_param)
    assert response.status_code == 200
    assert type(response.json()) == list

def test_get_properties_city_param():
    """
    Test if get properties endpoint is successful
    setting the city param
    """
    city_param = {
        "city": "bogota"
    }
    response = client.get(url, params=city_param)
    assert response.status_code == 200

def test_get_properties_multiple_params():
    """
    Test if get properties endpoint is successful
    setting the multiple params
    """
    params = {
        "year": 2000,
        "city": "bogota"
    }
    response = client.get(url, params=params)
    assert response.status_code == 200

def test_get_properties_invalid_param():
    """
    Test if get properties endpoint is a bad
    request setting an invalid param
    """
    invalid_param = {
        "state": "mexico"
    }
    response = client.get(url, params=invalid_param)
    assert response.status_code == 400

def test_get_properties_error_detail():
    """
    Test if get properties endpoint
    return the detail error response
    """
    invalid_param = {
        "state": "mexico"
    }
    error_detail = {
        "detail": "The state field does not "
                  "exist in db thus this "
                  "parameter is not valid"
    }
    response = client.get(url, params=invalid_param)
    assert response.status_code == 400
    assert response.json() == error_detail


from geometry import *
from physical import *
from ydata_profiling import ProfileReport
from pathlib import Path
import sweetviz as sv
import os
import seaborn as sns
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from matplotlib.colors import ListedColormap, Normalize

def list2str(lst):
    if(isinstance(lst, list)):
        num_str = ''.join(str(num) for num in lst)
        return num_str
    else:
        return lst

def one_hot_encode(lst):
    num = 0
    for i in range(6):
        if(lst[i]): num += 2**i
    return num

def color_map(data):
    fig = plt.figure()
    ax = sns.heatmap(data, cmap='coolwarm')
    # 设置标题、坐标轴标签、颜色条标签等
    plt.title('Correlation Heatmap')
    plt.xlabel('X Label')
    plt.ylabel('Y Label')
    # plt.colorbar(label='Correlation')
    ax.set_xticklabels(ax.get_xticklabels(), rotation=-45)
    plt.tight_layout()
    fig.savefig("data_process/picture/output.svg")
    plt.show()

def plotegv(all_df):
    eg_df = all_df[['E11','G12']]
    egv_df = all_df[['E11','G12','V12']]
    corr_matrix = eg_df.corr()
    # 创建图表和子图
    fig = plt.figure(figsize=(12, 4))

    # 创建第一个子图
    ax1 = fig.add_subplot(231)
    ax1.scatter(egv_df['E11'], egv_df['G12'], s=3, alpha=0.2)
    ax1.set_xlabel('E11')
    ax1.set_ylabel('G12')

    # 创建第二个子图
    ax2 = fig.add_subplot(232)
    ax2.scatter(egv_df['E11'], egv_df['V12'], s=3, alpha=0.2)
    ax2.set_xlabel('E11')
    ax2.set_ylabel('V12')

    # 创建第三个子图
    ax3 = fig.add_subplot(233)
    ax3.scatter(egv_df['G12'], egv_df['V12'], s=3, alpha=0.2)
    ax3.set_xlabel('G12')
    ax3.set_ylabel('V12')

    # ax4 = fig.add_subplot(234)
    # ax4.scatter(all_df['E11'], all_df['Thickness'], s=3, alpha=0.2)
    # ax4.set_xlabel('E11')
    # ax4.set_ylabel('thickness')

    ax5 = fig.add_subplot(235)
    ax5.scatter(all_df['E11'], all_df['RelDensity'], s=3, alpha=0.2)
    ax5.set_xlabel('E11')
    ax5.set_ylabel('RelDensity')

    ax6 = fig.add_subplot(236)
    ax6.scatter(all_df['V12'], all_df['RelDensity'], s=3, alpha=0.2)
    ax6.set_xlabel('V12')
    ax6.set_ylabel('RelDensity')
    # 调整子图间距
    fig.subplots_adjust(wspace=0.4, hspace=0.4)
    plt.show()

def plotegv3D(all_df):
    egv_df = all_df[['E11', 'G12', 'V12']]

    # 创建三维散点图
    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    ax.scatter(egv_df['E11'], egv_df['G12'], egv_df['V12'], s=3, alpha=0.2)

    # 添加坐标轴标签
    ax.set_xlabel('E11')
    ax.set_ylabel('G12')
    ax.set_zlabel('V12')

    # 显示图形
    plt.show()

csv_file_path = 'data_process/all_data_diff_thick.csv'
if os.path.exists(csv_file_path):
    all_df = pd.read_csv(csv_file_path)
    row = all_df.iloc[0]
else:
    geo_df = read_geo_file()
    phy_df = read_phy_files()
    # find the geo data of every phy_data_frame

    geo_dict = {"ID":[], "Type":[], "Fp":[]}
    phy_drop = []
    # 注意，物理属性里面的名字比几何信息的多一个S
    for row in phy_df.itertuples(index=True):
        name = (row.Name).rstrip('S')
        geo_data = geo_df.loc[geo_df['Name'] == name]
        if(geo_data.empty):
            phy_drop.append(row.Index)
            # print(row)
        else:
            geo_dict["ID"].append(geo_data['ID'].values[0])
            geo_dict["Type"].append((geo_data['Type'].values[0]))
            # geo_dict["Type"].append(one_hot_encode(geo_data['Type'].values[0])) #encode to int
            geo_dict["Fp"].append(geo_data['Fp'].values[0])

    new_df = pd.DataFrame(geo_dict)
    phy_df = phy_df.drop(phy_drop, axis=0)
    phy_df = phy_df.reset_index(drop=True)
    all_df = pd.concat([phy_df, new_df], axis=1)
    all_df.to_csv(csv_file_path, index=False)

breakpoint()
plotegv(all_df)
# profile = ProfileReport(all_df, title="Data Report", explorative=True, minimal=True)

# 创建一个数据集比较器
# all_df = all_df.applymap(list2str)

# my_report = sv.compare([all_df, "Original"], [all_df, "New"])
# my_report.show_html(filepath="data_process/big_data_report.html")

# profile = ProfileReport(all_df, title="Data Report", explorative=True)
# profile.to_file(Path("data_process/big_data_report.html"))

'''
num_df = all_df.drop(columns=['Name','ID','Fp','Type'])
corr_matrix = num_df.corr()
color_map(corr_matrix)

profile = ProfileReport(num_df, title="Data Report", explorative=True)
profile.to_file(Path("data_process/big_data_report.html"))
image_ids = profile.get_unique_id()
breakpoint()
'''


import logging
from math import ceil
from pathlib import Path
from typing import Generator, List

from django.db import models
from django.conf import settings
from django.template import engines
from django.db.models import QuerySet
from django.core.validators import MinLengthValidator
from django.utils.translation import ugettext_lazy as _

from commons.models import UserCreatedDatetimeModel
from staticsites.models import IndexPage, StaticPageBase

logger = logging.getLogger(__name__)

class NewsPage(StaticPageBase):
    index = models.OneToOneField(
        IndexPage,
        on_delete=models.CASCADE,
    )

    def get_total_pages(self, items_per_page: int = 10) -> int:
        return ceil(NewsItem.objects.filter(is_published=True).order_by('-publish_on').count() / items_per_page)

    def get_newsitems_pages(self, items_per_page: int = 10) -> Generator[QuerySet, None, None]:
        qs = NewsItem.objects \
            .filter(
                newspage=self,
                is_published=True,
            )\
            .order_by('-publish_on')

        for page_number in range(self.get_total_pages(items_per_page)):
            offset = items_per_page * page_number
            yield qs[offset: offset + items_per_page]

    def get_latest_n_published(self, n: int = 6) -> QuerySet:
        return NewsItem.objects.filter(is_published=True).order_by('-publish_on')[:n]

    def instantiate(self, root_directory: Path, items_per_page: int = settings.NEWS_ITEMS_PER_PAGE) -> List[dict]:
        """
        Create instantiated HTML and images in given root_directory
        """
        # prepare template
        django_engine = engines['django']
        template = django_engine.from_string(self.template)

        result_page_data = []
        for page_count, page_queryset in enumerate(self.get_newsitems_pages(items_per_page)):
            page_numbered_filename = str(self.filename.format(page_count))
            page_newsitems = list(page_queryset)
            relative_filepath = Path(str(self.relative_path), page_numbered_filename)
            absolute_filepath = root_directory / relative_filepath

            # make directories for target news html file
            logger.info(f'Writing ({relative_filepath}) to {root_directory} ...')
            absolute_filepath.parent.mkdir(parents=True, exist_ok=True)
            with absolute_filepath.open('w', encoding='utf8') as html_out:
                html = template.render(context={
                    self.index.newsitems_template_variablename: page_newsitems
                })
                html_out.write(html)

            # instantiate newsitem.images to root_directory
            for newsitem in page_newsitems:
                image_relative_filepath = Path(str(newsitem.image_relpath), str(newsitem.image.name))
                image_absolute_filepath = root_directory / image_relative_filepath
                image_absolute_filepath.parent.mkdir(parents=True, exist_ok=True)
                logger.info(f'Writing ({image_relative_filepath}) to {root_directory} ...')
                with newsitem.image.open('rb') as image_out, image_absolute_filepath.open('wb') as image_in:
                    image_in.write(image_out.read())

            page_data = {
                'relative_path': relative_filepath,
                'absolute_path': absolute_filepath,
                'filename': page_numbered_filename,
                'page_count': page_count
            }
            result_page_data.append(page_data)
        return result_page_data

    def save(self, *args, **kwargs):
        self.filename = 'news_{}.html'  # TODO: May want multiple pages here
        self.relative_path = 'news'
        self.type = 'news'  # auto-populate 'type' field
        super().save(*args, **kwargs)

class NewsItem(UserCreatedDatetimeModel):
    newspage = models.ForeignKey(
        NewsPage,
        on_delete=models.CASCADE
    )
    publish_on = models.DateTimeField()
    is_published = models.BooleanField(
        default=False
    )
    image = models.ImageField(
        null=True,
        blank=True,
    )
    image_relpath = models.CharField(
        max_length=250,
        default='imgs/news'
    )
    title = models.CharField(
        max_length=150,
        help_text=_('News Item Title')
    )
    text = models.TextField(
        validators=[MinLengthValidator(limit_value=10)]
    )


#/usr/bin/env python
#-*- coding:utf-8 -*-

import logging

from datetime import datetime

import threading

import os

from Common import ReadConfig

class Log:

    def __init__(self):

        global logPath,resultPath,rootPath,reportPath

        rootPath = ReadConfig.rootDir
        resultPath = os.path.join(rootPath,"resultReport")

        logPath = os.path.join(resultPath,str(datetime.now().strftime("%Y年%m月%d日%H时%M分%S秒")))

        reportPath = os.path.join(logPath,"reprot.html")

        logInfoPath = os.path.join(logPath,"out.txt")
        if not os.path.exists(resultPath):
            os.mkdir(resultPath)
        if not os.path.exists(logPath):
            os.mkdir(logPath)
        self.logger = logging.getLogger()

        self.logger.setLevel(logging.INFO)

        handler = logging.FileHandler(logInfoPath)

        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')

        handler.setFormatter(formatter)

        self.logger.addHandler(handler)
    def getLogger(self):
        return self.logger
    def getReprotPath(self):
        return reportPath
    def getResultPath(self):
        return resultPath

class MyLog:
    log = None
    mutext = threading.Lock()
    def __init__(self):
        pass
    @staticmethod
    def get_log():
        if MyLog.log is None:
            MyLog.mutext.acquire()
            MyLog.log = Log()
            MyLog.mutext.release()
        return MyLog.log


import json
import os
import re
import numpy as np
import webview

from pysom.graph import GraphCompileError
from pysom.nodes.calibrate import Calibrate
from pysom.nodes.som import SOM
import traceback
from collections import Counter, defaultdict

# Model training service

class SomViewService:
    def __init__(self, ds):
        self.ds = ds
        self.input_key = None
        self.som = None
        self.cal = None

        self.links = None
        self.nodes = None

        self.palette = np.array([[0.12156863, 0.46666667, 0.70588235],
                                 [0.68235294, 0.78039216, 0.90980392],
                                 [1., 0.49803922, 0.05490196],
                                 [1., 0.73333333, 0.47058824],
                                 [0.17254902, 0.62745098, 0.17254902],
                                 [0.59607843, 0.8745098, 0.54117647],
                                 [0.83921569, 0.15294118, 0.15686275],
                                 [1., 0.59607843, 0.58823529],
                                 [0.58039216, 0.40392157, 0.74117647],
                                 [0.77254902, 0.69019608, 0.83529412],
                                 [0.54901961, 0.3372549, 0.29411765],
                                 [0.76862745, 0.61176471, 0.58039216],
                                 [0.89019608, 0.46666667, 0.76078431],
                                 [0.96862745, 0.71372549, 0.82352941],
                                 [0.49803922, 0.49803922, 0.49803922],
                                 [0.78039216, 0.78039216, 0.78039216],
                                 [0.7372549, 0.74117647, 0.13333333],
                                 [0.85882353, 0.85882353, 0.55294118],
                                 [0.09019608, 0.74509804, 0.81176471],
                                 [0.61960784, 0.85490196, 0.89803922]])
        self.num_colors = len(self.palette)

    def generate_nodes(self):
        def rgb2hex(x):
            return "#{:02x}{:02x}{:02x}".format(
                *np.clip(np.array((x * 255.0), dtype=np.int32), 0, 255))

        def fmt(x):
            return ','.join([f'{i}' for i in x.keys()])

        def mix(w, m):
            return np.sum(m * w[:, np.newaxis], axis=0)

        labelmap = None
        if self.cal is not None:
            labelmap = {k[0] * self.som.size + k[1]: v for k,
                        v in self.cal.get_output(slot=1).items()}
        else:
            return [{'id': i} for i in range(self.som.size ** 2)]

        cmap = np.array([self.palette[i % self.num_colors]
                        for i in range(len(labelmap))])
        logit = self.cal.logit()
        # self.nodes = [{
        #     'id': i,
        #     'l': fmt(labelmap.get(i, Counter())),
        #     'c': rgb2hex(colored.get(i, np.array([0.1, 0.1, 0.1])))
        # } for i in range(self.som.size ** 2)]

        self.nodes = [{
            'id': i,
            'l': fmt(labelmap.get(i, Counter())),
            'c': rgb2hex(mix(logit[i], cmap))
        } for i in range(self.som.size ** 2)]

    def rect_links(self):
        dim = self.som.size
        w = self.som.get_weights()
        links = []
        for y in range(dim):
            for x in range(dim - 1):
                row = y * dim
                l = {'source': row + x, 'target': row + x + 1,
                     'value': np.linalg.norm(w[row + x] - w[row + x + 1]) * 100}
                links.append(l)

        for y in range(dim - 1):
            for x in range(dim):
                row = y * dim
                l = {'source': row + x, 'target': row + x + dim,
                     'value': np.linalg.norm(w[row + x] - w[row + x + dim]) * 100}
                links.append(l)

        return links

    def set_input(self, key):
        self.input_key = key

        data = self.ds.get_object_data(self.input_key)
        if data is None:
            return {'status': False, 'msg': 'Input data does not exist.'}

        if isinstance(data, SOM):
            self.som = data
        elif isinstance(data, Calibrate):
            self.cal = data
            self.som = self.cal.get_input()
        else:
            return {'status': False, 'msg': 'Input data is neither a SOM node or Calibrate node.'}

        self.links = self.rect_links()

        try:
            self.generate_nodes()
        except:
            return {'status': False, 'msg': traceback.format_exc()}

        return {'status': True, 'msg': key}

    def get_som_viz_data(self):
        if self.nodes is None or self.links is None:
            return {'status': False, 'msg': 'Not available'}

        return {'status': True, 'msg': 'Ok', 'obj': {'links': self.links, 'nodes': self.nodes}}


from django.shortcuts import render
from django.shortcuts import redirect
from django.http import HttpResponse
from django import http
from django.template import RequestContext, loader
from django.core.serializers.json import DjangoJSONEncoder
from django.utils.safestring import mark_safe
from urllib.request import urlopen
import json
from json import JSONEncoder
from hnproj.models import HNUser
from hnproj.models import HNStory
from hnproj.models import HNTopStory
from hnproj.models import TopStoryIdsByTime

def home(request):
    HNUser.objects.all()
    return http.HttpResponse('Hello World test!')
def users(request):
    user_list = HNUser.objects.all()
    if not user_list:
        jh = HNUser(username = 'nkzednan', last_run_max_id = 8000000)
        jh.save()
        k = HNUser(username = 'kogir', last_run_max_id = 8000000);
        k.save()
    return render(request, 'hn_users.html', {'users': user_list});

def clear_out_users(request):
    users = HNUser.objects.all()
    for u in users:
        u.delete();
    return http.HttpResponse("Deleted all users.");

def get_max_item_id():
    maxItemUrl = 'https://hacker-news.firebaseio.com/v0/maxitem.json'
    maxItemData = json.load(urlopen(maxItemUrl))
    return long(maxItemData);

# Logic for page to view the stories submitted by users following.
def newsByUser(request):
    user_list = HNUser.objects.order_by("username").all()
    stories = HNStory.objects.order_by("-hnStoryId").all()

    usersToStories = {}

    usernames = []
    for user in user_list:
      userStories = []
      for story in stories:
        if story.hnUserId == user.user_id:
          userStories.append(json.loads(story.storyJSON))
      usersToStories[user.username] = userStories
      usernames.append(user.username)

    context = {
               'users': mark_safe(json.dumps(usernames, cls=DjangoJSONEncoder)),
               'storiesByUser': mark_safe(json.dumps(usersToStories, cls=DjangoJSONEncoder))}
    return render(request, 'userStories.html', context)

# requests the item with the given id, return the json object.
def get_item(item_id):
    url = 'https://hacker-news.firebaseio.com/v0/item/' + str(item_id) + '.json'
    return json.load(urlopen(url))

def is_story(item_json):
    return "story" == item_json.get('type')

def is_deleted(item_json):
    return True == item_json.get('deleted')

def get_item_list_since(last_id, userjson):
    items = []
    for id in userjson["submitted"]:
      if id > last_id:
        items.append(id)
    return items

def get_user_data(username):
       url = 'https://hacker-news.firebaseio.com/v0/user/' + username + '.json'
       userResp = urlopen(url)
       userjson = json.load(userResp)
       return userjson

def follow_user(request):
    usernames = request.POST.getlist('hnUsername')
    curr_max_id = get_max_item_id()
    users = ""
    for username in usernames:
        if ((not username) or username == ""):
            continue
        # start out a little bit before the current max to see items from recently
        hnuser = HNUser(username, curr_max_id - 20000);
        hnuser.save();
        users += username + ", "
    return http.HttpResponse("now following user(s): " + users);

def update_top_items(request):
    url = 'https://hacker-news.firebaseio.com/v0/topstories.json';
    data = json.load(urlopen(url));
    jsonStr = json.dumps(data);
    topIdsObj = TopStoryIdsByTime(storyIds = data);
    topIdsObj.save();
    return http.HttpResponse("updated");

# removes top stories from the db.
def remove_top_items(request):
    ids = request.POST.getlist('storyId');
    allTopStories = HNTopStory.objects.all();
    delCount = 0;
    i = "ids: ";
    for si in ids:
        i = i + " " + str(si) + " " + str(type(si)) + " ";
    for story in allTopStories:
        storyId = story.hnStoryId
        if str(storyId) in ids or storyId in ids:
            story.marked_deleted = True
            story.save()
            delCount = delCount + 1;
    return redirect("/topStories");


"""EC2 Tag Auditing

This purpose of this script is to provide a working example
of how to audit tags and enforce compliance through serverless
Lambda functions.

This script focuses on enforcing tagging policies. It checks for
required tags on all EC2 instances, checks for appropriate values,
will set tags on first offense and send reminder, then will destroy
instances if they continue to be noncompliant.
"""
from __future__ import print_function
from datetime import datetime, timedelta
import fnmatch
import json
import boto3

client = boto3.client('ec2')

def handler(event, context):

    ### PARAMETERS ###

    # time resources can be non-compliant in days before termination
    noncompliant_threshold = 7

    # required tags, and allowed values
    # allowed values use fnmatch/regex
    required_tags = {
        'environment': [
            'dev',
            'prod',
            'stage'
        ],
        'owner': ['*@gmail.com']
    }

    ### BEGIN AUDIT ###

    # audit resources
    noncompliant_instances = []
    compliant_instances = []
    for instance in get_instance_details(client):
        if not audit_tags(instance['tags'], required_tags):
            noncompliant_instances.append(instance)
        else:
            compliant_instances.append(instance)

    # get records broken down by tasks: terminate, set expiration, terminate in future
    assessed = parse_compliance_status(noncompliant_instances)

    # tag expiration dates
    if assessed['set_expiration']:
        expiration_date = datetime.now() + timedelta(days=noncompliant_threshold)
        client.create_tags(
            Resources=assessed['set_expiration'],
            Tags=[
                {
                    'Key': 'expiration',
                    'Value': expiration_date.strftime("%Y-%m-%d %H:%M:%S")
                }
            ]
        )

    # terminate instances that aren't compliant
    if assessed['terminate']:
        client.terminate_instances(
            InstanceIds=assessed['terminate']
        )

    print("Compliant: " + json.dumps(compliant_instances))
    print("Noncompliant: " + json.dumps(noncompliant_instances))
    print("Set Expiration Date: " + json.dumps(assessed['set_expiration']))
    print("To Terminate Soon: " + json.dumps(assessed['terminate_soon']))
    print("To Terminate: " + json.dumps(assessed['terminate']))

    #print(noncompliant_instances)
    print("Function Complete")

def get_instance_details(ec2_client):
    """ Get Instance Data

    Retrieve and compile just the data we need from EC2 instances
    """

    # find all instances that are not terminated
    response = ec2_client.describe_instances(
        Filters=[
            {
                'Name': 'instance-state-name',
                'Values': [
                    'running',
                    'shutting-down',
                    'stopped',
                    'stopping',
                    'pending'
                ]
            }
        ]
    )

    # aggregate results into easy format to work with
    results = []
    for instance in response['Reservations'][0]['Instances']:
        results.append({
            'id': instance.get('InstanceId'),
            'tags': instance.get('Tags', {})
        })

    return results

def parse_compliance_status(noncompliant_records):
    """Parse Compliance Status

    Assess non compliant record and determine
    whether they need to be terminated, future terminated,
    or expiration date set
    """
    to_terminate = []
    to_set_expiration = []
    to_terminate_soon = []

    for instance in noncompliant_records:
        if 'tags' in instance:
            tag_keys = [t['Key'] for t in instance['tags']]
            if node_first_offense(tag_keys):
                to_set_expiration.append(instance['id'])
            else:
                for tag in instance['tags']:
                    if tag['Key'] == 'expiration':
                        expiration = tag['Value']
                        break

                if node_past_due(expiration):
                    to_terminate.append(instance['id'])
                else:
                    to_terminate_soon.append(instance['id'])
        else:
            to_set_expiration.append(instance['id'])

    return {
        'terminate': to_terminate,
        'terminate_soon': to_terminate_soon,
        'set_expiration': to_set_expiration
    }

def audit_tags(given_tags, required_tags):
    """Audit All Tags

    Assesses the tags in as much detail as possible to reach a conclusion
    about the current resource.
    """
    # validate the keys of tags first
    if validate_tag_keys([t['Key'] for t in given_tags], required_tags.keys()):
        # validate values of each tag
        for tag in given_tags:
            # if tag exist in required_tags, lets check values
            if required_tags.get(tag['Key']):
                return validate_tag_value(tag['Value'], required_tags.get(tag['Key']))
    else:
        return False

def validate_tag_value(given_value, accepted_values):
    """Audit Tag Values

    Will assess the value passed in and determine if it meets the
    requirements that are contained in the `required_tags` array at
    the top of the script

    given_value: string
        The value of the tag we are auditing

    accepted_values: list
        All of the accepted values that are accepted, wildcards included
    """
    for value in accepted_values:
        if fnmatch.fnmatch(given_value, value):
            return True

    return False

def validate_tag_keys(given_tag_keys, required_tag_keys):
    """Validate Set Tags

    Audits the keys provided by the resource and compares them to
    the required tags.

    given_tag_keys: list

    required_tag_keys: list
    """
    return required_tag_keys == list(set(given_tag_keys).intersection(required_tag_keys))

def node_first_offense(given_tags):
    """Node First Offense

    If expiration date hasn't been previously set, retrun True
    """
    return 'expiration' not in given_tags

def node_past_due(expiration_date):
    """Node Past Due

    If an expiration date is set on a node, is it past due.
    If it is past due return true

    Timestamp example: 2016-12-22 09:44:41
    """
    return datetime.now() > datetime.strptime(expiration_date, "%Y-%m-%d %H:%M:%S")


from airflow import DAG
from airflow.utils.dates import days_ago
from airflow.operators.python import PythonOperator

# import tasks
from m1_script import task1
from m2_script import task2
from postgres_script import task3
from dash_script import task4

default_args = {
    "owner": "airflow",
    "depends_on_past": False,
    'start_date': days_ago(2),
    "retries": 1,
}

dag = DAG(
    'accidents_etl_pipeline',
    default_args=default_args,
    description='accidents etl pipeline',
)
with DAG(
        dag_id='accidents_etl_pipeline',
        schedule_interval='@once',
        default_args=default_args,
        tags=['accidents-pipeline'],
) as dag:
    cleaning = PythonOperator(
        task_id='cleaning',
        python_callable=task1,
    )
    add_feature = PythonOperator(
        task_id='add_feature',
        python_callable=task2,
    )
    load_to_postgres_task = PythonOperator(
        task_id='load_to_postgres',
        python_callable=task3,
    )
    create_dashboard_task = PythonOperator(
        task_id='create_dashboard_task',
        python_callable=task4,

    )

    cleaning >> add_feature >> create_dashboard_task
    add_feature >> load_to_postgres_task


# https://stackoverflow.com/questions/34581255/python-flask-socketio-send-message-from-thread-not-always-working
# Set this variable to "threading", "eventlet" or "gevent" to test the
# different async modes, or leave it set to None for the application to choose
# the best option based on available packages.
async_mode = None

if async_mode is None:
    try:
        import eventlet
        async_mode = 'eventlet'
    except ImportError:
        pass

    if async_mode is None:
        try:
            from gevent import monkey
            async_mode = 'gevent'
        except ImportError:
            pass

    if async_mode is None:
        async_mode = 'threading'

    print('async_mode is ' + async_mode)

# monkey patching is necessary because this application uses a background
# thread
if async_mode == 'eventlet':
    import eventlet
    eventlet.monkey_patch()
elif async_mode == 'gevent':
    from gevent import monkey
    monkey.patch_all()

import json
from flask import Flask, render_template
from flask_socketio import SocketIO
from kafka import KafkaConsumer
#因为第一步骤安装好了flask，所以这里可以引用

app = Flask(import_name=__name__,
            static_url_path='/static', # 配置静态文件的访问 url 前缀
            static_folder='./vueproj/dist/static',    # 配置静态文件的文件夹
            template_folder='./vueproj/dist/') # 配置模板文件的文件夹

app.config['SECRET_KEY'] = 'secret!'
socketio = SocketIO(app, async_mode=async_mode)
thread = None
# 实例化一个consumer，接收topic为wordStats的消息
consumer = KafkaConsumer('wordStats')

# 一个后台线程，持续接收Kafka消息，并发送给客户端浏览器
def background_thread():
    print("-- now run back ground thread ----")
    for msg in consumer:
        wordStats = msg.value.decode('utf8')
        print("-------------- stat data ------------")
        print(wordStats)
        socketio.emit('wordStats_msg',{'data':wordStats})

# 客户端发送connect事件时的处理函数
@socketio.on('test_connect')
def connect(message):
    print(message)
    global thread
    if thread is None:
        print("now start back ground task")
        # 单独开启一个线程给客户端发送数据
        thread = socketio.start_background_task(target=background_thread)
    socketio.emit('connected', {'data': 'Connected'})

# 通过访问http://127.0.0.1:5000/访问index.html
@app.route("/")
def handle_mes():
    return render_template("index.html")

# main函数
if __name__ == '__main__':
    socketio.run(app,debug=True)


__version__ = '0.2'
import asyncio
import functools
import inspect
import json
import os
import re
import sys
import logging
import pickle
import time
import traceback
import subprocess

from mybuiltins import *
from cached_property import cached_property

import pymake
from .pat import *
from .req import *
from .exceptions import *
from .util import *
from .result import *

logger = logging.getLogger(__name__)

def dict_get(d, k, de):
    if not k in d:
        d[k] = de
    return d[k]

class Rule_utilities:

    def write_object(self, file_out, o):
        pymake.makedirs(os.path.dirname(file_out))

        with open(file_out, 'wb') as f:
            pickle.dump(o, f)

    def write_json(self, d):
        self.req.write_json(d)

class _Rule(Rule_utilities):
    """
    Base class for rules.
    You must derive from this class and overwrite the following member functions:

    - ``f_in``
    - ``build``
    """
    def __init__(self):
        """
        """
        self.__rules = None
        self.up_to_date = False
        self.req_out = None

    async def build_requirements(self, makecall, f):
        raise Exception(repr(self.__class__))
        yield

    def build(self, makecall, _, f_in):
        """
        This function must be defined by a derived class.

        :param MakeCall makecall: makecall object
        :param _: DO NOT USE
        :param f_in: a list of ...
        """
        raise NotImplementedError()

    def _gen_rules(self, makecall):
        return
        yield

    def _rules(self, makecall=None):
        if self.__rules is None:
            self.__rules = list()
            for r in self._gen_rules(makecall):
                yield r
                self.__rules.append(r)
        else:
            yield from self.__rules

    def print_dep(self, makecall, indent):
        try:
            for f in self.f_in(makecall):
                makecall.makefile.print_dep(f, indent)
        except Exception as e:
            logging.error("error in print_dep of {}".format(repr(self)))
            logging.error(e)

    def output_exists(self):
        return False

    def output_mtime(self):
        return None

    async def make_ancestors(self, makecall, test):

        makecall2 = makecall.copy(test=test, force=False)

        async def func(req):
            if req is None:
                raise Exception("None in f_in {}".format(self))

            r = await makecall2.make(req, ancestor=self.req_out)

            if not isinstance(r, Result):
                logging.error(req)
                logging.error(r)
                raise Exception()

            return req

        logger.debug(crayons.red(self))

        async for r in self.build_requirements(makecall2, func):
            #print(crayons.red(self))
            yield (await r)

    async def get_requirements(self, makecall):

        async def func(req): return req

        async for r in self.build_requirements(makecall, func):
            yield (await r)

    async def check(self, makecall, test=False):

        f_in = [r async for r in self.make_ancestors(makecall, test)]

        if makecall.args.get('force', False):
            return True, 'forced'

        if not bool(f_in):
            breakpoint()
            raise Exception()

        b = self.output_exists()
        if not b:
            return True, "output does not exist"

        mtime = self.output_mtime()

        for f in f_in:
            if isinstance(f, Rule):
                continue

            if not isinstance(f, Req):
                raise Exception(
                    f'{self!r} f_in should return generator of Req objects, not {f!r}')

            if mtime is None:
                return True, '{} does not define mtime'.format(self.__class__.__name__)
            else:
                b = f.output_exists()
                if b:
                    mtime_in = f.output_mtime()

                    if mtime_in is None:
                        return True, 'input file {} does not define mtime'.format(repr(self))

                    if mtime_in > mtime:
                        return True, 'mtime of {} is greater'.format(f)

        return False, 'up to date'

    def msg(self, s):
        logger.warning(crayons.green(f'{s}: {self!r}'))
        return
        try:
            logger.info(crayons.green(f'{s}: {self.req.get_id()}'))
        except Exception as e:
            try:
                logger.info(crayons.green(f'{s}: {self.req!r}'))
            except:
                logger.info(crayons.green(f'{s}: {self!s}'))

    def print_long(self, req):
        print(type(self))
        req.print_long()

    async def _make(self, makecall, req):
        logger.debug(f'makecall. test = {makecall.args.get("test", False)}')

        if self.up_to_date:
            raise Exception()
            return

        if req:
            if req.would_touch(makecall):
                should_build, f = self.check(makecall, test=True)
                if should_build:
                    req.touch_maybe(makecall)
                    return ResultBuild()
                else:
                    return ResultNoBuild()

        should_build, f = await self.check(makecall, test=makecall.args.get('test', False))
        #self.msg(f'should build: {should_build!s:5}. {f}')

        try:
            f_in = [r async for r in self.get_requirements(makecall)]
        except Exception as e:
            logging.error(repr(self))
            traceback.print_exc()
            raise e

        if should_build:
            if makecall.args.get('test', False):

                logging.error(crayons.blue(f'Build because {f}:'))
                print_lines(
                        lambda s: logging.error(crayons.blue(s)),
                        functools.partial(self.print_long, req),)

                return ResultTestBuild()
            else:
                #blue('build {} because {}'.format(repr(self), f))
                try:
                    self._makecall = makecall
                    ret = await self._build(makecall, None, f_in)
                except Exception as e:
                    logger.error(crayons.red('error building {}: {}'.format(repr(self), repr(e))))
                    raise

                if ret is None:
                    return ResultBuild()
                elif ret != 0:
                    raise BuildError(str(self) + ' return code ' + str(ret))
        else:
            if makecall.args.get('test', False):
                #print('DONT build',repr(self))
                return ResultTestNoBuild()
            else:
                return ResultNoBuild()

        self.up_to_date = True

    def write_text(self, filename, s):
        # it appears that this functionality is actually not useful as currently
        # implemented. revisit later

        #if check_existing_binary_data(filename, s.encode()):
        if True:
            pymake.os0.makedirs(os.path.dirname(filename))
            with open(filename, 'w') as f:
                f.write(s)
        else:
            logging.info('binary data unchanged. do not write.')

    def write_pickle(self, o):
        b = pickle.dumps(o)
        self.write_binary(b)

    def write_binary(self, b):
        # it appears that this functionality is actually not useful as currently
        # implemented. revisit later

        #if check_existing_binary_data(filename, b):
        if True:
            pymake.makedirs(os.path.dirname(self.req.fn))
            with open(self.req.fn, 'wb') as f:
                f.write(b)
        else:
            logging.info('binary data unchanged. do not write.')

    def rules(self, makecall):
        yield self
        for r in self._rules(makecall):
            yield from r.rules(makecall)

    async def _build(self, makecall, *args):
        print_lines(
                lambda s: logger.info(crayons.yellow(s, bold=True)),
                lambda: print(f'Build force={makecall.args.get("force", False)}'),
                functools.partial(self.print_long, self.req))

        await self.build(makecall, *args)

class Rule(_Rule):
    """
    a simple file-based rule

    :param str f_out: the filename that this rule builds
    """
    def __init__(self, req):
        super(Rule, self).__init__()
        self.req = req

    def __repr__(self):
        return "<{}.{}>".format(
                self.__class__.__module__,
                self.__class__.__name__)

    def output_exists(self):
        return self.req.output_exists()

    def output_mtime(self):
        return self.req.output_mtime()

    async def test(self, mc, req):
        """
        :param req: a requirement object

        determine if this rule builds ``req``
        """
        if not isinstance(req, ReqFile): return None
        if self.req.fn == req.fn:
            return self
        return None

"""
a rule to which we can pass a static list of files for f_out and f_in
"""
class RuleStatic(_Rule):
    def __init__(self, static_f_out, static_f_in, func):

        self.static_f_in = static_f_in
        self.static_f_out = static_f_out

        self.build = func

    def f_out(self):
        return self.static_f_out

    def f_in(self, mc):
        yield self.static_f_in

class RuleRegex(_Rule):
    """
    A rule whose output filename is defined by a regex.
    You must derive from this class and define the follwing class attributes:

    - ``pat_out``

    :param str target: target
    :param list groups: regular expression groups
    """

    @classmethod
    async def test(cls, mc, req):
        if not isinstance(req, ReqFile): return None

        logger.debug('{} {}'.format(cls.pat_out, req.fn))

        if callable(cls.pat_out):
            pat = cls.pat_out()
        else:
            pat = cls.pat_out

        if pat is None: return

        try:
            m = pat.match(req.fn)
        except Exception as e:
            logging.error(crayons.red(e))
            logging.error(crayons.red(repr(req)))
            logging.error(crayons.red(repr(req.fn)))
            logging.error(crayons.red(repr(pat)))
            raise

        if m is None: return None
        return cls(req, m.groups())

    def output_exists(self):
        return os.path.exists(self.f_out)

    def output_mtime(self):
        return os.path.getmtime(self.f_out)

    def __init__(self, req, groups):
        self.req = req
        self.f_out = req.fn
        self.groups = groups

        _Rule.__init__(self)

    def __repr__(self):
        return '{}{}'.format(self.__class__.__name__, (self.f_out, self.groups))

class RuleRegexSimple(RuleRegex):
    async def build(self, makecall, _, f_in):

        pymake.makedirs(os.path.dirname(self.f_out))

        if not f_in:
            raise Exception(f'{self.__class__} f_in is empty')

        if f_in[0].fn[-3:] == ".py":
            cmd = [sys.executable, f_in[0].fn, self.f_out] + [a.fn for a in f_in[1:]]
        else:
            prog = f_in[0].fn
            out = os.path.abspath(self.f_out)
            cmd = [prog, out] + [os.path.abspath(a.fn) for a in f_in[1:]]

        logging.info(crayons.yellow("cmd = {}".format(' '.join(cmd)),'yellow',bold=True))

        subprocess.run(cmd, check=True)

class RuleRegexSimple2(RuleRegex):
    async def build(self, makecall, _, f_in):
        for f in f_in:
            logger.info(crayons.yellow(f'  {f!r}'))

        pymake.makedirs(os.path.dirname(self.f_out))

        assert f_in[0].fn[-3:] == ".py"

        av = [f_in[0].fn, self.f_out] + [a for a in f_in[1:]]

        s = f_in[0].fn[:-3].replace('/','.')
        m = __import__(s, fromlist=['main'])

        c = await m.main(makecall, av)

class RuleDoc(Rule):
    """
    a rule that defines a file descriptor pattern
    a file descriptor pattern is a dict in which the attributes are regular values or patterns that can be used to match regular values
    """

    @classmethod
    async def test(cls, mc, req):
        if not isinstance(req, ReqDoc): return None

        # a - this descriptor
        # b - req descriptor

        pat = cls.descriptor_pattern()

        a = pat
        b = dict(req.d)

        #breakpoint()

        if 'type' in pat:
            logger.debug(crayons.blue(f'pat type={pat["type"]}'))

        logger.debug(f'pat={pat}')
        logger.debug(f'dsc={req.d}')

        set_a = set(a.keys())
        set_b = set(b.keys())

        a_and_b = set_a & set_b

        just_pat = set_a - set_b
        just_dsc = set_b - set_a

        b0 = False #"type" in a_and_b and a["type"] == b["type"]
        with context_if(functools.partial(logger_level_context, logger, logging.DEBUG), b0):

            for k in a_and_b:
                if isinstance(a[k], Pat):
                    if not a[k].match(b[k]):
                        logger.debug(f'{cls} {k!r} does not match pattern {a[k]!r} {b[k]!r}')
                        return None

                    if isinstance(a[k], PatNullable) and b[k] is None:
                        b[k] = a[k].default

                else:
                    if not (a[k] == b[k]):
                        #logger.debug(f'{k!r} differs')
                        return None

            # attributes in the pattern but not in the descriptor must be nullable
            for k in just_pat:
                if not isinstance(pat[k], PatNullable):
                    logger.debug(f'{cls} {k!r} is in pattern but not descriptor and is not nullable')
                    return None
                else:
                    b[k] = pat[k].default

            # attributes in the descriptor but not in the pattern must be null
            for k in just_dsc:
                if b[k] is not None:
                    logger.debug(f'{cls} {k!r} is in descriptor but not in pattern and is not None')
                    return None

            #logger.debug(crayons.green(f'match {a} and {b}'))

        return cls(req, await mc.decoder.decode(b, mc.copy(force=False)))

    def __init__(self, req, descriptor):
        super(RuleDoc, self).__init__(req)
        self.descriptor = descriptor
        self.d = descriptor

    def graph_string(self):
        return json.dumps(self.descriptor, indent=4)

    def __repr__(self):
        return f'<{self.__class__} desc={self.descriptor!r} req={self.req!r}>'

class RuleDocSimple(RuleDoc):

    async def build(self, makecall, _, f_in):
        for f in f_in:
            logger.info(crayons.yellow(f'  {f!r}'))

        filename = self.req.fn

        pymake.makedirs(os.path.dirname(filename))

        assert f_in[0].fn[-3:] == ".py"

        av = [f_in[0].fn, filename] + [a.fn for a in f_in[1:]]

        s = f_in[0].fn[:-3].replace('/','.')
        m = __import__(s, fromlist=['main'])

        await m.main(makecall, av)

def _copy(src, dst):
    with src.open('r')as f0:
        s = f0.read()

    with dst.open('w') as f1:
        f1.write(s)

def _copy_binary(src, dst):
    with src.open('rb')as f0:
        s = f0.read()

    with dst.open('wb') as f1:
        f1.write(s)

class RuleDocCopy(RuleDoc):
    async def build(self, makecall, _, requirements):
        _copy(requirements[0], self.req)

class RuleDocCopyBinary(RuleDoc):
    async def build(self, makecall, _, requirements):
        _copy_binary(requirements[0], self.req)


#mcb.py - Saves and loads pieces of text to the clipboard.
#Usage: mcb.py save <keyword> - Saves clipboard to keyword.
#       mcb.py <keyword> - Loads keyword to clipboard.
#       mcb.py list - Loads all keywods to clipboard.

import shelve, pyperclip, sys

mcbShelf = shelve.open('mcb')

#Save clipboard content.
if len(sys.argv) == 3 and sys.argv[1].lower() == 'save':
	mcbShelf[sys.argv[2]] = pyperclip.paste()
elif len(sys.argv) == 2:
	#List keywords and load content.
	if sys.argv[1].lower() == 'list':
		pyperclip.copy(str(list(mcbShelf.keys())))
	elif sys.argv[1] in mcbShelf:
		pyperclip.copy(mcbShelf[sys.argv[1]])

mcbShelf.close()