import tensorflow as tf
import keras
import os
import pandas as pd
import numpy as np
# import matplotlib.pyplot as plt

from keras.models import Model, Input, load_model
from keras.layers import LSTM, Embedding, Dense, TimeDistributed, Dropout, Conv1D, Lambda, GRU
from keras.layers import Bidirectional, concatenate, SpatialDropout1D, GlobalMaxPooling1D
from keras_contrib.layers import CRF

from keras.preprocessing.sequence import pad_sequences
from sklearn_crfsuite.metrics import flat_classification_report
from sklearn.model_selection import train_test_split
from keras.utils import to_categorical

import tensorflow as tf
import tensorflow_hub as hub
from keras import backend as K

from keras.callbacks import ModelCheckpoint,TensorBoard,EarlyStopping
from keras import optimizers

import numpy as np
from scipy.spatial.distance import pdist, squareform
from collections import Counter
from gensim.models import FastText
import pickle

from sklearn.utils import shuffle
import re
import random
from itertools import product
import pickle as pkl
from funcy import flatten
from nltk import everygrams, ngrams
import json
from pprint import pprint

from pyemd import emd
from gensim.similarities import WmdSimilarity

# Hyperparams if GPU is available
if tf.test.is_gpu_available():
    BATCH_SIZE = 32  # Number of examples used in each iteration
    EPOCHS = 30  # Number of passes through entire dataset
    MAX_LEN = 30  # Max length of sentence (in words)
    EMBEDDING = 40  # Dimension of word/character embedding vector
    max_len_char = 10

# Hyperparams for CPU training
else:
    BATCH_SIZE = 16
    EPOCHS = 5
    MAX_LEN = 30
    EMBEDDING = 20
    max_len_char = 10

print("BATCH_SIZE is {} ".format(BATCH_SIZE))

class SentenceGetter(object):
    """Class to Get the sentence in this format:
    [(Token_1, Part_of_Speech_1, Tag_1), ..., (Token_n, Part_of_Speech_1, Upper or lower, Tag_1)]"""
    def __init__(self, data):
        """Args:
            data is the pandas.DataFrame which contains the above dataset"""
        self.n_sent = 1
        self.data = data
        self.empty = False
        agg_func = lambda s: [(w, p, t) for w, p, t in zip(s["Word"].values.tolist(),
                                                           s["POS"].values.tolist(),
                                                           s["Tag"].values.tolist())]

        self.grouped = self.data.groupby("Sentence_ID_Norm").apply(agg_func)
        self.sentences = [s for s in self.grouped]

def Padwords(X,max_len = MAX_LEN):
    new_X = []
    for seq in X:
        new_seq = []
        for i in range(max_len):
            try:
                new_seq.append(seq[i])
            except:
                new_seq.append(word2idx["PAD"])
        new_X.append(new_seq)

    return new_X

def CharGetter(sentences, mlength = max_len_char):
    X_char = []
    for sentence in sentences:
        sent_seq = []
        for i in range(MAX_LEN):
            word_seq = []
            for j in range(mlength):
                try:
                    word_seq.append(char2idx.get(sentence[i][0][j]))
                except:
                    word_seq.append(char2idx.get("PAD"))
            sent_seq.append(word_seq)
        X_char.append(np.array(sent_seq))

    return X_char

def Padtags(y,max_len = MAX_LEN):
    new_y = []
    for seq in y:
        new_seq = []
        for i in range(max_len):
            try:
                new_seq.append(seq[i])
            except:
                new_seq.append(tag2idx["PAD"])
        new_y.append(new_seq)

    return new_y

def softmax(X, theta = 1.0, axis = None):

    # find axis
    if axis is None:
        axis = next(j[0] for j in enumerate(X.shape) if j[1] > 1)

    # multiply y against the theta parameter,
    X = X * float(theta)
    X = np.exp(X)

    # take the sum along the specified axis
    ax_sum = np.expand_dims(np.sum(X, axis = axis), axis)
    # finally: divide elementwise
    p = X / ax_sum
    if len(X.shape) == 1: p = p.flatten()

    return p

def euclideanScore(M):
    #Euclidean distance
    euc_M = squareform(pdist(M, 'euclidean'))
    alpha_euc = softmax(euc_M,  axis = 0)
    return np.sum(alpha_euc,axis=1)

def manhattenScore(M):
    #Manhattan Distance
    man_M = squareform(pdist(M, 'cityblock'))
    alpha_man = softmax(man_M,  axis = 0)

    return np.sum(alpha_man,axis=1)

def CosineScore(M):
    cos_M = squareform(pdist(M, 'cosine'))
    alpha_cos = softmax(cos_M,  axis = 0)

    return np.sum(alpha_cos,axis=1)

# def get_class_weights(y):
#     counter = Counter(y)
#     majority = max(counter.values())
#     return  {cls: float(majority/count) for cls, count in counter.items()}

def build_vocab(all_tokens, tags):
    # Returns:
    # id2token: list of tokens, where id2token[i] returns token that corresponds to token i
    # token2id: dictionary where keys represent tokens and corresponding values represent indices
    token_counter = Counter(all_tokens)
    vocab, count = zip(*token_counter.most_common(max_vocab_size))
    vocab= list(set((list(vocab) + tags)))
    if 'UNK' in vocab:
        vocab.remove('UNK')
    if 'PAD' in vocab:
        vocab.remove('PAD')
    id2token = vocab
    token2id = dict(zip(vocab, range(2,2+len(vocab)))) #good way to creat new dic
    id2token = ["PAD", "UNK"] + id2token
    token2id["PAD"] = 0
    token2id["UNK"] = 1
    return token2id, id2token

def subdata_getter(pub_ids,data,batch_size = BATCH_SIZE, alpha_t = True):
    subdata = {}
    subdata['words'],subdata['chars'],subdata['tags'], subdata['alpha'] = {},{},{},{}
    for pub_id in pub_ids:
        eachpub_sentence = {}
        #sentences
        df = data.loc[data["Pub_id"]==pub_id]

        each_sentences = SentenceGetter(df).sentences
        padsent_num = int(np.ceil(len(each_sentences) /(batch_size))*batch_size - len(each_sentences))
        for i in range(padsent_num):
            each_sentences.append([('PAD', 'PAD', 'PAD')])
        #words
        each_words = [[word2idx[w[0]] for w in s] for s in each_sentences]

        subdata['words'][pub_id] = Padwords(each_words)
        #characters
        subdata['chars'][pub_id] = CharGetter(each_sentences)

        #tags
        y = [[tag2idx[w[2]] for w in s] for s in each_sentences]
        y = Padtags(y)
        #one hot encoder
        subdata['tags'][pub_id] = [to_categorical(i, num_classes=n_tags+1) for i in y]

        if alpha_t:
            #get alpha
            word_embedding_pub = [[embedding_matrix[j] for j in s] for s in subdata['words'][pub_id]]
            word_embedding_pub = np.array(word_embedding_pub).reshape(-1,100)

            #use ConsineScore
            alpha = CosineScore(word_embedding_pub)
    #         alpha = alpha_func.CosineScore
            # alpha = alpha.reshape(-1,MAX_LEN,1024)
            subdata['alpha'][pub_id] = alpha.tolist()

    return subdata

class DataGenerator(keras.utils.Sequence):
    'Generates data for Keras'
    def __init__(self, pub_IDs, data_forgene, batch_size=BATCH_SIZE,
                 shuffle_pub=True, shuffle_sent=True, Type = 'train', alpha = True):
        'Initialization'

        self.batch_size = batch_size
        self.pub_IDs = pub_IDs
        self.data_forgene = data_forgene
        self.shuffle_pub = shuffle_pub
        self.shuffle_sent = shuffle_sent
        self.type = Type
        self.alpha = alpha
        self.on_epoch_end()

    def __len__(self):
        'Denotes the number of batches per epoch'
        length = 0
        for i in self.pub_IDs:
            length = length + int(len(self.data_forgene['words'][i]) / self.batch_size)
        return length

    def __getitem__(self, index):
        'Generate one batch of data'
        # generator from self.shuffled_data from second epoch

#         if self.shuffled_data:
        X = self.shuffled_data['words'][index*self.batch_size:(index+1)*self.batch_size]
        char = self.shuffled_data['chars'][index*self.batch_size:(index+1)*self.batch_size]
        y = self.shuffled_data['tags'][index*self.batch_size:(index+1)*self.batch_size]
        y = np.array(y).reshape(len(y), MAX_LEN, 5)

        if self.alpha:
            alpha = self.shuffled_data['alpha'][index*self.batch_size:(index+1)*self.batch_size]
            input_data = [np.array(X), np.array(char), np.array(alpha)]

        else:
            input_data = [np.array(X), np.array(char)]

        if self.type == 'train':
            return input_data, y
        else:
            return input_data

    def on_epoch_end(self):
        'Updates indexes after each epoch'
        self.indexes = np.arange(len(self.pub_IDs))
        if self.shuffle_pub == True:
            np.random.shuffle(self.indexes)
            pub_IDs_temp = [self.pub_IDs[k] for k in self.indexes]
            #shuffle pubs
            shuffled_data = {}
            shuffled_data['words'],shuffled_data['chars'],shuffled_data['tags'], shuffled_data['alpha']= [], [], [],[]
            #sentences of each pubs can be moded by batch_size
            for i in pub_IDs_temp:
                batch_length = len(self.data_forgene['tags'][i])
                if self.shuffle_sent == True:
                    indexes = np.arange(batch_length)
                    np.random.shuffle(indexes)
                    shuffled_data['words'] = shuffled_data['words'] + np.take(self.data_forgene['words'][i], indexes, axis=0).tolist()
                    shuffled_data['chars']= shuffled_data['chars'] + np.take(self.data_forgene['chars'][i], indexes, axis=0).tolist()
                    shuffled_data['tags'] = shuffled_data['tags'] + np.take(self.data_forgene['tags'][i], indexes, axis=0).tolist()
                    if self.alpha:
                        shuffled_data['alpha'] = shuffled_data['alpha'] + np.take(self.data_forgene['alpha'][i], indexes, axis=0).tolist()
                else:
                    shuffled_data['words'] = shuffled_data['words']+self.data_forgene['words'][i]
                    shuffled_data['chars']= shuffled_data['chars']+self.data_forgene['chars'][i]
                    shuffled_data['tags'] = shuffled_data['tags']+self.data_forgene['tags'][i]
                    if self.alpha:
                        shuffled_data['alpha'] = shuffled_data['alpha']+self.data_forgene['alpha'][i]

        else:
            shuffled_data = self.data_forgene

        self.shuffled_data = shuffled_data

class NBatchLogger(keras.callbacks.Callback):
    """
    A Logger that log average performance per `display` steps and saved loss & acc by batches
    """
    def __init__(self, display=500):
        self.step = 0
        self.display = display
        self.val = val_generator
        self.metric_cache = {}
        self.train_log = {}
        self.val_log = {}

    def on_train_begin(self, logs={}):
        self.val_log['loss'],self.val_log['acc'] = [], []
        self.train_log['loss'],self.train_log['acc'] = [], []

    def on_batch_end(self, batch, logs={}):
        self.step += 1
        for k in self.params['metrics']:
            if k in logs:
                self.metric_cache[k] = self.metric_cache.get(k, 0) + logs[k]
        if self.step % self.display == 0:
            metrics_log = ''
            val_loss, val_acc = self.model.evaluate_generator(generator = val_generator)
            i = 0
            for (k, v) in self.metric_cache.items():
                val = v / self.display
                if abs(val) > 1e-3:
                    metrics_log += ' - %s: %.4f' % (k, val)
                else:
                    metrics_log += ' - %s: %.4e' % (k, val)
                if i == 0:
                    self.train_log['loss'].append(val)
                else:
                    self.train_log['acc'].append(val)
                i+=1

            self.val_log['loss'].append(val_loss)
            self.val_log['acc'].append(val_acc)

            print('step: {}/{} ... {}'.format(self.step,
                                          self.params['steps'],
                                          metrics_log))

            self.metric_cache.clear()

#Load Lookup table used in the training
####################################################################################################################################
data =  pd.read_csv('df_concat.csv', encoding="latin1").fillna(method="ffill")
pub_ids = list(set(data["Pub_id"].values))

data_tags = data.loc[(data['Tag'] == 'B'),"Word"].values.tolist() + \
           data.loc[(data['Tag'] == 'I'),"Word"].values.tolist() + \
              data.loc[(data['Tag'] == 'M'),"Word"].values.tolist()

all_words = data["Word"].values.tolist()
max_vocab_size = 50000

#words
n_words = max_vocab_size + 2
# Vocabulary Key:word -> Value:token_index
# The first 2 entries are reserved for PAD and UNK
word2idx, idx2word = build_vocab(all_words, data_tags)
words = idx2word
print("Number of words in the dataset: ", n_words)

#change all other words in dataframe as "UNK"
data.loc[~data['Word'].isin(words), 'Word'] = 'UNK'

#tags
tags = list(set(data["Tag"].values))
print("Tags:", tags)
n_tags = len(tags)
print("Number of Labels: ", n_tags)

#characters
chars = set([w_i for w in words for w_i in w])
n_chars = len(chars)
print("Number of Labels: ", n_chars)

tag2idx = {t: i+1 for i, t in enumerate(tags)}
tag2idx["PAD"] = 0
# Vocabulary Key:tag_index -> Value:Label/Tag
idx2tag = {i: w for w, i in tag2idx.items()}

# Char Key:char -> Value:token_index
char2idx = {c: i + 2 for i, c in enumerate(chars)}
char2idx["UNK"] = 1
char2idx["PAD"] = 0

#Load model and data
####################################################################################################################################
model = load_model("ner_ch_o.h5")

#Load test data
data_te =  pd.read_csv('df_concat_test.csv', encoding="latin1").fillna(method="ffill")
data_te.loc[~data_te['Word'].isin(words),'Word'] = "UNK"
te_pub = list(set(data_te["Pub_id"].values))
test = subdata_getter(te_pub,data_te)
te_generator =  DataGenerator(te_pub, test, Type = 'test')

#Eval of NER
####################################################################################################################################
#get true tags
y_temp = test['tags']
y_te = []
# y_index = {}
# word_temp = test['words']
# word_te = []
for pub_id in te_pub:
    y_te = y_te + y_temp[pub_id]

pred_cat = model.predict_generator(generator=te_generator, verbose=0)
pred = np.argmax(pred_cat, axis=-1)
y_te_true = np.argmax(y_te, -1)

# Convert the index to tag
pred_tag = [[idx2tag[i] for i in row] for row in pred]
y_te_true_tag = [[idx2tag[i] for i in row] for row in y_te_true]

report = flat_classification_report(y_pred=pred_tag, y_true=y_te_true_tag)
pickle.dump(prediction, open('report_ner', 'wb'))
print(report)

#Eval of Dataset ID
####################################################################################################################################
import string
#stopwords list
import nltk
from nltk.stem import WordNetLemmatizer
from nltk import pos_tag, word_tokenize
nltk.download("stopwords")
nltk.download('punkt')
nltk.download('averaged_perceptron_tagger')

words_fast =FastText.load('model_fast30/model_fast.model')

def tokenize(sent):
#     if (token.text not in punctuations)
    tokens_lower = [token.lower() for token in sent if (token not in punctuations)
                    and (token not in stop_words)]
    tokens_lemma = [wnl.lemmatize(word) for word in tokens_lower]
    return tokens_lemma

prediction = {}
Y = {}
Words = {}

#load whole dataset ID and mentionlist
mentionlist = {}
with open('./train_test/data_sets.json') as f:
    dataset = json.load(f)
for i in range(len(dataset)):
    mentionlist[dataset[i]["data_set_id"]]=dataset[i]["mention_list"]

mentionlist100 = {}
with open('./train_test/data_set_citations_100.json') as f:
    dataset100 = json.load(f)
for i in range(len(dataset100)):
    mentionlist100[dataset100[i]["publication_id"]]=dataset100[i]["data_set_id"]

ID_true = []
predicted_ID = []

for pub_id in te_pub:
    each_test = subdata_getter([pub_id],data_te)
    each_te_generator =  DataGenerator([pub_id], each_test, Type = 'test')
    pred_cat = model.predict_generator(generator=each_te_generator, verbose=0)
    pred = np.argmax(pred_cat, axis=-1)

    #get words
    word_temp = each_test['words']
    word_flatten = np.array(word_temp[pub_id]).flatten()
    y_flatten = pred.flatten()

    pred_ind_B = list(np.where(y_flatten == tag2idx['B']) [0])
    pred_ind_I = list(np.where(y_flatten == tag2idx['I']) [0])
    pred_ind = pred_ind_B + pred_ind_I

#   pred_ind = np.sort(np.array(pred_ind))
    prediction[pub_id] = {}
    if len(pred_ind) == 0:
        prediction[pub_id]['predict_word']='Can not find mention list'
    else:
        prediction[pub_id]['predict_word'] = [idx2word[i] for i in np.take(word_flatten,pred_ind)]

    Y[pub_id] = pred
    Words[pub_id] = word_temp[pub_id]
    #compute similarity between predicted mentionlist with the whole data set and find the maximum, assign
    #this dataset id as prediction
    max_score = 0
    for i in mentionlist:
        reference = mentionlist[i]
        if len(reference)>0:
            #remove stop words
            cleaned_pred = tokenize(prediction[pub_id]['predict_word'])
            cleaned_ref = tokenize(reference)
            for i in cleaned_ref:
                similarity = words_fast.wv.wmdistance(cleaned_pred, i)
                if similarity>max_score:
                    max_score = similarity
                    prediction[pub_id]['predict_id'] = i
                    predicted_ID.append(i)
    ID_true.append(mentionlist100[pub_id])

report_ID = flat_classification_report(y_pred=predicted_ID, y_true=ID_true)
print(report_ID)

pickle.dump(prediction, open('prediction', 'wb'))
pickle.dump(Y, open('Y_prediction', 'wb'))
pickle.dump(Words, open('Words_prediction', 'wb'))

#     y_te = y_te + y_temp[pub_id]
#     y_index[pub_id] = np.argmax(y_temp[pub_id], -1)
    #get true words


import matplotlib
matplotlib.use('Agg')
from numpy import loadtxt,diff#,savetxt,shape,argsort
from pylab import *
#from glob import glob
import os
import sys
from scipy import integrate
sys.path.append('/home/fbrasili/.python')
from init_file import *

def compute_pot(g):
    #g = where( g==0 , 1.e-18 , g )
    return -log(g)

def compute_Fr(r,g):
    #g = where( g==0 , 1.e-18 , g )
    return gradient(g)/gradient(r)/g

def compute_threshold_gr_pot(fol=None):
    if fol is None:
        os.system( "pwd >> path.dat" )
        fol = str( loadtxt( "path.dat" , dtype=str ) )
        os.system( "rm path.dat" )
    gr_path = os.path.join( fol , 'gr_ions_npart_nopbc.dat' )
    r,g = loadtxt( gr_path, skiprows=1 , unpack=True )
    cond = g != 0
    r,g = r[cond],g[cond]
    U = -log(g[1:])
    F = diff(g)/diff(r)/g[1:]
    r,g = r[1:],g[1:]
    savetxt( os.path.join( fol , 'Fr.dat' ) , c_[r,F] )
    cond = r<20
    r,F,U = r[cond],F[cond],U[cond]
    cond = ( r>max(r[F==F.min()]) ) #* ( U+1>0 )
    return r[cond].min()

def compute_threshold_gr_fact(frac=0.2 , fol=None):
    if fol is None:
        os.system( "pwd >> path.dat" )
        fol = str( loadtxt( "path.dat" , dtype=str ) )
        os.system( "rm path.dat" )
    gr_path = os.path.join( fol , 'gr_ions_npart_nopbc.dat' )

    init_fname = os.path.join( fol , "mgel_nanoparts_eq10e6.dat" )
    _,_,box,_,_ = read_init(init_fname)
    V = box[0]**3
    _,N = check_init(init_fname, retPartType=3, ret=True, verb=False)

    r,g = loadtxt( gr_path, skiprows=1 , unpack=True )
    cond = ( r > max(r[ g == g.max() ]) ) * ( g <= g.max() * frac )
    return r[cond].min()

def compute_threshold_gr_Coul(fol=None):
    gr_paths = {
            2 : 'gr_neg-cions_npart.dat',
            3 : 'gr_ions_npart_nopbc.dat',
            4 : 'gr_npart_npart_nopbc.dat',
            5 : 'gr_pos-cions_npart.dat'
            }
    if fol is None:
        os.system( "pwd >> path.dat" )
        fol = str( loadtxt( "path.dat" , dtype=str ) )
        os.system( "rm path.dat" )
    init_fname = os.path.join( fol , "mgel_nanoparts_eq10e6.dat" )
    gr_paths = { i: os.path.join( fol , p ) for i,p in gr_paths.items() }

    _,_,box,_,_ = read_init(init_fname)
    V = box[0]**3
    Zs,Ns = check_init(init_fname, retPartType="all", ret=True, verb=False)
    Zr = 0
    for key in gr_paths.keys():
        Z,N = Zs[key],Ns[key]
        if key==4: N=N-1
        r,g = loadtxt( gr_paths[key] , skiprows=1 , unpack=True )
        Zr += 4*pi * Z*N/V * integrate.cumtrapz( g * r**2 , r , initial=0 )
    Ir = integrate.cumtrapz( Zr/r**2 , r , initial=0 )
    Ur = Zs[4]/r - Ir + Ir[-1]

    figure()
    plot(r,Ur)
    savefig("fol"); colse()
    return r[Ur+1<0].max()

def myPlot(observable , system , fol=None , norm=False , ret=False):
    filename = {
            'profile': {
                'nanoparticles': 'nanoparts_profile.dat',
                'microgel': 'monomers_profile.dat',
                'charge': 'charge_profile.dat',
                'positive counterions': 'pos_cions_profile.dat',
                'negative counterions': 'neg_cions_profile.dat'
                },
            'g(R)': {
                'nanoparticle - nanoparticle': 'gr_npart_npart_nopbc.dat', #nr_npart_npart_pbc.dat
                'charged monomer - charged monomer': 'gr_ions_ions_nopbc.dat', #nr_ions_ions_pbc.dat
                'nanoparticle - charged monomer': 'gr_ions_npart_nopbc.dat', #nr_ions_npart_pbc.dat
                'nanoparticle - positive counterion': 'gr_pos-cions_npart.dat',
                'nanoparticle - negative counterion': 'gr_neg-cions_npart.dat',
                'charged monomer - negative counterion': 'gr_ions_neg-cions.dat',
                'charged monomer - positive counterion': 'gr_ions_pos-cions.dat',
                'positive counterion - negative counterion': 'gr_neg-cions_pos-cions.dat'
                },
            'gyration radius': {
                'chull': os.path.join( "chull_rh" , "rh.dat" ),
                'zeno': 'swelling'
                },
            'hydrodynamic radius': {
                'chull': os.path.join( "chull_rh" , "rh.dat" ),
                'zeno': 'swelling'
                }
            }
    args = {
            'profile': [4,(1,3)],
            'g(R)': [1,(0,1)],
            'gyration radius': [0,(0,1)],
            'hydrodynamic radius': [0,(0,2)]
            }

    if   observable == 'prof'    : observable = 'profile'
    elif observable == 'gr'      : observable = 'g(R)'
    elif observable == 'Rg'      : observable = 'gyration radius'
    elif observable == 'Rh'      : observable = 'hydrodynamic radius'

    if observable == 'profile':
        if system in [ 'nanoparticle' , 'nanoparts' , 'nanopart' , 'nps' , 'np' ]: system,retPartType = 'nanoparticles',4
        elif system in [ 'mgel' , 'monomer' , 'monomers' ]: system,retPartType = 'microgel',None
        elif system == 'pos_cions': system,retPartType = 'positive counterions',5
        elif system == 'neg_cions': system,retPartType = 'negative counterions',2
    elif observable == 'g(R)':
        if system in [ 'nanoparticle' , 'nanoparts' , 'nanopart' , 'nps' , 'np' ]:
            system = 'nanoparticle - nanoparticle'
            retPartType = 4
        elif system in [ 'ion' , 'ions' , 'monomer' , 'monomers' ]:
            system = 'charged monomer - charged monomer'
            retPartType = 3
        elif system == 'np-ion':
            system = 'nanoparticle - charged monomer'
            retPartType = 3
        elif system == 'np-pos_cion':
            system = 'nanoparticle - positive counterion'
            retPartType = 5
        elif system == 'np-neg_cion':
            system = 'nanoparticle - negative counterion'
            retPartType = 2
        elif system == 'ion-neg_cion':
            system = 'charged monomer - negative counterion'
            retPartType = 2
        elif system == 'ion-pos_cion':
            system = 'charged monomer - positive counterion'
            retPartType = 5
        elif system == 'pos_cion-neg_cion':
            system = 'positive counterion - negative counterion'
            retPartType = 2
    if system == 'swelling': system = 'chull'

    skiprows,usecols = args[observable]
    if observable == 'profile' and system == 'charge': skiprows = 1

    if fol is None:
        os.system( "pwd >> path.dat" )
        fol = str( loadtxt( "path.dat" , dtype=str ) )
        os.system( "rm path.dat" )
    if 'alpha' not in fol:
        if system in ['chull', 'zeno']: fol = os.path.join( fol , 'swelling' )
        else                          : fol = os.path.join( fol , 'alpha_0.00' )

    path = os.path.join( fol , filename[observable][system] )
    x,y = loadtxt( path , skiprows=skiprows , usecols=usecols , unpack=True )

    if system in ['chull', 'zeno']: label = observable
    else                          : label = system

    if type(norm) is float:
        y /= norm
    elif norm:
        init_fname = os.path.join( fol , "mgel_nanoparts_eq10e6.dat")
        if not os.path.isfile( init_fname ): init_fname = os.path.join( fol , "mgel_cions_eq10e6.dat")
        if system == 'microgel':
            _,Nions = check_init(init_fname, retPartType=3, ret=True, verb=False)
            _,Nneut = check_init(init_fname, retPartType=1, ret=True, verb=False)
            Nparts = Nions + Nneut
        else:
            _,Nparts = check_init(init_fname, retPartType=retPartType, ret=True, verb=False)
        #_,_,box,_,_ = read_init(init_fname)
        y /= Nparts

    if ret:
        return x, y, label
    else:
        plot(x, y, label=label)

'''
non ha senso scrivere una funzione per i due/tre sublplot
def plot_swellingCurve(fol='./', withZeno=False, ret=False):
    Xrg,Yrg,label = myPlot('Rg', 'swelilng' , fol=fol , ret=True)
    plot(x,y)
'''


from flask import Flask, jsonify, request
import sys
import os
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))  # append package folder to PYTHONPATH

import auth_service
import json

app = Flask(__name__)

def authenticate(**kwargs):
    """Validate user login

    Args:
    - **kwargs: holds login credentials/payload from API request

    Returns:
    - `list[str]`: return list of scopes
    """

    print("[DEBUG]", json.dumps(kwargs, indent=2))
    kwargs["username"]
    kwargs["password"]
    if "scope_demo" in kwargs:
        return kwargs["scope_demo"].split()
    return []

# initialize auth rules
auth_service.setup(app, authenticate, {
    "api": [
        r"^/api/.*$"
    ],
    "dev": [
        r"^/devpage$"
    ],
    "admin": [
        r"^/admin$"
    ],
})

@app.route("/test")
def hello():
    return jsonify({
        "message": "anyone can access this"
    })

@app.route("/devpage")
def developer():
    return jsonify({
        "message": f"accessible for 'dev' scope, your scopes: {auth_service.get_token_payload()['scope']}"
    })

@app.route("/api/test")
def api_test():
    return jsonify({
        "message": f"accessible for 'api' scope, your scopes: {auth_service.get_token_payload()['scope']}"
    })

@app.route("/admin")
def admin():
    return jsonify({
        "message": f"accessible for 'admin' scope, your scopes: {auth_service.get_token_payload()['scope']}"
    })

if __name__ == "__main__":
    app.run(
        host="0.0.0.0",
        port=8080,
        debug=True
    )


import requests
from bs4 import BeautifulSoup as bsp
import webbrowser
def printAllDeatils(urls , dataa):
    url=urls
    r=requests.get(url)
    htmlv=r.content
    soup=bsp(htmlv,"html.parser")
    value=soup.find("ol", class_="row collapsible classResource")
    allLi=value.find_all("li")
    dicMain={}

    for i in allLi:

        x=i.find("div",class_="collapsible-header bold")

        if x:
            dataa.write("\n")
            dataa.write(x.text.strip().replace("\n","").replace("\r","").strip())
            dataa.write("\n")
            print()
            print("                  ",x.text.strip().replace("\n","").replace("\r","").strip())
            print()

        y=i.find_all("div",class_="collapsible-body")
        dic={}
        count=1
        for j in y:

            listValue=j.find_all("li",class_="collection-item")

            for mainValue in listValue:
                dataa.write(str(count))
                dataa.write("  " + mainValue.text.strip())
                print(count,end=" ")
                print("   ",mainValue.text.strip())

                ans=mainValue.text.strip().replace(' ','-')
                dic[count]=f"{url}/{ans}"
                count+=1

            print()
        if x:
            dicMain[x.text.strip().replace("\n","").replace("\r","").strip()]=dic
    print("\n\n")

    value=input("Enter Value =")
    value1=int(input(f"In {value} where you want to go :-"))
    print("\n\n")
    url=dicMain[value][value1]
    r=requests.get(url)
    htmlv=r.content
    soup=bsp(htmlv,"html.parser")
    value=soup.find("ul", class_="collection")
    value2=value.find_all("li")
    count=1
    dict1={}
    print("Question In",value1," are\n")
    for i in value2:
        dataa.write(str(count))
        dataa.write(i.find("span",class_="name").text.strip())
        print(count,end=" ")
        x=i.find("a").get("href")

        print("  ",i.find("span",class_="name").text.strip())
        dict1[count]=x
        count+=1
    print("\n\n\n")
    choice=int(input("Which Question You Want To Do \n"))
    url3="https://pepcoding.com"+dict1[choice]
    print(url3)
    webbrowser.open_new_tab(url3)

# This is Main Function
url="https://pepcoding.com/resources/"
r=requests.get(url)
htmlv=r.content
soup=bsp(htmlv,"html.parser")
values=soup.find_all("div", class_="card free-resource-card show-on-hover border-radius")
count=1
dict={}
print("\n")
dataa = open("C:\\Users\\pc\\OneDrive\\Desktop\\data science\\web scraping\\pep\\PepCoding-scrapper-\DATA\\pepData.txt" , 'a')

for value in values:

    print(count,end="  ")
    dataa.write(value.find("h2").text.strip() + "\n")
    print(value.find("h2").text.strip())
    x=value.find("a").get("href")
    dict[count]=(f"https://pepcoding.com{x}")
    count+=1

print("\n\n\n")

choice=int(input("Enter Choice = "))
printAllDeatils(str(dict[choice]) , dataa)
dataa.close()

ML-AI/Web Scraper/Pep Coding Scraping/Python Code/pepcodingwebscrapping.py


from multiprocessing import Queue

import frida

from src.content_provider_fuzzing.execution_units.frida.frida_instrumentation_exec_unit import \
    FridaInstrumentationRequest, FridaInstrumentationExecUnit
from tests.sample_provider_tests.frida_tests.frida_test_setup import FridaTest

class TestFridaSystemServerInstrumentation(FridaTest):
    def test_frida_instrumentation(self):
        self.feedback_queue = Queue()

        instrumentation_request = FridaInstrumentationRequest(
            frida_device=frida.get_usb_device(),
            target_process_name='system_server',
            frida_js_script="""
Java.perform(function () {
    send('AAA');
});
""",
            on_message_function=self.__on_message
        )
        self.instrumentation_exec_unit = FridaInstrumentationExecUnit(instrumentation_request)
        self.instrumentation_exec_unit.setup()
        self.instrumentation_exec_unit.run()
        self.instrumentation_exec_unit.cleanup()

        assert self.feedback_queue.get(timeout=10) == {'type': 'send', 'payload': 'AAA'}

    def __on_message(self, message, data):
        self.feedback_queue.put(message)


from os import cpu_count
import numpy as np
import pyopencl as cl
from pyopencl.tools import get_test_platforms_and_devices
import sys
from time import time
from pprint import pprint

KERNELS_FILE: str
context: cl.Context
queue: cl.CommandQueue
program: None
tile_size = 64

def initOpenCL():
    global context, queue, program
    platforms = cl.get_platforms()
    devices = platforms[0].get_devices(cl.device_type.GPU)
    device = devices[0]
    global_mem = device.get_info(cl.device_info.GLOBAL_MEM_SIZE)
    local_mem = device.get_info(cl.device_info.LOCAL_MEM_SIZE)
    max_CU = device.get_info(cl.device_info.MAX_COMPUTE_UNITS)
    max_WGS = device.get_info(cl.device_info.MAX_WORK_GROUP_SIZE)
    info = """Device: {}:
Global mem size: {} Bytes
Local mem size: {} Bytes
Max CU: {}
Max work groub size: {} WIs\n""".format( device.name, global_mem, local_mem, max_CU, max_WGS)
    print(info)
    context = cl.Context(devices)#cl.create_some_context()
    queue = cl.CommandQueue(context,
        properties=cl.command_queue_properties.PROFILING_ENABLE)

    #print("Prop: {}".format(queue.set_property(cl.command_queue_properties.PROFILING_ENABLE, True)))

    with open(KERNELS_FILE, 'r') as kernel_file:
        program = cl.Program(context, kernel_file.read() % {'tile_size': tile_size}).build()
        kernel_file.close()

def host_mul(A, B):
    sum = np.int32(0)

    start = time()
    C = A @ B
    cpu_time = time() - start

    return C, cpu_time

def global_normilize(G, L):
    while not G % L == 0:
        G += 1
    return G

def normilize_sizes(M, N):
    local_size_M = 1
    i = 0
    barrier = 32
    while(local_size_M <= M and local_size_M <= barrier and 2 ** (i + 1) <= barrier and M > 2 ** (i + 1)):
        # print(1, local_size_M <= M, local_size_M <= barrier, M > 2 ** (i + 1))
        i += 1
        local_size_M = 2 ** i

    # print(local_size_M)

    M = global_normilize(M, local_size_M)
    # print(M)

    local_size_N = 1
    barrier = 1024 // local_size_M
    # print(barrier)
    i = 0

    while(local_size_N <= N and local_size_N <= barrier and 2 ** (i + 1) <= barrier and N > 2 ** (i + 1)):
        # print(2, local_size_N <= M, local_size_N <= barrier, N > 2 ** (i + 1))
        i += 1
        local_size_N = 2 ** i

    # print(local_size_N)
    N = global_normilize(N, local_size_N)
    # print(N)

    return M, N, local_size_M, local_size_N

def lab1(M, K, N, check_results, print_results):
    global context, queue, program, tile_size

    print(f"LAB 1\n[{M}x{K}] X [{K}x{N}]\n")

    # kernels = program.all_kernels()
    low = 100
    high = 1000

    A = np.random.randint(low, high, M * K).astype(np.int32).reshape(M, K)
    B = np.random.randint(low, high, K * N).astype(np.int32).reshape(K, N)

    if print_results:
        print(f"Matr A: \n {A}\n")
        print(f"Matr B: \n {B}\n")
    # print(matr.nbytes)
    host_result_matr, device_result_matr = None, np.empty((M * N)).astype(np.int32)
    cpu_time = 0

    if check_results:
        host_result_matr, cpu_time = host_mul(A, B)
        if print_results:
            print(f"Host result matr: \n {host_result_matr}\n")

    buffer_in_A = cl.Buffer(context, cl.mem_flags.READ_ONLY | cl.mem_flags.COPY_HOST_PTR, hostbuf=A)
    buffer_in_B = cl.Buffer(context, cl.mem_flags.READ_ONLY | cl.mem_flags.COPY_HOST_PTR, hostbuf=B)
    buffer_out_C = cl.Buffer(context, cl.mem_flags.WRITE_ONLY, device_result_matr.nbytes)

    # kernel_name = 'global_matr_mul'
    kernel = program.global_matr_mul

    kernel.set_args(buffer_in_A, buffer_in_B, buffer_out_C, M, K, N)

    # check_amount = 50
    gpu_time = 0

    m, n, l_m, l_n = normilize_sizes(M, N)
    global_size = (n, m)
    local_size = (l_n, l_m)

    print("Global size: {}\nLocal size: {}\n".format(global_size, local_size))
    # for j in range(check_amount):

    event = cl.enqueue_nd_range_kernel(queue, kernel, global_size, local_size)
    event.wait()
    gpu_time += (event.profile.end  - event.profile.start)
    # gpu_time /= check_amount

    cl.enqueue_copy(queue, device_result_matr, buffer_out_C)
    device_result_matr = device_result_matr.reshape(M, N)

    if print_results:
        print(f"Device result matr: \n {device_result_matr}\n")
    if check_results:
        comparison = host_result_matr == device_result_matr
        is_equal = comparison.all()
        print(f'Compare with host matr: {is_equal}')
        print('CPU_TIME: {} ms'.format(cpu_time * 1e3))
        # data.update({"CPU_TIME (s)": cpu_time, 'Compare with host': flag})

    # data.update({"GPU_TIME (ms)": gpu_time/1e6})
    print('GPU_TIME: {} ms\n'.format(gpu_time * 1e-6))

def main():
    global KERNELS_FILE

    argv = sys.argv

    check_results = False
    print_results = False

    if len(argv) == 1:
        print("Enter more arguments")
        return
    elif len(argv) == 5:
        pass
    elif len(argv) == 6:
        check_results = True if argv[5] == 'check' else False
    elif len(argv) == 7:
        check_results = True if argv[5] == 'check' else False
        print_results = True if argv[6] == 'print' else False
    else:
        print("Enter less arguments")
        return
    KERNELS_FILE = argv[1]
    M = np.int32(argv[2])
    K = np.int32(argv[3])
    N = np.int32(argv[4])

    # if (not (M % 32 == 0 and N % 32 == 0 ))

    initOpenCL()
    start = time()
    lab1(M, K, N, check_results, print_results)
    print("Program work time:{}".format(time() - start))

if __name__ == '__main__':
    main()


import typing
from typing import Dict, Optional

from monitoring.monitorlib.clients.scd_automated_testing import QueryError
from monitoring.monitorlib.scd_automated_testing.scd_injection_api import InjectFlightResponse
from monitoring.uss_qualifier.common_data_definitions import Severity
from monitoring.uss_qualifier.scd.configuration import SCDQualifierTestConfiguration
from monitoring.uss_qualifier.scd.data_interfaces import AutomatedTest, TestStep, FlightInjectionAttempt
from monitoring.uss_qualifier.scd.executor.errors import TestRunnerError
from monitoring.uss_qualifier.scd.executor.report_recorder import ReportRecorder
from monitoring.uss_qualifier.scd.executor.target import TestTarget
from monitoring.uss_qualifier.scd.executor.test_steps import inject_flight, delete_flight
from monitoring.uss_qualifier.scd.reports import Report, Issue, AutomatedTestContext, TestStepReference, TestPhase

# TODO: Replace print by logging
class TestRunner:
    """A class to run automated test steps for a specific combination of targets per uss role"""

    def __init__(self, context: AutomatedTestContext, automated_test: AutomatedTest, targets: Dict[str, TestTarget], dss_target: Optional[TestTarget], report: Report):
        self.context = context
        self.automated_test = automated_test
        self.targets = targets
        self.dss_target = dss_target
        self.report_recorder = ReportRecorder(report, self.context)

    def get_scd_configuration(self) -> SCDQualifierTestConfiguration:
        return SCDQualifierTestConfiguration(injection_targets=list(map(lambda t: t.config, self.targets.values())))

    def run_automated_test(self):
        for i, step in enumerate(self.automated_test.steps):
            print('[SCD]   Running step {}: {}'.format(i, step.name))
            self.execute_step(step, i)

    def teardown(self):
        """Delete resources created by this test runner."""
        print("[SCD]   Teardown {}".format(self.automated_test.name))

        cleanup_test_step = 0
        for role, target in self.targets.items():
            flight_names = target.managed_flights()
            for flight_name in flight_names:
                print("[SCD]    - Deleting {} flights for target {}.".format(len(flight_names), target.name))

                step_ref = TestStepReference(
                    name="Clean up flight {} in {}".format(flight_name, target.name),
                    index=cleanup_test_step,
                    phase=TestPhase.Cleanup
                )

                try:
                    resp, query = target.delete_flight(flight_name)
                    self.report_recorder.capture_interaction(step_ref, query, 'Remove flight during test cleanup')
                except QueryError as e:
                    interaction_id = self.report_recorder.capture_interaction(step_ref, e.query, 'Remove flight during test cleanup')
                    self.report_recorder.capture_deletion_unknown_issue(
                                    interaction_id=interaction_id,
                                    summary="Deletion request for flight {} was unsuccessful".format(flight_name),
                                    details="Deletion attempt failed with status {}.".format(e.query.status_code),
                                    flight_name=flight_name,
                                    target_name=target.name,
                                    uss_role=role
                            )
                    print("[SCD] Error: Unable to delete flight {} during teardown".format(flight_name))
                cleanup_test_step = cleanup_test_step + 1

    def execute_step(self, step: TestStep, step_index: int):
        target = self.get_target(step)
        if target is None:
            self.print_targets_state()
            raise RuntimeError("[SCD] Error: Unable to identify the target managing flight {}".format(
                step.inject_flight.name if 'inject_flight' in step else step.delete_flight.flight_name
            ))

        step_ref = TestStepReference(
            name=step.name,
            index=step_index,
            phase=TestPhase.Test
        )

        if 'inject_flight' in step:
            inject_flight.execute(self, step, step_ref, target)
        elif 'delete_flight' in step:
            delete_flight.execute(self, step, step_ref, target)
        else:
            raise RuntimeError("[SCD] Error: Unable to identify the action to execute for step {}".format(step.name))

        print("[SCD]     Step {} COMPLETED".format(step.name))

    def get_managing_target(self, flight_name: str) -> typing.Optional[TestTarget]:
        """Returns the managing target which created a flight"""
        for role, target in self.targets.items():
            if target.is_managing_flight(flight_name):
                return target
        return None

    def get_target(self, step: TestStep) -> typing.Optional[TestTarget]:
        """Returns the target which should be called in the TestStep"""
        if 'inject_flight' in step:
            return self.targets[step.inject_flight.injection_target.uss_role]
        elif 'delete_flight' in step:
            return self.get_managing_target(step.delete_flight.flight_name)
        else:
            raise NotImplementedError("Unsupported step. A Test Step shall contain either a inject_flight or a delete_flight object.")

    def get_target_role(self, target_name):
        results = list(filter(lambda x: x[1].name == target_name, self.targets.items()))
        return results[0] if len(results) > 0 else None

    def evaluate_inject_flight_response(self, interaction_id: str, target: TestTarget, attempt: FlightInjectionAttempt, resp: InjectFlightResponse) -> typing.Optional[Issue]:
        if resp.result not in attempt.known_responses.acceptable_results:
            print("[SCD]     Result: ERROR. Received {}, expected one of {}. Reason: {}".format(
                resp.result,
                attempt.known_responses.acceptable_results,
                resp.get('notes', None))
            )
            known_issue = attempt.known_responses.incorrect_result_details.get(resp.result, None)
            if known_issue:
                issue = self.report_recorder.capture_injection_issue(interaction_id=interaction_id, target_name=target.name, attempt=attempt, known_issue=known_issue)
                if known_issue.severity != Severity.Low:
                    raise TestRunnerError("Failed attempt to inject flight {}: {}".format(attempt.name, known_issue.summary), issue)
            else:
                issue = self.report_recorder.capture_injection_unknown_issue(
                    interaction_id=interaction_id,
                    summary="Injection request was unsuccessful",
                    details="Injection attempt failed with unknown response {}".format(resp.result),
                    target_name=target.name,
                    attempt=attempt
                )
                raise TestRunnerError("Unsuccessful attempt to inject flight {}".format(attempt.name), issue)
        return None

    def print_targets_state(self):
        print("[SCD] Targets States:")
        for name, target in self.targets.items():
            print(f"[SCD]   - {name}: {target.created_flight_ids}")


from datetime import date
from bs4 import BeautifulSoup
import asyncio
import aiohttp
import json
import hashlib
from .find_dates_in_string import find_date_in_string, normalise_str
from datetime import datetime

result = []

headers = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,'
              'application/signed-exchange;v=b3;q=0.9',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
                  'Chrome/109.0.0.0 Safari/537.36'
}

async def make_queue_vstu(un_id, url, filter_date):
    try:
        async with aiohttp.ClientSession() as session:
            tasks = []

            year = filter_date.year - 1
            while True:
                year += 1
                if year > date.today().year + 1:
                    break
                url_ = f'{url}/{year}/'
                try:
                    async with session.get(url=url_, headers=headers, timeout=20) as response:
                        if response.status == 404:
                            print(f'{response.status} - Нет такой страницы {url_}')
                            return
                        response_text = await response.text()
                        soup = BeautifulSoup(response_text, 'lxml')
                        main_container = soup.find('div', class_='content-wrapper').find('div', class_='unit-75')
                        try:
                            _ = int(soup.find('div', class_='page-header-wrapper').find('div', class_='unit-100').text)
                        except:
                            print(f'Конференции за {year} год отсутствуют.')
                            continue

                        if not main_container.find('dl', class_='conf'):
                            print(f'Конференции за {year} год отсутствуют.')
                            continue

                        print(f'{response.status} Обрабатываем страницу {year} года, {url_}')
                        conf_block = main_container.find_all('dl', class_='conf')

                        for n, conf in enumerate(conf_block):
                            title = normalise_str(conf.find('a').text)
                            # print(title)
                            if 'конфер' in title:
                                conf_url = f"https://www.vstu.ru{conf.find('a').get('href').strip()}"
                                # print(title, conf_url)
                                task = asyncio.create_task(parser_vstu_pages(session, un_id, conf_url, filter_date, n, year))
                                tasks.append(task)
                                # break

                except asyncio.TimeoutError as e:
                    print(f'Отказ по таймауту, страница {year} года, {url_}', e)
                except Exception as e:
                    raise Exception(f'Не удалось обработать ссылку {n} года {year} в {__name__} для {url_}\n{e}')

            await asyncio.gather(*tasks)

    except Exception as e:
        raise Exception(f'Не удалось обработать очередь для {__name__}\n{e}')

async def parser_vstu_pages(session, un_id, url, filter_date, n, year):
    def date_str_prep(str_):
        words = str_.split()
        for n, word in enumerate(words):
            if '—' in word:
                words.insert(n, str(year))
                break
        words.append(str(year))
        str_ = ' '.join(words)
        return str_

    try:
        async with session.get(url=url, headers=headers, timeout=20) as response:
            if response.status == 404:
                print(f'{response.status} - Нет такой страницы {url}')
                return
            response_text = await response.text()

            print(f'Выполняется ссылка {n + 1} года {year}, {url}')
            soup = BeautifulSoup(response_text, 'lxml')
            try:
                conf_block = soup.find('div', class_='content-wrapper').find('div', class_='unit-75')
            except:
                print(f'Не найден нужный блок в html, не обработана ссылка {url}')
                return

            un_name = 'Волгоградский государственный технический университет'

            conf_name = normalise_str(soup.find('div', class_='page-header-wrapper').find('div', class_='unit-100').text)
            local = False if 'международн' in conf_name.lower() else True

            conf_id = f"{un_id}_{url.split('/')[-2]}"
            hash_ = str(hashlib.md5(bytes(conf_id, 'utf-8')).hexdigest())
            conf_card_href = url

            prep_dates = date_str_prep(normalise_str(conf_block.find('p').text))
            # print(prep_dates)

            conf_date_begin = str(list(find_date_in_string(prep_dates))[0].date()) if \
                list(find_date_in_string(prep_dates)) else ''
            conf_date_end = str(list(find_date_in_string(prep_dates))[1].date()) if \
                len(list(find_date_in_string(prep_dates))) > 1 else ''
            # print(conf_date_begin, conf_date_end)

            conf_s_desc = conf_name
            reg_date_begin = ''
            reg_date_end = ''

            reg_href = ''
            conf_desc = ''
            org_name = ''
            themes = ''
            online = False
            conf_href = ''
            offline = False
            conf_address = ''
            contacts = ''
            rinc = False

            try:
                lines_tr = conf_block.find('table', id='table1').find_all(['tr'])
            except:
                lines_tr = []

            for tr in lines_tr:
                lines = tr.find_all(['td', 'li'])
                for line in lines:
                    # print(line.text)
                    if conf_s_desc == '':
                        conf_s_desc = normalise_str(line.text)

                    if ('заявк' in line.text.lower() or 'принимаютс' in line.text.lower() or 'участи' in line.text.lower()
                                or 'регистрац' in line.text.lower() or 'регистрир' in line.text.lower()) and reg_date_begin == '':
                        reg_date_begin = str(list(find_date_in_string(line.text.lower()))[0].date()) if \
                            list(find_date_in_string(line.text.lower())) else ''
                        reg_date_end = str(list(find_date_in_string(line.text.lower()))[1].date()) if \
                            len(list(find_date_in_string(line.text.lower()))) > 1 else ''

                    if not online and ('онлайн' in line.text.lower() or 'трансляц' in line.text.lower()):
                        online = True
                        conf_href = line.find('a').get('href') if line.find('a') else 'отсутствует'
                    if not offline and ('место' in line.text.lower() or 'адрес' in line.text.lower()):
                        offline = True
                        conf_address = normalise_str(line.get_text(separator=" "))

                    conf_desc = conf_desc + ' ' + normalise_str(line.get_text(separator=" "))

                    if reg_href == '' and ('регистрац' in line.text.lower() or 'зарегистр' in line.text.lower()
                                           or 'участия' in line.text.lower() or 'заявк' in line.text.lower()):
                        reg_href = line.find('a').get('href') if line.find('a') \
                                                                 and ('http:' in line.find('a').get('href') or
                                                                      'https:' in line.find('a').get('href')) and\
                                                                 ('.pdf' not in line.find('a').get('href') or
                                                                  '.doc' not in line.find('a').get('href') or
                                                                  '.xls' not in line.find('a').get('href')) else 'отсутствует'

                    if org_name == '' and 'организатор' in line.text.lower():
                        org_name = normalise_str(line.get_text(separator=" "))

                    if not online and ('онлайн' in line.text.lower() or 'трансляц' in line.text.lower() or
                                       'ссылка' in line.text.lower()):
                        conf_href = line.find('a').get('href') if line.find('a') else 'отсутствует'
                        online = True

                    if not offline and ('место' in line.text.lower() or 'адрес' in line.text.lower() or
                                        'очно' in line.text.lower()):
                        conf_address = normalise_str(line.text)
                        offline = True

                    if ('тел.' in line.text.lower() or 'контакт' in line.text.lower() or 'mail' in line.text.lower()
                            or 'почта' in line.text.lower() or 'почты' in line.text.lower()):
                        contacts = contacts + ' ' + normalise_str(line.text)

                    if line.find('a') and 'mailto' in line.find('a').get('href'):
                        contacts = contacts + ' ' + normalise_str(line.find('a').text)

                    if not rinc:
                        rinc = True if 'ринц' in line.text.lower() else False

            if (conf_date_begin != '' and datetime.strptime(conf_date_begin,
                                                            '%Y-%m-%d') >= filter_date) or conf_date_begin == '':
                result.append(
                        {'conf_id': conf_id,
                         'hash': hash_,
                         'un_name': un_name,
                         'local': local,
                         'reg_date_begin': reg_date_begin,
                         'reg_date_end': reg_date_end,
                         'conf_date_begin': conf_date_begin,
                         'conf_date_end': conf_date_end,
                         'conf_card_href': conf_card_href,
                         'reg_href': reg_href,
                         'conf_name': conf_name,
                         'conf_s_desc': conf_s_desc,
                         'conf_desc': conf_desc,
                         'org_name': org_name,
                         'themes': themes,
                         'online': online,
                         'conf_href': conf_href,
                         'offline': offline,
                         'conf_address': conf_address,
                         'contacts': contacts.strip(),
                         'rinc': rinc,
                         }
                    )

    except asyncio.TimeoutError as e:
        print(f'Отказ по таймауту, ссылка {n + 1} года {year}, {url}', e)

def parser_vstu(un_id, urls, date_):
    try:
        for url in urls:
            asyncio.run(make_queue_vstu(un_id, url, date_))
        with open(f'Conference_data/Parsers/JSON_log/{un_id}_{date.today()}.json', 'w', encoding="utf-8") as file:
            json.dump(result, file, indent=4, ensure_ascii=False)
        return result
    except Exception as e:
        print(e)

if __name__ == '__main__':
    print(parser_vstu('vstu', ['https://www.vstu.ru/nauka/konferentsii'], datetime.strptime('2023.01.01', '%Y.%m.%d')))


from keys import CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET, DROPBOX_ACCESS_TOKEN
from tweepy import Stream, OAuthHandler
from tweepy.streaming import StreamListener
import json
import dropbox
from datetime import datetime
from time import sleep
from netifaces import interfaces, ifaddresses, AF_INET
import RPi.GPIO as GPIO
from flask import Flask, render_template
import threading

webapp = Flask(__name__)

auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
TRIGGER_MESSAGE = open('trigger','r').readline().rstrip("\n")
print("Using", TRIGGER_MESSAGE, "as trigger message")

class myThread (threading.Thread):
    def __init__(self):
        threading.Thread.__init__(self)
    def run(self):
        uploadIPtoDropbox()
        listenForTweets()

class LightsController:
    STAR_LIGHTS = 18
    TREE_LIGHTS = 23
    BALL_LIGHTS = 24
    STATUS_LIGHT = 25

    def __init__(self):

        self.lightsBusy = False

        GPIO.setmode(GPIO.BCM)

        GPIO.setup(18, GPIO.OUT)

        GPIO.setup(self.STAR_LIGHTS, GPIO.OUT)
        GPIO.setup(self.TREE_LIGHTS, GPIO.OUT)
        GPIO.setup(self.BALL_LIGHTS, GPIO.OUT)
        # GPIO.setup(self.STATUS_LIGHT, GPIO.OUT)
        print("Initialised GPIO pins")
        self.flashAllTogether(5)
        self.off()

    def off(self):

        print("**All lights off**")

        GPIO.output(self.STAR_LIGHTS, False)
        GPIO.output(self.TREE_LIGHTS, False)
        GPIO.output(self.BALL_LIGHTS, False)
        # GPIO.output(self.STATUS_LIGHT, False)

    def treeOn(self):

        GPIO.output(self.TREE_LIGHTS, True)

    def treeOff(self):

        GPIO.output(self.TREE_LIGHTS, False)

    def starOn(self):

        GPIO.output(self.STAR_LIGHTS, True)

    def starOff(self):

        GPIO.output(self.STAR_LIGHTS, False)

    def ballsOn(self):

        GPIO.output(self.BALL_LIGHTS, True)

    def ballsOff(self):

        GPIO.output(self.BALL_LIGHTS, False)

    def statusOn(self):

        print("**Status light on**")

        # GPIO.output(self.STATUS_LIGHT, True)

    def statusOff(self):

        print("**Status light off**")

        # GPIO.output(self.STATUS_LIGHT, False)

    def perm_on(self):
        self.lightsBusy = True
        self.treeOn()
        self.ballsOn()
        self.starOn()

    def on(self, duration):

        flashTime = 0.2
        numberFlashes = duration / flashTime

        self.lightsBusy = True

        print("**All lights on**")
        for n in range(0, int(numberFlashes)):
            # print("flash number: " + str(n))
            self.treeOn()
            self.starOn()
            self.ballsOn()
            sleep(flashTime)

        self.off()

    def flashTree(self, duration):

        flashTime = 0.2
        numberFlashes = duration / flashTime / 2.0

        self.lightsBusy = True

        for n in range(0, int(numberFlashes)):
            # print("flash number: " + str(n))
            self.treeOn()
            sleep(flashTime)
            self.treeOff()
            sleep(flashTime)

        self.lightsBusy = False

    def flashBalls(self, duration):

        flashTime = 0.2
        numberFlashes = duration / flashTime / 2.0

        self.lightsBusy = True

        for n in range(0, int(numberFlashes)):
            self.ballsOn()
            sleep(flashTime)
            self.ballsOff()
            sleep(flashTime)

        self.lightsBusy = False

    def flashStar(self, duration):

        flashTime = 0.2
        numberFlashes = duration / flashTime / 2.0

        self.lightsBusy = True

        for n in range(0, int(numberFlashes)):
            self.starOn()
            sleep(flashTime)
            self.starOff()
            sleep(flashTime)

        self.lightsBusy = False

    def flashStarAndBalls(self, duration):

        flashTime = 0.2
        numberFlashes = duration / flashTime / 2.0

        self.lightsBusy = True

        for n in range(0, int(numberFlashes)):
            self.starOn()
            self.ballsOn()
            sleep(flashTime)
            self.starOff()
            self.ballsOff()
            sleep(flashTime)

        self.lightsBusy = False

    def flashAllTogether(self, duration):

        flashTime = 0.2
        numberFlashes = duration / flashTime / 2.0

        self.lightsBusy = True

        for n in range(0, int(numberFlashes)):
            self.ballsOn()
            self.treeOn()
            self.starOn()
            sleep(flashTime)
            self.ballsOff()
            self.treeOff()
            self.starOff()
            sleep(flashTime)

        self.lightsBusy = False

    def flashAllSequence(self, duration):

        flashTime = 0.2
        numberFlashes = duration / flashTime / 3.0

        self.lightsBusy = True
        # print (datetime.datetime.now())

        for n in range(0, int(numberFlashes)):
            self.starOff()
            self.treeOn()
            sleep(flashTime)
            self.treeOff()
            self.ballsOn()
            sleep(flashTime)
            self.ballsOff()
            self.starOn()
            sleep(flashTime)
            self.off()

        # print (datetime.datetime.now())
        self.lightsBusy = False

# Set up Twitter listener
class listener(StreamListener):

    def on_data(self, data):
        d = json.loads(data)
        print("trigger message found in tweet: %s" % d['text'])
        lc.flashAllSequence(20)
        return True

    def on_error(self, status):
        print (status)
        return False

def uploadIPtoDropbox():

    # Establish dropbox connection - useful for uploading IP address to DB

    try:
        dbx = dropbox.Dropbox(DROPBOX_ACCESS_TOKEN)
        ip_message = "Socialite Christmas Jumper\n\n"

        # from http://stackoverflow.com/questions/166506/finding-local-ip-addresses-using-pythons-stdlib
        for ifaceName in interfaces():
            addresses = [i['addr'] for i in ifaddresses(ifaceName).setdefault(AF_INET, [{'addr':'No IP addr'}] )]
            ip_message = ip_message +  '%s: %s\n' % (ifaceName, ', '.join(addresses))

        ip_message = ip_message + "Updated: {1}".format("xx.xx.xx.xx", datetime.now())
        dbx.files_upload(ip_message.encode('utf-8'), '/jumper_ip.txt', mode=dropbox.files.WriteMode('overwrite'))
        print("IP address uploaded to Dropbox")

    except:
        print("Dropbox connection error. Attempting again in 10 seconds.")
        lc.on(10)
        uploadIPtoDropbox()

# Set up global lights controller object
lc = LightsController()

def listenForTweets():
    try:
        twitterStream = Stream(auth, listener())
        twitterStream.filter(track=[TRIGGER_MESSAGE])
        print("Twitter stream listener started.")

    except:
        print("Twitter connection error. Trying again in 10 seconds.")
        lc.on(10)
        listenForTweets()

# API routes
@webapp.route('/')
def index():
    return render_template('index.html')

@webapp.route('/on/')
def web_on():
    global lc
    lc.perm_on()
    return render_template('index.html', message="Message received! You made my lights shine!")

@webapp.route('/off/')
def web_off():
    global lc
    lc.off()
    return render_template('index.html', message="Message received! Had enough of my lights, I see...")

@webapp.route('/flash/')
def web_flash():
    global lc
    lc.flashAllTogether(20)
    return render_template('index.html', message="Message received! Flashing has commenced.")

@webapp.route('/chase/')
def web_chase():
    global lc
    lc.flashAllSequence(20)
    return render_template('index.html', message="Message received! You're into the fancy stuff I see.")

# Disabled due to SSL certificate error in school
#listenThread = myThread()
#listenThread.start()

if __name__ == '__main__':
    webapp.run(host="0.0.0.0", port=5000)


from django.http import HttpResponse
from django.shortcuts import render
from multiprocessing import context
from .models import Post
from . import forms

def index(request):
    db = Post.objects.all()
    context ={
        'title':'Blog',
        'heading':'Blog',
        'subheading': 'postingan',
        'post' : db,
    }
    # context = {
    #     'Judul' : 'AppBlog',
    #     'h1' : 'Django',
    #     'menu' : [['/', 'Home'], ['/blog/recent', 'Recent'], ['/blog/post', 'Post']]
    # }
    return render(request,'blog/index.html', context)

def recent(request):
    return HttpResponse("
RECENT BLOG
")

def post(request):
    return HttpResponse("
INI ISINYA POST
")

def about(request):
    return HttpResponse("
ABOUT BLOG
")

def form(request):
    classform = forms.namaclass()
    context = {
        'heading':'Home',
        'classform' : classform
    }

    # if request.method == 'POST':
    #     print("ini adalah method post")
    #     context['nama'] == request.POST['nama']
    #     context['alamat'] == request.POST['alamat']
    #     context['namac'] == request.POST['namac']
    #     context['alamat'] == request.POST['alamat']
    # else:
    if request.POST.get('nama') and request.POST.get('alamat') and request.POST.get('tgl_lahir') and request.POST.get('email'):
            post = Post()
            post.nama = request.POST.get('nama')
            post.alamat = request.POST.get('alamat')
            post.tgl_lahir = request.POST.get('tgl_lahir')
            post.email = request.POST.get('email')
            post.save()

            return render(request, 'form/index.html')
    else:
        print("ini adalah method get")
    return render(request, 'form.html', context)

Praktikum Modul 7 dan 8/Dev/tutor/blog/views.py


import aiohttp_jinja2
from aiohttp_session import get_session
from aiohttp import web
from aiohttp.typedefs import Handler
from aiohttp.web_middlewares import _Middleware
from manga_sales.containers import DatabaseContainer

async def handle_404(request: web.Request) -> web.Response:
    return aiohttp_jinja2.render_template("404.html", request, {}, status=404)

async def handle_500(request: web.Request) -> web.Response:
    return aiohttp_jinja2.render_template("500.html", request, {}, status=500)

def create_error_middleware(overrides: dict[int, Handler]) -> _Middleware:
    @web.middleware
    async def error_middleware(
        request: web.Request, handler: Handler
    ) -> web.StreamResponse:
        try:
            return await handler(request)
        except web.HTTPException as ex:
            override = overrides.get(ex.status)
            if override:
                return await override(request)

            raise

    return error_middleware

@web.middleware
async def session_middleware(
    request: web.Request, handler: Handler
) -> web.StreamResponse:
    request.session = await get_session(request)
    return await handler(request)

@web.middleware
async def db_middleware(request: web.Request, handler: Handler) -> web.StreamResponse:
    container = DatabaseContainer()
    response = await handler(request)
    await container.shutdown_resources()  # type: ignore # pylint: disable=no-member
    return response

def setup_middlewares(app: web.Application) -> None:
    error_middleware = create_error_middleware({404: handle_404, 500: handle_500})
    app.middlewares.append(error_middleware)
    app.middlewares.append(db_middleware)
    # app.middlewares.append(session_middleware)


import sys
import pygame
import config
import drawGame

class App:
    def __init__(self):
        pygame.init()
        self.window = pygame.display.set_mode(
            (config.CONSOLE_WIDTH, config.CONSOLE_HEIGHT))
        self.running = True
        self.grid = config.NEW_BOARD
        self.solution = config.SOLUTION
        self.selectedCell = []
        self.mousePos = None
        self.boardButtons = []
        self.lockedCells = []
        self.incorrectCells = []
        self.font = pygame.font.SysFont('arial', config.CELL_SIZE//2)
        self.solutionShown = False
        self.board = drawGame.DrawGame(self.window, self.grid, self.solution, self.selectedCell, self.boardButtons, self.lockedCells, self.incorrectCells, self.font, self.solutionShown)

    def play(self):
        while self.running:
            self.events()
            self.update()
            self.draw()
        pygame.quit()
        sys.exit()

##################################################################################################################################
##################################################### PLAY #######################################################################
##################################################################################################################################

    def events(self):

        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                self.running = False

            # mouse click
            if event.type == pygame.MOUSEBUTTONDOWN:
                cell = self.mouseonGrid()
                if cell:
                    self.selectedCell = cell

            # Key in number
            if event.type == pygame.KEYDOWN:
                if self.selectedCell != None and self.selectedCell not in self.lockedCells:
                    if self.isNumber(event.unicode):
                        # cell changed
                        self.incorrectCells = []
                        self.grid[self.selectedCell[1]][self.selectedCell[0]] = int(event.unicode)
                        if self.solution[self.selectedCell[1]][self.selectedCell[0]] != int(event.unicode):
                            if self.grid[self.selectedCell[1]][self.selectedCell[0]] == 0:
                                self.board.drawSelection(self.window, (self.selectedCell[1], self.selectedCell[0]))
                            else:
                                self.incorrectCells.append([self.selectedCell[0], self.selectedCell[1]])

    def update(self):
        self.mousePos = pygame.mouse.get_pos()
        for item in self.boardButtons:
            item.update(self.mousePos)

    def draw(self):
        self.board.selectedCell = self.selectedCell
        self.board.incorrectCells = self.incorrectCells
        self.board.draw()

#################################################################################################################################
############################################################ FUNCTION ###########################################################
#################################################################################################################################

    def mouseonGrid(self):
        if self.mousePos[0] < config.GRID_POS[0] or self.mousePos[1] < config.GRID_POS[1]:
            self.selectedCell = []
            return False
        if self.mousePos[0] > config.GRID_POS[0]+config.GRID_SIZE or self.mousePos[1] > config.GRID_POS[1]+config.GRID_SIZE:
            self.selectedCell = []
            return False
        return ((self.mousePos[0]-config.GRID_POS[0])//config.CELL_SIZE, (self.mousePos[1]-config.GRID_POS[1])//config.CELL_SIZE)

    def isNumber(self, string):
        try:
            int(string)
            return True
        except:
            return False


from django.conf.urls import url, include
from rest_framework import routers
from base import views
from django.contrib import admin
from rest_framework_jwt import views as jwt_views

router = routers.DefaultRouter()
router.register(r'users', views.UserViewSet)
router.register(r'groups', views.GroupViewSet)
router.register(r'gafs', views.GAFViewSet)
router.register(r'challenges', views.ChallengeViewSet)
router.register(r'assessments', views.AssessmentViewSet)
router.register(r'papers', views.PaperViewSet)
router.register(r'genes', views.GeneViewSet)
router.register(r'organisms', views.OrganismViewSet)
router.register(r'refseq', views.RefSeqViewSet)

urlpatterns = [
    url(r'^admin/', admin.site.urls),
    url(r'^', include(router.urls)),
    url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework')),
    url(r'^api-token-auth/', jwt_views.obtain_jwt_token),
    url(r'^api-token-auth-api/', views.ObtainAuthToken.as_view()),
    url(r'^api-token-auth-whoami/', views.WhoAmI.as_view()),
    url(r'^api-token-refresh/', jwt_views.refresh_jwt_token),
    url(r'^api-token-verify/', jwt_views.verify_jwt_token),
    url(r'^api/', include('stored_messages.urls')),
    url(r'^mark_all_read/$', views.mark_all_read, name='mark_all_read'),
]


# You are given a list of sentences as input. Your task is to create a program
# that counts the frequency of each word in the sentences and returns a dictionary
# with the word as the key and its frequency as the value. You should ignore case
# sensitivity and punctuation while counting the words.

import re
from collections import Counter

def count_word_frequency(sentences):
    word_frequency = Counter()
    for sentence in sentences:
        sentence = re.sub(r'[^\w\s]', '', sentence.lower())
        words = sentence.split()
        word_frequency.update(words)

    return dict(word_frequency)

# list of sentences
sentences = [
    "I went to college today morning with my Friend.",
    "Even though the food was very spicy, my friend managed to eat it somehow!",
    "I went to temple also!"
]

frequency = count_word_frequency(sentences)
print(frequency)


import binascii
import base58
import json

class Serializer:

    def __init__(self, Trans, sign=False):
        self.sign = sign
        vrt = (int(Trans.param['vertion'])).to_bytes(4, 'little')
        in_count = self.makeVar(Trans.param['tx_in count'])
        out_count = self.makeVar(Trans.param['tx_out count'])
        lkt = (int(Trans.param['lock_time'])).to_bytes(4, 'little').hex()
        self.param = {
            'vertion': vrt.hex(),
            'input count': in_count,
            'input': self.serialize_input(Trans.param['tx_in']),
            'output count': out_count,
            'output': self.serialize_output(Trans.param['tx_out']),
            'lockTime': lkt
        }

    def serialize_input(self, tx_in):
        in_rt = list()
        for i in range(len(tx_in)):
            str = (binascii.unhexlify(tx_in[i]['Previous txid'])[::-1].hex() +
                (int(tx_in[i]['Previous Tx Index']).to_bytes(4, 'little')).hex() +
                self.makeVar(tx_in[i]['Script Length']) +
                tx_in[i]['Signature Script'] + tx_in[i]['Sequence'])
            in_rt.append(str)
        return in_rt

    def serialize_output(self, tx_out):
        out = list()
        for i in range(len(tx_out)):
            str = ((int(tx_out[i]['value'])).to_bytes(8, 'little').hex() +
            (self.makeVar(tx_out[i]['Script Length'])) + tx_out[i]['Public Script'])
            out.append(str)
        return out

    def make(self):
        inp = ""
        out = ""
        add = "01000000" if (self.sign == True) else ""
        for i in range(len(self.param['input'])):
            inp = inp + self.param['input'][i]
        for i in range(len(self.param['output'])):
            out = out + self.param['output'][i]
        str = (self.param['vertion'] + self.param['input count'] +
                inp + self.param['output count'] + out + self.param['lockTime'] + add)
        return str

    def makeVar(self, strval):
        tmp = int(strval)
        if (tmp <= 255):
            bt = 1
            prf = ""
        elif tmp <= 65535:
            bt = 2
            prf = 'fd'
        elif tmp <= 4294967295:
            bt = 4
            prf = 'fe'
        else:
            bt = 8
            prf = 'ff'
        res = (int(strval)).to_bytes(bt, 'big')
        final = prf + res.hex()
        return final

class Deserializer:

    def __init__(self, str):
        inputCounterVal, inputCounterEnd = Deserializer.getVar(str[8:], 8)
        ilen, inp = self.getInput(inputCounterVal, str[inputCounterEnd:])

        outputCounterVal, outputCounterEnd = Deserializer.getVar(str[inputCounterEnd + ilen:], inputCounterEnd + ilen)
        olen, out = self.getOut(outputCounterVal, str[outputCounterEnd:])
        self.param = {
            'vertion': str[0 : 8],
            'tx_in count': inputCounterVal,
            'tx_in': inp,
            'tx_out count': outputCounterVal,
            'tx_out': out,
            'lock_time': int(str[-8 : ], 16)
        }

    @staticmethod
    def getVar(str, start):
        if str[:2] == 'fd':
            end = start + 6
            val = int(str[2: 6], 16)
        elif str[:2] == 'fe':
            end = start + 8
            val = int(str[2: 10], 16)
        elif str[:2] == 'ff':
            end = start + 16
            val = int(str[2: 18], 16)
        else:
            end = start + 2
            val = int(str[:2], 16)
        return val, end

    def getInput(self, len, str):
        inp = list()
        txid = 64
        vout = 8
        seqence = 8
        round = 0
        for i in range(len):
            scr_len, endScriptvar = Deserializer.getVar(str[(txid + vout) + round:], (txid + vout) + round)
            bytesScriptvar = endScriptvar - (txid + vout + round)
            inp.append({
                'Previous txid': str[round:round + txid],
                'Previous Tx Index': str[round + txid: round + txid + vout],
                'Script Length': scr_len,
                'Signature Script': str[round + txid + vout + bytesScriptvar: round + txid + vout + bytesScriptvar + scr_len * 2],
                'Sequence': str[round + txid + vout + bytesScriptvar + scr_len * 2: round + txid + vout + bytesScriptvar + scr_len * 2 + seqence]
            })
            round = round + txid + vout + bytesScriptvar + scr_len * 2 + seqence
        return round, inp

    def getOut(self, len, str):
        out = list()
        value = 16
        round = 0
        for i in range(len):
            scr_len, endScriptvar = Deserializer.getVar(str[value + round:], value + round)
            bytesScriptvar = endScriptvar - (value + round)
            print("-------", str[:round + value])
            out.append({
                'value': str[round:round + value],
                'Script Length': scr_len,
                'Public Script': str[round + value + bytesScriptvar: round + value + bytesScriptvar + scr_len * 2]
            })
            round = round + value + bytesScriptvar + scr_len * 2
        return round, out

    def make(self):
        self.param['vertion'] = int((binascii.unhexlify(self.param['vertion'])[::-1]).hex(), 16)
        for i in range(len(self.param['tx_in'])):
            self.param['tx_in'][i]['Previous txid'] = (binascii.unhexlify(self.param['tx_in'][i]['Previous txid'])[::-1]).hex()
            self.param['tx_in'][i]['Previous Tx Index'] = int((binascii.unhexlify(self.param['tx_in'][i]['Previous Tx Index'])[::-1]).hex(), 16)
        for i in range(len(self.param['tx_out'])):
            self.param['tx_out'][i]['value'] = int((binascii.unhexlify(self.param['tx_out'][i]['value'])[::-1]).hex(), 16)

    def toJSON(self):
        return json.dumps(self.param, default=lambda o: o.__dict__,
            sort_keys=True, indent=6)


import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from sklearn.cluster import Birch
import ast
import math

def tim(res):   #convert time list in 'Time' column into better format
    ans=[]
    res = ast.literal_eval(res)
    for i in range(len(res)):
        x=res[i]
        i=0
        hour=""
        minute=""
        while(x[i]!=":"):
            hour+=x[i]
            i+=1
        hour=int(hour)
        i+=1
        while(i<len(x)):
            minute+=x[i]
            i+=1
        minute=int(minute)
        ans.append(hour*60+minute)
    return ans

df1 = pd.read_csv(r'C:\Users\Ashish\The Last Credit\The Last Credit\staypoint.csv')

stored_data=[]   # storing the data in a well-format so that it could be used to retrieve information (note-it is different from the input for clustering)
for i in range(1000):
    for j in range(len(tim(df1['Time'][i]))):
        stored_data.append([df1['Vehicle No'][i],(df1['Latitude'][i],df1['Longitude'][i]),ast.literal_eval(df1['Time'][i])[j]])

# setting up distance coordinates
"""we know that distance between gps coordinates does not directly correspond to distance in kms
Therefore to scale that we should know how the distance is calculated.
In general the distance between two gps coordinates is defined as,
d = acos( sin φ1 ⋅ sin φ2 + cos φ1 ⋅ cos φ2 ⋅ cos Δλ ) ⋅ R

where,
φ1 = lat1 * Math.PI/180
φ2 = lat2 * Math.PI/180
Δλ = (lon2-lon1) * Math.PI/180
R = 6378.8*1000 metres

But in case of shorter distances, this formula reduces to,
d = R ⋅ √(x² + y²)

where,
x = (λ2-λ1) * Math.cos((φ1+φ2)/2)
x = (λ2-λ1) * 0.877 (approx.)
x = (λ2*0.877 - λ1* 0.877)
y = (φ2-φ1)

x(i) = R*λ(i)*0.877 = R*lon(i)*math.pi/180
y(i) = R*lat(i)*math.pi/180
d(i)=√({x(i)}²+{y(i)}²)
"""

# scaling of time wrt distance
"""Current scale is: 1 minute on time-axis is equivalent to 1 km on distance axis.
The current equation of sphere is (delta_x)^2 + (delta_y)^2 + (delta_t)^2 = (radius)^2
As the current radius of uncertainity wrt distance is 30 metres and current time uncertainity from mean time is 30 minutes.
Taking radius as 30 implies that maximum time variation from mean point at any moment will be 30 minutes according to
the scale we defined in the first line. We will get the maximum time variation at any instance when variation wrt to
distance is zero at that moment. Taking the radius as 30 also implies that maximum uncertainity wrt distance at any moment
is 30 metres and this uncertainity will only achieve if the time variation from the mean time is zero at that instance.

According to this scale (stated in first line) 'current radius of uncertainity wrt distance' and 'current time uncertainity
from mean time' will get an equal weightage.
"""

"""Changing the scale will alter the weightage of both the uncertainities.
"""
# The Scaling
current_radius_of_uncertainity = 30 # in metres
current_time_uncertainity = 30 # in minutes
scale = current_radius_of_uncertainity /float(current_time_uncertainity)   # this will scale the time axis wrt distance axes
radius = current_radius_of_uncertainity

data=[]    #creating  an array of the data to be clustered in the appropriate scaled format (note-it will be used as input for finding clusters)
for i in range(1000):
    for j in range(len(tim(df1['Time'][i]))):
        data.append([1000*6378.8*df1['Latitude'][i]*(math.pi/180),1000*6378.8*df1['Longitude'][i]*0.877*(math.pi/180),scale*(tim(df1['Time'][i])[j])])

# clustering process starts
brc = Birch(n_clusters=None,threshold=radius,compute_labels=True)   # setting up initial conditions for birch clustering
y=brc.fit(data)  # input data has  been fit for birch clustering
labels=y.labels_   # an array in which index is the staypoint no. and index value is its cluster no.

#storing the array(output) in a dictionary
k={}
for i,label in enumerate(labels):
    k[label]=k.get(label,[])
    k[label].append(i)

# writing a function to retrieve information from stored data which has been created in line no.30
def information(value):
    arr=[[],[],[]]
    for i in range(len(value)):
        I=value[i]
        arr[0].append(stored_data[I][0])
        arr[1].append(stored_data[I][1])
        arr[2].append(stored_data[I][2])
    return arr

# Changing the output format to same as previous one
centers=y.subcluster_centers_
arr=[]
for i in range(len(centers)):
    a=centers[i][0]/(6378.8*1000*(math.pi/180))
    b=centers[i][1]/(6378.8*1000*(math.pi/180)*0.877)
    c=str(int((centers[i][2]/scale)//60)).zfill(2)+":"+str(int((centers[i][2]/scale)%60)).zfill(2)
    arr.append([a,b,c])

# saving the modified data into final array
final_arr=[]
for key,value in k.items():
    arr[key].append(information(value))
    final_arr.append(arr[key])

df2=pd.DataFrame(columns=['Mean Latitude','Mean Longitude','Mean Time','Total Staypoints','Availability','Vehicles','Chances','Vehicle No.s','Vehicle Coordinates','Vehicle Stay Times','Vehicle Coordinates Mean','Vehicle Stay Times Mean'])

# writing down into the output into a new dataframe
for i in range(len(final_arr)):
    df2.loc[i]=None
    df2['Mean Latitude'][i]=final_arr[i][0]
    df2['Mean Longitude'][i]=final_arr[i][1]
    df2['Mean Time'][i]=final_arr[i][2]
    df2['Total Staypoints'][i]=len(final_arr[i][3][0])

    # for Availability and vehicles
    store=dict()
    for element in final_arr[i][3][0]:
        store[element]=store.get(element,0)+1

    chances=[]
    vehicles=[]
    for key,value in store.items():
        vehicles.append(key)
        chances.append(value)

    df2['Chances'][i]=chances
    df2['Availability'][i]=len(store)
    df2['Vehicles'][i]=vehicles
    df2['Vehicle No.s'][i]=final_arr[i][3][0]
    df2['Vehicle Coordinates'][i]=final_arr[i][3][1]
    df2['Vehicle Stay Times'][i]=final_arr[i][3][2]

    # for vehicle coordinates mean
    sum_lat=0
    sum_long=0
    for j in range(len(final_arr[i][3][1])):
        sum_lat+=float(final_arr[i][3][1][j][0])
        sum_long+=float(final_arr[i][3][1][j][1])
    mean_lat=sum_lat/len(final_arr[i][3][1])
    mean_long=sum_long/len(final_arr[i][3][1])

    df2['Vehicle Coordinates Mean'][i]=(mean_lat,mean_long)
    # for vehicle stay times mean
    minutes=0
    res = final_arr[i][3][2]
    for l in range(len(res)):
        x=res[l]
        j=0
        hour=""
        minute=""
        while(x[j]!=":"):
            hour+=x[j]
            j+=1
        hour=int(hour)
        j+=1
        while(j<len(x)):
            minute+=x[j]
            j+=1
        minute=int(minute)
        minutes+=hour*60+minute
    minutes=minutes/len(res)
    hour=int(minutes//60)
    minute=int(minutes%60)

    df2['Vehicle Stay Times Mean'][i]=str(hour)+":"+str(minute)

df2=df2.sort_values(by=['Availability'], ascending=False)           #sorting in accordance with no. of unique taxis in a cluster

df2.to_csv(r'C:\Users\Ashish\The Last Credit\The Last Credit\clust.csv')