# coding: utf-8

'''
	Codded By :

 █     █░ ██▓ ██▓    ▓█████▄  ▒█████   ███▄    █  ██▓ ▒█████   ███▄    █
▓█░ █ ░█░▓██▒▓██▒    ▒██▀ ██▌▒██▒  ██▒ ██ ▀█   █ ▓██▒▒██▒  ██▒ ██ ▀█   █
▒█░ █ ░█ ▒██▒▒██░    ░██   █▌▒██░  ██▒▓██  ▀█ ██▒▒██▒▒██░  ██▒▓██  ▀█ ██▒
░█░ █ ░█ ░██░▒██░    ░▓█▄   ▌▒██   ██░▓██▒  ▐▌██▒░██░▒██   ██░▓██▒  ▐▌██▒
░░██▒██▓ ░██░░██████▒░▒████▓ ░ ████▓▒░▒██░   ▓██░░██░░ ████▓▒░▒██░   ▓██

 --------------------------------------------
|          	     trainer class
| -------------------------------------------
| extract the latent space of training data
| using variational autoencoder
|
| access granted for : dataloader_ object
| save				 : save the trained model
| load               : load existing model if there is
| train              : train the model on dataloader objcet
| __call__           : get the latent space of the data
| decode             : decode the input data
| recons	     : reconstruction of input data
|

'''

from ._vae import VAE
import numpy as np
import os
import sys
import matplotlib.pyplot as plt
import torch
import torch.optim as optim
from torch.autograd import Variable

MODEL_PATH = os.path.dirname(os.path.abspath(__file__)) + '/utils/pc_model_vae.pth'

class trainer():

	def __init__(self, data, device, latent_dim=2, epoch=30):

		cuda = torch.cuda.is_available() if device is 'cuda' else None
		self.__device = torch.device("cuda" if cuda else "cpu")
		torch.backends.cudnn.benchmark = True

		if data is not None and type(data) is torch.utils.data.dataloader.DataLoader:
			self.dataloader_ = data
		else:
			print("[?] please specify a training pytorch dataloader object for training VAE model.")
			sys.exit(1)

		self.epoch = epoch
		self.loss_tracker = []
		self.loss = 0.0

		print("\n________dataset information during extracting features using VAE________\n")
		print(f"{self.dataloader_.dataset}\n")

		if os.path.exists(MODEL_PATH):
			print("\n________found existing pre-trained VAE model________\n")
			self.__load(latent_dim=latent_dim)

		else:
			# -------------------------------------------------
			#  training Variational Autoencoder Model
			# -------------------------------------------------
			print("\n________found no existing pre-trained model________\n")
			print(f"\t➢   training on latent space using VAE on {self.__device}\n")

			if self.epoch > 40:
				print("[?] please specify an epoch < 40 or at most 40.")
				sys.exit(1)
			else:
				self.vae_model = VAE(pc_features=self.__show_sample().shape[1], latent_dim=latent_dim).to(self.__device)
				self.__train(log_interval=500)

	def __train(self, log_interval):
		self.optimizer = torch.optim.Adam(self.vae_model.parameters(), lr=1e-3)
		for e in range(self.epoch):
			self.vae_model.train()
			for i_batch, sample_batch in enumerate(self.dataloader_):
				self.optimizer.zero_grad()
				reconstructed_batch, mu, log_variance = self.vae_model(sample_batch.float().to(self.__device))
				self.loss = self.vae_model.loss(reconstructed_batch, sample_batch, mu, log_variance)
				self.loss.backward() # calculate the gradient using computational graph for all weights
				self.optimizer.step() # update weights and other parameters like biases
				if (i_batch % log_interval == 0) and (e % 10 == 0):
					print("\t\tEpoch: {} [{}/{}]\tLoss: {:.6f}".format(e, i_batch, len(self.dataloader_), self.loss.data/len(sample_batch)))
			self.loss_tracker.append(self.loss.data)
			print(f"\t\t=====> Epoch: {e} done! - Batch shape: {sample_batch.shape}")

		checkpoint = {
			'model_state_dict': self.vae_model.state_dict(),
			'optimizer_state_dict': self.optimizer.state_dict(),
			'epoch': e+1,
			'loss': self.loss,
			'loss_tracker': self.loss_tracker
		}

		self.__save(checkpoint=checkpoint)

	def __show_sample(self):
		batch_index = torch.randint(len(self.dataloader_), (1,), device=self.__device)[0]
		for i_batch, sample_batch in enumerate(self.dataloader_):
			if i_batch == batch_index:
				break
		return sample_batch

	def __save(self, checkpoint):

		try:
			print("\n________saving trained VAE model________\n")
			torch.save(checkpoint, MODEL_PATH)
			print(f"\t➢   saved VAE model info at {MODEL_PATH}________\n")
		except IOError:
			print(f"\t➢   can't save VAE model at : {MODEL_PATH}\n")

	def __load(self, latent_dim):
		try:
			checkpoint = torch.load(MODEL_PATH)
			print(f"\t➢   loaded pre-trained model from {MODEL_PATH}\n")
		except IOError:
			print(f"\t➢   can't load pre-trained model from : {MODEL_PATH}\n")

		self.vae_model = VAE(pc_features=self.__show_sample().shape[1], latent_dim=latent_dim).to(self.__device)
		self.vae_model.load_state_dict(checkpoint['model_state_dict'])

		self.optimizer = optim.Adam(self.vae_model.parameters(), lr=1e-3)
		self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])

		self.epoch = checkpoint['epoch']
		self.loss = checkpoint['loss']
		self.loss_tracker = checkpoint['loss_tracker']

		self.vae_model.eval()

	def __call__(self, data):
		'''
			data -> encode -> mu, log_variance -> reparam
			return : numpyndarray
		'''
		data = Variable(torch.from_numpy(data), requires_grad=False)
		self.vae_model.eval()
		return self.vae_model.get_latent_z(data.float()).data.numpy()

	def decode(self, latent):
		'''
			reparam -> decode
			return : pytorch tensor
		'''
		latent = Variable(torch.from_numpy(latent), requires_grad=False)
		rp = self.vae_model.decode(latent.float())
		return rp

	def recons(self, data):
		'''
			data -> encode -> mu, log_variance -> reparam -> decode
			this method is the combination of decode and __call__ method.
			return : pytorch tensor
		'''
		self.vae_model.eval()
		data = torch.from_numpy(data)
		data = Variable(data, requires_grad=False)
		reconstructed_batch, mu, log_variance = self.vae_model(data.float().to(self.__device))
		return reconstructed_batch, mu, log_variance

	def plot_loss(self):
		print("\n________plotting VAE model training loss________\n")
		fig_path = os.path.dirname(os.path.abspath(__file__))+'/utils/pc_model_loss.png'
		plt.figure()
		plt.plot(np.array(self.loss_tracker), label='loss')
		plt.xlabel('epoch', fontsize=10)
		plt.ylabel('loss', fontsize=10)
		plt.legend()
		plt.savefig(fig_path)
		print(f"\t➢   plot saved at {fig_path}\n")


"""Make a scatterplot of male and female articles."""
import wikibios
from datetime import datetime
from matplotlib import pyplot

figure = pyplot.figure()

# get the current axes
axes = figure.gca()

firstedits_male = wikibios.columns_male['firstedit']
birth_years_male = wikibios.columns_male['birth_year']

# for articles about males, make a scatter plot of the first edit date vs birth year
# label the plot with parameter keyword label='Male', make the points green and very
# translucent (alpha=0.1).
axes.scatter(firstedits_male, birth_years_male, alpha=0.1, c='green', edgecolors='None', label='Male')

firstedits_female = wikibios.columns_female['firstedit']
birth_years_female = wikibios.columns_female['birth_year']
axes.scatter(firstedits_female, birth_years_female, alpha=0.1, c='orange', edgecolors='None', label='Female')
axes.set_xlabel('Article Date')
axes.set_ylabel('Birth Year')

# legend() displays the legend made up of the labels specified above.
axes.legend()

figure.savefig('scatter.png')


import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import networkx as nx
from scipy.stats import wishart, gamma
from math import pi
from sklearn import preprocessing

#import data

os.chdir('/home/david/Dropbox/3. MSc/Statistik Seminar/Graphical-Network-Models-for-Financial-Flows')

from src.lib import static_network_calculation
from src.lib import graph_decomposition
from pgmpy.models import JunctionTree
from src.lib import tools

df = pd.read_csv(os.getcwd() + '/data/raw/WEBSTATS_LBS_D_PUB_DATAFLOW_csv_col.csv')

#print(  df.head()  )

#cleaning dataframe

#available countrys
#print( np.unique( country ))

#selection: see paper
selected_countries = ['AT', 'BS', 'BH', 'BE', 'CA',
                      'KY', 'DK', 'FI', 'FR', 'DE',
                      'HK', 'IE', 'IT', 'JP', 'LU',
                      'NL', 'AN', 'NO', 'SG', 'ES',
                      'SE', 'CH', 'GB', 'US']

#select only selected_countries that transfer money to selected countries
if True:
	df = df[ df['L_REP_CTY'].isin(selected_countries) &
		df['L_CP_COUNTRY'].isin(selected_countries) ]

#select only cross border fincancial flows and only all sector in Counterparty sector
#
df = df[ (df['L_POS_TYPE']=='N') &  ( df['Counterparty sector']=='All sectors')
           & (df['Type of instruments']=='All instruments') & (df['L_MEASURE']== 'F')
	&(df['Balance sheet position']=='Total liabilities') ]

#subset on time scale
df = df.iloc[:, list(range(0,24)) + list(range(df.columns.get_loc("1983-Q4"),
		df.columns.get_loc("2011-Q3")+1)) ]

#check if data is complete
#df['L_REP_CTY'].value_counts()
#df['L_REP_CTY'].unique().shape

df.to_csv('data/intermediate/cleaned_data.csv', sep = ",")

#aggregate data by row wise summation for each country
df_aggr_sum = df.groupby(['L_REP_CTY'], as_index=True)[df.filter(like='Q').columns].sum()

#plot missing values
plt.clf()
df_missing = df_aggr_sum.transpose()
missing = df_missing.isnull().sum()
missing.plot(kind = 'bar')
plt.xlabel('country')
plt.ylabel('number of missing values')
plt.title('number of missing values per country')
plt.savefig('results/outputs/missing_value_statistics.png')
plt.show()

#omit countries where no data is available
#partially complete data
df_partial = df_aggr_sum.dropna(axis = 0, how = 'all')

#complete data
df_complete = df_aggr_sum.dropna(axis = 0, how = 'any')

#test whether data is normally distributed
from scipy.stats.mstats import normaltest
k2, p_values = normaltest(df_complete, axis = 1)
#add nodenames for p_values
df_test_statistic = pd.DataFrame(p_values, columns=['p-value'], index = df_complete.index)
df_test_statistic['test-statistic'] = np.asarray(k2)
df_test_statistic.iloc[:,0] = df_test_statistic.iloc[:,0].round(4)
df_test_statistic.iloc[:,1] = df_test_statistic.iloc[:,1].round(4)
df_test_statistic.to_csv(os.getcwd()+'/results/outputs/presentation/normal_test_before.csv')

#output preprocessed data to file

#transform time series so that it is normally distributed
#take log difference
#add constant value to data series so every value becomes positive
lag = 1
def log_lag_transform(series, lag):
	return np.log(series +100000000).diff(periods = lag)  #+2000 necessary so that log can be performed (nonnegative values required)

list_series = [log_lag_transform(df_complete.loc[i,:], lag = 1) for i in df_complete.index]

df_complete_transformed = pd.DataFrame()

for i in range(0, len(list_series)):
	df_complete_transformed = df_complete_transformed.append(list_series[i])

#remove the columns with na
df_complete_transformed = df_complete_transformed.dropna(axis=1)

#scale data series
df_complete_transformed = tools.scale_input_matrix(df_complete_transformed)

#test for normality after transformation
k2, p_values = normaltest(df_complete_transformed, axis = 1)
#add nodenames for p_values
df_test_statistic = pd.DataFrame(p_values, columns=['p-value'], index = df_complete.index)
df_test_statistic['test-statistic'] = np.asarray(k2)
df_test_statistic.iloc[:,0] = df_test_statistic.iloc[:,0].round(4)
df_test_statistic.iloc[:,1] = df_test_statistic.iloc[:,1].round(4)
df_test_statistic.to_csv(os.getcwd()+'/results/outputs/presentation/normal_test_after_transform.csv')

df_complete.to_csv(os.getcwd() +'/data/intermediate/complete_preprocessed_data.csv')

#partial complete data
# from sklearn.preprocessing import Imputer
# imp = Imputer(strategy="mean", axis=0) #impute missing values
# df_partial = pd.DataFrame(imp.fit_transform(df_partial), columns= df_partial.columns, index = df_partial.index)
# df_partial = tools.scale_input_matrix(df_partial)
df_partial.to_csv(os.getcwd() +'/data/intermediate/partially_preprocessed_data.csv')

X = df_complete.transpose()

full_node_names = ['Belgiun', 'Switzerland', 'Germany', 'Denmark', 'France',
                   'United Kingdom', 'Ireland', 'Japan', 'Luxenburg', 'Netherlands',
                   'Sweden', 'United States']
# full_node_names = ['Bahamas', 'Finland', 'France', 'Hong Kong', 'Bahrain', 'Cayman Islands', 'Denmark',
#  'Ireland', 'Austria', 'Belgium', 'Canada', 'Germany']

# #write covariance matrix to file
# df_cov = X.cov()
# df_cov.to_csv(os.getcwd() +'/data/intermediate/sample_covariance_matrix.csv')

#output cleaned data to csv file
#X.to_csv('data/intermediate/preprocessed_data.csv', sep = ",")
marginal_correlation_matrix = static_network_calculation.calculate_marginal_correlation_matrix(X)

partial_correlation_matrix = static_network_calculation.calculate_partial_correlation_matrix(X)

#plotting covariance matrix as network graph
adjacency_matrix_marginal = static_network_calculation.calculate_adjacency_matrix(marginal_correlation_matrix)
adjacency_matrix_partial = static_network_calculation.calculate_adjacency_matrix(partial_correlation_matrix)

#create graph for marginal correlation analysis
G_marginal = nx.from_numpy_matrix(adjacency_matrix_marginal)
mapping = dict(zip(list( range(0,len(selected_countries)) ), selected_countries)) #rename nodes
G_marginal = nx.relabel_nodes(G_marginal,mapping)

#plotting graph
# image = static_network_calculation.show_graph_with_labels(G)
# image.savefig('results/outputs/example_network_structure.png')

# print( nx.connected_components(G) )

# sorted(nx.degree(G).values())

#development of a systemic risk measure
#calculate eigenvector centrality -> calculates the importance of a node in a network
eigenvector_centrality =static_network_calculation.nx.eigenvector_centrality(G_marginal)
#assign eigenvector centrality as attribute to the nodes in the graph
nx.set_node_attributes(G_marginal, 'eigenvector_centrality', eigenvector_centrality)
color_values = list( range(0,len(G_marginal.nodes() )))
nodes_dict = dict(zip(G_marginal.nodes(), full_node_names))

#plot graph with different node size dependent on eigenvector centrality
plt.clf()
nx.draw_networkx(G_marginal, node_size=[v * 5000 for v in eigenvector_centrality.values()], with_labels=True,
                 node_color=color_values, alpha = 0.9, edge_color='b',  cmap=plt.get_cmap('Dark2'), labels = nodes_dict)
plt.axis('off')
plt.title('marginal correlation graph')
plt.savefig('results/outputs/marginal_correlation_graph.png')
plt.show()

#output preprocessed graph
nx.write_adjlist(G_marginal, path = os.getcwd() + '/data/intermediate/marginal_adjacency.txt', encoding='utf-8')

#output partial correlation graph
G_partial = nx.from_numpy_matrix(adjacency_matrix_partial)
mapping = dict(zip(list( range(0,len(selected_countries)) ), selected_countries)) #rename nodes
G_partial = nx.relabel_nodes(G_partial,mapping)

#plotting graph
# image = static_network_calculation.show_graph_with_labels(G_partial)
# image.savefig('results/outputs/example_network_structure_partial.png')

#development of a systemic risk measure
#calculate eigenvector centrality -> calculates the importance of a node in a network
eigenvector_centrality =static_network_calculation.nx.eigenvector_centrality(G_partial)
#assign eigenvector centrality as attribute to the nodes in the graph
nx.set_node_attributes(G_partial, 'eigenvector_centrality', eigenvector_centrality)

#plot graph with different node size dependent on eigenvector centrality
color_values = list( range(0,len(G_partial.nodes() )))

nodes_dict = dict(zip(G_partial.nodes(), full_node_names))

plt.clf()
#plt.figure(figsize=(20,10))
nx.draw_networkx(G_partial, node_size=[v * 5000 for v in eigenvector_centrality.values()], with_labels=True,
                 node_color=color_values, alpha = 0.9, edge_color='b',  cmap=plt.get_cmap('Dark2'), labels = nodes_dict)
plt.title('partial correlation graph')
plt.axis('off')
plt.savefig('results/outputs/partial_correlation_graph.png')
plt.show()

#output preprocessed graph
# nx.write_adjlist(G_partial, path = os.getcwd() + '/data/intermediate/network_adjacency_partial.txt', encoding='utf-8')


"""
Created on Sun Jul 29 18:02:55 2018

Exécutable pour pre

@author: Claire
"""
# coding: utf-8
import numpy as np
import pandas as pd
import re
from collections import Counter
import copy
#%%

#Une liste de contractions par défaut
contractions_default = {
    "j'" : "je ",
    "t'" : "tu ",
    "ns ": "nous ",
    "vs ": "vous ",
    "pk " : "pourquoi ",
    "pq " : "pourquoi ",
    "asap" : "dès que possible",
    "cad " : "c'est à dire",
    "slt" : "salut",
    "cc" : "coucou",
    "jtm" : "je t'aime",
    "svt" : "souvent",
    "sis" : "ma soeur",
    "bro" : "mon frère",
    "omg" : "oh mon dieu",
    "dar " : "bien",
    "irl " : "dans la vraie vie ",
    "dm " : "message direct",
    "c " : "c'est",
    "mme" : "madame",
    "mr" : "monsieur",
    "mlle " : "mademoiselle ",
    "tt " : "tout ",
    "ts " : "tous ",
    "bcp " : "beaucoup ",
    "trp " : "trop "
}
#%%
def nettoyer(text, to_replace=None, stopwords = None):
    '''
    Arguments :
        - une chaine de caractères à nettoyer
        - un dictionnaire de caractères à remplacer , default = contractions
        - une liste de mots à supprimer

    Retourne :
        - la chaine nettoyée : caractères spéciaux, chiffres (et stopwords) enlevés
       '''

    if to_replace == 'default':
        to_replace = contractions_default

    # Convert words to lower case
    text = text.lower()

    # Replace contractions with their longer forms

    if to_replace :
        text = text.split()
        new_text = []
        for word in text:
            if word in to_replace:
                new_text.append(to_replace[word])
            else:
                new_text.append(word)
        text = " ".join(new_text)

    # Format words and remove unwanted characters
    text = re.sub(r'https?:\/\/.*[\r\n]*', '', text, flags=re.MULTILINE) #urls
    text = re.sub(r'\<a href', ' ', text) #balises
    text = re.sub(r'&', '', text) #&
    text = re.sub(r'[_"«»\-;%()|+&=*%.,!?:#$@\[\]/]', ' ', text) #caractères spéciaux
    text = re.sub(r'<br />', ' ', text) #balises
    text = re.sub(r'\'', ' ', text)

    # Optionally, remove stop words
    if stopwords:
        text = text.split()
        stops = stopwords
        text = [w for w in text if not w in stops]
        text = " ".join(text)

    return text
#%%
def clean_data(data, to_replace = None, stopwords = None):
    '''
    data : liste ou série de textes à nettoyer
    Retourne une liste de textes nettoyés
    '''
    clean_texts = []
    for text in data:
        clean_texts.append(nettoyer(text, to_replace, stopwords))

    return clean_texts
#%%
def count_words(compteur, text):
    '''Count the number of occurrences of each word in a set of text'''
    for sentence in text:
        for word in sentence.split():
            if word not in compteur:
                compteur[word] = 1
            else:
                compteur[word] += 1

#%%
def clean_and_count(data, to_replace, stopwords):

    # Nettoyons tout ça :
    clean_texts = clean_data(data.texte, to_replace, stopwords = None)
    print('Textes cleaned')
    clean_resumes = clean_data(data.resume, to_replace)
    print('Resumes cleaned')

    #Stats
    compteur = Counter()
    count_words(compteur, clean_texts)
    count_words(compteur, clean_resumes)
    nb_mots= len(compteur)
    print("Nombre de mots : ", nb_mots)

    return clean_texts, clean_resumes, compteur

#%%
def get_embeddings(compteur, seuil, embeddings_dim, pre_embeddings=None):
    '''
    Cette fonction récupère :
        - compteur : dictionnaire de type {'mot':fréquence}
        - seuil : seuil de fréquence au dessus duquel un mot est pertinent
        - pre_embeddings : dictionnaire d'embeddings pré-entraînés de type {'mot': array}

    Et retourne :
        - une matrice d'embeddings , np.array
        - un dictionnaire associant chaque indexe de la matrice au mot correspondant
        - un dictionnaire associant chaque mot à l'indexe correspondant dans la matrice
    '''
    #Mapping
    word2int={}
    value = 1
    for mot, compte in compteur.items():
        if compte > seuil or mot in pre_embeddings:
            word2int[mot] = value
            value+=1

    #Tokens spécifiques à seq2seq à ajouter au vocabulaire :
    #UNK : unknown
    #PAD : remplir le vide dans le batch
    #EOS : fin de la phrase (end of sentence)
    #GO : premier mot envoyé dans le batch
    codes = ["<UNK>","<EOS>","<GO>"]

    #Add PAD
    word2int["<PAD>"] = 0
    # Add codes to vocab
    for code in codes:
        word2int[code] = len(word2int)

    #Mapping inverse : d'un entier à un mot :
    int2word={}
    for mot, value in word2int.items():
        int2word[value] = mot

    assert len(int2word) == len(word2int)

    vocab_size = len(word2int)
    print('Taille du vocabulaire: ',vocab_size)
    print('Pourcentage de mots utilisés:' ,round(len(int2word)/len(compteur)*100, 2), '%')

    words_embeddings_matrix = np.zeros((vocab_size, embeddings_dim ), dtype=np.float32)

    for mot, int_value in word2int.items():
        if mot in pre_embeddings:
            words_embeddings_matrix[int_value] = pre_embeddings[mot]
            #On attribue au vecteur au hasard si on n'en trouve pas dans notre table
        else :
            new_vecteur = np.random.uniform(-1,1, embeddings_dim)
            words_embeddings_matrix[int_value] = new_vecteur
            pre_embeddings[mot] = new_vecteur

    return words_embeddings_matrix, word2int, int2word

#%%
'''
Transforme chaque donnée d'entrée en la suite du numéro onehot qui lui correspond
'''

def text2int(text, unk_count, word2int, eos=False, start_token=False):
    word_count=0
    text_int =[]
    for phrase in text:
        phrase_int=[]
        for mot in phrase.split():
            if mot in word2int:
                phrase_int.append(word2int[mot])
            else:
                phrase_int.append(word2int['<UNK>'])
                unk_count+=1
            word_count+=1
        if eos:
            phrase_int.append(word2int['<EOS>'])
        text_int.append(phrase_int)
    return text_int, word_count, unk_count
#%%

def encode_texts(clean_texts, clean_resumes, word2int):
    unk_count=0

    text_int, word_count_text, unk_count = text2int(clean_texts, unk_count, word2int, eos=False)
    resumes_int, word_count_resume, unk_count = text2int(clean_resumes, unk_count, word2int, eos=True)

    word_count_total = word_count_text + word_count_resume

    unk_percent = round(unk_count/word_count_total,4)*100
    reduction_percent = round(word_count_resume/word_count_text,2)*100

    print("Nombe total de mots dans les textes : ", word_count_text)
    print("Nombe total de mots dans les résumés : ", word_count_resume)
    print("Rapport du nb de mots de résumé à texte", reduction_percent, '%')
    print("Nombe total de mots ", word_count_total)
    print("Nombre total de UNKs :", unk_count)
    print("Pourcentage de mots UNK: {}%".format(unk_percent))

    return text_int, resumes_int
#%%

def unk_counter(sentence, unkchar_id):
    '''Counts the number of time UNK appears in a sentence.'''
    unk_count = 0
    for word in sentence:
        if word == unkchar_id:
            unk_count += 1
    return unk_count

def create_lengths(text, unkchar_id):

    '''Create a data frame of the sentence lengths from a text'''
    lengths = []
    nb_unks= []
    for sentence in text:
        lengths.append(len(sentence))
        nb_unks.append(unk_counter(sentence , unkchar_id))
    return lengths, nb_unks

def filter_data(X, Y, unkchar_id, text_len, resume_len, text_unk_limit, resume_unk_limit):

    sorted_X=[]
    sorted_Y=[]

    '''
    text_bracket : tuple de taille 2 avec longueurs max et min acceptées pour les textes
    resume_brakcet : tuple avec longueurs max et min acceptées pour les résumés
    text_unk_limit : nombre de unk acceptés pour les textes
    resume_unk_limit : nombre de unk acceptés pour les résumés

    '''

    lengths_text, nb_unks_text = create_lengths(X, unkchar_id)
    lengths_resume, nb_unks_resume = create_lengths(Y, unkchar_id)

    data = {'lengths_text':lengths_text,
            'nb_unks_text' :nb_unks_text,
            'lengths_resumes': lengths_resume,
            'nb_unks_resumes': nb_unks_resume
           }

    len_data = pd.DataFrame(data=data)
    print('Avant filtrage')
    print(len_data.describe())

    filtre = (len_data['nb_unks_resumes']<=resume_unk_limit) & (len_data['nb_unks_text'] <= text_unk_limit)
    filtre = filtre & (len_data['lengths_text'] >= text_len[0]) & (len_data['lengths_text'] <= text_len[1])
    filtre = filtre & (len_data['lengths_resumes'] >= resume_len[0]) & (len_data['lengths_resumes']  <= text_len[1])

    filtered_len = len_data[filtre]
    print('Après filtrage')
    print(filtered_len.describe())

    sorted_len = filtered_len.sort_values('lengths_text')

    for i in sorted_len.index:
        sorted_X.append(X[i])
        sorted_Y.append(Y[i])

    return sorted_X, sorted_Y
#%%
def gen_decoder_inputs_from_target(Y, go_char_id):
    '''
    Y : liste de tokenized sequences
    go_char_id : l'id du caractère de début de phrase
    '''
    decoder_inputs = copy.deepcopy(Y)
    for target in decoder_inputs:
        target.pop()
        target.insert(0,go_char_id)
    return decoder_inputs


import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import GaussianNB
from sklearn import preprocessing
import argparse

ap = argparse.ArgumentParser()
ap.add_argument("-train", "--trainingdataset", required=True)
ap.add_argument("-test", "--testingdataset")
ap.add_argument("-attri", "--attributes", type=str, required=True)
ap.add_argument("-cat", "--category", type=str)
ap.add_argument("-c", "--class", type=int, required=True)
args = vars(ap.parse_args())
le = preprocessing.LabelEncoder()

balance_data = pd.read_csv(args['trainingdataset'], sep=',')

X = balance_data.values[:, int(args['attributes'].split(':')[0]):int(args['attributes'].split(':')[1])]
Y = balance_data.values[:, args['class']]
# Y = Y.astype('int')

if args['category']:
    for i in range(int(args['category'].split(':')[0]), int(args['category'].split(':')[1])):
        col = int(args['category'].split(':')[0])
        le.fit(list(set(X[:, i - col])))
        X[:, i - col] = le.transform(X[:, i - col])

if args['testingdataset']:
    testing_data = pd.read_csv(args['testingdataset'], sep=',')
    X_test = testing_data.values[:, int(args['attributes'].split(':')[0]):int(args['attributes'].split(':')[1])]
    y_test = testing_data.values[:, args['class']]
    X_train = X
    y_train = Y
else:
    X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.3, random_state=100)

clf=GaussianNB()
clf.fit(X_train,y_train)
predicted_proba=clf.predict_proba(X_test)
acc = clf.score(X_test, y_test)
print("accuracy: " + str(acc))

Tradeoff between Performance and Explanation of Classification Models/naive_bayes.py


import numpy as np
import time
import matplotlib.pyplot as plt
import numpy.linalg as la
from scipy.spatial.distance import pdist
from scipy.spatial.distance import squareform
# 计算X矩阵的距离矩阵
def compute_squared_EDM(X):
    return squareform(pdist(X, metric='euclidean'))
# 将分类的数据不同颜色显示。
def plotFeature(data, labels_):
    clusterNum=len(set(labels_))
    fig = plt.figure()
    scatterColors = ['black', 'blue', 'green', 'yellow', 'red', 'purple', 'orange', 'brown','#BC8F8F','#8B4513','#FFF5EE']
    ax = fig.add_subplot(111)
    for i in range(-1,clusterNum):
        colorSytle = scatterColors[i % len(scatterColors)]
        subCluster = data[np.where(labels_==i)]
        ax.scatter(subCluster[:,0], subCluster[:,1], c=colorSytle, s=12)
    plt.show()
# 获得新簇集
def getNewCluster(classStart,mindisIndex,dataLength,disLength):
    # classStart[mindisIndex]是新簇集的开始坐标
    # 如果距离最小值索引是距离集合的最后一个
    # 新簇集结束坐标是data的最后一个值
    if (mindisIndex == disLength - 1):
        return classStart[mindisIndex],dataLength
    # 如果距离最小值索引不是距离集合的最后一个
    # 新簇集结束坐标为mindisIndex + 2，因为Python切片不包含索引为mindisIndex + 2的数据
    # Python切片，包含前面，不包含后面
    return classStart[mindisIndex],classStart[mindisIndex + 2]
# 获得新簇集的上一个簇集
def getPreCluster(classStart,mindisIndex):
    # 如果距离最小值索引是距离集合的第0个元素
    # 新簇集不存在上一个集合，因此返回None
    if (mindisIndex == 0):
        return None
    # 如果距离最小值索引不是距离集合的第0个元素
    # 新簇集即为：classStart[mindisIndex - 1]:classStart[mindisIndex]
    return classStart[mindisIndex - 1],classStart[mindisIndex]
# 获得新簇集的下一个簇集
def getNextCluster(classStart,mindisIndex,dataLength,disLength):
    # 如果距离最小值索引是距离集合的最后一个元素
    # 新簇集不存在下一个集合，因此返回None
    if (mindisIndex == disLength - 1):
        return None
    # 如果距离最小值索引是距离集合的倒数第二个元素
    # 新簇集的下一个集合应该为簇集集合中的最后一个簇集
    elif (mindisIndex == disLength - 2):
        return classStart[mindisIndex + 2],dataLength
    # 如果距离最小值索引不是距离集合的倒数第二个元素，也不是倒数第一个
    # 新簇集即为：classStart[mindisIndex + 2]:classStart[mindisIndex + 3]
    else:
        return classStart[mindisIndex + 2],classStart[mindisIndex + 3]
def updateNextCluster(dataDisMat,classDis,classStart,mindisIndex,dataLength,disLength):
    # 获得新产生的簇集集合
    newCluster = getNewCluster(classStart,mindisIndex,dataLength,disLength)
    # 获得与新簇集相邻的下一个簇集
    nextCluster = getNextCluster(classStart,mindisIndex,dataLength,disLength)
    # 如果preCluster为None，说明新簇集是最后一个簇集，不存在下一个簇集
    if(nextCluster is None):
        return classDis
    # 如果preCluster不为None，查找新簇集与后一个簇集的距离矩阵
    nextDisMat = dataDisMat[newCluster[0]:newCluster[1], nextCluster[0]:nextCluster[1]]
    n, m = nextDisMat.shape
    # 计算两个簇集的平均距离（这里也可以是最小距离，也可以是最大距离）
    nextDis = np.sum(nextDisMat) / (n * m)
    # 将新簇集与后一个簇集的距离赋值给mindisIndex + 1的位置上
    classDis[mindisIndex + 1] = nextDis
    return classDis
def updatePreCluster(dataDisMat,classDis,classStart,mindisIndex,dataLength,disLength):
    # 获得新产生的簇集集合
    newCluster = getNewCluster(classStart,mindisIndex,dataLength, disLength)
    # 获得新簇集的前一个集合
    preCluster = getPreCluster(classStart,mindisIndex)
    # 如果preCluster为None，说明新簇集就是第一个簇集
    if (preCluster is None):
        return classDis
    # 如果preCluster不为None，查找新簇集与前一个簇集的距离矩阵
    preDisMat = dataDisMat[preCluster[0]:preCluster[1],newCluster[0]:newCluster[1]]
    n, m = preDisMat.shape
    # 计算两个簇集的平均距离（这里也可以是最小距离，也可以是最大距离）
    preDis = np.sum(preDisMat) / (n * m)
    # 将新簇集与前一个簇集的距离赋值给mindisIndex - 1的位置上
    classDis[mindisIndex - 1] = preDis
    return classDis
def ST_AGNES(data,classNum):
    # 将data沿着y轴滚动一次
    rollData = np.roll(data, 1, axis=0)
    # 计算向量之间的距离矩阵
    dataDisMat = compute_squared_EDM(data)
    # 计算一共有多少条数据
    dataLength = len(data)
    # la.norm(rollData - data, axis=1) 求相邻数据之间的欧式距离，默认为2范数
    # np.delete()删除距离数组中的第一个元素
    # classDis中的第i个元素代表着第i与i+1类别之间的距离
    classDis = np.delete(la.norm(rollData - data, axis=1), 0)
    # 记录每一个簇集起始点的索引
    # classStart数组的长度一定比classDis数组的长度多1
    # classStart数组的长度等于簇集的个数
    # 初始化簇集起始点的索引，起始点，每一个点都是一个簇集
    classStart = np.arange(0, len(data))
    # 当簇集合并到classNum数量时，聚类过程结束
    while (len(classStart) > classNum):
        # 寻找到最小距离的id
        # 需要合并的簇集mindisIndex 和mindisIndex + 1
        mindisIndex = np.argmin(classDis)
        disLength=len(classDis)
        # 簇集向前扩展，即合并后的簇集，重新计算新簇集与前一个簇集的距离
        classDis = updatePreCluster(dataDisMat, classDis, classStart, mindisIndex, dataLength,disLength)
        # 簇集向后扩展，即合并后的簇集，重新计算新簇集与后一个簇集的距离并
        classDis = updateNextCluster(dataDisMat, classDis, classStart, mindisIndex,dataLength,disLength)
        # 在簇集合并当中mindisIndex + 1的簇集被mindisIndex的簇集合并掉
        # 因此删掉第mindisIndex + 1簇集的初始索引
        classStart = np.delete(classStart, mindisIndex + 1)
        # 在簇集合并当中mindisIndex + 1的簇集被mindisIndex的簇集合并掉
        # 并将新的簇集距离赋值给mindisIndex - 1 和 mindisIndex + 1
        # 因此去掉索引为mindisIndex的距离
        classDis = np.delete(classDis, mindisIndex)
    return classStart
# 此方法通过classStart 生成相应的簇集
# classStart的长度标志着簇集的个数
# classStart中每一个元素都标志着簇集的开始点坐标
def extract_clusters(data,classStart):
    # 将每一个簇集的类别初始化为-1
    labels=np.full((len(data),),-1)
    # 循环给簇集的每一个元素赋值
    for i in range(len(classStart)):
        if(i==(len(classStart)-1)):
            labels[classStart[i]:] = i
        else:
            labels[classStart[i]:classStart[i+1]]=i
    return labels
# 加载聚类数据
data = np.loadtxt("data/cluster.csv", delimiter=",")
# 执行聚类算法，得到聚类类别的初始点坐标索引
start = time.clock()
classStart=ST_AGNES(data,4)
end = time.clock()
# 获得每一个点的聚类类别
labels=extract_clusters(data,classStart)
# 显示图形
print('finish all in %s' % str(end - start))
plotFeature(data,labels)


import time

from django.conf import settings
from django.db.models.signals import pre_save
from django.utils.functional import curry
from django.db.models.loading import get_model
from auditlog.models import LogEntry

class AuditlogMiddleware(object):
    """
    Middleware to couple the request's user to log items. This is accomplished by currying the signal receiver with the
    user from the request (or None if the user is not authenticated).
    """

    def process_request(self, request):
        """
        Gets the current user from the request and prepares and connects a signal receiver with the user already
        attached to it.
        """
        if hasattr(request, 'user') and hasattr(request.user, 'is_authenticated') and request.user.is_authenticated():
            user = request.user
            request.auditlog_ts = time.time()
            set_actor = curry(self.set_actor, user)
            pre_save.connect(set_actor, sender=LogEntry, dispatch_uid=(self.__class__, request.auditlog_ts), weak=False)

    def process_response(self, request, response):
        """
        Disconnects the signal receiver to prevent it from staying active.
        """
        # Disconnecting the signal receiver is required because it will not be garbage collected (non-weak reference)
        if hasattr(request, 'auditlog_ts'):
            pre_save.disconnect(sender=LogEntry, dispatch_uid=(self.__class__, request.auditlog_ts))

        return response

    def process_exception(self, request, exception):
        """
        Disconnects the signal receiver to prevent it from staying active in case of an exception.
        """
        if hasattr(request, 'auditlog_ts'):
            pre_save.disconnect(sender=LogEntry, dispatch_uid=(self.__class__, request.auditlog_ts))

        return None

    @staticmethod
    def set_actor(user, sender, instance, **kwargs):
        """
        Signal receiver with an extra, required 'user' kwarg. This method becomes a real (valid) signal receiver when
        it is curried with the actor.
        """
        try:
            app_label, model_name = settings.AUTH_USER_MODEL.split('.')
            auth_user_model = get_model(app_label, model_name)
        except ValueError:
            auth_user_model = get_model('auth', 'user')
        if sender == LogEntry and isinstance(user, auth_user_model) and instance.actor is None:
            instance.actor = user


# -*- coding: utf-8 -*-
from collections import OrderedDict
import requests

from django_filters.rest_framework import DjangoFilterBackend
from rest_framework import viewsets
from rest_framework.decorators import action
from rest_framework.exceptions import APIException
from rest_framework.filters import SearchFilter
from rest_framework.response import Response
from rest_framework.status import HTTP_200_OK

from . import filters, models, serializers, services

class ArticleViewSet(viewsets.ModelViewSet):
    """ViewSet for viewing and editing Articles."""

    queryset = models.Article.objects.fast()
    serializer_class = serializers.ArticleSerializer
    filter_backends = (DjangoFilterBackend, SearchFilter)
    filterset_class = filters.ArticleFilter
    search_fields = ('description', )

    @action(detail=True, methods=['get'], url_path='eutils')
    def eutils(self, request, pk=None):
        instance = self.get_object()
        efetch = requests.get(instance.fetch_url)
        esummary = requests.get(instance.summary_url)

        if efetch.status_code != 200:
            raise APIException(
                detail='Could not fetch from eutils',
                code=efetch.status_code,
            )

        if esummary.status_code != 200:
            raise APIException(
                detail='Could not fetch from eutils',
                code=esummary.status_code,
            )

        data = services.parse_efetch(
            efetch.text,
            esummary.json(),
            instance.identifier
        )

        return Response(OrderedDict(data), HTTP_200_OK)


# coding: utf8
import telebot
import func
import connector
from multiprocessing import Process
from message_templates import *
import re
import imgur

commands_validate_pattern = re.compile(r'/[A-Za-z_]+')
processes = {}
token = '638360024:AAGohOi4-86lRzixBH1rK4JQmgKwRo_R4M4'
bot = telebot.TeleBot(token)
client_status = {}

def dispatch(user_id, text, _bot=None, for_all=False):
    if not for_all:
        users = DB.get_all_users(user_id=user_id, bot=_bot)
        bot_token = DB.get_one_time_token(user_id=user_id, bot_name=_bot)[0][0]
        __bot = telebot.TeleBot(bot_token)
        for user in users[0][0].split():
            print('Sending to user {}, message: {}'.format(user, text))
            __bot.send_message(chat_id=user, text=text)
    else:
        data = DB.select_info_for_all_dispatch(user_id=user_id)
        for _token in data:
            __bot = telebot.TeleBot(_token)
            for user in data[_token]:
                __bot.send_message(chat_id=user, text=text)

def get_bot_name(_token):
    return telebot.TeleBot(_token).get_me().username

def check_token(_token):
    driver = telebot.TeleBot(_token)
    try:
        driver.get_me()
        return True
    except:
        return False

@bot.message_handler(commands=['start', 'run'])
def greeting(message):
    markup = telebot.types.ReplyKeyboardMarkup(one_time_keyboard=False, resize_keyboard=True)
    markup.row('Добавить бота', 'Мои боты')
    markup.row('Как это работает?', 'Поддержка')
    bot.send_message(message.chat.id,
                     text=how,
                     reply_markup=markup,
                     parse_mode='Markdown')

@bot.message_handler(content_types=["text", 'photo'])
def add_bot(message):
    if message.text == 'Добавить бота':
        reset_option(message)
        markup = telebot.types.InlineKeyboardMarkup()
        cancel_button = telebot.types.InlineKeyboardButton(text='Отмена', callback_data='Отмена')
        markup.add(cancel_button)
        bot.send_message(message.chat.id, text='Отправьте токен, или перешлите сообщение с токеном от @BotFather.',
                         reply_markup=markup)
        client_status[message.from_user.id] = {'token': 'wait'}
    elif message.text == 'Мои боты':
        reset_option(message)
        bot_settings(msg=message)
    elif message.text == 'Как это работает?':
        reset_option(message)
        bot.send_message(message.chat.id, text=how,
                         parse_mode='Markdown')
    elif message.text == 'Поддержка':
        reset_option(message)
        keyboard = telebot.types.InlineKeyboardMarkup()
        donate = telebot.types.InlineKeyboardButton(text='Донат', callback_data='donate')
        keyboard.add(donate)
        bot.send_message(message.chat.id, text=support, reply_markup=keyboard)
    elif message.from_user.id in client_status and 'token' in client_status[message.from_user.id]:
        add_new_bot(message)
    elif message.from_user.id in client_status and 'option' in client_status[message.from_user.id]:
        bot_name = client_status[message.from_user.id]['bot_name']
        main_bots_settings(message=message, user_id=message.from_user.id,
                           option=client_status[message.from_user.id]['option'],
                           value=message.text,
                           bot_name=bot_name if bot_name != 'all' else False)

def add_new_bot(message):
    if client_status[message.from_user.id]['token'] == 'wait':
        if len(message.text.split()) == 1:
            _token = message.text
        else:
            try:
                _token = message.text.split('\n\n')[1].split('API:\n')[1]
            except IndexError:
                return bot.send_message(chat_id=message.from_user.id, text='Токен необнаружен. Попробуйте снова.')
        client_status[message.from_user.id]['token'] = _token
        try:
            if client_status[message.from_user.id]['token'] not in DB.get_all_tokens(extend=True):
                if check_token(client_status[message.from_user.id]['token']):
                    process = Process(target=func.main, args=(_token, message.from_user.id))
                    name = get_bot_name(_token)
                    if str(message.from_user.id) not in processes:
                        processes[str(message.from_user.id)] = {name: process}
                    else:
                        processes[str(message.from_user.id)].update({name: process})
                    process.start()
                    DB.insert_new_bot(user=str(message.from_user.id),
                                      bot_name=get_bot_name(client_status[message.from_user.id]['token']),
                                      token=client_status[message.from_user.id]['token'])
                    bot.send_message(message.chat.id, text='*Бот создан.*', parse_mode='Markdown')
                    bot_settings(message)
                else:
                    bot.send_message(message.chat.id, text='Токен невалидный. Попробуйте снова.')
                    client_status[message.from_user.id]['token'] = 'wait'
            else:
                client_status[message.from_user.id]['token'] = 'wait'
                bot.send_message(message.chat.id, text='Бот с таким токеном уже существует. Попробуйте снова.')
        except Exception as bot_create_error:
            bot.send_message(message.chat.id, bot_create_error)

def reset_option(message):
    if message.from_user.id in client_status:
        user = client_status[message.from_user.id]
        if 'option' in user:
            del client_status[message.from_user.id]['option']
        if 'token' in user:
            del client_status[message.from_user.id]['token']

def validate_commands(commands):
    result = commands_validate_pattern.findall(commands)
    if result:
        return len(result) > 1, result

def add_more_commands(commands, message):
    commands_set = []
    for command in commands:
        if command not in [x[0] for x in DB.get_commands(_all=True, user_id=message.from_user.id)]:
            print(command)
            DB.add_command(user_id=message.from_user.id, command=command, msg='empty')
        else:
            commands_set.append(command)
    if commands_set:
        message_text = '('+', '.join(commands_set).strip(', ')+')'
        bot.send_message(chat_id=message.from_user.id,
                         text='Некоторые из команд у Вас уже есть. {}\nОни небыли добавлены.'.format(message_text))
    else:
        bot.send_message(chat_id=message.from_user.id, text='Команды добавлены.')
        return True
    return False

def main_bots_settings(message, user_id, option, bot_name=False, value=False):
    if bot_name:
        if option == 'first':
            DB.reset_greeting(user_id=user_id, bot_name=client_status[user_id]['bot_name'],
                              new_greeting=value)
            del client_status[user_id]['option']
            return set_greetings(chat_id=user_id)
        elif option == 'delay':
            if value.isdigit():
                DB.set_greeting_delay(user_id=user_id, value=value,
                                      bot_name=client_status[user_id]['bot_name'])
                del client_status[user_id]['option']
                bot.send_message(user_id, text='Готово.')
            else:
                bot.send_message(message.from_user.id, text='Введите задержку в секундах, например: *18*',
                                 parse_mode='Markdown')
                return
        elif option == 'second':
            DB.reset_greeting(second=True, user_id=user_id, new_greeting=value, bot_name=bot_name)
            bot.send_message(user_id, text='Готово.')
            del client_status[user_id]['option']
            return set_greetings(chat_id=user_id)
        elif option == 'dispatch':
            dispatch(_bot=bot_name, user_id=user_id, text=message.text)
            bot.send_message(message.chat.id, text='Рассылка окончена.')
    else:
        if option in ['first', 'delay', 'second']:
            if option == 'first':
                DB.reset_greeting(user_id=user_id, new_greeting=value)
                bot.send_message(user_id, text='Готово.')
                del client_status[user_id]['option']
                return set_greetings(chat_id=user_id, _all=True)
            elif option == 'delay':
                if value.isdigit():
                    DB.set_greeting_delay(user_id=user_id, value=value, for_all=True)
                else:
                    bot.send_message(user_id, text='Введите задержку в секундах, например: *18*',
                                     parse_mode='Markdown')
                    return
            elif option == 'second':
                DB.reset_greeting(second=True, user_id=user_id, new_greeting=value)
                bot.send_message(user_id, text='Готово.')
                del client_status[user_id]['option']
                return set_greetings(chat_id=user_id, _all=True)
        elif option == 'dispatch':
            dispatch(for_all=True, user_id=user_id, text=message.text)
            bot.send_message(message.chat.id, text='Рассылка окончена.')
        elif option == 'add':
            validate = validate_commands(message.text)
            if not validate[0]:
                keyboard = telebot.types.InlineKeyboardMarkup()
                cancel = telebot.types.InlineKeyboardButton(text='Отмена', callback_data='cancel')
                keyboard.add(cancel)
                if message.text not in [x[0] for x in DB.get_commands(_all=True, user_id=message.from_user.id)]:
                    if message.text.startswith('/'):
                        client_status[user_id]['command'] = validate[1][0]
                        client_status[user_id]['option'] = 'set_value'
                        bot.send_message(chat_id=user_id, text='Укажите значение для команды *(только текст или смайлы)*',
                                         parse_mode='Markdown', reply_markup=keyboard)
                    else:
                        bot.send_message(user_id, text='Ошибочный формат команды. Попробуйте снова.',
                                         reply_markup=keyboard)
                    return
                return bot.send_message(message.chat.id, text='У Вас уже есть такая команда. Укажите другое имя.',
                                        reply_markup=keyboard)
            else:
                add_more_commands(validate[1], message)
        elif option == 'set_value':
            keyboard = telebot.types.InlineKeyboardMarkup()
            next_step = telebot.types.InlineKeyboardButton(text='Завершить', callback_data='next')
            keyboard.add(next_step)
            client_status[user_id]['value'] = message.text
            client_status[user_id]['option'] = 'set_image'
            bot.send_message(chat_id=user_id, text='Отправьте фото. (нажмите завершить чтобы пропустить этот шаг)',
                             reply_markup=keyboard)
            return
        elif option == 'set_image':
            command = client_status[user_id]['command']
            value = client_status[user_id]['value']
            _file_id = bot.get_file(message.photo[-1].file_id)
            path = _file_id.file_path
            image_url = str(imgur.upload_image(path))
            DB.add_command(user_id=user_id, command=command, msg=value, image=image_url)
        elif option == 'edit':
            msg = message.text
            DB.update_command(user_id=user_id, command=client_status[user_id]['command'], value=msg)
            bot.send_message(user_id, text='Готово.')
            commands_settings(chat_id=user_id, user_id=user_id, get=True)
            del client_status[user_id]['option']
            return
        del client_status[user_id]['option']
        bot.send_message(user_id, text='Готово.')
    get_two_level_settings(send=True, message=message, bot_name=client_status[message.from_user.id]['bot_name'])

def bot_settings(msg, edit=False):
    if msg.from_user.id in client_status:
        del client_status[msg.from_user.id]
    keyboard = telebot.types.InlineKeyboardMarkup()
    bots = DB.get_bots(user_id=str(msg.from_user.id))
    if bots:
        for i in bots:
            url_button = telebot.types.InlineKeyboardButton(text=i[0], callback_data=i[0])
            keyboard.add(url_button)
        change_all_greetings = telebot.types.InlineKeyboardButton(text='Все боты',
                                                                  callback_data='all')
        keyboard.add(change_all_greetings)
        if not edit:
            bot.send_message(msg.chat.id, 'Настройка ботов.', reply_markup=keyboard)
        else:
            bot.edit_message_text(chat_id=msg.message.chat.id,
                                  message_id=msg.message.message_id, text="Настройка ботов.",
                                  reply_markup=keyboard)
    else:
        bot.send_message(msg.chat.id, text='У Вас нет ботов.')

def get_two_level_settings(message, bot_name, send=False):
    client_status[message.from_user.id] = {'bot_name': bot_name}
    keyboard = telebot.types.InlineKeyboardMarkup()
    if bot_name != 'all':
        ad = DB.manage_ad(get=True, bot_name=bot_name, user_id=message.from_user.id)[0][0]
        second_status = DB.get_second_greeting_status(user_id=message.from_user.id,
                                                      bot_name=client_status[message.from_user.id]['bot_name'])[0][0]
        ad_status = ('✖ Реклама', 'on_ad') if not ad else ('✔ Реклама', 'off_ad')
        bot_status = ('✖ Приветствие', 'on') if DB.off_greeting(get_status=True,
                                                                 user_id=message.from_user.id,
                                                                 bot_name=client_status[message.from_user.id]['bot_name']) == '0' else ('✔ Приветствие', 'off')
        second_greeting = ('✖ Второе приветствие', 'on_second') if not second_status else ('✔ Второе приветствие',
                                                                                'off_second')
    else:
        second_greeting = ('✖ Второе приветствие', 'on_second')
        bot_status = ('✖ Приветствие', 'on')
        ad_status = ('✖ Реклама', 'on_ad')
        for i in DB.get_bots(user_id=message.from_user.id):
            if DB.off_greeting(get_status=True, user_id=message.from_user.id, bot_name=i[0]) == '1':
                bot_status = ('✔ Приветствие', 'off')
                break
        for i in DB.get_bots(user_id=message.from_user.id):
            ad = DB.manage_ad(get=True, bot_name=i[0], user_id=message.from_user.id)[0][0]
            if ad:
                ad_status = ('✔ Реклама', 'off_ad')
                break
        for i in DB.get_second_greeting_status(for_all=True, user_id=message.from_user.id):
            if i[0] == 1:
                second_greeting = ('✔ Второе приветствие', 'off_second')
                break
    text = 'Настройка всех ботов' if bot_name == 'all' else 'Настройка бота @{}'.format(bot_name)
    set_ad = telebot.types.InlineKeyboardButton(text=ad_status[0], callback_data=ad_status[1])
    set_greeting_button = telebot.types.InlineKeyboardButton(text='Настройка приветствий', callback_data='greeting')
    set_delay_button = telebot.types.InlineKeyboardButton(text='Установить задержку', callback_data='delay')
    on_greeting_button = telebot.types.InlineKeyboardButton(text=bot_status[0], callback_data=bot_status[1])
    back_button = telebot.types.InlineKeyboardButton(text='Назад', callback_data='back')
    users = telebot.types.InlineKeyboardButton(text='Пользователи', callback_data='users')
    spam_list = telebot.types.InlineKeyboardButton(text='Чёрный список', callback_data='spam')
    delete_bot = telebot.types.InlineKeyboardButton(text='Удалить бота' if bot_name != 'all' else 'Удалить ботов',
                                                    callback_data='delB')
    second_greeting_active = telebot.types.InlineKeyboardButton(text=second_greeting[0],
                                                                callback_data=second_greeting[1])
    my_buttons = telebot.types.InlineKeyboardButton(text='Мои кнопки', callback_data='m_buttons')
    add_button = telebot.types.InlineKeyboardButton(text='Добавить кнопку', callback_data='a_button')
    keyboard.add(on_greeting_button, second_greeting_active)
    if bot_name == 'all':
        commands_button = telebot.types.InlineKeyboardButton(text='Мои команды', callback_data='get_commands_list')
        add_command_button = telebot.types.InlineKeyboardButton(text='Добавить команду', callback_data='add_command')
        keyboard.add(set_ad, commands_button)
        keyboard.add(add_command_button, set_delay_button)
        keyboard.add(set_greeting_button, delete_bot)
        keyboard.add(my_buttons, add_button)
        keyboard.add(back_button, users)
    if bot_name != 'all':
        keyboard.add(set_ad, set_delay_button)
        keyboard.add(set_greeting_button, spam_list)
        keyboard.add(users, delete_bot)
        keyboard.add(my_buttons, add_button)
        keyboard.add(back_button)
    if not send:
        bot.edit_message_text(chat_id=message.message.chat.id, message_id=message.message.message_id,
                              text=text,
                              reply_markup=keyboard)
    else:
        bot.send_message(message.from_user.id, text=text, reply_markup=keyboard)

def add_button_callback(call):
    pass

def commands_settings(chat_id, user_id, msg_id='', get=False):
    keyboard = telebot.types.InlineKeyboardMarkup()
    delete_button = telebot.types.InlineKeyboardButton(text='Удалить', callback_data='delete_command')
    edit_button = telebot.types.InlineKeyboardButton(text='Изменить значение', callback_data='edit_command')
    show_message_button = telebot.types.InlineKeyboardButton(text='Показать значение',
                                                             callback_data='show_command')
    back_btn = telebot.types.InlineKeyboardButton(text='Назад', callback_data='get_commands_list')
    keyboard.add(edit_button, show_message_button)
    keyboard.add(back_btn, delete_button)
    if not get:
        bot.edit_message_text(chat_id=chat_id, message_id=msg_id,
                              text='Настройка команды {}'.format(client_status[user_id]['command']),
                              reply_markup=keyboard)
    else:
        bot.send_message(chat_id=chat_id, text='Настройка команды {}'.format(client_status[user_id]['command']),
                         reply_markup=keyboard)

def get_commands_list(call):
    commands = DB.get_commands(_all=True, user_id=call.from_user.id)
    if commands:
        keyboard = telebot.types.InlineKeyboardMarkup()
        for i in commands:
            command_btn = telebot.types.InlineKeyboardButton(text=i[0], callback_data=i[0])
            keyboard.add(command_btn)
        cancel = telebot.types.InlineKeyboardButton(text='Назад', callback_data='cancel')
        keyboard.add(cancel)
        return bot.edit_message_text(chat_id=call.from_user.id, message_id=call.message.message_id,
                                     text='Список Ваших команд', reply_markup=keyboard)
    bot.send_message(call.from_user.id, text='У Вас нет команд')

def user_commands_callback(call):
    if call.from_user.id in client_status:
        client_status[call.from_user.id]['command'] = call.data
        commands_settings(chat_id=call.message.chat.id, user_id=call.from_user.id, msg_id=call.message.message_id)

def cancel_callback(call):
    if call.from_user.id in client_status:
        if 'bot_name' in client_status[call.from_user.id]:
            get_two_level_settings(call, client_status[call.from_user.id]['bot_name'])
            if 'option' in client_status[call.from_user.id]:
                del client_status[call.from_user.id]['option']

def second_greeting_callback(call):
    keyboard = telebot.types.InlineKeyboardMarkup()
    back_button = telebot.types.InlineKeyboardButton(text='Назад', callback_data='greeting')
    keyboard.add(back_button)
    try:
        client_status[call.from_user.id]['option'] = call.data
    except KeyError:
        pass
    if call.from_user.id in client_status and client_status[call.from_user.id]['bot_name'] == 'all':
        bot.edit_message_text(chat_id=call.message.chat.id, message_id=call.message.message_id,
                              text='Установить второе приветствие для всех ботов.',
                              reply_markup=keyboard)
    else:
        if call.from_user.id in client_status and 'bot_name' in client_status[call.from_user.id]:
            _bot = client_status[call.from_user.id]['bot_name']
            bot.edit_message_text(chat_id=call.message.chat.id, message_id=call.message.message_id,
                                  text='Установить второе приветствие для: @{}'.format(_bot),
                                  reply_markup=keyboard)

def on_first_greeting_callback(call):
    if call.from_user.id in client_status and 'bot_name' in client_status[call.from_user.id]:
        bot_name = client_status[call.from_user.id]['bot_name']
        if bot_name != 'all':
            DB.off_greeting(user_id=call.from_user.id, bot_name=client_status[call.from_user.id]['bot_name'],
                            value='1')
        else:
            for i in DB.get_bots(user_id=call.from_user.id):
                DB.off_greeting(user_id=call.from_user.id, bot_name=i[0],
                                value='1')
        get_two_level_settings(call, client_status[call.from_user.id]['bot_name'])

def off_first_greeting_callback(call):
    if call.from_user.id in client_status and 'bot_name' in client_status[call.from_user.id]:
        bot_name = client_status[call.from_user.id]['bot_name']
        if bot_name != 'all':
            DB.off_greeting(user_id=call.from_user.id, bot_name=client_status[call.from_user.id]['bot_name'],
                            value='0')
        else:
            for i in DB.get_bots(user_id=call.from_user.id):
                DB.off_greeting(user_id=call.from_user.id, bot_name=i[0],
                                value='0')
        get_two_level_settings(call, client_status[call.from_user.id]['bot_name'])

def set_option_callback(call):
    if call.from_user.id in client_status and 'bot_name' in client_status[call.from_user.id]:
        keyboard = telebot.types.InlineKeyboardMarkup()
        back_button = telebot.types.InlineKeyboardButton(text='Назад', callback_data='cancel')
        keyboard.add(back_button)
        client_status[call.from_user.id]['option'] = call.data
        if client_status[call.from_user.id]['bot_name'] == 'all':
            bot.edit_message_text(chat_id=call.message.chat.id, message_id=call.message.message_id,
                                  text=option_text['for_all'][call.data],
                                  reply_markup=keyboard)
        else:
            bot.edit_message_text(chat_id=call.message.chat.id, message_id=call.message.message_id,
                                  text=option_text['for_one'][call.data].format(
                                      '@' + client_status[call.from_user.id]['bot_name']),
                                  reply_markup=keyboard)

def add_command_callback(call):
    if call.from_user.id in client_status:
        keyboard = telebot.types.InlineKeyboardMarkup()
        cancel = telebot.types.InlineKeyboardButton(text='Назад', callback_data='cancel')
        keyboard.add(cancel)
        bot.edit_message_text(chat_id=call.message.chat.id, message_id=call.message.message_id,
                              text='Укажите название команды.\nНапример:\n/price\n/my_shop\n/pay',
                              reply_markup=keyboard)
        client_status[call.from_user.id]['option'] = 'add'

def ad_trigger_callback(call):
    if call.from_user.id in client_status and 'bot_name' in client_status[call.from_user.id]:
        bot_name = client_status[call.from_user.id]['bot_name']
        if call.data == 'off_ad':
            if bot_name != 'all':
                DB.manage_ad(bot_name=bot_name, user_id=call.from_user.id,
                             ad_status=0)
            else:
                DB.manage_ad(for_all=True, user_id=call.from_user.id, ad_status=0)
        elif call.data == 'on_ad':
            if bot_name != 'all':
                DB.manage_ad(bot_name=bot_name, user_id=call.from_user.id,
                             ad_status=1)
            else:
                DB.manage_ad(for_all=True, user_id=call.from_user.id, ad_status=1)
        get_two_level_settings(call, bot_name)

def delete_command_callback(call):
    if call.from_user.id in client_status and 'command' in client_status[call.from_user.id]:
        DB.delete_command(user_id=call.from_user.id, command=client_status[call.from_user.id]['command'])
        get_commands_list(call)

def show_command_value_callback(call):
    if call.from_user.id in client_status and 'command' in client_status[call.from_user.id]:
        info = DB.get_commands(user_id=call.from_user.id, command=client_status[call.from_user.id]['command'])
        bot.send_message(call.from_user.id, text=info)

def edit_command_callback(call):
    if call.from_user.id in client_status:
        keyboard = telebot.types.InlineKeyboardMarkup()
        cancel = telebot.types.InlineKeyboardButton(text='Назад', callback_data='back_cm')
        keyboard.add(cancel)
        client_status[call.from_user.id]['option'] = 'edit'
        bot.edit_message_text(chat_id=call.message.chat.id, message_id=call.message.message_id,
                              text='Укажите новое значение для команды.', reply_markup=keyboard)

def dispatch_callback(call):
    if call.from_user.id in client_status:
        client_status[call.from_user.id]['option'] = 'dispatch'
        keyboard = telebot.types.InlineKeyboardMarkup()
        cancel = telebot.types.InlineKeyboardButton(text='Назад', callback_data='cancel')
        keyboard.add(cancel)
        bot.edit_message_text(chat_id=call.message.chat.id, message_id=call.message.message_id,
                              text='Введите сообщение. *(только текст и/или смайлы)*',
                              parse_mode='Markdown',
                              reply_markup=keyboard)

def users_callback(call):
    if call.from_user.id in client_status and 'bot_name' in client_status[call.from_user.id]:
        bot_name = client_status[call.from_user.id]['bot_name']
        if bot_name == 'all':
            msg = 'Вы можете отправить сообщение пользователям всех Ваших ботов'
        else:
            msg = 'Вы можете отослать сообщение всем пользователям бота @{}'.format(bot_name)
        keyboard = telebot.types.InlineKeyboardMarkup()
        dispatch_button = telebot.types.InlineKeyboardButton(text='Рассылка', callback_data='dispatch')
        cancel = telebot.types.InlineKeyboardButton(text='Назад', callback_data='cancel')
        keyboard.add(dispatch_button)
        keyboard.add(cancel)
        bot.edit_message_text(chat_id=call.message.chat.id, message_id=call.message.message_id,
                              text=msg,
                              reply_markup=keyboard)

def delete_bot_menu_callback(call):
    if call.from_user.id in client_status and 'bot_name' in client_status[call.from_user.id]:
        _bot_name = client_status[call.from_user.id]['bot_name']
        keyboard = telebot.types.InlineKeyboardMarkup()
        yes = telebot.types.InlineKeyboardButton(text='Да', callback_data='delete_bot')
        no = telebot.types.InlineKeyboardButton(text='Нет', callback_data='cancel')
        keyboard.add(yes)
        keyboard.add(no)
        if _bot_name != 'all':
            msg = 'Вы уверены что хотите удалить бота @{}?'.format(_bot_name)
        else:
            msg = 'Вы уверены что хотите удалить всех ботов?'
        bot.edit_message_text(chat_id=call.message.chat.id,
                              message_id=call.message.message_id,
                              text=msg,
                              reply_markup=keyboard)

def delete_bot_callback(call):
    if call.from_user.id in client_status and 'bot_name' in client_status[call.from_user.id] and str(
            call.from_user.id) in processes:
        _bot_name = client_status[call.from_user.id]['bot_name']
        if _bot_name != 'all':
            DB.delete_bot(user_id=call.from_user.id, bot_name=_bot_name)
            bot_process = processes[str(call.from_user.id)][_bot_name]
            if bot_process.is_alive():
                bot_process.terminate()
            del processes[str(call.from_user.id)][_bot_name]
            bot.answer_callback_query(text='Бот отключён и удалён', callback_query_id=call.id, show_alert=True)
            print('BOT {} was deleted.'.format(_bot_name))
        else:
            DB.delete_all_bots(user_id=call.from_user.id)
            for i in processes[str(call.from_user.id)].values():
                if i.is_alive():
                    i.terminate()
            del processes[str(call.from_user.id)]
            bot.answer_callback_query(text='Боты отключёны и удалёны', callback_query_id=call.id,
                                      show_alert=True)
            print('All user {} bots was deleted.'.format(call.from_user.id))
        bots = DB.get_bots(user_id=call.from_user.id)
        if not bots:
            bot.edit_message_text(chat_id=call.message.chat.id,
                                  message_id=call.message.message_id,
                                  text='У Вас нет ботов.')
        else:
            bot_settings(msg=call, edit=True)

def set_greetings_callback(call):
    if call.from_user.id in client_status and 'bot_name' in client_status[call.from_user.id]:
        pointer = True if client_status[call.from_user.id]['bot_name'] == 'all' else False
        set_greetings(call.message.chat.id, call.message.message_id, edit=True, _all=pointer)

def second_greeting_activate_callback(call):
    bot_name = client_status[call.from_user.id]['bot_name']
    if bot_name != 'all':
        DB.activate_second_greeting(user_id=call.from_user.id, bot_name=bot_name)
    else:
        if call.data == 'off_second':
            value = 0
        else:
            value = 1
        DB.activate_second_greeting(user_id=call.from_user.id, for_all=True, value=value)
    get_two_level_settings(call, bot_name)

def donate_callback(call):
    bot.send_message(chat_id=call.message.chat.id,
                     text='📥 Вы можете поддержать нас отправив Bitcoin\n\n`Средства поступят через 1 подтверждение сети.`',
                     parse_mode='Markdown')
    bot.send_message(chat_id=call.message.chat.id,
                     text='*17xDn9FC5BRHSPPEvP6bFT6znVBJLm5msX*',
                     parse_mode='Markdown')
    bot.send_photo(call.message.chat.id, open(r"QR.jpg", 'rb').read())

@bot.callback_query_handler(func=lambda call: True)
def callback_inline(call):
    if call.message:
        if call.data == 'cancel':
            cancel_callback(call)
        elif call.data == 'second':
            second_greeting_callback(call)
        elif call.data in ['off_second', 'on_second']:
            second_greeting_activate_callback(call)
        elif call.data == 'next':
            command = client_status[call.from_user.id]['command']
            value = client_status[call.from_user.id]['value']
            DB.add_command(user_id=call.from_user.id, command=command, msg=value)
            del client_status[call.from_user.id]['option']
            bot.send_message(call.from_user.id, text='Готово.')
            get_two_level_settings(send=True, message=call, bot_name=client_status[call.from_user.id]['bot_name'])
        elif call.data == 'on':
            on_first_greeting_callback(call)
        elif call.data == 'off':
            off_first_greeting_callback(call)
        elif call.data in ['on_ad', 'off_ad']:
            ad_trigger_callback(call)
        elif call.data in ['first', 'delay']:
            set_option_callback(call)
        elif call.data == 'back':
            bot_settings(call, edit=True)
        elif call.data == 'Отмена':
            if call.from_user.id in client_status:
                del client_status[call.from_user.id]
                bot.edit_message_text(chat_id=call.message.chat.id, message_id=call.message.message_id,
                                      text='Отменено')
        elif call.data == 'Назад':
            if call.from_user.id in client_status:
                del client_status[call.from_user.id]
            bot_settings(msg=call, edit=True)
        elif call.data == 'get_commands_list':
            get_commands_list(call)
        elif call.data == 'add_command':
            add_command_callback(call)
        elif call.data == 'delete_command':
            delete_command_callback(call)
        elif call.data == 'donate':
            donate_callback(call)
        elif call.data == 'show_command':
            show_command_value_callback(call)
        elif call.data == 'edit_command':
            edit_command_callback(call)
        elif call.data == 'back_cm':
            reset_option(call)
            commands_settings(chat_id=call.message.chat.id, user_id=call.from_user.id, msg_id=call.message.message_id)
        elif call.data == 'dispatch':
            dispatch_callback(call)
        elif call.data == 'users':
            users_callback(call)
        elif call.data == 'spam':
            get_spam_list(call)
        elif call.data == 'delB':
            delete_bot_menu_callback(call)
        elif call.data == 'delete_bot':
            delete_bot_callback(call)
        elif call.data == 'a_button':
            pass
        elif call.data == 'm_buttons':
            pass
        elif call.data == 'greeting':
            set_greetings_callback(call)
        else:
            try:
                user_commands = [x[0] for x in DB.get_commands(_all=True, user_id=call.from_user.id)]
            except TypeError:
                user_commands = []
            if call.data in user_commands:
                user_commands_callback(call)
            elif call.from_user.id in client_status:
                if 'bot_name' in client_status[call.from_user.id]:
                    if call.data in DB.get_banned_users(user_id=call.from_user.id,
                                                        bot_name=client_status[call.from_user.id]['bot_name']).split():
                        DB.clear_ban_user(user_id=call.from_user.id, banned_user=call.data,
                                          bot_name=client_status[call.from_user.id]['bot_name'])
                        bot.answer_callback_query(text='Пользователь разблокирован', callback_query_id=call.id)
                        get_two_level_settings(call, client_status[call.from_user.id]['bot_name'])
            elif call.data == 'all':
                get_two_level_settings(call, 'all')
            else:
                get_two_level_settings(call, call.data)

def get_spam_list(call):
    if call.from_user.id in client_status and 'bot_name' in client_status[call.from_user.id]:
        keyboard = telebot.types.InlineKeyboardMarkup()
        spam = DB.get_banned_users(user_id=call.from_user.id, bot_name=client_status[call.from_user.id]['bot_name']).split()
        if spam:
            for i in spam:
                name = bot.get_chat_member(chat_id=int(i), user_id=int(i)).user.first_name
                user_button = telebot.types.InlineKeyboardButton(text=name, callback_data=i)
                keyboard.add(user_button)
            back = telebot.types.InlineKeyboardButton(text='Назад', callback_data='cancel')
            keyboard.add(back)
            bot.edit_message_text(chat_id=call.from_user.id,
                                  text='Заблокированные пользователи. Нажмите на пользователя для разблокировки.',
                                  reply_markup=keyboard,
                                  message_id=call.message.message_id)
        else:
            bot.answer_callback_query(text='Список пуст', callback_query_id=call.id)

def set_greetings(chat_id, message_id='', edit=False, _all=False):
    if not _all:
        current_first_greeting = DB.get_greeting(user_id=chat_id, bot_name=client_status[chat_id]['bot_name'])
        current_second_greeting = DB.get_greeting(second=True, user_id=chat_id,
                                                  bot_name=client_status[chat_id]['bot_name'])
        msg = 'Настройка приветствий бота: @{}\n' \
              'Текущее первое приветствие: {}\n' \
              'Текущее второе приветствие: {}'.format(client_status[chat_id]['bot_name'],
                                                      current_first_greeting[0] if current_first_greeting is not None else 'None',
                                                      current_second_greeting[0] if current_second_greeting is not None else 'None')
    else:
        msg = 'Настройка приветствий всех ботов.'
    keyboard = telebot.types.InlineKeyboardMarkup()
    first = telebot.types.InlineKeyboardButton(text='Первое приветствие', callback_data='first')
    second = telebot.types.InlineKeyboardButton(text='Второе приветствие', callback_data='second')
    cancel = telebot.types.InlineKeyboardButton(text='Назад', callback_data='cancel')
    keyboard.add(first)
    keyboard.add(second)
    keyboard.add(cancel)
    if not edit:
        bot.send_message(chat_id=chat_id, text=msg, reply_markup=keyboard)
    else:
        bot.edit_message_text(chat_id=chat_id, message_id=message_id, text=msg, reply_markup=keyboard)

def run_bots():
    print('Запуск пользовательских ботов.')
    counter = 0
    for i in DB.get_all_tokens():
        if check_token(i[0]):
            counter += 1
            print('Запуск бота под номером: {}.'.format(counter))
            name = get_bot_name(i[0])
            process = Process(target=func.main, args=(i[0], i[1]))
            if i[1] not in processes:
                processes[i[1]] = {name: process}
            else:
                processes[i[1]].update({name: process})
            try:
                process.start()
            except RuntimeError:
                print('Can\'t start new process')
        else:
            print('Token {} was deleted'.format(i[0]))
            DB.delete_items(token=i[0], user_id=i[1])

def main():
    run_bots()
    print('Основной бот запущен.')
    bot.polling(none_stop=True)

if __name__ == '__main__':
    DB = connector.DataBaseConnect()
    main()


#itertools-用于操作迭代对象的函数
import itertools

#无限迭代器
natuals = itertools.count(1)
for n in natuals:
	print(n)

#无限循环序列
cs = itertools.cycle('ABC')
for c in cs:
	print(c)

#循环元素，可限定次数
ns = itertools.repeat('A', 3)
for n in ns:
	print(n)

#截取有序序列
natuals = itertools.count(1)
ns = itertools.takewhile(lambda x:x<=10,natuals)
list(ns)

#串联迭代对象
for c in itertools.chain('ABC','XYZ'):
	print(c)

#把迭代器中相邻的重复元素挑出来放在一起
for key,group in itertools.groupby('AAABBCCAA',lambda c:c.upper()):
	print(key,list(group))

#计算圆周率
def pi(N):
	#setp 1:创建一个奇数数列
	natuals = itertools.count(1,2)
	#setp 2:取该序列的前N项
	ns = itertools.takewhile(lambda x:x<=(2*N-1),natuals)
	#setp 3:添加正负符号并用4除 4/1 -4/3 4/5
	#setp 4:求和
	res = 0
	for k,v in enumerate(list(ns)):
	    res += 4/v if k%2==0 else -4/v
	return res

print(pi(10))
print(pi(100))
print(pi(1000))
print(pi(10000))
assert 3.04 < pi(10) < 3.05
assert 3.13 < pi(100) < 3.14
assert 3.140 < pi(1000) < 3.141
assert 3.1414 < pi(10000) < 3.1415
print('ok')


__author__ = 'Ryan'
from bs4 import *
import pandas as pd
import requests
import pickle
import datetime
import math
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import os
import time
import random
from urllib.request import urlretrieve
import sys
import shutil

class FG_Scrape:
    def __init__(self):
        self.addr = '/projections/'
        self.zips_ros_b = 'http://www.fangraphs.com/projections.aspx?pos=all&stats=bat&type=rzips&team=0&lg=all&players=0'
        self.zips_ros_p = 'http://www.fangraphs.com/projections.aspx?pos=all&stats=pit&type=rzips&team=0&lg=all&players=0'
        self.steamer_ros_b = 'http://www.fangraphs.com/projections.aspx?pos=all&stats=bat&type=steamerr&team=0&lg=all&players=0'
        self.steamer_ros_p = 'http://www.fangraphs.com/projections.aspx?pos=all&stats=pit&type=steamerr&team=0&lg=all&players=0'
        self.steamer_b = 'http://www.fangraphs.com/projections.aspx?pos=all&stats=bat&type=steamer&team=0&lg=all&players=0'
        self.steamer_p = 'http://www.fangraphs.com/projections.aspx?pos=all&stats=pit&type=steamer&team=0&lg=all&players=0'
        self.dc_ros_p = 'http://www.fangraphs.com/projections.aspx?pos=all&stats=bat&type=rfangraphsdc&team=0&lg=all&players=0'
        self.dc_ros_b = 'http://www.fangraphs.com/projections.aspx?pos=all&stats=pit&type=rfangraphsdc&team=0&lg=all&players=0'
        # self.link_list = [self.zips_ros_b, self.zips_ros_p, self.steamer_ros_b, self.steamer_ros_p]#, self.dc_ros_p, self.dc_ros_b]
        self.link_list = [self.steamer_b, self.steamer_p]
        self.name_list = ['zips_ros_b', 'zips_ros_p', 'steamer_ros_b', 'steamer_ros_p']  # , 'dc_ros_p', 'dc_ros_b']
        chromedriver = 'D:\downloads/chromedriver'
        os.environ['webdriver.chrome.driver'] = chromedriver
        self.driver = webdriver.Chrome(chromedriver)
        self.dl_path = 'C:\\Users\Ryan\Downloads'
        self.dest_path = 'C:\\Users\Ryan\PycharmProjects\\NCB_Stats\projections'

    def scrape(self):
        for l, n in zip(self.link_list, self.name_list):
            self.driver.get(url=l)
            time.sleep(5)
            print('link up')
            link = self.driver.find_element_by_id('ProjectionBoard1_cmdCSV').click()
            time.sleep(5)
            print('clicked')
            d = os.curdir
            for filename in os.listdir(self.dl_path):
                print(filename)
                if filename == 'FanGraphs Leaderboard.csv':
                    print('rename')
                    src_path = os.path.join(self.dl_path, filename)
                    dest_path = os.path.join(self.dest_path, n + '.csv')
                    os.rename(src_path, dest_path)
                    print('rename done')
                    # shutil.copy(filename, self.addr)

    def clear_old(self):
        for filename in os.listdir(self.dl_path):
            if 'FanGraphs Leaderboard' in filename:
                os.remove(filename)
        for filename in os.listdir(self.dest_path):
            os.remove(filename)

    def run_scrape(self):
        self.clear_old()
        self.scrape()

def main():
    scrape = FG_Scrape()
    scrape.run_scrape()

main()


from django.test import TestCase
from productos.models import Product
from django.urls import reverse
from django.core.management import call_command

class VerProductosTestCase(TestCase):
    def setUp(self):
        Product.objects.create(original_code='qwerewetret', product_code='sahsakjhaksjh', name='capot',
                               description='blanco', car_brand='honda', car_model='civic',
                               car_year='2020', public_price='200', card_price='250', master_price='100',
                               wholesale_price='150', dozen_price='190')

    def test_url_correct(self):
        response = self.client.get(reverse('shop:ver_catalogo'))
        self.assertEqual(response.status_code, 200)

    def test_view(self):
        response = self.client.get(reverse('shop:ver_catalogo'))
        self.assertContains(response, 'capot')


import copy
import importlib
import itertools

import six
import yaml

from ioc.schema.dependency import Dependency
from ioc.schema.exc import InvalidDeclaration
from ioc.schema.adapters import SymbolDependencyAdapter
from ioc.schema.adapters import LiteralDependencyAdapter
from ioc.schema.adapters import NestedDependencyAdapter
from ioc.schema.adapters import DependencyCollectionAdapter
from ioc.schema.base import Schema

class SchemaParser(object):
    """Parses the dependency configuration schema."""

    def __init__(self, provider=None, override=False):
        self.provider = provider or importlib.import_module('ioc.provider')
        self.override = override

    def load(self, dependencies):
        """Inspect the dependency configuration and validate the declared
        items.
        """
        schema = Schema(self.provider, override=self.override)

        # Check that each dependency has at least a name.
        names = set()
        for dep in dependencies:
            name = dep.get('name')
            if name is None:
                raise InvalidDeclaration(dep, "No name specified")

            if not isinstance(name, six.string_types):
                raise InvalidDeclaration(dep, "Invalid name specified: %s" % repr(name))
            if name in names:
                raise InvalidDeclaration(dep, "Duplicate name: %s" % name)

            names.add(name)

        # For each dependency, try to load them as either a Simple, Literal
        # or NestedDependency.
        adapters = [
            NestedDependencyAdapter(),
            SymbolDependencyAdapter(),
            LiteralDependencyAdapter(),
            DependencyCollectionAdapter(),
        ]

        for dep in dependencies:
            cleaned_dep = None
            for adapter in adapters:
                cleaned_dep, errors = adapter.load(copy.deepcopy(dep))
                if not errors:
                    break

                cleaned_dep = None

            # If there is still no dependency loaded at this point, this
            # means the declaration was invalid.
            if cleaned_dep is None:
                raise InvalidDeclaration(dep, "Invalid dependency declaration")

            assert issubclass(type(cleaned_dep), Dependency),\
                "load() must return a Dependency implementation, got %s" %\
                repr(cleaned_dep)
            schema.add(cleaned_dep)

        return schema


from django.urls import path

from . import views

app_name = 'slots'
urlpatterns = [
    path('', views.index_view, name='index'),

    path('login/', views.login_view, name='login'),
    path('logout/', views.logout_view, name='logout'),
    path('signup/', views.signup_view, name='signup'),

    path('shops/', views.ShopsView.as_view(), name='shops'),
    path('shops/<int:pk>/', views.ShopView.as_view(), name='view-shop'),
    path('shops/<int:shop>/<str:method>/<int:slot>/',
        views.slot_manage_view,
        name='manage-slot'
    ),
    path('shops/<int:shop>/modify/', views.modify_view, name='modify'),

    path('me/', views.user_view, name='user'),
    path('me/slots/', views.SlotsView.as_view(), name='slots'),
    path('me/settings/', views.user_settings_view, name='settings'),
]


from __future__ import print_function
import serial, time, threading
import serial.tools.list_ports
import sys, glob, math
# from ftdiManager import FTDIManager

class Statistics():
    dongleWakeup = 0
    dongleBytesDropped = 0
    radioPackagesSentOk = 0
    radioPackagesSentErr = 0
    radioPackagesReceiveOk = 0
    radioPackagesReceiveErr = 0
    def toString(self):
        statStr = "dongleWakeup ="+str(self.dongleWakeup)+"\n"
        statStr += "dongleBytesDropped ="+str(self.dongleBytesDropped)+"\n"
        statStr += "radioPackagesSentOk ="+str(self.radioPackagesSentOk)+"\n"
        statStr += "radioPackagesSentErr ="+str(self.radioPackagesSentErr)+"\n"
        statStr += "radioPackagesReceiveOk ="+str(self.radioPackagesReceiveOk)+"\n"
        statStr += "radioPackagesReceiveErr ="+str(self.radioPackagesReceiveErr)+"\n"
        return statStr

class Dongle():
    def __init__(self):
        self.ser = None
        self.stat = Statistics
        self.lock = threading.Lock()
        self.autoconnectThread = None
        pass

    def connect(self, portNr):
        self.lock.acquire()
        try:#actual BAUD rate on arduino 368 @8MHz => 8*10^6/(8*(8+1)) = 111111 (instead of 115200)
            if sys.platform.startswith('linux'):
                portList = glob.glob ('/dev/tty[A-Za-z]*')
                self.ser = serial.Serial(portList[portNr], 500000, timeout = 0.005)  # 0.015
            elif sys.platform.startswith('win32'):
                self.ser = serial.Serial(portNr, 500000, timeout = 0.015, xonxoff=False, rtscts=False, dsrdtr=False)
            elif sys.platform.startswith('cygwin'):
                print("Warning: Cygwin environment not yet tested")
                portList = glob.glob ('/dev/tty[A-Za-z]*')
                self.ser = serial.Serial(portList[portNr], 500000, timeout = 0.015)
            elif sys.platform.startswith('darwin'):
                #portsList = glob.glob('/dev/cu.usbserial-A602N31Z')
                portsList =glob.glob('/dev/cu.usb*')
                print("Warning: Mac OS X environment not yet tested")
                self.ser = serial.Serial(portsList[portNr], 500000, timeout = 0.015)
            # FTDIManager.setLatency(portNr, 1) #TODO: check that it is a dongle first ##########
            self.lock.release()
            time.sleep(2)
            return self.ping()
        except Exception: # serial.SerialException eller ValueError
            self.lock.release()
            print("Connection Exception ", sys.exc_info())
            return False

    class DongleConnectThread(threading.Thread):
        def __init__(self, dongle, period = 1):
            threading.Thread.__init__(self)
            self._dongle = dongle
            self._period = period
            self._connected = False

        def reconnect(self):
            self._connected = False

        def isConnected(self):
            return self._connected

        def run(self):
            while True:
                try:
                    if not self._connected:
                        if self._autoconnect():
                            self._connected = True
                    time.sleep(self._period)
                except Exception:
                    print("Exception in autoconnect thread! ", sys.exc_info())

        def _autoconnect(self):
            print("Trying to autoconnect dongle:")
            if self._dongle.ping() == True:
                print(" already connected")
                return True

            self._dongle.close() # close open port if any
            for port, _, _ in sorted(serial.tools.list_ports.comports()):
                print(" ",port, "", end='')
                res = self._dongle.connect(port)
                if res == True:
                    if self._dongle.ping() == True:
                        print("...success")
                        return True
                print("...failure")
                self._dongle.close()
            return False

    def autoconnectStart(self):
        if self.autoconnectThread == None:
            self.autoconnectThread= self.DongleConnectThread(self, period = 1)
            self.autoconnectThread.setName('Dongle Connect Thread')
            self.autoconnectThread.start()
        else:
            self.autoconnectThread.reconnect()

    def isConnected(self):
            return self.autoconnectThread.isConnected()

    def _isACK(self,reply):
        return (len(reply) != 0 and reply[0]==ord('A'))

    def _isSerialReady(self):
        if self.ser == None: return False
        try:
            self.ser.inWaiting()
            self.ser.write([])
            self.ser.read(size=0)
        except Exception:
            print("Dongle not ready! ", sys.exc_info())
            return False
        return True

    def wakeup(self):
        #if self.ser == None: return False
        if not self._isSerialReady(): return False
        self.lock.acquire()
        self.stat.dongleWakeup += 1
        for i in range(10):
            self.ser.write(bytes([0xff]))
        self.ser.flush()
        time.sleep(0.1)
        self.ser.read(self.ser.inWaiting())
        self.lock.release()
        return self.ping()

    def checkConnection(self, returnPing=True):
        if not self._isSerialReady():
            self.autoconnectStart()
            return False
        if returnPing:
            return self.ping()
        else:
            return True

    def ping(self):
        if not self._isSerialReady(): return False
        self.lock.acquire()
        cTime = time.clock()
        self.ser.write(bytes([0xff]))
        self.ser.flush()
        reply = self.ser.read(1)
        lag = 1000*(time.clock()-cTime)
        if lag > 5:
            print("Dongle seems slow (%s ms), change latency timer to 1 ms in devicemanager..." % int(lag))
        self.lock.release()
        return self._isACK(reply)

    def setLed(self,val):
        if not self._isSerialReady(): return False
        self.lock.acquire()
        self.ser.write(bytes([0xfb]))
        self.ser.write(bytes([val]))
        self.ser.flush()
        reply = self.ser.read(1)
        self.lock.release()
        return self._isACK(reply)

    def setRGBLed(self,r,g,b):
        if not self._isSerialReady(): return False
        self.lock.acquire()
        self.ser.write(bytes([0xfa]))
        self.ser.write(bytes([r]))
        self.ser.write(bytes([g]))
        self.ser.write(bytes([b]))
        self.ser.flush()
        reply = self.ser.read(1)
        self.lock.release()
        return self._isACK(reply)

    def scanForModules(self):
        if not self._isSerialReady(): return False
        self.lock.acquire()
        self.ser.write(bytes([0xf4]))
        self.ser.flush()
        time.sleep(0.2)
        print("self.ser.inWaiting()=",self.ser.inWaiting())
        reply = self.ser.read(self.ser.inWaiting())
        print(reply)
        for x in reply:
            print(x)
        self.lock.release()

    def setDongleBuzzer(self, tone):
        if not self._isSerialReady(): return False
        tone = int(tone)
        b0 = int(tone & 0xff)
        b1 = int((tone >> 8) & 0xff)
        self.lock.acquire()
        self.ser.write(bytes([0xf9]))
        self.ser.write(bytes([b1]))
        self.ser.write(bytes([b0]))
        self.ser.flush()
        reply = self.ser.read(1)
        res = self._isACK(reply)
        self.lock.release()
        return res

    def getDongleTime(self):
        if not self._isSerialReady(): return False
        self.lock.acquire()
        self.ser.write(bytes([0xf8]))
        self.ser.flush()
        reply = self.ser.read(1)
        if (not self._isACK(reply)):
            self.lock.release()
            return -1
        b = self.ser.read(4)
        res = b[0] + (b[1]<<8) + (b[2]<<16) + (b[3]<<24)
        self.lock.release()
        return res

    def stopDongle(self):
        if not self._isSerialReady(): return False
        self.lock.acquire()
        self.ser.write(bytes([0xf7]))
        self.ser.flush()
        reply = self.ser.read(1)
        res = self._isACK(reply)
        self.lock.release()
        return res

    def writeRadioPacket(self, data):
        if not self._isSerialReady(): return False
        self.lock.acquire()
        if not self.ser.inWaiting() == 0:
            self.stat.dongleBytesDropped += self.ser.inWaiting()
            print("Help!!!, throw away=", self.ser.read( self.ser.inWaiting() ), " Port latency must be lower (i.e 1ms)")
        buffer = []
        buffer.append(0xfe)
        buffer.append(len(data))
        for b in data:
            buffer.append(b)
        self.ser.write(bytes(buffer))
        self.ser.flush()
        reply = self.ser.read(1)
        success = self._isACK(reply)
        if (success):
            self.stat.radioPackagesSentOk +=1
        else:
            self.stat.radioPackagesSentErr +=1
            if not self.ser.inWaiting() == 0:
                self.stat.dongleBytesDropped += self.ser.inWaiting()
                data = self.ser.read( self.ser.inWaiting() )
                print("Help!!!, throw away=", data, " Port latency must be lower (i.e 1ms)")
                if len(data)!=0 and data[len(data)-1]==ord('A'):
                    success = True
        self.lock.release()
        return success

    def readPacket(self, nBytes, nTimeout=1):
        if not self._isSerialReady():
            print("not ready")
            return
        self.lock.acquire()
        for _ in range(nTimeout):
            packet = self.ser.read(nBytes)
            if len(packet) == nBytes: break
        if len(packet) == nBytes:
            self.stat.radioPackagesReceiveOk +=1
        else:
            # print("timeout")
            print("timeout, packet lenght = ", len(packet), "required length = ", nBytes)
            self.stat.radioPackagesReceiveErr +=1
        self.lock.release()
        return packet

    def close(self):
        if self.ser == None:
            return
        self.lock.acquire()
        self.ser.close()
        self.ser = None
        self.lock.release()

    def getStatistics(self):
        return self.stat