import os
import importlib.util
import logging
import Framework.Utility as Utility
import Framework.JlscaBruteForceCalculation as JlscaBruteForceCalculation
from python_log_indenter import IndentedLoggerAdapter

logger = IndentedLoggerAdapter(logging.getLogger())

def cw_to_trs(root_dir, cw_dir, cw_date_string, trs_file, num_traces_to_test):
    cmd = "julia "
    cmd += os.path.join(root_dir, 'Framework', 'cw_to_trs.jl') + " "
    cmd += "--cw_date_string {} ".format(cw_date_string)
    cmd += "--cw_trace_location {} ".format(cw_dir)
    cmd += "--num_traces {} ".format(num_traces_to_test)
    cmd += trs_file
    Utility.execute(cmd)

def attack_inc_cpa(root_dir, trs_file, attack_params, jlsca_log_file):
    cmd = "julia" + " "
    cmd += os.path.join(root_dir, 'Framework', 'inc_cpa.jl') + " "
    for key, value in attack_params.items():
        cmd += str(key)+ " "
        cmd += str(value) + " "
    cmd += trs_file
    Utility.execute(cmd, jlsca_log_file)

def jlsca_analysis(root_dir, ProjectDir, local_setup):
    logger = IndentedLoggerAdapter(logging.getLogger())
    logger.add()
    # Load LocalSetup from user input
    spec = importlib.util.spec_from_file_location(local_setup, os.path.join(os.getcwd(), local_setup))
    LocalSetup = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(LocalSetup)
    logger.info("Using parameters from LocalSetup:")
    logger.add()
    for key, value in LocalSetup.AttackParameters.items():
        logger.info(key + " " + str(value))
    logger.sub()
    # Prepare cw files to trs files if not already converted
    for num_traces_to_test in LocalSetup.TraceNumList:
        # Trace file to be generated and used in attack
        trs_file = os.path.join(
            LocalSetup.TraceDir,
            LocalSetup.num_samples + "_samples_" + num_traces_to_test + "_traces_data.trs")

        # Log file to be generated from attack and used in brute force calculation
        jlsca_log_file = os.path.join(
            LocalSetup.ProjectDir,
            '_log',
            'jlsca_' + num_traces_to_test + '_traces.log'
        )
        logger.info("Jlsca attack using {:>6} traces:".format(num_traces_to_test))
        logger.add()
        if not os.path.isfile(trs_file):
            logger.info("Converting CW project to JlSca trs format")
            cw_to_trs(root_dir, LocalSetup.CWFilesDir, LocalSetup.date_string, trs_file, num_traces_to_test)
            logger.info("CW files converted into .trs file")

        logger.info("Running attack Incremental Correlation Power Analysis using Julia")
        attack_inc_cpa(root_dir, trs_file, LocalSetup.AttackParameters, jlsca_log_file)
        logger.info("Attack Incremental Correlation Power Analysis completed")

        logger.info("Running calculation on number of guesses needed based on brute force algorithm")
        num_of_guesses = JlscaBruteForceCalculation.calculate(jlsca_log_file,
                                                              LocalSetup.CWKnownKey,
                                                              "--show-plots" in LocalSetup.AttackParameters)

        logger.info("Number of guesses calculation completed")
        logger.info("Number of guesses needed using {:>6} traces: {}".format(num_traces_to_test, num_of_guesses))
        logger.sub()
    logger.sub()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
from django.conf import settings

class Migration(migrations.Migration):

    dependencies = [
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
        ('main', '0032_answer'),
    ]

    operations = [
        migrations.CreateModel(
            name='Comment',
            fields=[
                ('id', models.AutoField(primary_key=True, auto_created=True, verbose_name='ID', serialize=False)),
                ('ip', models.CharField(max_length=30)),
                ('text', models.CharField(max_length=500)),
                ('date', models.DateTimeField()),
                ('page', models.ForeignKey(to='main.Page')),
                ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
            ],
        ),
    ]


import multiprocessing
import os
from argparse import ArgumentParser

import gensim
import tensorflow as tf
from tqdm import tqdm

import constants as const
import models
import preprocess

def parse_args():
    parser = ArgumentParser()
    parser.add_argument("input_path", help="The path of the input file")
    parser.add_argument("output_path", help="The path of the output file")
    parser.add_argument(
        "resources_path", help="The path of the resources needed to load your model"
    )

    return parser.parse_args()

def predict(input_path, output_path, resources_path):
    """
    This is the skeleton of the prediction function.
    The predict function will build your model, load the weights from the checkpoint and write a new file (output_path)
    with your predictions in the BIES format.

    The resources folder should contain everything you need to make the predictions. It is the "resources" folder in
    your submission.

    N.B. DO NOT HARD CODE PATHS IN HERE. Use resource_path instead, otherwise we will not be able to run the code.

    :param input_path: the path of the input file to predict.
    :param output_path: the path of the output file (where you save your predictions)
    :param resources_path: the path of the resources folder containing your model and stuff you might need.
    :return: None
    """

    # tf configurations
    config = tf.ConfigProto()
    config.intra_op_parallelism_threads = multiprocessing.cpu_count()
    config.inter_op_parallelism_threads = multiprocessing.cpu_count()
    tf.Session(config=config)

    # resources
    weights_path = os.path.join(resources_path, "model_weights.h5")
    embedding = os.path.join(resources_path, "bigram_unigram300.txt")

    # Load the vocab
    print("Computing vocab...")
    w2v = gensim.models.KeyedVectors.load_word2vec_format(embedding)
    vocab = preprocess.vocab_from_w2v(w2v)

    # Build the model
    print("loading model...")
    model = models.build_model(
        n_classes=4,
        dropout=0.45,
        recurrent_dropout=0.2,
        learning_rate=0.0025,
        word2vec=w2v,
        word2vec_bi=w2v,
    )
    # Load the weights
    model.load_weights(weights_path)

    # Load test txt
    features_test = preprocess.read_dataset(input_path)
    with open(output_path, mode="w", encoding="utf8") as out_file:
        for line in tqdm(features_test):
            line = line.strip()

            to_predict_uni = preprocess.compute_X([line], vocab, pad=False)
            to_predict_bi = preprocess.compute_X(
                [preprocess.compute_bigrams(line)], vocab, pad=False
            )

            prediction = model.predict(
                [to_predict_uni, to_predict_bi], workers=multiprocessing.cpu_count()
            )

            id_prediction = prediction.argmax(-1)[0]
            bies_prediction = [const.INVERSE_LABELS[p] for p in id_prediction]
            out_file.write("".join(bies_prediction) + "\n")

    return

if __name__ == "__main__":
    args = parse_args()
    predict(args.input_path, args.output_path, args.resources_path)


# 5LSL0 Final Assignment main code for mfcc based model

## Import arbitrary libraries
import numpy as np
import matplotlib.pyplot as plt
import os
import openpyxl
import sys

## Import tensorflow and keras
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.layers import Input, Dense, Conv1D, Conv2D, MaxPooling1D, LeakyReLU, AvgPool2D, UpSampling2D, ReLU, MaxPooling2D, Reshape, Flatten, Activation
from tensorflow.keras.losses import MSE, categorical_crossentropy
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.optimizers import Adam
from sklearn.metrics import accuracy_score
from keras.models import load_model
import bisect

## Import selfmade modules
from generator_mfcc import DataGenerator
from model_mfcc import deep_cnn
sys.path.insert(0, 'Data')
import createNoise
from datasplitter import datasplitter

def getDivisors(x):
    divisors = []
    i = 1
    for i in range(1,x):
        if(x%i == 0):
            divisors.append(i)
        else:
            continue
        i+=1
    return divisors

def find_le(a, x):
    'Find rightmost value less than or equal to x'
    i = bisect.bisect_right(a, x)
    if i:
        return a[i-1]
    raise ValueError

## Labels
labels=("yes", "no", "up", "down", "left", "right", "on", "off", "stop", "go", "silence", "unknown")
num_classes = len(labels)
training_batch_size = 128
# test_batch_size = 28
final_test_batch_size = 6

## normalization options
normalization = {"subtract_mean": True,
                 "epsilon": 1e-8,           # prevent division by 0
                 "normalize": "95",        # "none", "var", "std", "95", "97"
                 "type": "sample"}          # "sample", "class"

## evaluation metrics
metrics=['acc']

## create silence
USE_SILENCE = True
N_SILENCE = 2000
if USE_SILENCE:
    createNoise.create(N_SILENCE)

## create lists
datasplitter()

## paths
data_path = 'Data/train/audio/'
data_lists = 'Data/train/lists/'
final_test_path = 'Data/test/audio/'

## Create generator
traingen1 = DataGenerator(data_path=data_path,
                         data_listing=data_lists+'train_set.txt',
                         batch_size=training_batch_size,
                         dims_in=(16000,1),
                         dims_out=(13,32),
                         labels=labels)

validgen1 = DataGenerator(data_path=data_path,
                         data_listing=data_lists+'validation_set.txt',
                         batch_size=training_batch_size,
                         dims_in=(16000,1),
                         dims_out=(13,32),
                         labels=labels)

## Create test model
shape = (13,32,1)
model = deep_cnn(shape, num_classes)

#optimizer = Adam(lr = 0.0001)
model.compile(optimizer='Adam', loss = categorical_crossentropy, metrics = metrics)

# Train model on dataset
num_lines_train = sum(1 for line in open(data_lists+'train_set.txt'))
num_lines_valid = sum(1 for line in open(data_lists+'validation_set.txt'))

history = model.fit_generator(generator=traingen1,
                              epochs = 15,
                              verbose = 1,
                              validation_data=validgen1)

# summarize history for acc
plt.plot(history.history['acc'])
plt.plot(history.history['val_acc'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'valid'], loc='upper left')
plt.savefig('results/mfcc/model_spectrum_norm_acc.pdf')
plt.close()

# summarize history for loss
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'valid'], loc='upper left')
plt.savefig('results/mfcc/model_spectrum_norm_loss.pdf')

# save model
model.save('results/mfcc/model_spectrum_norm.h5')

## test data
from generator_mfcc_test import DataGenerator

with open(data_lists+'test_set.txt') as f:
    content = f.readlines()

divisors_test = getDivisors(len(content))
test_batch_size = find_le(divisors_test, 100)

testgen = DataGenerator(data_path=data_path,
                         data_listing=data_lists+'test_set.txt',
                         batch_size=test_batch_size,
                         dims_in=(16000,1),
                         dims_out=(13,32),
                         labels=labels)

y_pred_proba = model.predict_generator(generator=testgen,
                                     verbose=1)

y_pred = np.argmax(y_pred_proba, axis=1)

y_true = []
# Added modulo to test batch size to make sure len(y_true) == len(y_pred)
for lbl in range(len(content)):
    label = content[lbl].split('/')[0]
    if label in labels:
        y_true.append(labels.index(label))
    else:
        y_true.append(labels.index('unknown'))

acc_score = accuracy_score(y_true, y_pred)
print(acc_score)

## test final data for submission
from generator_mfcc_finaltest import DataGenerator

y_files = os.listdir(final_test_path)

finaltestgen = DataGenerator(data_path=final_test_path,
                             dataset = y_files,
                             batch_size=final_test_batch_size,
                             dims_in=(16000,1),
                             dims_out=(13,32),
                             labels=labels)

y_pred_proba = model.predict_generator(generator = finaltestgen,
                                     verbose=1)
y_pred = np.argmax(y_pred_proba, axis=1)

# Save computed probabilities so that they can be used with the other models
np.savetxt('results/mfcc/probabilities_spectrum_norm.csv', y_pred_proba, delimiter=',')

# Save the final outcome
wb = openpyxl.Workbook()
sheet = wb.active
sheet.cell(row=1, column=1).value='fname,label'
for y in range(len(y_pred)):
    sheet.cell(row=y+2, column=1).value=y_files[y]+','+labels[y_pred[y]]

wb.save('results/mfcc/submission_spectrum_norm.xlsx')


"""
#---------------------------------------------------------------------------
# Projeto 2 - Cinemática de colisões relativísticas (Aniquilação elétron-pósitron)
# Aluno: Gabriel Moreira e Victor Augustus
# Professoras: Helena Malbouisson e Clemencia Mora
# Disciplina: Introdução a Python - UERJ 2018.2
#---------------------------------------------------------------------------

Script desenvolvido para simulações de colisões relativísticas com foco em aniquilação elétron-pósitron.
Com a colisão de um pósitron de momentum randômico com um elétron em repouso no referencial do laboratório foi possível calcular a energia final dos fótons resultandos
	pelo princípio de conservação de energia com base em um distribuição de ângulos finais de um dos fótons.

Source: https://arxiv.org/pdf/1011.1943.pdf

Obs:
O script tem dependências com os packages numpy, pandas e openpyxl.
Caso o script não funcione executando no terminal com --python3 projeto2.py é preciso instalar esses packages pelo terminal da seguinte maneira:

sudo apt install python3-pip
pip3 install pandas
pip3 install openpyxl
"""

# Módulos utilizados
from math 					import *
from particula 				import *
from random 				import *
import matplotlib.pyplot	as plt
import numpy 				as np
import pandas 				as pd
from pandas 				import ExcelWriter

# Definindo as funções
def aniquilacao_ep(Ep,Ee,theta):
	k = sqrt((Ep - Ee)/(Ep + Ee))
	Ef = Ee/(1-k*cos(radians(theta)))
	return Ef

def plot(dict,x_label,y_label,title):
	for i in range(1,n_colisoes+1):
		data_x = np.array(dict['Colisão ' + str(i)][0])
		data_y = np.array(dict['Colisão ' + str(i)][1])
		plt.plot(data_x,data_y, label = 'Colisão ' + str(i))
		plt.xlabel(x_label, fontsize = 12)
		plt.ylabel(y_label + '1 (MeV)', fontsize = 12)

	plt.title(title, fontsize = 14)
	plt.legend(bbox_to_anchor = (1.04,1), loc = 'upper left')
	plt.tight_layout(rect = [0,0,0.75,1])
	plt.savefig('foton1.png')
	plt.show()
	plt.close()

	for i in range(1,n_colisoes+1):
		data_x = np.array(dict['Colisão ' + str(i)][0])
		data_y = np.array(dict['Colisão ' + str(i)][2])
		plt.plot(data_x,data_y, label = 'Colisão ' + str(i))
		plt.xlabel(x_label, fontsize = 12)
		plt.ylabel(y_label + '2 (MeV)', fontsize = 12)

	plt.title(title, fontsize = 14)
	plt.legend(bbox_to_anchor = (1.04,1), loc = "upper left")
	plt.tight_layout(rect = [0,0,0.75,1])
	plt.savefig('foton2.png')
	plt.show()
	plt.close()

# Apresentando o programa ao usuário e coletando o número de colisões
print('\nProjeto de Colisões Relativísticas, com foco em Aniquilação Elétron-Pósitron \n Foi utilizado um sistema de unidades naturais onde c = 1')
print('\bO programa irá calcular a energia final dos fótons resultantes da colisão de um elétron com momentum p = 0 e um pósitron com momentum p randômico entre 1.0 e 100.0.')
print('No final do processo, iniciará a plotagem dos gráficos da distribuição do ângulo resultante do fóton 1 pela distribuição da energa final dos fótons 1 e 2, além disso, o script também irá armazenar os dados.')
n_colisoes = int(input('\nDigite o número de colisões: '))
while n_colisoes <= 0:
	n_colisoes = int(input('Essa resposta não é válida! Digite um número > 0: '))

# Definindo o dicionário que irá armazenar os dados
dist = {}

for i in range(1,n_colisoes+1):
	dist['Colisão ' + str(i)] = [[] for i in range(6)]

# Criando um loop com a variável loop_question para reiniciar o programa depedendo da resposta do usuário
loop_question = 'sim'
while loop_question == 'sim':

	# Loop
	for i in range(1,n_colisoes+1):
		p = uniform(1.0,100.0)
		e = Particula(0.511)
		pos = Particula(0.511,p)

		theta = 0
		while theta <= 360:
			Ef = aniquilacao_ep(pos.energia(),e.energia(),theta)
			fot1 = Particula(0,Ef)
			fot2 = Particula(0,(pos.energia()+e.energia())-Ef)

			dist['Colisão ' + str(i)][0].append(theta)
			dist['Colisão ' + str(i)][1].append(fot1.energia())
			dist['Colisão ' + str(i)][2].append(fot2.energia())
			dist['Colisão ' + str(i)][3].append(e.energia())
			dist['Colisão ' + str(i)][4].append(pos.energia())
			dist['Colisão ' + str(i)][5].append(p)

			theta += 1

	# Gráfico
	plot(dist,'\u03B8 (\u00B0)', 'Energia \u03B3_', 'Aniquilação Elétron-Pósitron')

	# Criando um arquivo .txt e armazenando os dados calculados
	arq = open('dados_aniquilação_ep.txt', 'w')
	for i in range(1,n_colisoes+1):
		arq.write('\n############### Colisão ' + str(i) + ' ###############')
		arq.write('\np_pos(MeV/c)		\u03B8 (\u00B0)		E_e(MeV)		E_pos(MeV)		E_e+p(MeV)		E_\u03B3_1(MeV)		E_\u03B3_2(MeV)		E_\u03B3_1+\u03B3_2(MeV)')
		arq.write('\n')
		for j in range(0,len(dist['Colisão 1'][0])):
			arq.write(str("%.4f" % dist['Colisão ' + str(i)][5][j]))
			arq.write('			')
			arq.write(str("%.4f" % dist['Colisão ' + str(i)][0][j]))
			arq.write('			')
			arq.write(str("%.4f" % dist['Colisão ' + str(i)][3][j]))
			arq.write('			')
			arq.write(str("%.4f" % dist['Colisão ' + str(i)][4][j]))
			arq.write('			')
			arq.write(str("%.4f" % (dist['Colisão ' + str(i)][3][j] + dist['Colisão ' + str(i)][4][j])))
			arq.write('			')
			arq.write(str("%.4f" % dist['Colisão ' + str(i)][1][j]))
			arq.write('			')
			arq.write(str("%.4f" % dist['Colisão ' + str(i)][2][j]))
			arq.write('			')
			arq.write(str("%.4f" % (dist['Colisão ' + str(i)][1][j] + dist['Colisão ' + str(i)][2][j])))
			arq.write('\n')
	arq.close()

	# Criando um arquivo .xlsx e armazenando os dados cada sheet do excel
	writer = ExcelWriter("dados_aniquilação_ep.xlsx")

	Df_List = []

	for i in range(1, n_colisoes + 1):

		theta 		= np.array(dist['Colisão ' + str(i)][0])
		E_fot1_ene	= np.array(dist['Colisão ' + str(i)][1])
		E_fot2_ene 	= np.array(dist['Colisão ' + str(i)][2])
		E_ene		= np.array(dist['Colisão ' + str(i)][3])
		P_ene		= np.array(dist['Colisão ' + str(i)][4])
		P			= np.array(dist['Colisão ' + str(i)][5])

		My_data_frame = pd.DataFrame({ "p_pos(MeV/c)": P,
										"\u03B8 (\u00B0)": theta,
										"E_e(MeV)": E_ene,
										"E_pos(MeV)": P_ene,
										"E_e+p(MeV)": E_ene + P_ene,
										"E_\u03B3_1(MeV)": E_fot1_ene,
										"E_\u03B3_2(MeV)": E_fot2_ene,
										"E_\u03B3_1+\u03B3_2(MeV)": E_fot1_ene + E_fot2_ene
									})

		Df_List.append(My_data_frame)

	for i in range(len(Df_List)):
		#print (Df_List)
		col_string = 'Sheet_' + str(i+1)
		Df_List[i].to_excel(writer, col_string, index=False)
		writer.save()

	# Update da loop_question para saber se o usuário quer realizar outras simulações
	loop_question = input('\nDeseja simular outras colisões? [sim/não] ')
	while loop_question not in {'sim', 'não'}:
		loop_question = input('Essa não é uma resposta válida! Responda [sim] ou [não]. ')

	# Atualizando a condição inicial do número de colisões para o caso que o usuário deseja realizar outras simulações
	if loop_question == 'sim':
		n_colisoes = int(input('Digite o número de colisões: '))
		while n_colisoes <= 0:
			n_colisoes = int(input('Essa resposta não é válida! Digite um número > 0: '))

	# Limpando o dicionário que irá armazenar os dados para o caso que o usuário deseja realizar outras simulações
		dist = {}

		for i in range(1,n_colisoes+1):
			dist['Colisão ' + str(i)] = [[] for i in range(6)]

print('\nEncerrando o programa!')


"""
xkcd module by @Protoh
"""
from telegram import Bot, Update
from urllib.parse import quote
import requests

import core
PLUGINVERSION = 2
# Always name this variable as `plugin`
# If you dont, module loader will fail to load the plugin!
plugin = core.Plugin(name="xkcd")

TEXT = """
*title:* %(safe_title)s
*number:* %(num)s
*date*`(yyyy-mm-dd for christ's sake)`*:* %(year)s-%(month)s-%(day)s
*alt:* _%(alt)s_
*link:* xkcd.com/%(num)s
"""

@plugin.command(command="/xkcd",
                description="Sends xkcd comics in the chat! usage: '/xkcd', '/xkcd <number>', or '/xkcd <query>'",
                inline_supported=True,
                hidden=False)
def xkcd(bot: Bot, update: Update, user, args): # pylint: disable=W0613
    """
    Example usage:
    User:
    /xkcd 1

    OctoBot:
    title: Barrel - Part 1
    number: 1
    date(yyyy-mm-dd for christ's sake): 2006-01-01
    alt: Don't we all.
    link: xkcd.com/1
    """
    argument = " ".join(args)
    id = ""
    if not argument:
        id = -1
    else:
        if argument.isdigit():
            id = argument
        else:
            queryresult = requests.get('https://relevantxkcd.appspot.com/process?',params={"action":"xkcd","query":quote(argument)}).text
            id = queryresult.split(" ")[2].lstrip("\n")
    data = ""
    if id == -1:
        data = requests.get("https://xkcd.com/info.0.json").json()
    else:
        r = requests.get("https://xkcd.com/{}/info.0.json".format(id))
        if r.ok:
            data = r.json()
        else:
            return core.message("xkcd n.{} not found!".format(id), failed=True)
    #msg.reply_photo(photo = data['img']) Cause telegram makes preview of comic, we dont need it anymore
    data['month'] = data['month'].zfill(2)
    data['day'] = data['day'].zfill(2)
    return core.message(TEXT % data, parse_mode="MARKDOWN")


import numpy as np
import scipy.interpolate as interp
from datetime import datetime, timedelta
from typing import Dict

from resistics.time.data import TimeData
from resistics.common.math import intdiv
from resistics.common.print import errorPrint

def interpolateToSecond(timeData: TimeData) -> TimeData:
    """Interpolate data to be on the second

    Some formats of time data (e.g. SPAM) do not start on the second with their sampling. This method interpolates so that sampling starts on the second and improves interoperability with other recording formats. As an example, consider recording at 10 Hz, with the first sample at 0.05 seconds. Then the sample times will be:

    .. code-block::text

        0.05 0.15 0.25 0.35 0.45 0.55 0.65 0.75 0.85 0.95 1.05 1.15 1.25 ...

    Interpolating to the second will change the sample times to:

    .. code-block::text

        0.10 0.20 0.30 0.40 0.50 0.60 0.70 0.80 0.90 1.00 1.10 1.20 ...

    .. warning::

        Do not use this method on data recording with a sampling frequency of less than 1Hz

    Parameters
    ----------
    timeData : TimeData
        Time data to interpolate onto the second

    Returns
    -------
    TimeData
        A new TimeData object interpolated to start on the second
    """
    startTimeInterp, numSamplesInterp, dataInterp = interpolateToSecondData(
        timeData.data, timeData.sampleFreq, timeData.startTime
    )
    # calculate the stop time
    sampleFreqInterp = timeData.sampleFreq
    stopTimeInterp = startTimeInterp + timedelta(
        seconds=(1.0 / sampleFreqInterp) * (numSamplesInterp - 1)
    )
    # the comments
    newcomment = "Time data interpolated to nearest second. New start time {}, new end time {}, new number of samples {}".format(
        startTimeInterp, stopTimeInterp, numSamplesInterp
    )
    commentsInterp = timeData.comments + [newcomment]
    # create a new time data object
    return TimeData(
        sampleFreqInterp, startTimeInterp, stopTimeInterp, dataInterp, commentsInterp
    )

def interpolateToSecondData(
    data: Dict[str, np.ndarray], sampleFreq: float, startTime: datetime
) -> Dict[str, np.ndarray]:
    """Interpolate data to be on the second

    Interpolates the sampling so that it coincides with full seconds. The function also shifts the start point to the next full second. This function will truncate the data to the previous full second.

    .. warning::

        Do not use this method on data recording with a sampling frequency of less than 1Hz

    Parameters
    ----------
    data : Dict
        Dictionary with channel as keys and data as values
    sampleFreq : float
        Sampling frequency of the data
    startTime : datetime
        Time of first sample

    Returns
    -------
    data : Dict
        Dictionary with channel as keys and data as values

    todo:
    This function needs to be more robust for low (< 1Hz) sample frequencies as the use of microseconds and seconds makes no sense for this
    """
    # data properties
    chans = list(data.keys())
    samplePeriod = 1.0 / sampleFreq
    # set initial vals
    numSamples = data[chans[0]].size

    # now caluclate the interpolation
    microseconds = startTime.time().microsecond
    # check if the dataset already begins on a second
    if microseconds == 0:
        return startTime, numSamples, data  # do nothing, already on the second
    # now turn microseconds into a decimal
    microseconds = microseconds / 1000000.0
    # now calculate the number of complete samples till the next second
    eps = 0.000000001
    test = microseconds
    samplesToDrop = 0
    # this loop will always either calculate till the full second or the next sample passed the full second
    while test < 1.0 - eps:
        test += samplePeriod
        samplesToDrop += 1

    # if this is exact, i.e. integer number of samples to next second, just need to drop samples
    multiple = (1.0 - microseconds) / samplePeriod
    if np.absolute(multiple - samplesToDrop) < eps:  # floating point arithmetic
        dataInterp = {}  # create a new dictionary for data
        for chan in chans:
            dataInterp[chan] = data[chan][samplesToDrop:]
        # update the other data
        numSamplesInterp = numSamples - samplesToDrop
        startTimeInterp = startTime + timedelta(
            seconds=1.0 * samplesToDrop / sampleFreq
        )
        return startTimeInterp, numSamplesInterp, dataInterp

    # if here, then we have calculated one extra for samplesToDrop
    samplesToDrop -= 1

    # now the number of samples to the next full second is not an integer
    # interpolation will have to be performed
    shift = (multiple - samplesToDrop) * samplePeriod
    sampleShift = shift / samplePeriod
    x = np.arange(0, numSamples)
    xInterp = np.arange(samplesToDrop, numSamples - 1) + sampleShift
    # calculate return vars
    numSamplesInterp = xInterp.size
    startTimeInterp = (
        startTime
        + timedelta(seconds=1.0 * samplesToDrop / sampleFreq)
        + timedelta(seconds=shift)
    )

    # interpolation
    dataInterp = {}
    for chan in chans:
        tck = interp.splrep(x, data[chan], s=0)
        dataInterp[chan] = interp.splev(xInterp, tck, der=0)

    return startTimeInterp, numSamplesInterp, dataInterp

def fillGap(timeData1: TimeData, timeData2: TimeData) -> TimeData:
    """Fill gap between time series

    Fill gaps between two different recordings. The intent is to fill the gap when recording has been interrupted and there are two data files. Both times series must have the same sampling frequency. The missing timestamps and samples will be added to the timeseries and filled in using interpolation.

    Parameters
    ----------
    timeDat1 : TimeData
        Time series data
    timeData2 : TimeData
        Time series data

    Returns
    -------
    TimeData
        Time series data with gap filled
    """
    from resistics.time.clean import removeNansChan

    if timeData1.sampleFreq != timeData2.sampleFreq:
        errorPrint(
            "fillGap",
            "fillGap requires both timeData objects to have the same sample rate",
            quitrun=True,
        )
        return False
    sampleFreq = timeData1.sampleFreq
    sampleRate = 1.0 / sampleFreq
    timeDataFirst = timeData1
    timeDataSecond = timeData2
    if timeData1.startTime > timeData2.stopTime:
        timeDataFirst = timeData2
        timeDataSecond = timeData1
    # time data start and end times are inclusive
    gapStart = timeDataFirst.stopTime + timedelta(seconds=sampleRate)
    gapEnd = timeDataSecond.startTime - timedelta(seconds=sampleRate)
    # number of samples in the gap - add 1 because inclusive
    numSamplesGap = int(round((gapEnd - gapStart).total_seconds() * sampleFreq)) + 1
    # interpolate
    newData = {}
    for chan in timeDataFirst.chans:
        missing = np.empty(shape=(numSamplesGap))
        missing[:] = np.nan
        combined = np.concatenate([timeDataFirst[chan], missing, timeDataSecond[chan]])
        newData[chan] = removeNansChan(combined)
    comment = (
        ["-----------------------------", "TimeData1 comments"]
        + timeDataFirst.comments
        + ["-----------------------------", "TimeData2 comments"]
        + timeDataSecond.comments
    )
    comment += ["-----------------------------"] + [
        "Gap filled from {} to {}".format(gapStart, gapEnd)
    ]
    return TimeData(
        sampleFreq, timeDataFirst.startTime, timeDataSecond.stopTime, newData, comment,
    )


# uncompyle6 version 3.7.4
# Python bytecode 2.7 (62211)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: /tmp/pip-install-n_sfyb/Django/django/core/cache/backends/memcached.py
# Compiled at: 2019-02-14 00:35:17
"""Memcached cache backend"""
import pickle, re, time, warnings
from django.core.cache.backends.base import DEFAULT_TIMEOUT, BaseCache
from django.utils import six
from django.utils.deprecation import RemovedInDjango21Warning
from django.utils.encoding import force_str
from django.utils.functional import cached_property

class BaseMemcachedCache(BaseCache):

    def __init__(self, server, params, library, value_not_found_exception):
        super(BaseMemcachedCache, self).__init__(params)
        if isinstance(server, six.string_types):
            self._servers = re.split('[;,]', server)
        else:
            self._servers = server
        self.LibraryValueNotFoundException = value_not_found_exception
        self._lib = library
        self._options = params.get('OPTIONS') or {}

    @property
    def _cache(self):
        """
        Implements transparent thread-safe access to a memcached client.
        """
        if getattr(self, '_client', None) is None:
            self._client = self._lib.Client(self._servers, **self._options)
        return self._client

    def get_backend_timeout(self, timeout=DEFAULT_TIMEOUT):
        """
        Memcached deals with long (> 30 days) timeouts in a special
        way. Call this function to obtain a safe value for your timeout.
        """
        if timeout == DEFAULT_TIMEOUT:
            timeout = self.default_timeout
        if timeout is None:
            return 0
        else:
            if int(timeout) == 0:
                timeout = -1
            if timeout > 2592000:
                timeout += int(time.time())
            return int(timeout)

    def make_key(self, key, version=None):
        return force_str(super(BaseMemcachedCache, self).make_key(key, version))

    def add(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
        key = self.make_key(key, version=version)
        return self._cache.add(key, value, self.get_backend_timeout(timeout))

    def get(self, key, default=None, version=None):
        key = self.make_key(key, version=version)
        val = self._cache.get(key)
        if val is None:
            return default
        else:
            return val

    def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
        key = self.make_key(key, version=version)
        if not self._cache.set(key, value, self.get_backend_timeout(timeout)):
            self._cache.delete(key)

    def delete(self, key, version=None):
        key = self.make_key(key, version=version)
        self._cache.delete(key)

    def get_many(self, keys, version=None):
        new_keys = [ self.make_key(x, version=version) for x in keys ]
        ret = self._cache.get_multi(new_keys)
        if ret:
            _ = {}
            m = dict(zip(new_keys, keys))
            for k, v in ret.items():
                _[m[k]] = v

            ret = _
        return ret

    def close(self, **kwargs):
        self._cache.disconnect_all()

    def incr(self, key, delta=1, version=None):
        key = self.make_key(key, version=version)
        if delta < 0:
            return self._cache.decr(key, -delta)
        else:
            try:
                val = self._cache.incr(key, delta)
            except self.LibraryValueNotFoundException:
                val = None

            if val is None:
                raise ValueError("Key '%s' not found" % key)
            return val

    def decr(self, key, delta=1, version=None):
        key = self.make_key(key, version=version)
        if delta < 0:
            return self._cache.incr(key, -delta)
        else:
            try:
                val = self._cache.decr(key, delta)
            except self.LibraryValueNotFoundException:
                val = None

            if val is None:
                raise ValueError("Key '%s' not found" % key)
            return val

    def set_many(self, data, timeout=DEFAULT_TIMEOUT, version=None):
        safe_data = {}
        for key, value in data.items():
            key = self.make_key(key, version=version)
            safe_data[key] = value

        self._cache.set_multi(safe_data, self.get_backend_timeout(timeout))

    def delete_many(self, keys, version=None):
        self._cache.delete_multi(self.make_key(key, version=version) for key in keys)

    def clear(self):
        self._cache.flush_all()

class MemcachedCache(BaseMemcachedCache):
    """An implementation of a cache binding using python-memcached"""

    def __init__(self, server, params):
        import memcache
        super(MemcachedCache, self).__init__(server, params, library=memcache, value_not_found_exception=ValueError)

    @property
    def _cache(self):
        if getattr(self, '_client', None) is None:
            client_kwargs = dict(pickleProtocol=pickle.HIGHEST_PROTOCOL)
            client_kwargs.update(self._options)
            self._client = self._lib.Client(self._servers, **client_kwargs)
        return self._client

class PyLibMCCache(BaseMemcachedCache):
    """An implementation of a cache binding using pylibmc"""

    def __init__(self, server, params):
        import pylibmc
        super(PyLibMCCache, self).__init__(server, params, library=pylibmc, value_not_found_exception=pylibmc.NotFound)
        legacy_behaviors = {}
        for option in list(self._options):
            if option not in ('behaviors', 'binary', 'username', 'password'):
                warnings.warn('Specifying pylibmc cache behaviors as a top-level property within `OPTIONS` is deprecated. Move `%s` into a dict named `behaviors` inside `OPTIONS` instead.' % option, RemovedInDjango21Warning, stacklevel=2)
                legacy_behaviors[option] = self._options.pop(option)

        if legacy_behaviors:
            self._options.setdefault('behaviors', {}).update(legacy_behaviors)

    @cached_property
    def _cache(self):
        return self._lib.Client(self._servers, **self._options)

    def close(self, **kwargs):
        pass


import requests
import json
import pandas as pd
import yaml
from datetime import datetime

with open('G:\WEBSCRAP\REAL_ESTATE\SgpRealEstateSecrets.yaml','r') as masked_data:
    secrets = yaml.load(masked_data, Loader=yaml.FullLoader)

myAccessKey=secrets['AccessKey']
URL_token = "https://www.ura.gov.sg/uraDataService/insertNewToken.action"
tokenResponse = requests.get(URL_token, headers={'AccessKey':myAccessKey})
mytoken= (tokenResponse.text.split(":"))[3].split('}')[0].replace('"','')

outputDir = 'I:\\REAL_ESTATE_DATA\\DATA_TO_PROCESS'

for i in range(1,5) :
    transactionFileName = 'TransactionData_' + datetime.today().strftime('%d_%m_%Y') + '_' + str(i) + '.csv'
    condoFileName = 'CondoData_' + datetime.today().strftime('%d_%m_%Y') + '_' + str(i) + '.csv'
    df_CondoDetails = pd.DataFrame()
    df_Transactions = pd.DataFrame()
    URL_privateResidenceTransaction = 'https://www.ura.gov.sg/uraDataService/invokeUraDS?service=PMI_Resi_Transaction&batch=' + str(i)
    Response_transaction = requests.get(URL_privateResidenceTransaction, headers={'AccessKey':myAccessKey, 'token':mytoken})
    Response_transaction = Response_transaction.text
    transactions = json.loads(Response_transaction)
    transactions = transactions['Result']

    for transaction_rec in transactions :
        CondoName = transaction_rec['project']
        Street = transaction_rec['street']
        MarketSegment = transaction_rec['marketSegment']
        District = transaction_rec['transaction'][0]['district']
        Tenure = transaction_rec['transaction'][0]['tenure']
        PropertyType = transaction_rec['transaction'][0]['propertyType']
        condoDict = {'CondoName': [CondoName],
                     'Street': [Street],
                     'MarketSegment': [MarketSegment],
                     'District': [District],
                     'PropertyType': [PropertyType]
                    }
        tempdf_CondoDetails = pd.DataFrame.from_dict(condoDict )

        trans = transaction_rec['transaction']
        tempdf_Transactions = pd.DataFrame(trans)
        tempdf_Transactions['CondoName'] = CondoName
        if df_CondoDetails.empty == True:
            df_CondoDetails = tempdf_CondoDetails.copy()
        df_CondoDetails = df_CondoDetails.append(tempdf_CondoDetails)

        if df_Transactions.empty == True:
            df_Transactions = tempdf_Transactions.copy()
        df_Transactions = df_Transactions.append(tempdf_Transactions)

    df_Transactions['contractDate_mm'] = df_Transactions['contractDate'].str[:2]
    df_Transactions['contractDate_yy'] = df_Transactions['contractDate'].str[2:]
    if 'nettPrice' in df_Transactions :
        df_Transactions = df_Transactions.drop(columns=['nettPrice'])
    df_CondoDetails = df_CondoDetails.set_index('CondoName')
    df_Transactions = df_Transactions.set_index('CondoName')

    df_Transactions.to_csv(outputDir + '\\' + transactionFileName )
    df_CondoDetails.to_csv(outputDir + '\\' + condoFileName )


# ==== Parameters =============================================================
# --filtered_dmoz /home/olga/Schreibtisch/MASTER__ARBEIT/RESOURCES/filtered_dmoz.txt
# --corpus_dir /home/olga/Schreibtisch/MASTER__ARBEIT/RESOURCES
# --log_dir /home/olga/master_arbeit/logger/logs
# --corpus_size 40000
# =============================================================================

import logging
import argparse
import os.path
import random
from time import gmtime, strftime
from time import time

from loaders.doc_corpus_loader import DocCorpusLoader
from logger.logging import init_logging
from scripts.dataset_processing.filter_valid_urls import create_dir
from scripts.dataset_processing.corpus_builder import save_text, page_content_crawlen

def config_arg_parser():
    parser = argparse.ArgumentParser(description='Build corpus of URLs')
    parser.add_argument('--filtered_dmoz', required=True,
                        help='a path to the file with a list of filtered URLs and categories')
    parser.add_argument('--corpus_dir', required=True, help='a target directory to save corpus of valid URLs')
    parser.add_argument('--log_dir', required=True, help='path to logging directory')
    parser.add_argument('--corpus_size', required=True, help='size of corpus')
    return parser.parse_args()

logger = logging.getLogger()
loader = DocCorpusLoader()

def select_random_pages(pages_list, number):
    return random.sample(pages_list, number)

class DMOZCorpus():
    def __init__(self):
        self.filtered_dmoz = arguments.filtered_dmoz
        self.corpus_dir = ''
        self.corpus_size = int(arguments.corpus_size)

    def build_corpus(self, corpus_dir):
        self.corpus_dir = corpus_dir
        corpus_size = self.get_corpus_size()
        if corpus_size != self.corpus_size:
            self.append_corpus()

    def append_corpus(self, id=1):
        number_of_pages = self.get_number_of_pages()
        logging.info('Number of missing pages: {0}'.format(number_of_pages))
        file = open(self.filtered_dmoz, "r")
        lines = file.readlines()
        random_lines = select_random_pages(lines, number_of_pages)

        for line in random_lines:
            splitted_line = line.split
            url = splitted_line[0]
            label = splitted_line[1].replace('\n', '')
            logger.info('Check: {0}) {1}'.format(id, url))
            page_content = page_content_crawlen(url)

            if (page_content != '') and (page_content is not None):
                f = '{0}/{1}___{2}.txt'.format(corpus_dir, url, label)
                save_text(page_content, f)
            else:
                filename = '{0}/invalid_urls.txt'.format(self.corpus_dir)
                with open(filename, 'a') as file:
                    file.write('{0},{1}\n'.format(url, label))
                    file.close()
            id += 1
        corpus_size = self.get_corpus_size()
        logging.info('Current corpus size: {0}'.format(corpus_size))
        if corpus_size != self.corpus_size:
            self.append_corpus(id)

    def get_number_of_pages(self):
        corpus_size = self.get_corpus_size()
        if corpus_size != self.corpus_size:
            return self.corpus_size - corpus_size

    def get_corpus_size(self):
        return len(os.listdir(self.corpus_dir))

if __name__ == "__main__":
    start_time = time()
    arguments = config_arg_parser()
    logger.info("Started building DMOZ Corpus of URLs")

    log_file = os.path.join(arguments.log_dir,
                            '__log__dmoz_corpus_builder__{0}.log'.format(strftime("%Y-%m-%d   %H:%M:%S", gmtime())))
    init_logging(arguments.log_dir, log_file)

    corpus_dir = os.path.join(arguments.corpus_dir, 'DMOZ_CORPUS')
    create_dir(corpus_dir)
    dmoz = DMOZCorpus()
    dmoz.build_corpus(corpus_dir)

    logger.info("Total execution time of URL Corpus building is %0.4fs" % (time() - start_time))
    logger.info('Corpus of URLs is saved to : {0}'.format(corpus_dir))
    logger.info("="*60)


import time
from selenium import webdriver
from selenium.webdriver.support.ui import Select
import math

def calc(x):
  return str(math.log(abs(12*math.sin(int(x)))))

browser = webdriver.Chrome()

browser.get("http://suninjuly.github.io/alert_accept.html")
time.sleep(1)

submit_button = browser.find_element_by_css_selector('[type="submit"]')
submit_button.click()

alert = browser.switch_to.alert
alert.accept()

time.sleep(1)

x = browser.find_element_by_id("input_value").text

input_text = browser.find_element_by_id("answer")
submit_button = browser.find_element_by_css_selector('[type="submit"]')

input_text.send_keys(calc(x))

submit_button.click()
time.sleep(15)

# browser.execute_script("alert('Robots at work');")
# browser.execute_script('document.title="Script executing";')
# element = browser.execute_script('document.getElementsByName("name")')
# getElementById
# getElementsByTagName
# getElementsByClassName
# querySelector - для CSS
# querySelectorAll - для CSS (находит все совпадения)
# button = browser.find_element_by_tag_name("button")
# browser.execute_script("return arguments[0].scrollIntoView(true);", button)
# button.click()
# button = document.getElementsByTagName("button")[0];
# button.scrollIntoView();
#
# human_radio = browser.find_element_by_id("humanRule")
# human_checked = human_radio.get_attribute("checked")
# print("value of human radio: ", human_checked)
# assert human_checked is not None, "Human radio is not selected by default"

# browser.find_element_by_tag_name("select").click()
# browser.find_element_by_css_selector("option:nth-child(2)").click()
# Последняя строчка может выглядеть и так:
#
# browser.find_element_by_css_selector("[value='1']").click()
# Это не самый удобный способ, так как нам приходится делать лишний клик для открытия списка.
#
# Есть более удобный способ, для которого используется специальный класс Select из библиотеки WebDriver.
# Вначале мы должны инициализировать новый объект, передав в него WebElement с тегом select.
# Далее можно найти любой вариант из списка с помощью метода select_by_value(value):
#
# from selenium.webdriver.support.ui import Select
# select = Select(browser.find_element_by_tag_name("select"))
# select.select_by_value("1") # ищем элемент с текстом "Python"
#
# select.select_by_visible_text("text")
# select.select_by_index(index)
# select.select_by_visible_text("Python")
#
# import os
#
# current_dir = os.path.abspath(os.path.dirname(__file__))    # получаем путь к директории текущего исполняемого файла
# file_path = os.path.join(current_dir, 'file.txt')           # добавляем к этому пути имя файла
# element.send_keys(file_path)
# alert = browser.switch_to.alert
# alert.accept()
# Чтобы получить текст из alert, используйте свойство text объекта alert:
#
# alert = browser.switch_to.alert
# alert_text = alert.text
# метод для отказа: alert.dismiss()
#
# Третий вариант модального окна — prompt — имеет дополнительное поле для ввода текста. Чтобы ввести текст, используйте метод send_keys():
#
# prompt = browser.switch_to.alert
# prompt.send_keys("My answer")
# prompt.accept()

# После выполнения всех действий мы не должны забыть закрыть окно браузера
browser.quit()


import os
import traceback
import discord
from discord.ext import commands
from urllib import parse, request
import re
import datetime
import pytz
from bs4 import BeautifulSoup

bot = commands.Bot(command_prefix = '/' , description="This is a Helper Bot" )
token = os.environ['DISCORD_BOT_TOKEN']
TEST_ID = 737270360277254164

@bot.command()
async def ping(ctx):
    await ctx.send('pong')

@bot.command()
async def dii(ctx):
    await ctx.send('バカ')

@bot.command()
async def pict(ctx):
    await ctx.send('http://www.no1game.net/games/escapemen/game0056.html')

@bot.command()
async def clear(ctx, number = 1):
    mgs = []
    number = int(number) #Converting the amount of messages to delete to an integer
    counter = 0

    async for x in ctx.message.channel.history(limit = number+1):
         if counter < number+1:
             print(x.content)
             mgs.append(x)
             counter += 1
    await x.channel.delete_messages(mgs)
    delmsg = await ctx.send(str(counter-1) + "件削除しました")
    print(delmsg.content)
    await delmsg.delete(delay=2)

@bot.command()
async def youtube(ctx, *, search):
    query_string = parse.urlencode({'search_query': search})
    html_content = request.urlopen('http://www.youtube.com/results?' + query_string)
    OBJ = BeautifulSoup(html_content,"html.parser")
    html_content.close()
    search_results = re.findall('"videoId":"(.{11})"' , OBJ.decode())
    if search_results :
        pass
    else:
        print('nasi')
    # I will put just the first result, you can loop the response to show more results
    await ctx.send('https://www.youtube.com/watch?v='+ search_results[0])
    await ctx.send('https://www.youtube.com/watch?v='+ search_results[4])

@bot.event
async def on_ready():
   channel1 = bot.get_channel(TEST_ID)
   await channel1.send("起動")

@bot.event
async def on_voice_state_update(member, before, after):
    if before.channel == after.channel:
        print('STATS変更')
        return
    if after.channel is None:
        print('退出')
        return
    if  after.channel is not None:
        TIME = datetime.datetime.now(pytz.timezone('Asia/Tokyo'))
        STR_TIME = TIME.strftime('%Y/%m/%d %H:%M')
        print(member.display_name + ' ' + after.channel.name)
        channel2 = bot.get_channel(TEST_ID)
        await channel2.send("[" + STR_TIME +  "]" + member.display_name + "が" + after.channel.name + "にきたぞ")

bot.run(token)


import pygame
import random
WIDTH = 800
HEIGHT = 600

class Projectile:
    def __init__(self,x,y,radius,color):
        self.x = x
        self.y = y
        self.radius = radius
        self.color = color
        self.vel = 7

    def draw(self,win):
        pygame.draw.circle(win, self.color, (self.x,self.y), self.radius)


import graphviz as graphviz
import numpy as np
import pandas as pd
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier, export_graphviz

'''
 Author: Cristiano Casadei
'''

# carico il dataset
indianLiver = pd.read_csv("./liver.csv")
print("Diamo uno sguardo alla struttura dati")
print(indianLiver.info())
print()
print("Diamo uno sguardo ai dati")
print(indianLiver.head())
print()

# codifico le colonne non numeriche con delle colonne booleane dummy
# in questo caso la colonna "gender" verrà sostituita da due colonne
# "gender_female" e "gender_male" che indicheranno l'appartenenza al genere
indianLiver = pd.get_dummies(indianLiver)
print("Il dataset modificato con le colonne dummies del sesso")
print(indianLiver.head())
print()

# creo i dataset delle proprietà X e del target Y
X = indianLiver.drop("LABEL", axis=1).values
Y = indianLiver["LABEL"].values

# elimino le righe dove compare un NaN nei valori (darebbe errore in fase di elaborazione)
Y = Y[~np.isnan(X).any(axis=1)]
X = X[~np.isnan(X).any(axis=1)]

# suddivido il dataset in dataset di train e di test
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.3, random_state=0)

# NOTA: gli alberi decisionali non richiedono normalizzazione o standardizzazione!

# istanzio la classe di creazione di alberi decisionali, utilizzando l'algoritmo Gini per calcolare l'impurità
decTree = DecisionTreeClassifier(criterion="gini")
decTree.fit(X_train, Y_train)

# calcolo le predizioni su train set e test set
Y_pred_train = decTree.predict(X_train)
Y_pred_test = decTree.predict(X_test)

# calcolo l'accuracy
accuracy_train = accuracy_score(Y_train, Y_pred_train)
accuracy_test = accuracy_score(Y_test, Y_pred_test)

print("Uso l'algoritmo 'gini' per l'impurità")
print("Profondità albero:", decTree.tree_.max_depth)
print("Accuracy train set:", accuracy_train)
print("Accuracy test set:", accuracy_test)
print()

# metto in grafica l'albero decisionale e creo un pdf
dot_data = export_graphviz(decTree, out_file=None,
                           feature_names=indianLiver.columns.drop("LABEL"))
graph = graphviz.Source(dot_data)
graph.render("liver_tree_gini", view=True)

# rifaccio usando l'entropia
decTree = DecisionTreeClassifier(criterion="entropy")
decTree.fit(X_train, Y_train)

# calcolo le predizioni su train set e test set
Y_pred_train = decTree.predict(X_train)
Y_pred_test = decTree.predict(X_test)

# calcolo l'accuracy
accuracy_train = accuracy_score(Y_train, Y_pred_train)
accuracy_test = accuracy_score(Y_test, Y_pred_test)

print("Uso l'algoritmo 'entropy' per l'impurità")
print("Profondità albero:", decTree.tree_.max_depth)
print("Accuracy train set:", accuracy_train)
print("Accuracy test set:", accuracy_test)
print()

# metto in grafica l'albero decisionale e creo un pdf
dot_data = export_graphviz(decTree, out_file=None,
                           feature_names=indianLiver.columns.drop("LABEL"))
graph = graphviz.Source(dot_data)
graph.render("liver_tree_entropy", view=True)

# rifaccio usando nuovamente Gini, ma limitando la profondità dell'albero
decTree = DecisionTreeClassifier(criterion="gini", max_depth=2)
decTree.fit(X_train, Y_train)

# calcolo le predizioni su train set e test set
Y_pred_train = decTree.predict(X_train)
Y_pred_test = decTree.predict(X_test)

# calcolo l'accuracy
accuracy_train = accuracy_score(Y_train, Y_pred_train)
accuracy_test = accuracy_score(Y_test, Y_pred_test)

print("Uso l'algoritmo 'gini' per l'impurità e limito a 2 la profondità dell'albero")
print("Profondità albero:", decTree.tree_.max_depth)
print("Accuracy train set:", accuracy_train)
print("Accuracy test set:", accuracy_test)
print()

# metto in grafica l'albero decisionale e creo un pdf
dot_data = export_graphviz(decTree, out_file=None,
                           feature_names=indianLiver.columns.drop("LABEL"))
graph = graphviz.Source(dot_data)
graph.render("liver_tree_gini_depth2", view=True)


#!/usr/bin/python

import cv2
import numpy as np

def filter_region(image, vertices):
    mask = np.zeros_like(image)
    if len(mask.shape) == 2:
        cv2.fillPoly(mask, vertices, 255)
    else:
        cv2.fillPoly(mask, vertices, (255,)*mask.shape[2])
    return cv2.bitwise_and(image, mask)

kernel = np.ones((5,5),np.uint8)
cap = cv2.VideoCapture(0)
ret,frame = cap.read()
rows, cols = frame.shape[:2]

bottom_left = [cols*.1, rows]
top_left = [cols*0.4, rows*0.6]
bottom_right = [cols*0.9, rows]
top_right = [cols*0.6, rows*0.6]
vertices = np.array([[bottom_left, top_left, top_right, bottom_right]], dtype = np.int32)

while True:
    ret,frame = cap.read()
    hsv = cv2.cvtColor(filter_region(frame, vertices) , cv2.COLOR_BGR2HSV)
    ymn = np.array([20,100,100])
    ymx = np.array([50,255,255])

    mask = cv2.inRange(hsv, ymn, ymx)
    res = cv2.bitwise_and(frame, frame, mask = mask)
    cv2.imshow('res', res)
    cv2.imshow('mask', mask)

    threshold = []
    for i in range(479):
        if mask[i][240] != 0:
            threshold.append(255)
        else:
            threshold.append(0)
    print(threshold)

    edge = cv2.Canny(mask, 50, 150, apertureSize= 3)
    cv2.imshow('edges', edge)

    minLineLength = 100
    maxLineGap = 10
    lines = cv2.HoughLinesP(edge,1,np.pi/180,100,minLineLength,maxLineGap)
    if lines is not None:
        for x1,y1,x2,y2 in lines[0]:
            cv2.line(res,(x1,y1),(x2,y2),(0,255,0),2)

        cv2.imshow('houghlines2',res)

    a = cv2.waitKey(5)
    if a == 20:
        break
cv2.destroyAllWindows()
cap.release()


""" Module for testing CoinBasse API """

import json
from sys import path
from unittest import TestCase

from hokonui.exchanges.base import Exchange as base
from hokonui.exchanges.coinbase import CoinBase as cba
from hokonui.utils.helpers import docstring_parameter as docparams

LIBPATH = "../hokonui"
if LIBPATH not in path:
    path.append(LIBPATH)

class TestCoinbase(TestCase):
    """Class for testing Coinbase API"""

    @classmethod
    @docparams(cba.__name__, "setup")
    def setup(cls):
        """{0}.{1}"""
        print(__name__, ": TestClass.setup_class() ----------")

    @classmethod
    @docparams(cba.__name__, "teardown")
    def teardown(cls):
        """{0}.{1}"""
        print(__name__, ": TestClass.teardown_class() -------")

    @classmethod
    @docparams(cba.__name__, "name")
    def test_name(cls):
        """{0}.{1}"""
        assert cba.NAME == cls.__name__.replace("Test", "")

    @classmethod
    @docparams(cba.__name__, "price")
    def test_price(cls):
        """{0}.{1}"""
        assert float(cba.get_current_price(base.CCY_DEFAULT, None, None, cba.HEADER)) > 0.00

    @classmethod
    @docparams(cba.__name__, "bid")
    def test_bid(cls):
        """{0}.{1}"""
        assert float(cba.get_current_bid(base.CCY_DEFAULT, None, None, cba.HEADER)) > 0.00

    @classmethod
    @docparams(cba.__name__, "ask")
    def test_ask(cls):
        """{0}.{1}"""
        assert float(cba.get_current_ask(base.CCY_DEFAULT, None)) > 0.00

    @classmethod
    @docparams(cba.__name__, "ticker")
    def test_ticker(cls):
        """{0}.{1}"""
        data = json.loads(cba.get_current_ticker(base.CCY_DEFAULT, None))
        assert data["pair"] == base.CCY_DEFAULT, "pair should be base.CCY_DEFAULT"
        assert float(data["ask"]) > 0.00, "ask should not be empty"
        assert float(data["bid"]) > 0.00, "bid should not be empty"
        assert float(data["bid"]) <= float(data["ask"]), "bid should be < ask"
        assert float(data["timestamp"]) > 0, "Timestamp should be > zero"

    @classmethod
    @docparams(cba.__name__, "orders")
    def test_orders(cls):
        """{0}.{1}"""
        orders = cba.get_current_orders(base.CCY_DEFAULT)
        assert len(orders["asks"]) > 0, "Asks array should not be empty"
        assert len(orders["bids"]) > 0, "Bids array should not be empty"
        assert orders["source"] == "Coinbase", "Source should be 'Coinbase'"
        assert float(orders["timestamp"]) > 0, "Timestamp should be > zero"

if __name__ == "__main__":
    pass


# coding: utf-8

# flask
from flask import Flask
# request param
from flask import request
# GPIO
import RPi.GPIO as GPIO
import time
###########################################################

# GPIO 핀번호 할당
GPIO.setmode(GPIO.BOARD)

# pin 번호 chanel
LED = 19
#LED_2= 16
# 핀 설정
GPIO.setup(LED, GPIO.OUT, initial=GPIO.LOW)
#GPIO.setup(LED_2, GPIO.OUT, initial=GPIO.LOW)

# PWM 객체 생성 : 11번 핀, 주파수 - 100Hz
p_1 = GPIO.PWM(LED, 100)
#p_2= GPIO.PWM(LED_2, 100)

# PWM 신호 출력
p_1.start(0)
#p_2.start(0)
def change_duty_1(c):
    p_1.ChangeDutyCycle(c)

#def change_duty_2(c):
#    p_2.ChangeDutyCycle(c)

####################################################

app = Flask(__name__)

@app.route('/')

def index():

   while 1:
       dc=[10,20,30,40,50]
       for value in dc:

            value =  request.args.get('value', '0')
            #val_2 = request.args.get('val_2', '0')

            change_duty_1(int(value))
            time.sleep(1)
    #change_duty_2(int(val_2))

            return value

if __name__ == '__main__':
    # host
    # 로컬 테스트 시에는 개발하는 PC의 IP로 설정
    # 라즈베리파이 에서 실행을 할 경우 라즈베리파이 IP를 설정
    app.run(host='192.168.0.68', port='5000')


#!/usr/bin/env python
#
# Copyright 2011 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# vim: set ts=4 sw=4 et tw=79:

"""Implementation of a counter with persistent storage."""

__author__ = 'Greg Darke <darke@google.com>'

import logging
import math
import time

from google.appengine.api import backends
from google.appengine.api import runtime
from google.appengine.api import taskqueue
from google.appengine.ext import db
from google.appengine.ext import webapp
from google.appengine.ext.webapp.util import run_wsgi_app

_MEMORY_LIMIT = 128  # The memory limit for a B1 server
_CULL_AMOUNT = 0.15

def _get_taskqueue_target():
    return '%d.%s' % (backends.get_instance(), backends.get_backend())

class CounterModel(db.Model):
    value = db.IntegerProperty(default=0)
    _dirty = False
    _last_accessed = None

    @classmethod
    def get_or_new(cls, name):
        model = cls.get_by_key_name(name)
        if model is None:
            model = cls(key_name=name)
        return model

class CounterStore(object):
    def __init__(self):
        self._store = {}
        self._has_shutdown = False
        self._dirty = False
        self._batch_size = 100

    def get_value(self, name):
        self._ensure_within_memory_limit()

        if name not in self._store:
            model = CounterModel.get_or_new(name)
            self._store[name] = model
        else:
            model = self._store[name]

        model._last_accessed = time.time()
        return model

    def inc_value(self, name, delta):
        if not self._dirty:
            # Enqueue a task with a 'target' specifying traffic be sent to this
            # instance of the backend.
            taskqueue.add(url='/backend/counter/flush',
                          target=_get_taskqueue_target(),
                          countdown=2)
            self._dirty = True

        model = self.get_value(name)
        model.value += delta
        model._dirty = True
        if self._has_shutdown:
            # Since the shutdown hook may be called at any time, we need to
            # protect ourselves against this.
            model.put()
        return model

    def _ensure_within_memory_limit(self):
        memory_limit = _MEMORY_LIMIT * 0.8

        memory_usage = runtime.memory_usage().current()
        if memory_usage >= memory_limit:
            # Create a list of candidate counters to remove. We remove counters
            # that have not been modified before those that have been modified,
            # then order them by the last time they were accessed.
            counters = self._store.values()
            counters.sort(key=lambda counter: (counter._dirty,
                                               counter._last_accessed))
            counters_to_cull = int(math.ceil(len(counters) * _CULL_AMOUNT))
            counters = counters[:counters_to_cull]

            logging.info('Removing %d entries as we are over the memory limit '
                         'by %dMB.',
                         counters_to_cull, memory_limit - memory_usage)

            self._write_in_batches(counters)
            for counter in counters:
                del self._store[counter.key().name()]

    def _put_counters(self, counters):
        db.put(counters)
        for counter in counters:
            counter._dirty = False

    def _write_in_batches(self, counters):
        """Write out the dirty entries from 'counters' in batches.

        The batch size is determined by self._batch_size.

        Args:
          counters: An iterable containing instances of CounterModel.
        """
        to_put = []
        for counter in counters:
            if counter._dirty:
                to_put.append(counter)

                if len(to_put) >= self._batch_size:
                    self._put_counters(to_put)
                    to_put = []

        if to_put:
            self._put_counters(to_put)

    def flush_to_datastore(self):
        """Write the dirty entries from _store to datastore."""
        self._write_in_batches(self._store.itervalues())
        self._dirty = False

    def shutdown_hook(self):
        """Ensures all counters are written to datastore."""
        if self._dirty:
            self.flush_to_datastore()
        self._has_shutdown = True

class StartHandler(webapp.RequestHandler):
    """Handler for '/_ah/start'.

    This url is called once when the backend is started.
    """
    def get(self):
        runtime.set_shutdown_hook(_counter_store.shutdown_hook)

class FlushHandler(webapp.RequestHandler):
    """Handler for counter/flush.

    This handler is protected by login: admin in app.yaml.
    """
    def post(self):
        _counter_store.flush_to_datastore()

class CounterHandler(webapp.RequestHandler):
    """Handler for counter/{get,inc,dec}.

    This handler is protected by login: admin in app.yaml.
    """

    def _write_error(self, error_message):
        self.response.error(400)
        self.response.out.write(error_message)

    def post(self, method):
        """Handler a post to counter/{get,inc,dec}.

        The 'method' parameter is parsed from the url by a regex capture group.

        Args:
          method: The type of operation to perform against the counter store.
              It must be one of 'get', 'inc' or 'dec'.
        """
        key_name = self.request.get('name')
        delta = self.request.get('delta')

        if not key_name:
            self._write_error('Request did not have a "key_name" parameter.')
            return

        if method == 'get':
            model = _counter_store.get_value(key_name)
        else:
            if not delta:
                self._write_error('Request did not have a "delta" parameter.')
                return
            try:
                delta = int(delta)
            except ValueError:
                self._write_error('"delta" is not an integer.')
                return

            if method == 'inc':
                model = _counter_store.inc_value(key_name, delta)
            elif method == 'dec':
                model = _counter_store.inc_value(key_name, -delta)

        self.response.headers['Content-Type'] = 'text/plain'
        self.response.out.write('%d' % model.value)

_handlers = [(r'/_ah/start', StartHandler),
             (r'/backend/counter/flush$', FlushHandler),
             (r'/backend/counter/(get|inc|dec)$', CounterHandler)]

application = webapp.WSGIApplication(_handlers)
_counter_store = CounterStore()

def main():
    run_wsgi_app(application)

if __name__ == '__main__':
    main()


"""
Test cases for the metadata reading/writing of pyslim.
"""
import os

import numpy as np
import pytest
import msprime
import tskit
import pyslim

import tests
from .recipe_specs import recipe_eq

class TestMetadataSchemas(tests.PyslimTestCase):

    def validate_table_metadata(self, table):
        ms = table.metadata_schema
        for j, row in enumerate(table):
            a = table.metadata_offset[j]
            b = table.metadata_offset[j+1]
            raw_md = table.metadata[a:b]
            # this checks to make sure metadata follows the schema
            enc_md = ms.validate_and_encode_row(row.metadata)
            assert bytes(raw_md) == enc_md

    def test_slim_metadata(self, recipe):
        tables = recipe["ts"].dump_tables()
        for t in (tables.populations, tables.individuals, tables.nodes, tables.edges,
                  tables.sites, tables.mutations, tables.migrations):
            self.validate_table_metadata(t)

    def test_default_metadata(self):
        for k in pyslim.slim_metadata_schemas:
            schema = pyslim.slim_metadata_schemas[k]
            entry = pyslim.default_slim_metadata(k)
            encoded = schema.validate_and_encode_row(entry)
            decoded = schema.decode_row(encoded)
            if entry is None:
                assert decoded is None
            else:
                assert entry == decoded

    def test_slim_metadata_schema_equality(self, recipe):
        t = recipe["ts"].dump_tables()
        assert t.metadata_schema == pyslim.slim_metadata_schemas['tree_sequence']
        assert t.edges.metadata_schema == pyslim.slim_metadata_schemas['edge']
        assert t.sites.metadata_schema == pyslim.slim_metadata_schemas['site']
        assert t.mutations.metadata_schema == pyslim.slim_metadata_schemas['mutation']
        assert t.nodes.metadata_schema == pyslim.slim_metadata_schemas['node']
        assert t.individuals.metadata_schema == pyslim.slim_metadata_schemas['individual']
        assert t.populations.metadata_schema == pyslim.slim_metadata_schemas['population']

class TestTreeSequenceMetadata(tests.PyslimTestCase):
    arbitrary_recipe = [next(recipe_eq())]  # for testing any one recipe

    def validate_slim_metadata(self, t):
        # t could be tables or a tree sequence
        schema = t.metadata_schema.schema
        assert 'SLiM' in schema['properties']
        assert 'SLiM' in t.metadata
        for k in pyslim.default_slim_metadata('tree_sequence')['SLiM']:
            assert k in schema['properties']['SLiM']['properties']
            assert k in t.metadata['SLiM']

    def validate_model_type(self, ts, model_type):
        assert ts.metadata['SLiM']['file_version'] == pyslim.slim_file_version
        assert ts.metadata['SLiM']['model_type'] == model_type
        assert ts.metadata['SLiM']['generation'] > 0
        assert ts.metadata['SLiM']['generation'] >= np.max(ts.tables.nodes.time)

    @pytest.mark.parametrize('recipe', arbitrary_recipe, indirect=True)
    def test_set_tree_sequence_metadata_errors(self, recipe):
        tables = recipe["ts"].dump_tables()
        tables.metadata_schema = tskit.MetadataSchema(None)
        assert len(tables.metadata) > 0
        with pytest.raises(ValueError):
            pyslim.set_tree_sequence_metadata(tables, "nonWF", 0)

    @pytest.mark.parametrize('recipe', arbitrary_recipe, indirect=True)
    def test_set_tree_sequence_metadata_keeps(self, recipe):
        # make sure doesn't overwrite other stuff
        dummy_schema = tskit.MetadataSchema({
                'codec': 'json',
                'type': 'object',
                'properties': { 'abc': { 'type': 'string' } }
                })
        dummy_metadata = { 'abc': 'foo' }
        tables = recipe["ts"].dump_tables()
        tables.metadata_schema = dummy_schema
        tables.metadata = dummy_metadata
        pyslim.set_tree_sequence_metadata(tables, "nonWF", 0)
        schema = tables.metadata_schema.schema
        for k in dummy_metadata:
            assert k in schema['properties']
            assert k in tables.metadata
            assert tables.metadata[k] == dummy_metadata[k]
        self.validate_slim_metadata(tables)
        assert tables.metadata['SLiM']['model_type'] == "nonWF"
        assert tables.metadata['SLiM']['generation'] == 0

    @pytest.mark.parametrize('recipe', arbitrary_recipe, indirect=True)
    def test_set_tree_sequence_metadata(self, recipe):
        tables = recipe["ts"].dump_tables()
        pyslim.set_tree_sequence_metadata(
                tables, "WF", 99,
                spatial_dimensionality='xy',
                spatial_periodicity='y',
                separate_sexes=False,
                nucleotide_based=True)
        self.validate_slim_metadata(tables)
        assert tables.metadata['SLiM']['model_type'] == "WF"
        assert tables.metadata['SLiM']['generation'] == 99
        assert tables.metadata['SLiM']['spatial_dimensionality'] == 'xy'
        assert tables.metadata['SLiM']['spatial_periodicity'] == 'y'
        assert tables.metadata['SLiM']['separate_sexes'] == False
        assert tables.metadata['SLiM']['nucleotide_based'] == True

    @pytest.mark.parametrize('recipe', recipe_eq("WF"), indirect=True)
    def test_WF_model_type(self, recipe):
        self.validate_model_type(recipe["ts"], "WF")

    @pytest.mark.parametrize('recipe', recipe_eq("nonWF"), indirect=True)
    def test_nonWF_model_type(self, recipe):
        self.validate_model_type(recipe["ts"], "nonWF")

    @pytest.mark.parametrize(
        'recipe', recipe_eq(exclude="user_metadata"), indirect=True)
    def test_recover_metadata(self, recipe):
        # msprime <=0.7.5 discards metadata, but we can recover it from provenance
        ts = recipe["ts"]
        tables = ts.dump_tables()
        tables.metadata_schema = tskit.MetadataSchema(None)
        tables.metadata = b''
        new_ts = pyslim.load_tables(tables)
        assert new_ts.metadata == ts.metadata

    @pytest.mark.parametrize('recipe', recipe_eq("user_metadata"), indirect=True)
    def test_user_metadata(self, recipe):
        ts = recipe["ts"]
        md = ts.metadata["SLiM"]
        #print(md)
        assert "user_metadata" in md
        assert md['user_metadata'] == {
                "hello" : ["world"],
                "pi" : [3, 1, 4, 1, 5, 9]
                }

class TestDumpLoad(tests.PyslimTestCase):
    '''
    Test reading and writing.
    '''

    def verify_times(self, ts, slim_ts):
        gen = slim_ts.slim_generation
        assert ts.num_nodes == slim_ts.num_nodes
        # verify internal consistency
        for j in range(slim_ts.num_nodes):
            assert slim_ts.node(j).time == slim_ts.tables.nodes.time[j]
        # verify consistency between tree sequences
        for n1, n2 in zip(ts.nodes(), slim_ts.nodes()):
            assert n1.time == n2.time

    def test_load_tables(self, recipe):
        ts = recipe["ts"]
        assert isinstance(ts, pyslim.SlimTreeSequence)
        tables = ts.dump_tables()
        new_ts = pyslim.load_tables(tables)
        assert isinstance(new_ts, pyslim.SlimTreeSequence)
        new_tables = new_ts.dump_tables()
        assert tables == new_tables

    def test_load(self, recipe):
        fn = recipe["path"]["ts"]
        # load in msprime then switch
        msp_ts = tskit.load(fn)
        assert isinstance(msp_ts, tskit.TreeSequence)
        # transfer tables
        msp_tables = msp_ts.dump_tables()
        new_ts = pyslim.load_tables(msp_tables)
        assert isinstance(new_ts, pyslim.SlimTreeSequence)
        self.verify_times(msp_ts, new_ts)
        new_tables = new_ts.dump_tables()
        self.assertTableCollectionsEqual(msp_tables, new_tables)
        # convert directly
        new_ts = pyslim.SlimTreeSequence(msp_ts)
        assert isinstance(new_ts, pyslim.SlimTreeSequence)
        self.verify_times(msp_ts, new_ts)
        new_tables = new_ts.dump_tables()
        self.assertTableCollectionsEqual(msp_tables, new_tables)
        # load to pyslim from file
        slim_ts = pyslim.load(fn)
        assert isinstance(slim_ts, pyslim.SlimTreeSequence)
        slim_tables = slim_ts.dump_tables()
        self.assertTableCollectionsEqual(msp_tables, slim_tables)
        assert slim_ts.slim_generation == new_ts.slim_generation

    def test_dump_equality(self, recipe, tmp_path):
        """
        Test that we can dump a copy of the specified tree sequence
        to the specified file, and load an identical copy.
        """
        tmp_file = os.path.join(tmp_path, "test_dump.trees")
        ts = recipe["ts"]
        ts.dump(tmp_file)
        ts2 = pyslim.load(tmp_file)
        assert ts.num_samples == ts2.num_samples
        assert ts.sequence_length == ts2.sequence_length
        assert ts.tables == ts2.dump_tables()
        assert ts.reference_sequence == ts2.reference_sequence

class TestAlleles(tests.PyslimTestCase):
    '''
    Test nothing got messed up with haplotypes.
    '''

    def test_haplotypes(self, recipe):
        slim_ts = recipe["ts"]
        tables = slim_ts.dump_tables()
        ts = tables.tree_sequence()
        self.verify_haplotype_equality(ts, slim_ts)

class TestNucleotides(tests.PyslimTestCase):
    '''
    Test nucleotide support
    '''

    def test_nucleotides(self, recipe):
        '''
        Check that nucleotides are all valid, i.e.,
        -1, 0, 1, 2, or 3.
        '''
        ts = recipe["ts"]
        for mut in ts.mutations():
            # print(mut)
            for u in mut.metadata['mutation_list']:
                assert u["nucleotide"] >= -1
                assert u["nucleotide"] <= 3

class TestDecoding(tests.PyslimTestCase):
    '''
    Test by comparing decoding to our previous direct implementation of struct decoding.
    '''

    def verify_decoding(self, t, decoder):
        ms = tskit.MetadataSchema(None)
        nt = t.copy()
        nt.metadata_schema = ms
        for a, b in zip(t, nt):
            md = a.metadata
            with pytest.warns(FutureWarning):
                omd = decoder(b.metadata)
            if md is None:
                assert omd is None
            else:
                assert md == omd.asdict()

    def verify_mutation_decoding(self, t):
        ms = tskit.MetadataSchema(None)
        nt = t.copy()
        nt.metadata_schema = ms
        for a, b in zip(t, nt):
            md = a.metadata
            with pytest.warns(FutureWarning):
                omd = pyslim.decode_mutation(b.metadata)
            assert md == {"mutation_list": [u.asdict() for u in omd]}

    def test_decoding(self, recipe):
        tables = recipe["ts"].dump_tables()
        self.verify_decoding(tables.populations, pyslim.decode_population)
        self.verify_decoding(tables.individuals, pyslim.decode_individual)
        self.verify_decoding(tables.nodes, pyslim.decode_node)
        self.verify_mutation_decoding(tables.mutations)

@pytest.mark.parametrize('recipe', [next(recipe_eq())], indirect=True)
class TestMetadataAttributeError(tests.PyslimTestCase):
    """
    These are all only tested on a single recipe, the first grabbed from recipe
    """

    def test_population_error(self, recipe):
        ts = recipe["ts"]
        for x in ts.populations():
            if x.metadata is not None:
                with pytest.raises(AttributeError) as exec_info:
                    _ = x.metadata.slim_id
                assert 'legacy' in str(exec_info)
                with pytest.raises(AttributeError) as exec_info:
                    _ = x.metadata.selfing_fraction
                assert 'legacy' in str(exec_info)
                with pytest.raises(AttributeError) as exec_info:
                    _ = x.metadata.sex_ratio
                assert 'legacy' in str(exec_info)
            with pytest.raises(AttributeError) as exec_info:
                _ = x.metadata.ping
            assert "has no attribute 'ping'" in str(exec_info)
            break

    def test_individual_error(self, recipe):
        ts = recipe["ts"]
        for x in ts.individuals():
            with pytest.raises(AttributeError) as exec_info:
                _ = x.metadata.pedigree_id
            assert 'legacy' in str(exec_info)
            with pytest.raises(AttributeError) as exec_info:
                _ = x.metadata.age
            assert 'legacy' in str(exec_info)
            with pytest.raises(AttributeError) as exec_info:
                _ = x.metadata.subpopulation
            assert 'legacy' in str(exec_info)
            with pytest.raises(AttributeError) as exec_info:
                _ = x.metadata.sex
            assert 'legacy' in str(exec_info)
            with pytest.raises(AttributeError) as exec_info:
                _ = x.metadata.flags
            assert 'legacy' in str(exec_info)
            with pytest.raises(AttributeError) as exec_info:
                _ = x.metadata.pong
            assert "has no attribute 'pong'" in str(exec_info)
            break

    def test_node_error(self, recipe):
        ts = recipe["ts"]
        for x in ts.nodes():
            if x.metadata is not None:
                with pytest.raises(AttributeError) as exec_info:
                    _ = x.metadata.slim_id
                assert 'legacy' in str(exec_info)
                with pytest.raises(AttributeError) as exec_info:
                    _ = x.metadata.is_null
                assert 'legacy' in str(exec_info)
                with pytest.raises(AttributeError) as exec_info:
                    _ = x.metadata.genome_type
                assert 'legacy' in str(exec_info)
            with pytest.raises(AttributeError) as exec_info:
                _ = x.metadata.pang
            assert "has no attribute 'pang'" in str(exec_info)
            break

    def test_mutation_error(self, recipe):
        ts = recipe["ts"]
        for x in ts.mutations():
            with pytest.raises(KeyError) as exec_info:
                _ = x.metadata[0]
            assert 'legacy' in str(exec_info)
            with pytest.raises(KeyError) as exec_info:
                _ = x.metadata[999]
            assert 'legacy' in str(exec_info)
            with pytest.raises(KeyError):
                _ = x.metadata['ping']
            break


import psycopg2

class Database():
    '''Create a psql database and execute the schema file'''

    def __init__(self, db_name, schema_file):
        self.db_name = db_name
        self.schema_file = schema_file
        conn = psycopg2.connect(dbname='postgres')
        conn.autocommit = True
        cursor = conn.cursor()
        cursor.execute("DROP DATABASE IF EXISTS " + db_name)
        cursor.execute("CREATE DATABASE " + db_name)
        cursor.close()
        conn.commit()
        conn.close()
        conn = psycopg2.connect("dbname="+db_name)
        cursor = conn.cursor()
        cursor.execute(open(self.schema_file, 'r').read())
        cursor.close()
        conn.commit()
        conn.close()

class Cursor(Database):
    '''Create a psycopg2 connection and cursor object

    Args: none

    Returns:
        An open cursor to the database "tournament".

    This class offers a context manager through __enter__ and __exit__ methods.
    Usage:
        with Cursor() as my_cursor:
            my_cursur.execute(some_sql)'''

    def __init__(self, database):
        self.db_name = database.db_name

    def __enter__(self):
        self.conn = psycopg2.connect("dbname=" + self.db_name)
        self.cursor = self.conn.cursor()
        return self.cursor

    def __exit__(self, exec_type, exec_val, exec_tb):
        self.conn.commit()
        self.cursor.close()
        self.conn.close()


#      888 ,e,                                     d8   888
#  e88 888  "   dP"Y 888 888 8e   ,"Y88b 888 8e   d88   888  ,e e,
# d888 888 888 C88b  888 888 88b "8" 888 888 88b d88888 888 d88 88b
# Y888 888 888  Y88D 888 888 888 ,ee 888 888 888  888   888 888   ,
#  "88 888 888 d,dP  888 888 888 "88 888 888 888  888   888  "YeeP"

# Copyright 2021 Gary Stidston-Broadbent
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =============================================================================
import shutil
from pathlib import Path
import pytest

@pytest.fixture(scope='function')
def datadir(tmpdir, request):
    """Fixture responsible for searching a folder with the same name of test
    module and, if available, moving all contents to a temporary directory so
    tests can use them freely.
    """
    filename = Path(request.module.__file__)
    test_dir = Path(filename.parent, 'data', filename.stem)

    if test_dir.is_dir():
        shutil.rmtree(str(tmpdir), ignore_errors=True)
        shutil.copytree(test_dir, str(tmpdir))

    return tmpdir

@pytest.fixture(scope="session")
def httpserver_listen_address():
    """Use port 9090 to ensure not to conflict with Area28s main port."""
    return ("127.0.0.1", 9090)


import datetime
import pickle
from os import path

import numpy as np
import tensorflow as tf
from keras import Model
from keras.callbacks import EarlyStopping, ModelCheckpoint, CSVLogger
from sklearn.metrics import confusion_matrix

from constants import EMOTIONS, WEEK, PKL_ROOT

def train(model: Model, x: [], y: [], EPOCHS: int, batch_size=4, early_stopping=True,
          save_history=True, TIME=datetime.datetime.now().strftime("%Y%m%d-%H%M%S")):
    print("Start Training")

    log_dir = "logs/week_{}/fit_{}_class/{}_{}_{}".format(WEEK, len(EMOTIONS), model.name,
                                                          type(model.optimizer).__name__,
                                                          TIME)
    tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir)

    callback_list = [
        ModelCheckpoint(
            filepath=model.name + '.h5',
            monitor='val_acc',
            save_best_only='True',
            verbose=1,
            mode='max'
        ), tensorboard_callback]

    if early_stopping:
        callback_list.append(
            EarlyStopping(
                monitor='val_loss',
                patience=10,
                verbose=1,
                mode='min'
            )
        )

    if save_history:
        history_file = "history/week_{}/{}_{}_{}.csv".format(WEEK, model.name, type(model.optimizer).__name__,
                                                             TIME)
        callback_list.append(
            CSVLogger(filename=history_file)
        )

    tic = datetime.datetime.now()
    history = model.fit(x, y,
                        batch_size=batch_size, epochs=EPOCHS,
                        validation_split=0.2,
                        verbose=True,
                        callbacks=callback_list)
    toc = datetime.datetime.now()

    diff = toc - tic
    print("Finished Training: Took : {} Seconds".format(diff.total_seconds()))

    return history, model, diff.total_seconds()

def test(model: Model, x: [], y: []):
    matrices = model.evaluate(x, y)

    test_results = {}
    for i in range(len(model.metrics_names)):
        print("{} : \t {}".format(model.metrics_names[i], matrices[i]))
        test_results[model.metrics_names[i]] = matrices[i]

    return test_results

def randomize_split(data, split_ratio=0.8):
    # shuffle the dataset
    np.random.shuffle(data)

    # divide training and testing dataset
    training_count = int(len(data) * split_ratio)

    training_data = data[:training_count]
    testing_data = data[training_count:]
    return training_data, testing_data

def plot_model(model, filename):
    filename = 'model_plots/{}.png'.format(filename)
    print("Plotting the model. Saving at: {}".format(filename))
    tf.keras.utils.plot_model(
        model,
        to_file=filename,
        show_shapes=True,
        show_layer_names=True
    )

def plot_confusion_matrix(cm,
                          target_names,
                          filename,
                          title='Confusion matrix',
                          cmap=None,
                          normalize=True):
    """
    given a sklearn confusion matrix (cm), make a nice plot

    Arguments
    ---------
    cm:           confusion matrix from sklearn.metrics.confusion_matrix

    target_names: given classification classes such as [0, 1, 2]
                  the class names, for example: ['high', 'medium', 'low']

    title:        the text to display at the top of the matrix

    cmap:         the gradient of the values displayed from matplotlib.pyplot.cm
                  see http://matplotlib.org/examples/color/colormaps_reference.html
                  plt.get_cmap('jet') or plt.cm.Blues

    normalize:    If False, plot the raw numbers
                  If True, plot the proportions

    Usage
    -----
    plot_confusion_matrix(cm           = cm,                  # confusion matrix created by
                                                              # sklearn.metrics.confusion_matrix
                          normalize    = True,                # show proportions
                          target_names = y_labels_vals,       # list of names of the classes
                          title        = best_estimator_name) # title of graph

    Citiation
    ---------
    http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html

    """
    import matplotlib.pyplot as plt
    import numpy as np
    import itertools

    accuracy = np.trace(cm) / float(np.sum(cm))
    misclass = 1 - accuracy

    if cmap is None:
        cmap = plt.get_cmap('Blues')

    plt.figure()
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title, pad=6)
    plt.colorbar()

    if target_names is not None:
        tick_marks = np.arange(len(target_names))
        plt.xticks(tick_marks, target_names, rotation=45)
        plt.yticks(tick_marks, target_names)

    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]

    thresh = cm.max() / 1.5 if normalize else cm.max() / 2
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        if normalize:
            plt.text(j, i, "{:0.4f}".format(cm[i, j]),
                     horizontalalignment="center",
                     color="white" if cm[i, j] > thresh else "black")
        else:
            plt.text(j, i, "{:,}".format(cm[i, j]),
                     horizontalalignment="center",
                     color="white" if cm[i, j] > thresh else "black")

    # plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label\naccuracy={:0.4f}; misclass={:0.4f}'.format(accuracy, misclass))
    plt.savefig(filename)
    print("Saved Confusion Matrix to: {}".format(filename))

def get_confusion_matrix(model: Model, x_test, y_test, prediction_index=None):
    predictions = model.predict(x_test)
    if prediction_index is None:
        prediction_classes = np.argmax(predictions[0], axis=1)
    else:
        prediction_classes = np.argmax(predictions[prediction_index], axis=1)

    return confusion_matrix(np.argmax(y_test, axis=1), prediction_classes)

def get_dataset(filename='signal-dataset.pkl'):
    if not path.exists(PKL_ROOT + filename):
        download(filename)

    with open(PKL_ROOT + filename, 'rb') as f:
        data = pickle.load(f)
        return data

def download(filename, base_url='https://s3-ap-southeast-1.amazonaws.com/usq.iothealth/iemocap/'):
    import urllib.request

    url = base_url + filename

    print('Beginning file download {}'.format(url))

    store_file = PKL_ROOT + filename
    urllib.request.urlretrieve(url, store_file)

    print("Downloaded and saved to file: {}".format(store_file))


"""Minimization of Homogenous Loss Functions
Author: Gabriel Fernandez
Date: 09/21/2018
git: gabriel80808
"""

import argparse
import numpy as np
import tensorflow as tf
import schema.loss_functions as lf

# Uncomment bottom out after debugging
tf.enable_eager_execution

def parse():
    """Pass in arguments form user for experiments"""
    parser = argparse.ArgumentParser()
    parser.add_argument('--initial_guess', help='Initial values or guess where minimum', default=None)
    parser.add_argument('--loss_func', type=str, help='Name of loss function to be minimized', default=None)
    parser.add_argument('--exp_name', '-n', type=str)
    args = parser.parse_args()

    return args

# def min_itr():
#     algo = JOpt(
#         x0,
#         loss_function,
#         gamma0=2,
#         flb=2)
#     algo.minimize()

if __name__ == '__main__':
    args = parse()
    loss_func = lf.HomogeneousQuartic(
        x0=np.array([5, 5, 5, 5]), flb=0)
    f = loss_func.eval_func(np.array([5.0, 7.1, 5.3, 6.5])[np.newaxis, :])
    g = loss_func.eval_grad(np.array([5, 7, 5, 6]))
    # min_itr()


import requests
from requests_toolbelt.multipart.encoder import MultipartEncoder
import os

PATH_TEST = os.path.dirname(os.path.realpath(__file__))

url = "http://127.0.0.1:8000/alignments/"
f = open(os.path.join(PATH_TEST, 'example/umbrella_line.txt'), 'rb')

alignment_data = MultipartEncoder(
    fields={
            # a file upload field
            'lyrics': ('test_long_lyrics.txt', f, 'text/plain'),
            # plain text fields
            'title': 'new composition',
            'accompaniment': '2',
            'level':'1'

           }
    )

response = requests.post(url, data=alignment_data, headers={'Content-Type': alignment_data.content_type})
response.raise_for_status()
print(response.text)


# -*- coding: utf-8 -*-
"""

    walle-web

    :copyright: © 2015-2017 walle-web.io
    :created time: 2017-03-25 11:15:01
    :author: wushuiyong@walle-web.io
"""

from flask import request
from walle.api.api import SecurityResource
from walle.form.server import ServerForm
from walle.model.server import ServerModel
from walle.service.extensions import permission
from walle.service.rbac.role import *

class ServerAPI(SecurityResource):
    def get(self, id=None):
        """
        fetch environment list or one item
        /environment/<int:env_id>

        :return:
        """
        super(ServerAPI, self).get()

        return self.item(id) if id else self.list()

    def list(self):
        """
        fetch environment list

        :return:
        """
        page = int(request.args.get('page', 0))
        page = page - 1 if page else 0
        size = int(request.args.get('size', 10))
        kw = request.values.get('kw', '')

        server_model = ServerModel()
        server_list, count = server_model.list(page=page, size=size, kw=kw)
        return self.list_json(list=server_list, count=count, enable_create=permission.enable_role(MASTER))

    def item(self, id):
        """
        获取某个用户组

        :param id:
        :return:
        """

        server_model = ServerModel(id=id)
        server_info = server_model.item()
        if not server_info:
            return self.render_json(code=-1)
        return self.render_json(data=server_info)

    def post(self):
        """
        create a environment
        /environment/

        :return:
        """
        super(ServerAPI, self).post()

        form = ServerForm(request.form, csrf_enabled=False)
        if form.validate_on_submit():
            server_new = ServerModel()
            id = server_new.add(name=form.name.data, host=form.host.data)
            if not id:
                return self.render_json(code=-1)

            return self.render_json(data=server_new.item(id))
        else:
            return self.render_json(code=-1, message=form.errors)

    def put(self, id):
        """
        update environment
        /environment/<int:id>

        :return:
        """
        super(ServerAPI, self).put()

        form = ServerForm(request.form, csrf_enabled=False)
        form.set_id(id)
        if form.validate_on_submit():
            server = ServerModel(id=id)
            ret = server.update(name=form.name.data, host=form.host.data)
            return self.render_json(data=server.item())
        else:
            return self.render_json(code=-1, message=form.errors)

    def delete(self, id):
        """
        remove an environment
        /environment/<int:id>

        :return:
        """
        super(ServerAPI, self).delete()

        server_model = ServerModel(id=id)
        server_model.remove(id)

        return self.render_json(message='')


import os
import json

folders = os.listdir(".")
folders = [f for f in folders if not f.startswith('.')]
folders = [f for f in folders if not f.endswith('.py')]

data = {}

for f in folders:
    files = os.listdir(f)
    files = [f + os.sep + fi for fi in files]
    files = [fi for fi in files if os.path.splitext(fi)[1].lower() in [".png", ".gif", ".jpeg", ".jpg"]]
    data[f] = files

print(data)

with open('images.json', 'w') as fp:
    json.dump(data, fp)


from django.db import models
from django.contrib.auth.models import User
from jdirs.models import Category, Reltype, Region
from django.utils.translation import ugettext_lazy as _
from django_bleach.models import BleachField

from django.shortcuts import render_to_response
from django.core.context_processors import csrf

class Client(models.Model):
    owner 		= models.ForeignKey(User, null=False, blank=False, related_name='client_owner')
    parent      = models.ForeignKey('self', null=True, blank=True, related_name='client_parent', verbose_name=_('Parent'))
    isfolder 	= models.BooleanField(default=False)
    code 		= models.CharField(max_length=9, null=False, unique=False, blank=False, verbose_name = _('Code'))
    name 		= BleachField(max_length=255, null=True, unique=False, blank=True, verbose_name = _('Name'))
    iscompany 	= models.BooleanField(default=True)
    INN 		= models.CharField(max_length=12, null=True, unique=False, blank=True)
    KPP 		= models.CharField(max_length=12, null=True, unique=False, blank=True)
    email 		= models.EmailField()
    uradress	= BleachField(max_length=150, null=True, unique=False, blank=True,  verbose_name = _('Uradress'))
    factadress	= models.CharField(max_length=150, null=True, unique=False, blank=True, verbose_name = _('Factadress'))
    phone		= BleachField(max_length=30, null=True, unique=False, blank=True, verbose_name = _('Phone'))
    fax			= models.CharField(max_length=30, null=True, unique=False, blank=True)
    site		= models.CharField(max_length=50, null=True, unique=False, blank=True)
    isdeleted 	= models.BooleanField(default=False)
    region 		= models.ForeignKey(Region, null=True, blank=True, related_name='client_region')
    categoty	= models.ForeignKey(Category, null=True, blank=True, related_name='client_category')
    reltype		= models.ForeignKey(Reltype, null=True, blank=True, related_name='client_reltype')

    def __unicode__(self):
        return '(' + str(self.id) + ')'+ self.code + ' ' + self.name

class Note(models.Model):
    code 		= models.CharField(max_length=9, null=False, unique=False, blank=False)
    client 		= models.ForeignKey(Client, null=False, blank=False, related_name='Note_client')
    content 	= models.TextField(null=True, unique=False, blank=True, default='')
    isdeleted 	= models.BooleanField(default=False)
    user 		= models.ForeignKey(User, null=False, blank=False, related_name='Note_user')

    def __unicode__(self):
        if len(self.content) > 25:
            cont = '...'
        else:
            cont = ''
        return self.client.code + ' ' + self.client.name + '   ' + self.content[:25] + cont