import pandas as pd
import itertools
import numpy as np
from utils import load_fasta_seq

"""
@author Jack Ringer
Date: 3/2/2023
Description:
    File containing various calculations used for part 1a of the assignment.
"""

def averaged_non_synom_freqs():
    """
    Calculate averaged frequencies of non-synonymous changes and
    save result to a csv file.
    :return: Non
    """
    df = pd.read_csv("../data/codons.txt", sep='\t')
    df = df.rename({'    AminoAcid': "AminoAcid"}, axis="columns")
    non_synom_cnts = {}
    synom_cnts = {}
    nucleotides = ["A", "C", "G", "T"]
    for base, mutant in itertools.product(nucleotides, nucleotides):
        if base == mutant:
            continue
        synom = 0
        non_synom = 0
        for i in range(0, 3):
            df_sub = df[
                df["Codon"].map(lambda x: True if x[i] == base else False)]
            # df_sub = df[df["Codon"].str.startswith(base)]
            df_mut = pd.DataFrame({})
            df_mut["Codon"] = df_sub["Codon"].map(
                lambda x: x[:i] + x[i].replace(base, mutant) + x[i + 1:])
            # see where amino acid changed
            for j in range(len(df_mut["Codon"])):
                cdn = df_mut["Codon"].iloc[j]
                new_acid = df[df["Codon"] == cdn]["AminoAcid"].iloc[0]
                old_acid = df_sub["AminoAcid"].iloc[j]
                if new_acid == old_acid:
                    synom += 1
                else:
                    non_synom += 1
        non_synom_cnts[base] = non_synom_cnts.get(base, 0) + non_synom
        synom_cnts[base] = synom_cnts.get(base, 0) + synom
    freq_mp = {}
    for nt in nucleotides:
        ns_cnt = non_synom_cnts[nt]
        s_cnt = synom_cnts[nt]
        freq_mp[nt] = ns_cnt / (ns_cnt + s_cnt)
    df = pd.DataFrame(freq_mp.items(), columns=["Nucleotide", "Frequency"])
    df["Frequency"] = df["Frequency"].map(lambda x: np.round(x, decimals=4))
    df.to_csv("../data/batch_non_synom_freqs.csv", index=False)

def cov19_nt_freqs(fasta_pth: str):
    """
    Get frequency of nucleotides (nt) in original Covid-19 genome.
    :param fasta_pth: path to .fasta file containing Covid-19 genome.
    :return: dict, where dict[nt] = frequency of nt in genome
    """
    genome = load_fasta_seq(fasta_pth)
    genome_len = len(genome)
    freqs = {}
    total = 0
    for nt in ['A', 'C', 'T', 'G']:
        freqs[nt] = genome.count(nt) / genome_len
        total += freqs[nt]
    assert total == 1.0
    df = pd.DataFrame(freqs.items(), columns=["Nucleotide", "Frequency"])
    df.to_csv("../results/covid_freqs.csv", index=False)
    return freqs

def cov19_rbd_nt_freqs(fasta_pth: str):
    """
    Get frequency of nucleotides (nt) in original Covid-19 RBD. (same as above function with different path.)
    :param fasta_pth: path to .fasta file containing Covid-19 RBD.
    :return: dict, where dict[nt] = frequency of nt in genome
    """

    # first 1269 (0:1268) nt's are not part of it,
    # the following 669 are the RBD

    genome = load_fasta_seq(fasta_pth)
    RBD_genome = genome[1269:1938]
    genome_len = len(RBD_genome)
    print(genome_len)
    freqs = {}
    total = 0
    for nt in ['A', 'C', 'T', 'G']:
        freqs[nt] = RBD_genome.count(nt) / genome_len
        total += freqs[nt]
    assert total == 1.0
    df = pd.DataFrame(freqs.items(), columns=["Nucleotide", "Frequency"])
    df.to_csv("../results/covid_rbd_freqs.csv", index=False)
    return freqs

def q2_main():
    """
    Script used to answer question 2 of part 1a on the report
    :return:
    """
    n_muts = 89709
    covid_freq_df = pd.read_csv("../results/covid_freqs.csv")
    nonsynom_freq_df = pd.read_csv("../results/batch_non_synom_freqs.csv")
    probs_sum = np.dot(covid_freq_df["Frequency"],
                       nonsynom_freq_df["Frequency"])
    non_synom_muts = np.round(n_muts * probs_sum, decimals=0)
    synom_muts = n_muts - non_synom_muts
    print("# of non-synonymous mutations:", non_synom_muts)
    print("# of synonymous mutations:", synom_muts)

if __name__ == "__main__":
    averaged_non_synom_freqs()
    # calc_non_synom_freqs()
    freqs = cov19_nt_freqs("../results/covid_seq.fasta")
    freqs_rbd = cov19_rbd_nt_freqs("../results/cov_spike_nt_genome.fasta")

    rounded_freqs = {k: np.round(v, decimals=4) for k, v in freqs.items()}
    rounded_rbd_freqs = {k: np.round(v, decimals=4) for k, v in freqs_rbd.items()}
    print(rounded_freqs)
    print(rounded_rbd_freqs)


from telnetlib import Telnet
import ast

# Assume passed in tn is in this state such that it will pass bytearrays like so:
# b"Test number: 1/50\n"
# b"array = [1, 5, 7, 3, 1]\n"
# b"k1 = 4\n"
# b"k2 = 2\n"
def parseBiggestLowest(tn):
    array = []
    k1 = 0
    k2 = 0
    answer = bytearray()

    # Read in the next line, b"array = []\n"
    line = tn.read_until(b"\n")
    if b'array = ' in line:
        # Parse the bytearray into an integer array, then sort it
        array = ast.literal_eval(line.decode("utf-8").split('array = ')[1])
        array.sort()
        # Read in the next line, b"k1 = x\n"
        line = tn.read_until(b"\n")
        k1 = int(line.decode("utf-8").split('k1 = ')[1])
        # Read in the next line, b"k2 = y\n"
        line = tn.read_until(b"\n")
        k2 = int(line.decode("utf-8").split('k2 = ')[1])

    # Assemble answer of a comma separated list of first k1 elements increasing,
    # a semicolon, then a comma separated list of the last k2 elements decreasing
    for i in range(0, k1):
        answer.extend(str(array[i]).encode())
        if i != (k1 - 1):
            answer += b", "
    answer += b"; "

    for i in range(0, k2):
        index = i * (-1) - 1
        answer.extend(str(array[index]).encode())
        if i != (k2 - 1):
            answer += b", "

    answer += b"\n"
    return answer

with Telnet('challs.xmas.htsp.ro', 6051) as tn:
    while True:
        line = tn.read_until(b"\n")
        if b'Test number: ' in line:
            answer = parseBiggestLowest(tn)
            tn.write(answer)
        # Print out the flag when we're done solving
        if b'X-MAS{' in line:
            print(line)
            break


import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.svm import SVC

# Load the dataset
data = pd.read_excel('embeddingsdata.xlsx')

# 'embed_0' and 'embed_1' are the features and 'Label' is the target variable
features = data[['embed_0', 'embed_1']]
target = data['Label']

# Split the data into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(features, target, test_size=0.2, random_state=42)

# Train and test SVM with 'linear' kernel
clf_linear = SVC(kernel='linear')
clf_linear.fit(X_train, y_train)
accuracy_linear = clf_linear.score(X_test, y_test)
print(f'Accuracy with linear kernel: {accuracy_linear}')

# Train and test SVM with 'poly' kernel
clf_poly = SVC(kernel='poly')
clf_poly.fit(X_train, y_train)
accuracy_poly = clf_poly.score(X_test, y_test)
print(f'Accuracy with poly kernel: {accuracy_poly}')

# Train and test SVM with 'rbf' kernel
clf_rbf = SVC(kernel='rbf')
clf_rbf.fit(X_train, y_train)
accuracy_rbf = clf_rbf.score(X_test, y_test)
print(f'Accuracy with rbf kernel: {accuracy_rbf}')

# Train and test SVM with 'sigmoid' kernel
clf_sigmoid = SVC(kernel='sigmoid')
clf_sigmoid.fit(X_train, y_train)
accuracy_sigmoid = clf_sigmoid.score(X_test, y_test)
print(f'Accuracy with sigmoid kernel: {accuracy_sigmoid}')


from datetime import datetime
from pydantic import UUID4
from typing import Optional
from uuid import uuid4

from sqlalchemy import delete, insert
from sqlalchemy.orm import Session

from app.models.question import Question
from tests.utils.kitchen_sink import random_lower_string

def create_test_question(
    db: Session,
    answer_options: list[dict],
    correct_answer: int,
    question: str,
    question_type: str,
    topic_id: UUID4,
    id: Optional[UUID4] = None,
    is_deleted: Optional[bool] = False,
) -> Question:
    if not id:
        id = uuid4()

    result = db.execute(
        insert(Question)
        .values(
            id=id,
            created_at=datetime.utcnow(),
            last_modified_at=None,
            answer_options=answer_options,
            correct_answer=correct_answer,
            is_deleted=is_deleted,
            question=question,
            question_type=question_type,
            topic_id=topic_id,
        )
        .returning(
            Question.id,
            Question.answer_options,
            Question.correct_answer,
            Question.question,
            Question.question_type,
            Question.topic_id,
        )
    )

    db.commit()
    return result.first()

def delete_test_questions(db: Session) -> None:
    db.execute(delete(Question))
    db.commit()

# Dummy data to be used in conftest
random_answer_options = [
    {"id": 1, "option_1": random_lower_string()},
    {"id": 2, "option_2": random_lower_string()},
    {"id": 3, "option_3": random_lower_string()},
    {"id": 4, "option_4": random_lower_string()},
]


# coding: utf-8

import requests, re
import pandas as pd

def split_talk(talk):
    lines = talk.split("
")
    match_lines = [re.search("「(.*?)」$",i) for i in lines]
    check_chara = ["y" if i.find("彡") else "g" for i in lines]
    return [(c, re.sub("「|」","",i.group())) for c, i in zip(check_chara,match_lines) if i is not None]

def make_pair(pair, talk_list):
    if talk_list:
        talk = talk_list.pop(0)
        tmp = [(talk[i-1], talk[i]) for i in range(len(talk)) if i > 0]
        pair.extend([i for i in tmp if i[0][0] == "y" and i[1][0] == "g"])
        return make_pair(pair, talk_list)
    else:
        return pair

def scraping(url):
    html = requests.get(url).text
    comments = [i for i in html.split("\n") if i.find("<dd class=") != -1]
    talk = [i.encode('utf-8') for i in comments if i.find(u"彡") != -1 and i.find(u"ω")]
    return [split_talk(i) for i in talk]

def generate_corpus(url, title):
    talk_list = scraping(url)
    pair_list = make_pair([], talk_list)
    out = {"0_原住民":[],"1_やきう民":[]}
    for pair in pair_list:
        out["0_原住民"].append(pair[0][1])
        out["1_やきう民"].append(pair[1][1])
    pd.DataFrame(out).to_csv('./data/corpus/'+title+'.csv', index=False)

if __name__=="__main__":
    url_data = pd.read_csv("./data/url_list.csv")
    url_data = zip(list(url_data.iloc[:,0]),list(url_data.iloc[:,1]))
    for u,t in url_data:
        generate_corpus(u,t)


from ..legacytorch.field import Field, LabelField
from ..legacytorch.dataset import TabularDataset
from ..legacytorch.iterator import BucketIterator
from ..utils.utils import init_tokenizer
from abc import ABC, abstractclassmethod

class DataPreparation(ABC):

    tokenizer = None
    max_input_length = None
    train_eval_split = None

    '''
    Personalize dataset creation
    '''
    def __init__(self, tokenizer, device, train_eval_split, batch_size):
        self.tokenizer = tokenizer if type(tokenizer) is not str else init_tokenizer(tokenizer)
        self.device = device
        self.train_eval_split = train_eval_split
        self.__class__.train_eval_split = self.train_eval_split
        self.batch_size = batch_size
        self.init_token_idx = self.tokenizer.cls_token_id
        self.eos_token_idx = self.tokenizer.sep_token_id
        self.pad_token_idx = self.tokenizer.pad_token_id
        self.unk_token_idx = self.tokenizer.unk_token_id
        self.init_token = self.tokenizer.cls_token
        self.eos_token = self.tokenizer.sep_token
        self.pad_token = self.tokenizer.pad_token
        self.unk_token = self.tokenizer.unk_token
        self.max_input_length = self.tokenizer.max_model_input_sizes['bert-base-uncased']
        self.__class__.tokenizer = self.tokenizer
        self.__class__.max_input_length = self.max_input_length
        self.LABEL = None

    @classmethod
    def tokenize_bert(cls, sentence):
        tokens = cls.tokenizer.tokenize(sentence)
        return tokens

    @classmethod
    def split_and_cut(cls, sentence):
        tokens = sentence.strip().split(" ")
        tokens = tokens[:cls.max_input_length-1]
        return tokens

    @classmethod
    def trim_sentence(cls, sent):
        try:
            sent = sent.split()
            sent = sent[:128]
            return " ".join(sent)
        except:
            return sent

    @classmethod
    def get_sent1_token_type(cls, sent):
        try:
            return [0]* len(sent)
        except:
            return []

    @classmethod
    def get_sent2_token_type(cls, sent):
        try:
            return [1]* len(sent)
        except:
            return []

    @classmethod
    def combine_seq(cls, seq):
        return " ".join(seq)

    @classmethod
    def combine_mask(cls, mask):
        mask = [str(m) for m in mask]
        return " ".join(mask)

    @classmethod
    @abstractclassmethod
    def _prepare_df(cls, df_train, df_test):
        pass

    def output_dim(self):
        if self.LABEL is not None:
            return len(self.LABEL.vocab)
        else:
            return 2

    def labels(self):
        if self.LABEL is not None:
            return len(self.LABEL.vocab.itos)
        else:
            return 2

    def nli_labels(self):
        return {"entailment": 0, "contradiction": 1} if self.LABEL is None \
            else [v for v in self.LABEL.vocab.itos.values()][0] if type(self.LABEL.vocab.itos) is dict \
            else dict(zip(self.LABEL.vocab.itos, [0, 1, 2]))

    '''
    Preparing dataset for NLI training through tokenization, mask creation and adding of BERT tokens [CLS], [SEP], ...
    '''
    @classmethod
    def _prepare_dataset(cls, df_train, df_test):
        df_train = df_train.sample(frac=1).reset_index(drop=True)

        df_train, df_dev, df_test = cls._prepare_df(df_train, df_test)

        df_train['sent1_t'] = df_train['sent1'].apply(cls.tokenize_bert)
        df_train['sent2_t'] = df_train['sent2'].apply(cls.tokenize_bert)
        df_dev['sent1_t'] = df_dev['sent1'].apply(cls.tokenize_bert)
        df_dev['sent2_t'] = df_dev['sent2'].apply(cls.tokenize_bert)
        df_test['sent1_t'] = df_test['sent1'].apply(cls.tokenize_bert)
        df_test['sent2_t'] = df_test['sent2'].apply(cls.tokenize_bert)

        df_train['sent1_token_type'] = df_train['sent1_t'].apply(cls.get_sent1_token_type)
        df_train['sent2_token_type'] = df_train['sent2_t'].apply(cls.get_sent2_token_type)
        df_dev['sent1_token_type'] = df_dev['sent1_t'].apply(cls.get_sent1_token_type)
        df_dev['sent2_token_type'] = df_dev['sent2_t'].apply(cls.get_sent2_token_type)
        df_test['sent1_token_type'] = df_test['sent1_t'].apply(cls.get_sent1_token_type)
        df_test['sent2_token_type'] = df_test['sent2_t'].apply(cls.get_sent2_token_type)

        df_train['sequence'] = df_train['sent1_t'] + df_train['sent2_t']
        df_dev['sequence'] = df_dev['sent1_t'] + df_dev['sent2_t']
        df_test['sequence'] = df_test['sent1_t'] + df_test['sent2_t']

        df_train['attention_mask'] = df_train['sequence'].apply(cls.get_sent2_token_type)
        df_dev['attention_mask'] = df_dev['sequence'].apply(cls.get_sent2_token_type)
        df_test['attention_mask'] = df_test['sequence'].apply(cls.get_sent2_token_type)

        df_train['token_type'] = df_train['sent1_token_type'] + df_train['sent2_token_type']
        df_dev['token_type'] = df_dev['sent1_token_type'] + df_dev['sent2_token_type']
        df_test['token_type'] = df_test['sent1_token_type'] + df_test['sent2_token_type']

        df_train['sequence'] = df_train['sequence'].apply(cls.combine_seq)
        df_dev['sequence'] = df_dev['sequence'].apply(cls.combine_seq)
        df_test['sequence'] = df_test['sequence'].apply(cls.combine_seq)

        df_train['attention_mask'] = df_train['attention_mask'].apply(cls.combine_mask)
        df_dev['attention_mask'] = df_dev['attention_mask'].apply(cls.combine_mask)
        df_test['attention_mask'] = df_test['attention_mask'].apply(cls.combine_mask)

        df_train['token_type'] = df_train['token_type'].apply(cls.combine_mask)
        df_dev['token_type'] = df_dev['token_type'].apply(cls.combine_mask)
        df_test['token_type'] = df_test['token_type'].apply(cls.combine_mask)

        df_train = df_train[['gold_label', 'sequence', 'attention_mask', 'token_type']]
        df_dev = df_dev[['gold_label', 'sequence', 'attention_mask', 'token_type']]
        df_test = df_test[['gold_label', 'sequence', 'attention_mask', 'token_type']]

        df_train = df_train.loc[df_train['gold_label'].isin(['entailment','contradiction','neutral'])]
        df_dev = df_dev.loc[df_dev['gold_label'].isin(['entailment','contradiction','neutral'])]
        df_test = df_test.loc[df_test['gold_label'].isin(['entailment','contradiction','neutral'])]

        df_train.to_csv('train.csv', index=False)
        df_dev.to_csv('dev.csv', index=False)
        df_test.to_csv('test.csv', index=False)

        return df_train, df_dev, df_test

    def convert_to_int(self, tok_ids):
        tok_ids = [int(x) for x in tok_ids]
        return tok_ids

    def create_iterator(self, df_train, df_test):
        df_train, df_dev, df_test = self._prepare_dataset(df_train, df_test)

        TEXT = Field(batch_first = True,
                        use_vocab = False,
                        tokenize = self.split_and_cut,
                        preprocessing = self.tokenizer.convert_tokens_to_ids,
                        pad_token = self.pad_token_idx,
                        unk_token = self.unk_token_idx)

        self.LABEL = LabelField()

        ATTENTION = Field(batch_first = True,
                        use_vocab = False,
                        tokenize = self.split_and_cut,
                        preprocessing = self.convert_to_int,
                        pad_token = self.pad_token_idx)

        TTYPE = Field(batch_first = True,
                        use_vocab = False,
                        tokenize = self.split_and_cut,
                        preprocessing = self.convert_to_int,
                        pad_token = 1)

        fields = [('label', self.LABEL), ('sequence', TEXT), ('attention_mask', ATTENTION), ('token_type', TTYPE)]

        train_data, valid_data, test_data = TabularDataset.splits(
                                                path ='',
                                                train = 'train.csv',
                                                validation = 'dev.csv',
                                                test = 'test.csv',
                                                format = 'csv',
                                                fields = fields,
                                                skip_header = True)

        self.LABEL.build_vocab(train_data)

        train_iterator, valid_iterator, test_iterator = BucketIterator.splits(
            (train_data, valid_data, test_data),
            batch_size = self.batch_size,
            sort_key = lambda x: len(x.sequence),
            sort_within_batch = False,
            device = self.device)

        return train_iterator, valid_iterator, test_iterator


import os
import pandas as pd
import geopandas as gpd
import matplotlib as mpl
import matplotlib.pyplot as plt
import contextily as ctx

mpl.use('TkAgg')

"""
script to plot average error at all save points
"""

sp2coord = os.path.join(os.getcwd(), '..', 'data',
				'NACCS_SavePts_18977_ConversionKey_MSL_to_NAVD_meters.csv')
sp2coord = pd.read_csv(sp2coord, skiprows=[1])
sp2coord['SP_Longitude'] = pd.to_numeric(sp2coord['SP_Longitude'])
sp2coord['SP_Latitude'] = pd.to_numeric(sp2coord['SP_Latitude'])

path_to_models = os.path.join(os.getcwd(), 'LSTM_training_results',
						'Model_results')
models = [
			"LSTM_B15_h100_l1_bbS",
			"LSTM_LL1_B15_h256_l1_bbS",
			"LSTM_LL2_B15_h256_l1_bbS",

			# "LSTM_LL1_B15_h100_l1_bbM",
			# "LSTM_LL2_B15_h100_l1_bbM",

			# "LSTM_LL1_B15_h100_l1_bbL",
			# "LSTM_LL2_B15_h100_l1_bbL",
			  ]

model_names = [
			'LSTM',
			'LSTM + Linear Layer',
			'LSTM + Norm Layer + Linear Layer',

			'LSTM + Linear Layer',
			'LSTM + Norm Layer + Linear Layer',

			'LSTM + Linear Layer',
			'LSTM + Norm Layer + Linear Layer'
   			]

bb = ['Small', 'Small', 'Small',
	  'Medium', 'Medium',
	  'Large', 'Large' ]

cbar_axes = [
			[0.92, 0.105, 0.02, 0.78],		# small bounding box
			[0.92, 0.105, 0.02, 0.78],
			[0.92, 0.105, 0.02, 0.78],

			[0.92, 0.25, 0.02, 0.5],		# med bounding box
			[0.92, 0.25, 0.02, 0.5],

			[0.83, 0.105, 0.02, 0.78],		# lg. bounding box
			[0.83, 0.105, 0.02, 0.78]
			]
cbar_max = [1.0, 1.0, 1.0,
			1.0, 1.0,
			1.0, 1.0
			]

for i, model in enumerate(models):
	predict = model + '_predict.csv'
	predict = os.path.join(path_to_models, predict)
	predict = pd.read_csv(predict).mean(axis=0)

	target = model + '_target.csv'
	target = os.path.join(path_to_models, target)
	target = pd.read_csv(target).mean(axis=0)

	error = predict-target
	print(error.max(), error.min())
	sp = error.index.to_list()
	sp = [(i.split('_')[1]) for i in sp]
	sp2coord_temp = sp2coord[sp2coord['SavePointID'].isin(sp)]
	sp2coord_temp['error'] = error.values

	gdf = gpd.GeoDataFrame(sp2coord_temp,
    					   geometry=gpd.points_from_xy(
    					   			sp2coord_temp.SP_Longitude,
    					   			sp2coord_temp.SP_Latitude
    					   			)
    					   )

	gdf.crs = {'init': 'epsg:4326'}
	gdf = gdf.to_crs({'init': 'epsg:3857'})

	fig, axes = plt.subplots(1,1, figsize = (12, 10))
	gdf.plot(column=error, cmap = 'bwr', vmin=-cbar_max[i], vmax=cbar_max[i],
			ax=axes, legend = False, edgecolor = 'k', linewidth = 0.3)
	ctx.add_basemap(axes, url=ctx.providers.Stamen.TonerLite)
	axes.set_xticks([])
	axes.set_yticks([])
	axes.set_title('Model: {}\nBounding Box: {}' .format(model_names[i], bb[i]))

	cbar_ax = fig.add_axes(cbar_axes[i])
	norm = mpl.colors.Normalize(vmin=-cbar_max[i],vmax=cbar_max[i])

	sm = plt.cm.ScalarMappable(cmap='bwr', norm=norm)
	sm.set_array([])
	plt.colorbar(sm, cax=cbar_ax)

	path_out = os.path.join(os.getcwd(), 'spatial_plots', model)
	plt.savefig(path_out)


from . settings import _CLASS_LABELS,_DATA_PATH,mean_signal_length

import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)
import joblib
import os
import sys
from typing import Tuple
#from scipy.io import wavfile
from sklearn.metrics import roc_curve, auc, classification_report,accuracy_score,confusion_matrix

import numpy as np
import scipy.io.wavfile as wav

from speechpy.feature import mfcc
import numpy as np
from sklearn.model_selection import train_test_split

"""
This example demonstrates how to use `CNN` model from
`speechemotionrecognition` package
"""
from keras.utils import np_utils
import matplotlib.pyplot as plt
import librosa
import soundfile
import tensorflow as tf

def load_data(load_from_file,name):
    if load_from_file:
        current_dir = os.getcwd()
        subdir = '\\training_data\\'
        save_dir = current_dir + subdir

        x_train = joblib.load(os.path.join(save_dir, 'x_train_'+name+'.joblib'))
        x_test = joblib.load(os.path.join(save_dir, 'x_test_'+name+'.joblib'))
        y_train = joblib.load(os.path.join(save_dir, 'y_train_'+name+'.joblib'))
        y_test = joblib.load(os.path.join(save_dir, 'y_test_'+name+'.joblib'))
        num_labels = joblib.load(os.path.join(save_dir, 'num_labels_'+name+'.joblib'))
        #data = joblib.load(os.path.join(save_dir, 'data_'+name+'.joblib'))
        #labels = joblib.load(os.path.join(save_dir, 'labels_'+name+'.joblib'))
    else:
        x_train, x_test, y_train, y_test, num_labels = extract_data()
        current_dir = os.getcwd()
        subdir = '\\training_data\\'
        save_dir = current_dir + subdir
        joblib.dump(x_train, os.path.join(save_dir, 'x_train_'+name+'.joblib'))
        joblib.dump(x_test, os.path.join(save_dir, 'x_test_'+name+'.joblib'))
        joblib.dump(y_train, os.path.join(save_dir, 'y_train_'+name+'.joblib'))
        joblib.dump(y_test, os.path.join(save_dir, 'y_test_'+name+'.joblib'))
        joblib.dump(num_labels, os.path.join(save_dir, 'num_labels_'+name+'.joblib'))
        #joblib.dump(data, os.path.join(save_dir, 'data_'+name+'.joblib'))
        #joblib.dump(labels, os.path.join(save_dir, 'labels_'+name+'.joblib'))
    return x_train,x_test,y_train,y_test,num_labels#,data, labels

def initialise():
    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
    os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'
    os.environ['TF_ENABLE_GPU_GARBAGE_COLLECTION'] = 'false'
    os.system('cls' if os.name == 'nt' else 'clear')
    """
    gpus = tf.config.list_physical_devices('GPU')
    tf.config.experimental.set_virtual_device_configuration(
            gpus[0],
            [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=4096)])

    from keras import backend as K
    K.clear_session()

    from numba import cuda
    device = cuda.get_current_device()
    device.reset()
    """

def extract_data():
    data, labels = get_data(_DATA_PATH, class_labels=_CLASS_LABELS)

    x_train, x_test, y_train, y_test = train_test_split(
        data,
        labels,
        test_size=0.2,
        random_state=42)
    return np.array(x_train), np.array(x_test), np.array(y_train), np.array(
        y_test), len(_CLASS_LABELS)#,data, labels

def get_data(data_path: str, flatten: bool = True, mfcc_len: int = 40,
             class_labels: Tuple = _CLASS_LABELS):
             #-> \        Tuple[np.ndarray, np.ndarray]:

    data = []
    labels = []
    names = []
    cur_dir = os.getcwd()
    sys.stderr.write('curdir: %s\n' % cur_dir)
    data_path = _DATA_PATH
    ##data_path = 'D:\\PhD\\EmoDB'
    os.chdir(data_path)
    tempcount=0
    #count total files
    file_count = sum(len(files) for _, _, files in os.walk(data_path))
    #count total files
    for i, directory in enumerate(class_labels):
        sys.stderr.write("started reading folder %s\n" % directory)
        os.chdir(directory)
        for filename in os.listdir('.'):
            tempcount+=1
            print("** "+str(tempcount)+" / "+str(file_count)+" **")
            filepath = os.getcwd() + '/' + filename
            #feature_vector=extract_feature(filepath, mfcc=False, chroma=True, mel=False,contrast=False,tonnetz=False)
            #************************************************************************************************************
            #************************************************************************************************************
            #************************************************************************************************************
            feature_vector=extract_feature(filepath, mfcc=True, chroma=True, mel=True,contrast=True,tonnetz=True,num_features=40)
            #************************************************************************************************************
            #************************************************************************************************************
            #************************************************************************************************************
            data.append(feature_vector)
            #print("****************************************")
            #print("Start:  " + str(feature_vector) + "End:  ")
            #print("****************************************")
            #print(str(feature_vector.shape)+"\n")
            #print("****************************************")
            labels.append(i)
            print("Procesing:  " + str(filename))
            #print("****************************************")
            names.append(filename)
        sys.stderr.write("ended reading folder %s\n" % directory)
        os.chdir('..')
    os.chdir(cur_dir)
    return np.array(data), np.array(labels)

def extract_feature(file_name, **kwargs):
    """
    Extract feature from audio file `file_name`
        Features supported:
            - MFCC (mfcc)
            - Chroma (chroma)
            - MEL Spectrogram Frequency (mel)
            - Contrast (contrast)
            - Tonnetz (tonnetz)
        e.g:
        `features = extract_feature(path, mel=True, mfcc=True)`
    """
    mfcc = kwargs.get("mfcc")
    chroma = kwargs.get("chroma")
    mel = kwargs.get("mel")
    contrast = kwargs.get("contrast")
    tonnetz = kwargs.get("tonnetz")
    num_features = kwargs.get("num_features")
    with soundfile.SoundFile(file_name) as sound_file:
        X = sound_file.read(dtype="float32")
        ###################################
        s_len = len(X)
        if s_len < mean_signal_length:
            pad_len = mean_signal_length - s_len
            pad_rem = pad_len % 2
            pad_len //= 2
            X = np.pad(X, (pad_len, pad_len + pad_rem),
                            'constant', constant_values=0)
        else:
            pad_len = s_len - mean_signal_length
            pad_len //= 2
            X = X[pad_len:pad_len + mean_signal_length]
        ###################################
        sample_rate = sound_file.samplerate
        if chroma or contrast:
            stft = np.abs(librosa.stft(X))
        result = np.empty((num_features,0))
        if mfcc:
            mfcc = np.array(librosa.feature.mfcc(y=X, sr=sample_rate, n_mfcc=num_features))
            #print(str(mfcc.shape))
            result = np.hstack((result, mfcc))
            #print("MFCC Result:"+str(result) +"\n   Shape" + str(result.shape))
        if chroma:
            chroma = np.array(librosa.feature.chroma_stft(S=stft, sr=sample_rate,n_chroma=num_features))
            #print(str(chroma.shape))
            result = np.hstack((result, chroma))
            #print("CHROMA Result:"+str(result) +"\n  Shape" + str(result.shape))
        if mel:
            mel = np.array(librosa.feature.melspectrogram(X, sr=sample_rate,n_mels=num_features))
            result = np.hstack((result, mel))
            #print("MEL Result:"+str(result) +"\n  Shape" + str(result.shape))
        if contrast:
            contrast = np.array(librosa.feature.spectral_contrast(S=stft, sr=sample_rate))
            result = np.hstack((result, np.pad(contrast,(0,num_features-7),'constant', constant_values=(0, 0))))
            #print("CONTRAST Result:"+str(result) +"\n  Shape" + str(result.shape))
        if tonnetz:
            tonnetz = np.array(librosa.feature.tonnetz(y=librosa.effects.harmonic(X), sr=sample_rate))
            result = np.hstack((result, np.pad(tonnetz,(0,num_features-6),'constant', constant_values=(0, 0))))
            #print("TONNETZ Result:"+str(result) +"\n  Shape" + str(result.shape))
    return result


import random
import time

import PySimpleGUI as gui
import simpleaudio as sa

from .musique import Musique
from .difficulte import Difficulte
from .questions import Questions
from .images import Images
from .indicateurs import Indicateur

SON_VICTOIRE = sa.WaveObject.from_wave_file('audios/effet sonores/522243__dzedenz__result-10.wav')
SON_ERREUR = sa.WaveObject.from_wave_file('audios/effet sonores/409282__wertstahl__syserr1v1-in_thy_face_short.wav')
SON_FIN_PARTIE = sa.WaveObject.from_wave_file('audios/effet sonores/173859__jivatma07__j1game_over_mono.wav')

TITRE = 'Monsieur Tartempion'
TEMPS = 'TEMPS'
OU = 'OU'

BOUTON_GAUCHE = 'BOUTON_GAUCHE'
BOUTON_DROIT = 'BOUTON_DROIT'
BOUTON_COMMENCER = 'BOUTON_COMMENCER'
BOUTON_QUITTER = 'BOUTON_QUITTER'
COMBO_DIFFICULTE = 'COMBO_DIFFICULTE'
COMBO_MUSIQUE = 'COMBO_MUSIQUE'

QUESTION = 'QUESTION'
INDICATEUR = 'INDICATEUR'
BOUTON_ACTION = 'BOUTON_ACTION'
IMAGE_BOUTON_INACTIF = 'IMAGE_BOUTON_INACTIF'

# Définir les styles de police pour différents éléments
police_title: tuple = (gui.DEFAULT_FONT, 40, 'italic')
police_etiquettes: tuple = (gui.DEFAULT_FONT, 20, 'normal')
police_temps: tuple = (gui.DEFAULT_FONT, 50, 'normal')
police_question: tuple = (gui.DEFAULT_FONT, 30, 'normal')
police_reponses: tuple = (gui.DEFAULT_FONT, 20, 'normal')
police_ou: tuple = (gui.DEFAULT_FONT, 20, 'italic')

class Partie:
    def __init__(self, difficulte_choisie: Difficulte, musique_choisie: Musique, images: Images):

        self.difficulte_choisie = difficulte_choisie
        self.musique_choisie = sa.WaveObject.from_wave_file(musique_choisie.chemin_fichier)
        self.musique_questions_controles = None

        self.images = images
        self.questions = Questions().choisir_questions(self.difficulte_choisie.nombre_questions)
        self.reponses = self._melanger_reponses()

        self.decompte_actif = False

        self.temps_restant = self.difficulte_choisie.temps

        self.prochaine_question = 0
        self.position_meilleure_tentative = 0

        self.fenetre = self._fenetre_de_jeu()

    def _fenetre_de_jeu(self) -> gui.Window:
        """Afficher l'interface du jeu"""

        temps = [[gui.Text('Temps restant', font=police_etiquettes, size=70, justification='center')], [
            gui.Text(str(self.difficulte_choisie.temps), key=TEMPS, font=police_temps)]]

        boutons_reponse = [gui.Column(
            [[gui.Button(key=BOUTON_GAUCHE, font=police_reponses, button_color=('white', gui.theme_background_color()),
                         border_width=0, disabled=True, visible=True),
              gui.Text(' ou ', key=OU, font=police_ou,
                       text_color=gui.theme_background_color()),
              gui.Button(key=BOUTON_DROIT, font=police_reponses, button_color=('white', gui.theme_background_color()),
                         border_width=0, disabled=True, visible=True)]], element_justification='center')]

        question = [gui.Text(' ', key=QUESTION, font=police_question)]

        action = [gui.Button(image_data=self.images.bouton_jouer_base64(), key=BOUTON_ACTION,
                             border_width=0, button_color=(gui.theme_background_color(), gui.theme_background_color()),
                             pad=(0, 10)),
                  gui.Image(data=self.images.bouton_inactif_base64(), key=IMAGE_BOUTON_INACTIF, visible=False, pad=(0, 10))]

        indicateurs = [*[gui.Image(data=self.images.indicateur_vide_base64(), key=f'{INDICATEUR}-{i}', pad=(
            4, 10)) for i in range(self.difficulte_choisie.nombre_questions)]]

        fenetre = gui.Window(TITRE, [temps, boutons_reponse, question, action, indicateurs], keep_on_top=True,
                             element_padding=(0, 0),
                             element_justification='center', resizable=False, finalize=True)

        return fenetre

    def _melanger_reponses(self) -> list:
        """fonction pour mélanger les réponses"""

        reponses = [(question[0][1], question[0][2]) for question in self.questions]

        for position_couple_reponses in range(len(reponses)):
            reponses[position_couple_reponses] = (
                reponses[position_couple_reponses][0], reponses[position_couple_reponses][1]) if bool(
                random.getrandbits(1)) else (
                reponses[position_couple_reponses][1], reponses[position_couple_reponses][0])
        return reponses

    def __splasher_echec(self, temps_ms: int = 3000) -> None:
        """afficher un écran d'échec"""

        gui.Window(TITRE, [[gui.Image(data=self.images.echec_base64())]], transparent_color=gui.theme_background_color(),
                   no_titlebar=True, keep_on_top=True).read(timeout=temps_ms, close=True)

    def __splasher_succes(self, temps_ms: int = 3000) -> None:
        """afficher quand le joueur gagne"""

        gui.Window(TITRE, [[gui.Image(data=self.images.succes_base64())]], transparent_color="maroon2",
                   no_titlebar=True, keep_on_top=True).read(timeout=temps_ms, close=True)

    def __afficher_question(self, question: tuple, reponses: tuple) -> None:
        """afficher la question"""
        fenetre = self.fenetre

        fenetre[QUESTION].update(question[0])
        fenetre[BOUTON_GAUCHE].update(reponses[0], disabled=False, visible=True)
        fenetre[OU].update(text_color='white')
        fenetre[BOUTON_DROIT].update(reponses[1], disabled=False, visible=True)

    def __effacer_question(self) -> None:
        """Effacer la question affichée dans la fenêtre"""

        self.fenetre[QUESTION].update('')
        self.fenetre[BOUTON_GAUCHE].update('', disabled=True, visible=True)
        self.fenetre[OU].update(text_color=gui.theme_background_color())
        self.fenetre[BOUTON_DROIT].update('', disabled=True, visible=True)

    def _fin_partie(self, est_un_echec: bool) -> None:

        self.decompte_actif = False
        self.fenetre.hide()
        self.__effacer_question()
        for i in range(self.difficulte_choisie.nombre_questions):
            self.fenetre[f'{INDICATEUR}-{i}'].update(data=self.images.indicateur_vide_base64())

            if not est_un_echec:
                self.questions[i][1] = Indicateur.VIDE

        self.musique_questions_controles.stop()

        if est_un_echec:
            SON_FIN_PARTIE.play()
            self.__splasher_echec()
        else:
            SON_VICTOIRE.play()
            self.__splasher_succes()

        self.fenetre.close()

    def __reprendre_partie(self):
        self.fenetre[BOUTON_ACTION].update(disabled=True, visible=False)
        self.fenetre[IMAGE_BOUTON_INACTIF].update(visible=True)
        self.__afficher_question(self.questions[self.prochaine_question][0], self.reponses[self.prochaine_question])

        # set le temps
        self.temps_actuel = round(time.time())
        self.decompte_actif = True

        # controleur de la musique qui joue
        self.musique_questions_controles = self.musique_choisie.play()

    def commencer(self) -> None:
        """commencer la partie"""
        fenetre = self.fenetre

        quitter = False
        # Tant qu'on ne quitte pas le jeu, fait cela
        while not quitter:
            evenement, valeurs = fenetre.read(timeout=10)

            # Si le décompte est actif, on diminue le temps restant et on met à jour le UI
            if self.decompte_actif:

                dernier_temps = self.temps_actuel
                self.temps_actuel = round(time.time())

                if dernier_temps != self.temps_actuel:
                    self.temps_restant -= 1
                    print(self.temps_restant)
                    fenetre[TEMPS].update(str(self.temps_restant))

                    # Si le temps est écoulé, affiche l'écran d'échec
                    if self.temps_restant == 0:
                        # arrete la partie
                        self._fin_partie(True)

            # Si on clique sur le bouton pour commencer le jeu, on affiche les questions
            if evenement == BOUTON_ACTION:

                # set le temps
                self.temps_actuel = round(time.time())
                self.decompte_actif = True

                # demarrer ou reprendre la partie
                self.__reprendre_partie()

            # Si on clique sur une des deux réponses (gauche ou droite)
            elif evenement == BOUTON_GAUCHE or evenement == BOUTON_DROIT:

                # Si le joueur a choisi la bonne réponse
                if (evenement == BOUTON_GAUCHE and fenetre[BOUTON_GAUCHE].get_text() != self.questions[self.prochaine_question][0][
                    1]) or \
                        (evenement == BOUTON_DROIT and fenetre[BOUTON_DROIT].get_text() != self.questions[
                            self.prochaine_question][0][
                            1]):

                    fenetre[f'{INDICATEUR}-{self.prochaine_question}'].update(data=self.images.indicateur_vert_base64())
                    self.questions[self.prochaine_question][1] = Indicateur.VERT
                    self.prochaine_question += 1

                    if self.prochaine_question < self.difficulte_choisie.nombre_questions:
                        self.__afficher_question(self.questions[self.prochaine_question][0], self.reponses[
                            self.prochaine_question])

                    # quand le joueur gagne
                    elif self.difficulte_choisie.nombre_questions <= self.prochaine_question:
                        self._fin_partie(False)

                # Sinon, le joueur a choisi la mauvaise réponse
                else:
                    self.decompte_actif = False
                    self.__effacer_question()

                    for i in range(self.prochaine_question):
                        fenetre[f'{INDICATEUR}-{i}'].update(
                            data=self.images.indicateur_jaune_base64())
                        self.questions[i][1] = Indicateur.JAUNE

                    if self.prochaine_question > self.position_meilleure_tentative or self.position_meilleure_tentative == 0:
                        self.position_meilleure_tentative = self.prochaine_question
                        fenetre[f'{INDICATEUR}-{self.position_meilleure_tentative}'].update(
                            data=self.images.indicateur_rouge_base64())
                        self.questions[self.position_meilleure_tentative][1] = Indicateur.ROUGE

                    fenetre[BOUTON_ACTION].update(disabled=False, visible=True)
                    fenetre[IMAGE_BOUTON_INACTIF].update(visible=False)

                    self.prochaine_question = 0

                    self.musique_questions_controles.stop()
                    SON_ERREUR.play()

            # Si on ferme le jeu
            elif evenement == gui.WIN_CLOSED:
                if self.musique_questions_controles:
                    self.musique_questions_controles.stop()

                self.decompte_actif = False
                quitter = True

        fenetre.close()


import time
import os
import requests
import re
import nltk
import string
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from nltk.tokenize import word_tokenize
from nltk.tokenize import sent_tokenize
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import TruncatedSVD
from sklearn.decomposition import NMF

chromedriver = "/Applications/chromedriver" # path to the chromedriver executable
os.environ["webdriver.chrome.driver"] = chromedriver

class GetArticles:
    """This class provides a pipeline to scrape articles from a website

    Args:
        driver (:obj): This is an instance of a selenium driver
        url (str): The url of the site to be scraped

    """

    def __init__(self):

        self.driver = None
        self.url = None

    def create_driver(self, url: str):
        """Creates a selenium driver and gets the url

        Args:
            url (str): site to be scraped

        Returns:
            Nothing

        """

        self.url = url
        driver = webdriver.Chrome(chromedriver)
        self.driver = driver
        return self.driver.get(url)

    def scroll(self):
        """Scrolls to the bottom of a long web page for a max of 30 seconds"""

        scroll_pause_time = 4

        # Get scroll height
        last_height = self.driver.execute_script("return document.body.scrollHeight")

        t_end = time.time() + 30

        while time.time() < t_end:
            # Scroll down to bottom
            self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")

            # Wait to load page
            time.sleep(scroll_pause_time)

            # Calculate new scroll height and compare with last scroll height
            new_height = self.driver.execute_script("return document.body.scrollHeight")
            if new_height == last_height:
                break
            last_height = new_height

    def get_site_links(self):
        """ Creates an article list from the home page of a website

        Returns:
            A list of article urls

        """
        soup = BeautifulSoup(self.driver.page_source, "lxml")
        article_list = []
        for link in soup.find_all('a'):
            try:
                if self.url in link['href']:
                    article_list.append(link['href'])
            except:
                continue
        article_list = list(set(article_list))

        return article_list

    def get_articles(self, article_list: list):
        """returns the article text for each article in a list of articles

                Args:
                    article_list(list): list of urls to get article text from

                Returns:
                    A dictionary of of the articles

        """
        article_dict = {}

        #The slicing was added below to test code
        for article in article_list[0:10]:

            response = requests.get(article)
            page = response.text
            soup = BeautifulSoup(page, "lxml")
            article_title = soup.find('h1')
            article_text = soup.find_all(['h2', 'p'])

            if len(article_text) > 3:

                article_dict[article_title] = article_text

        return article_dict

    def create_article_dict(self, url: str):
        """Creates a dictionary of all of the articles from a web page

                Args:
                    url(str): the url for the web page

                Returns:
                    A dictionary of of the articles

        """

        self.create_driver(url)
        self.scroll()
        article_list = self.get_site_links()

        return self.get_articles(article_list)

class NLPPipeline:
    """This class provides a pipeline for NLP models

    Args:
        nltk_stopwords (list): A list of all of the stop words
        topic_model (:obj): Sklearn topic model
        model (:obj): fit topic model to a vectorized corpus
        vectorizer (sklearn.feature_extraction.text.CountVectorizer): Sklearn vectorizer object
        vectorized_corpus (scipy.sparse.csr.csr_matrix): The vectorized corpus

    """

    def __init__(self, vectorizer=None, topic_model=None):

        self.nltk_stop_words = set(stopwords.words('english'))
        if not vectorizer:
            vectorizer = CountVectorizer(stop_words=self.nltk_stop_words, min_df=15, max_df=0.25, ngram_range=(1, 3))
        self.topic_model = None
        self.model = None
        self.vectorizer = vectorizer
        self.vectorized_corpus = None

    def remove_html(self, article: str):
        """Removes html from an article

                Args:
                    article(str): the raw html from a web page

                Returns:
                    str: Article text

        """

        text = article['text']
        soup = BeautifulSoup(text, "lxml")
        article_text = soup.get_text()[1:-1]

        return article_text

    def text_cleaning(self, article: str):
        """Cleans a string of text for tokenization

                Args:
                    article(str): the raw text from a web page

                Returns:
                    str: Article text

        """

        clean_text = re.sub('[%s]' % re.escape(string.punctuation), ' ', article)
        clean_text = re.sub('\w*\d\w*', ' ', clean_text)
        clean_text = clean_text.lower()

        return clean_text

    def text_lemmatizing(self, article):
        """Lemmatizes a string of text

                Args:
                    article(str): Raw text

                Returns:
                    str: raw text

        """

        lemmatized_word_list = []
        words = word_tokenize(article)
        wordnet_lemmatizer = WordNetLemmatizer()
        for word in words:
            lemmatized_word = wordnet_lemmatizer.lemmatize(word, pos='v')
            lemmatized_word_list.append(lemmatized_word)

        lemmatized_word_string = ' '.join(lemmatized_word_list)

        return lemmatized_word_string

    def vectorize(self, articles: str):
        """Vectorizes a string of text

                 Args:
                     articles(str): Raw text

         """

        cleaned_article_corpus = []
        for article in articles:

            article_no_html = self.remove_html(article)
            clean_article = self.text_cleaning(article_no_html)
            lemmatized_article = self.text_lemmatizing(clean_article)
            cleaned_article_corpus.append(lemmatized_article)

        self.vectorized_corpus = self.vectorizer.fit_transform(cleaned_article_corpus)

    def fit(self, topic_model=None):
        """Performs topic modeling on a vectorized text corpus

                 Args:
                     topic_model(:obj): Sklearn topic model

         """

        if not topic_model:
            topic_model = self.topic_model
        else:
            self.topic_model = topic_model

        self.model = self.topic_model.fit_transform(self.vectorized_corpus)

    def display_topics(self, no_top_words: int, topic_names=None):
        """Displays the topics for a model

                 Args:
                     no_top_words (int): The number of top words in each category to display
                     topic_names (list): A list of the topic names to display

                 Returns:
                     Prints the topics

         """
        for ix, topic in enumerate(self.topic_model.components_):
            if not topic_names or not topic_names[ix]:
                print("\nTopic ", ix)
            else:
                print("\nTopic: '",topic_names[ix],"'")
            print(", ".join([self.vectorizer.get_feature_names()[i]
                            for i in topic.argsort()[:-no_top_words - 1:-1]]))


import inkex

from ..elements import EmptyDObject, SatinColumn, Stroke
from ..i18n import _
from ..svg.tags import ORIGINAL_D, PATH_EFFECT, SODIPODI_NODETYPES
from .base import InkstitchExtension

class StrokeToLpeSatin(InkstitchExtension):
    """Convert a satin column into a running stitch."""

    def __init__(self, *args, **kwargs):
        InkstitchExtension.__init__(self, *args, **kwargs)
        self.arg_parser.add_argument("--lpe_satin", type=str, default=None)
        self.arg_parser.add_argument("--options", type=str, default=None)
        self.arg_parser.add_argument("--info", type=str, default=None)

        self.arg_parser.add_argument("-p", "--pattern", type=str, default="normal", dest="pattern")
        self.arg_parser.add_argument("-i", "--min-width", type=float, default=1.5, dest="min_width")
        self.arg_parser.add_argument("-a", "--max-width", type=float, default=7, dest="max_width")
        self.arg_parser.add_argument("-l", "--length", type=float, default=15, dest="length")
        self.arg_parser.add_argument("-t", "--stretched", type=inkex.Boolean, default=False, dest="stretched")
        self.arg_parser.add_argument("-r", "--rungs", type=inkex.Boolean, default=False, dest="add_rungs")
        self.arg_parser.add_argument("-s", "--path-specific", type=inkex.Boolean, default=True, dest="path_specific")

    def effect(self):
        if not self.svg.selection or not self.get_elements():
            inkex.errormsg(_("Please select at least one stroke."))
            return

        if not any((isinstance(item, Stroke) or isinstance(item, SatinColumn)) for item in self.elements):
            # L10N: Convert To Satin extension, user selected one or more objects that were not lines.
            if any(isinstance(item, EmptyDObject) for item in self.elements):
                inkex.errormsg(_("This element has lost its path information. Please move the element slightly back and forth before you try again."))
            else:
                inkex.errormsg(_("Please select at least one stroke to convert to a satin column."))
            return

        pattern = self.options.pattern
        if pattern not in satin_patterns:
            inkex.errormsg(_("Could not find the specified pattern."))
            return

        # convert user input values to the units of the current svg
        min_width = inkex.units.convert_unit(str(max(self.options.min_width, 0.5)) + 'mm', self.svg.unit)
        max_width = inkex.units.convert_unit(str(max(self.options.max_width, 0.5)) + 'mm', self.svg.unit)
        length = inkex.units.convert_unit(str(self.options.length) + 'mm', self.svg.unit)

        # get pattern path and nodetypes
        pattern_obj = satin_patterns[pattern]
        pattern_path = pattern_obj.get_path(self.options.add_rungs, min_width, max_width, length, self.svg.unit)
        pattern_node_type = pattern_obj.node_types
        copy_type = 'repeated' if self.options.stretched is False else 'repeated_stretched'

        if not self.options.path_specific:
            lpe = self._create_lpe_element(pattern, pattern_path, pattern_node_type, copy_type)

        for element in self.elements:
            if self.options.path_specific:
                lpe = self._create_lpe_element(pattern, pattern_path, pattern_node_type, copy_type, element)
            if isinstance(element, SatinColumn):
                self._process_satin_column(element, lpe, pattern_path, copy_type)
            elif isinstance(element, Stroke):
                self._process_stroke(element, lpe)

    def _create_lpe_element(self, pattern, pattern_path, pattern_node_type, copy_type, element=None):
        # define id for the lpe path
        if not element:
            lpe_id = f'inkstitch-effect-{pattern}'
        else:
            lpe_id = f'inkstitch-effect-{pattern}-{element.id}'

        # it is possible, that there is already a path effect with this id, if so, use it
        previous_lpe = self.svg.getElementById(lpe_id)
        if previous_lpe is not None:
            return previous_lpe

        lpe = inkex.PathEffect(attrib={'id': lpe_id,
                                       'effect': "skeletal",
                                       'is_visible': "true",
                                       'lpeversion': "1",
                                       'pattern': pattern_path,
                                       'copytype': copy_type,
                                       'prop_scale': "1",
                                       'scale_y_rel': "false",
                                       'spacing': "0",
                                       'normal_offset': "0",
                                       'tang_offset': "0",
                                       'prop_units': "false",
                                       'vertical_pattern': "false",
                                       'hide_knot': "false",
                                       'fuse_tolerance': "0.02",
                                       'pattern-nodetypes': pattern_node_type})
        # add the path effect element to the defs section
        self.svg.defs.add(lpe)
        return lpe

    def _process_stroke(self, element, lpe):
        element = self._ensure_path_element(element, lpe)

        previous_effects = element.node.get(PATH_EFFECT, None)
        if not previous_effects:
            element.node.set(PATH_EFFECT, lpe.get_id(as_url=1))
            element.node.set(ORIGINAL_D, element.node.get('d', ''))
        else:
            url = previous_effects + ';' + lpe.get_id(as_url=1)
            element.node.set(PATH_EFFECT, url)
        element.node.pop('d')
        element.set_param('satin_column', 'true')

        element.node.style['stroke-width'] = self.svg.viewport_to_unit('0.756')

    def _ensure_path_element(self, element, lpe):
        # elements other than paths (rectangle, circles, etc.) can be handled by inkscape for lpe
        # but they are way easier to handle for us if we turn them into paths
        if element.node.TAG == 'path':
            return element

        path_element = element.node.to_path_element()
        parent = element.node.getparent()
        parent.replace(element.node, path_element)
        return Stroke(path_element)

    def _process_satin_column(self, element, lpe, pattern_path, copy_type):
        current_effects = element.node.get(PATH_EFFECT, None)
        # there are possibly multiple path effects, let's check if inkstitch-effect is among them
        if not current_effects or 'inkstitch-effect' not in current_effects:
            # it wouldn't make sense to apply it to a normal satin column without the inkstitch-effect
            inkex.errormsg(_('Cannot convert a satin column into a live path effect satin. Please select a stroke.'))
            return

        # get the inkstitch effect
        current_effects = current_effects.split(';')
        inkstitch_effect_position = [i for i, effect in enumerate(current_effects) if 'inkstitch-effect' in effect][0]
        inkstitch_effect = current_effects[inkstitch_effect_position][1:]
        old_effect_element = self.svg.getElementById(inkstitch_effect)
        old_pattern_type = inkstitch_effect[17:]

        # update pattern if it is equal to the previous pattern
        if not self.options.path_specific and old_pattern_type == self.options.pattern:
            old_effect_element.set('pattern', pattern_path)
            old_effect_element.set('copytype', copy_type)
            element.node.pop('d')
            return

        # remove the old inkstitch-effect if it was path specific
        if inkstitch_effect.endswith(element.id):
            if old_pattern_type.startswith(self.options.pattern) and self.options.path_specific:
                old_effect_element.set('pattern', pattern_path)
                old_effect_element.set('copytype', copy_type)
            else:
                old_effect_element.getparent().remove(old_effect_element)

        # update path effect link
        current_effects[inkstitch_effect_position] = lpe.get_id(as_url=1)
        element.node.set(PATH_EFFECT, ';'.join(current_effects))
        element.node.pop('d')

class SatinPattern:
    def __init__(self, path=None, node_types=None, flip=True, rung_node=1):
        self.path: str = path
        self.node_types: str = node_types
        self.flip: bool = flip
        self.rung_node: int = rung_node

    def get_path(self, add_rungs, min_width, max_width, length, to_unit):
        # scale the pattern path to fit the unit of the current svg
        scale_factor = scale_factor = 1 / inkex.units.convert_unit('1mm', f'{to_unit}')
        pattern_path = inkex.Path(self.path).transform(inkex.Transform(f'scale({scale_factor})'), True)

        # create a path element
        el1 = inkex.PathElement(attrib={'d': str(pattern_path),
                                        SODIPODI_NODETYPES: self.node_types})

        # transform to fit user input size values
        bbox = el1.bounding_box()
        scale_x = length / max(bbox.width, 0.1)
        if bbox.height == 0:
            scale_y = 1
        else:
            scale_y = (max_width - min_width) / (bbox.height * 2)
        el1.transform = inkex.Transform(f'scale({scale_x}, {scale_y})')
        el1.apply_transform()
        path1 = el1.get_path()

        # copy first path and (optionally) flip it to generate the second satin rail
        el2 = el1.copy()
        if self.flip:
            el2.transform = inkex.Transform(f'scale(1, -1) translate(0, {min_width})')
        else:
            el2.transform = inkex.Transform(f'translate(0, {-min_width})')
        el2.apply_transform()
        path2 = el2.get_path()

        # setup a rung
        point1 = list(path1.end_points)[self.rung_node]
        point2 = list(path2.end_points)[self.rung_node]

        rungs = ''
        if add_rungs:
            rungs = f' M {point1[0]} {point1[1] + 0.1} L {point2[0]} {point2[1] - 0.2}'

        return str(path1) + str(path2) + rungs

satin_patterns = {'normal': SatinPattern('M 0,0 H 4 H 8', 'cc'),
                  'pearl': SatinPattern('M 0,0 C 0,0.22 0.18,0.4 0.4,0.4 0.62,0.4 0.8,0.22 0.8,0', 'csc'),
                  'diamond': SatinPattern('M 0,0 0.4,0.2 0.8,0', 'ccc'),
                  'triangle': SatinPattern('M 0,0 0.4,0.1 0.78,0.2 0.8,0', 'cccc'),
                  'square': SatinPattern('M 0,0 H 0.2 0.4 L 0.47,0.2 H 0.8 L 0.86,0', 'ccccc'),
                  'wave': SatinPattern('M 0,0 C 0.2,0.01 0.29,0.2 0.4,0.2 0.51,0.2 0.58,0.01 0.8,0', 'cac'),
                  'arch': SatinPattern('M 0,0.25 C 0,0.25 0.07,0.05 0.4,0.05 0.7,0.05 0.8,0.25 0.8,0.25', 'czcczc', False)}


#!/usr/bin/python

from __future__ import print_function
import os
import sys
import cv2
import numpy as np
import json

info_path = "/home/bingbing/git/dilation/datasets/cityscapes.json"
with open(info_path, 'r') as fp:
    info = json.load(fp)
palette = np.array(info["palette"], dtype=np.uint8)
black = np.array([[0, 0, 0]], dtype=np.uint8)
palette = np.concatenate([palette, black], axis=0)

def id2color(id_img):
    h, w = id_img.shape
    color_img = palette[id_img.ravel()].reshape((h, w, 3))
    color_img = cv2.cvtColor(color_img, cv2.COLOR_RGB2BGR)
    return color_img

if __name__ ==  "__main__":

    img_id = cv2.imread(sys.argv[1], 0)
    img_color = id2color(img_id)

    img_name = sys.argv[1].split('/')[-1]
    path = img_name
    # path = img_name[:-6]+"Color.png"
    print("Writing: ", path)
    cv2.imwrite(path, img_color)


import spotipy
from spotipy import SpotifyOAuth
from config.var_env import *

spotify = spotipy.Spotify(auth_manager=SpotifyOAuth(client_id=client,
                                                    client_secret=secret,
                                                    redirect_uri=url_direct))

def get_artists_from_playlist(playlist_uri):
    """

    :param playlist_uri: Playlist to analyse
    :return: A dictionary(artist uri : artist name) of all primary artists in a playlist.
    """
    artists = {}
    playlist_tracks = spotify.playlist_tracks(playlist_id=playlist_uri)
    for song in playlist_tracks['items']:
        if song['track']:
            print(song['track']['artists'][0]['name'])
            artists[song['track']['artists'][0]['uri']] = song['track']['artists'][0]['name']
    return artists


import curses
import shutil
from bson import ObjectId
from datetime import datetime
from journal.dayentries import Day
from persistence import entry_collection

MONTHS = ["n/a", "January", "February", "March", "April",
          "May", "June", "July", "August",
          "September", "October", "November", "December"]

DAYS = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"]

date_format = """{0} {1} {2}, {3}"""

header_format = """{hdr_char:{hdr_char}>100}
{date:^100}
{hdr_char:{hdr_char}>100}
"""

entry_by_day_format = """>{hour:>02}:{minute:>02}
id: {id}
category: {category}
tags: {tags}
 {entry_content}"""

entry_format = """{hdr_char:{hdr_char}>100}
> {weekday} {day}, {month} {year} at {hour:>02}:{minute:>02}
id: {id}
category: {category}
tags: {tags}
{hdr_char:{hdr_char}>100}
 {entry_content}"""

def get_hdr_view(day):
    date_view = date_format.format(
        DAYS[day.date.weekday()],
        MONTHS[day.date.month],
        day.date.day,
        day.date.year
    )
    return header_format.format(date=date_view, hdr_char='-')

def get_entry_by_day_view(entry):
    entry_date = datetime.fromtimestamp(entry.timestamp)
    return entry_by_day_format.format(
        id=entry._id,
        hour=entry_date.hour,
        minute=entry_date.minute,
        category=entry.category,
        tags=entry.tags,
        entry_content=entry.content
    )

def get_entry_view(entry):
    entry_date = datetime.fromtimestamp(entry.timestamp)
    return entry_format.format(
        id=entry._id,
        hour=entry_date.hour,
        minute=entry_date.minute,
        weekday=DAYS[entry_date.weekday()],
        day=entry_date.day,
        month=MONTHS[entry_date.month],
        year=entry_date.year,
        category=entry.category,
        tags=entry.tags,
        entry_content=entry.content,
        hdr_char='-'
    )

class JournalView:
    def __init__(self, entry_list):
        self.entry_list = entry_list

    def print_contents(self, display, use):
        if use == 'pager':
            self.print_with_pager()
        elif display == 'plain':
            self.print_plain()
        else:
            self.print_all_days()

    def print_all_days(self):
        day = Day()
        for entry in self.entry_list:
            if not day.belongs_to_this_day(entry):
                day.set_date(datetime.fromtimestamp(entry.timestamp))
                hdr_view = get_hdr_view(day)
                print(hdr_view)
            entry_view = get_entry_by_day_view(entry)
            print(entry_view)

    def print_plain(self):
        for entry in self.entry_list:
            entry_view = get_entry_view(entry)
            print(entry_view)

    def print_with_pager(self):
        curses.wrapper(func=self.curses_pager)

    def curses_pager(self, win):
        i = 0
        entry_list = self.entry_list
        entry_info_str = "Entry {} [h]previous; [l]next; [d]elete\n"

        curses.start_color()
        curses.init_pair(1, curses.COLOR_WHITE, curses.COLOR_BLACK)

        t_cols = shutil.get_terminal_size()[0]
        t_rows = shutil.get_terminal_size()[1]

        entry_pad = curses.newpad(t_rows, t_cols)
        entry_pad.bkgd(curses.color_pair(1))
        entry_pad.nodelay(False)
        entry_list_delimiter = 0
        entry_pad.clear()
        entry_pad.refresh(0, 0, 0, 0, t_rows, t_cols)

        entry_changed = True
        num_lines = 0

        while 1:
            if entry_changed:
                entry_pad.clear()
                entry_pad.refresh(0, 0, 0, 0, t_rows, t_cols)
                if len(entry_list) == 0:
                    empty_msg = "No entries found"
                    entry_pad.addstr(0, 0, empty_msg)
                    entry_pad.refresh(0, 0, 0, 0, t_rows, t_cols)
                    key = entry_pad.getkey()
                    break

                current_entry = entry_list[i]
                entry_view = get_entry_view(entry_list[i])
                entry_lines = entry_view.splitlines(keepends=True)
                entry_lines.insert(0, entry_info_str.format(i))

                num_lines = len(entry_lines)
                if num_lines <= (t_rows - 1):
                    entry_pad.addstr(entry_info_str.format(i))
                    entry_pad.addstr(entry_view)
                else:
                    for j in range(0, (t_rows - 1)):
                        entry_pad.addstr(entry_lines[j])
                entry_pad.refresh(0, 0, 0, 0, t_rows, t_cols)
                entry_list_delimiter = 0
                entry_changed = False

            key = entry_pad.getkey()
            if key == 'l' and i+1 in range(len(entry_list)):
                i = i+1
                entry_changed = True
            elif key == 'h' and i-1 in range(len(entry_list)):
                i = i - 1 if i-1 >= 0 else i
                entry_changed = True
            elif key == 'L':
                i = len(entry_list) - 1
                entry_changed = True
            elif key == 'H':
                i = 0
                entry_changed = True
            elif key == 'j' and num_lines > (t_rows - 1) and (t_rows - 1)+entry_list_delimiter+1 <= num_lines:
                entry_list_delimiter += 1
                entry_pad.clear()
                entry_pad.refresh(0, 0, 0, 0, t_rows, t_cols)
                for j in range(entry_list_delimiter, (t_rows - 1)+entry_list_delimiter):
                    entry_pad.addstr(entry_lines[j])
                entry_pad.refresh(0, 0, 0, 0, t_rows, t_cols)
                continue
            elif key == 'k' and num_lines > (t_rows - 1) and entry_list_delimiter-1 >= 0:
                entry_list_delimiter -= 1
                entry_pad.clear()
                entry_pad.refresh(0, 0, 0, 0, t_rows, t_cols)
                for j in range(entry_list_delimiter, (t_rows - 1)+entry_list_delimiter):
                    entry_pad.addstr(entry_lines[j])
                entry_pad.refresh(0, 0, 0, 0, t_rows, t_cols)
                continue
            elif key == 'd' and entry_list_delimiter == 0:
                msg = "Are you sure you want to delete this entry? [Y/n]"
                entry_pad.addstr(0, 0, msg)
                entry_pad.refresh(0, 0, 0, 0, t_rows, t_cols)
                while 1:
                    confirm = entry_pad.getkey()
                    if confirm == 'Y':
                        _entry_collection = entry_collection.EntryCollection()
                        _entry_collection.delete_one({'_id': ObjectId(current_entry._id)})
                        entry_list.pop(i)
                        i = len(entry_list)-1 if i == len(entry_list) else i
                        entry_changed = True
                        break
                    elif confirm == 'n' or 'N':
                        entry_pad.addstr(0, 0, entry_lines[0])
                        entry_pad.refresh(0, 0, 0, 0, t_rows, t_cols)
                        entry_changed = False
                        break
            if key == 'q':
                break


#!/usr/bin/env python3
import sys
sys.setrecursionlimit(4100000)
import math
def error(*args, end="\n"): print("[stderr]", *args, end=end, file=sys.stderr)
from bisect import bisect, bisect_left, bisect_right
from collections import defaultdict, deque
MOD = 998244353
INF = float("inf")
MINF = -float("inf")

N, K = map(int, input().split())
A = list(map(int, input().split()))
#print(A)

# init
cnt = 0
X = 0
# 周期計算用
d = defaultdict(lambda:-1)
loop_length = 0
loop_sum = 0
loop_times = 0

for ii in range(K):
    # update
    X = cnt % N
#    print("--", X, A[X])
#    if False:
    if d[X] != -1:
        #print("cnt", cnt)
        #print(ii, d[X], X)
        # loop detected
        loop_length = ii - d[X]
        loop_times = (K - d[X]) // loop_length
        loop_times_mod = (K - d[X]) % loop_length
        # loop sum
        tmp_cnt = X
        for jj in range(loop_length):
            tmp_X = tmp_cnt % N
            a = A[tmp_X]
            tmp_cnt += a
        loop_sum = tmp_cnt - X
        # cnt
        #print(loop_length)
        #print(loop_times)
        #print(loop_times_mod)
        #print(loop_sum)
        #print("cnt", cnt)
        cnt += loop_sum * (loop_times-1)
        #print("cnt", cnt)
        # 余り
        tmp_cnt = X
        for jj in range(loop_times_mod):
            tmp_X = tmp_cnt % N
            # add candy
            a = A[tmp_X]
            tmp_cnt += a
        cnt += (tmp_cnt - X)
        break
    else:
        d[X] = ii

    # add candy
    a = A[X]
    cnt += a

print(cnt)


# -*- coding: utf-8 -*-
import logging
import datetime
from odoo import models, fields, api

_logger = logging.getLogger(__name__)

class product_configurator_check(models.Model):
    _inherit = 'product.template'

    check = fields.Selection([
        ('no', '未確認'),
        ('yes', '確認済'),
    ], default='no',
        string="確認",
        tracking=True)

    def write(self, values):
        flg = self._origin.check
        print(values)
        # _logger.debug('Create a %s with vals %s', self._name, values)
        if self._origin.check == 'no' and values.get("check") == 'yes':
            values['check'] = 'yes'
        else:
            if 'taxes_id' in values or 'image_1920' in values or 'sequence' in values:
                values['check'] = 'yes'
            else:
                values['check'] = 'no'
        res = super(product_configurator_check, self).write(values)
        return res


from selenium import webdriver
from time import sleep
from PIL import Image
from pytesseract import image_to_string
import base64

class Bot:
    def __init__(self):
        self.driver = webdriver.Firefox(executable_path='files/geckodriver.exe')
        self.navigate()

    def take_screenshot(self):
        self.driver.save_screenshot('files/avito_screenshot.png')

    def tel_recon(self):
        image = Image.open('files/tel.gif')
        self.number_phone = image_to_string(image).strip()

    def crop(self, location, size):
        image = Image.open('files/avito_screenshot.png')
        x = location['x']
        y = location['y']
        width = size['width']
        height = size['height']
        image.crop((x, y, x+width, y+height)).save('files/tel.gif')
        self.tel_recon()

    def navigate(self):
        self.driver.get('https://www.avito.ru/rostov-na-donu/predlozheniya_uslug/evakuator_1187800124')

        button = self.driver.find_element_by_xpath('//a[@class="button item-phone-button js-item-phone-button button-origin contactBar_greenColor button-origin_full-width button-origin_large-extra item-phone-button_hide-phone item-phone-button_card js-item-phone-button_card contactBar_height"]')

        button.click()
        sleep(3)
        self.take_screenshot()

        image = self.driver.find_element_by_xpath('//div[@class="item-phone-big-number js-item-phone-big-number"]//*')
        location = image.location
        size = image.size
        self.crop(location, size)

if __name__ == '__main__':
    Bot().number_phone


# -*- coding:utf-8 -*-
#version 5

import csv, json, collections

day_list	  = collections.defaultdict(list)
Mary_day_list = collections.defaultdict(list)
John_day_list = collections.defaultdict(list)

with open('log_data.csv', 'r') as csvfile:

	# 將file轉換成dict
	for date_time, name in csv.reader(csvfile):
		data = {name:{date_time.split(' ')[0]:date_time.split(' ')[1]}}

		# 設user為key的字典
		for key, value in data.items():
			day_list[key].append(value)

	# 將user內的log分離日期與時間
	for m in day_list['Mary']: # {'2021-01-05': ['23:28:14'...],...
		for day, time in m.items():
			Mary_day_list[day].append(time)

	for j in day_list['John']: # {'2021-01-06': ['15:49:01', '15:49:01']}...
		for day, time in j.items():
			John_day_list[day].append(time)

# 將user與日期時間資料裝進字典
user = {}
user.update({'Mary_logs': Mary_day_list, 'John_logs': John_day_list})

# 取出user的最早最晚log_day
Mary_start_day = [day for day in Mary_day_list][0]
for d in [day for day in Mary_day_list]:
	if Mary_start_day > d:
		Mary_start_day = d
	else:
		Mary_end_day = d

John_start_day = [day for day in John_day_list][0]
for d in [day for day in John_day_list]:
	if John_start_day > d:
		John_start_day = d
	else:
		John_end_day = d

# 求解組成清單印出結果
Mary = []
John = []

Mary.append(['logs of day',       user['Mary_logs']])
Mary.append(['start time of day', Mary_start_day])
Mary.append(['end time of day',   Mary_end_day])
Mary.append(['total days',        len(user['Mary_logs'])])
Mary.append(['total logs',        len(day_list['Mary'])])
Mary.append(['avg log of day',    len(day_list['Mary']) / len(user['Mary_logs'])])

John.append(['logs of day',       user['John_logs']])
John.append(['start time of day', John_start_day])
John.append(['end time of day',   John_end_day])
John.append(['total days',        len(user['John_logs'])])
John.append(['total logs',        len(day_list['John'])])
John.append(['avg log of day',    len(day_list['John']) / len(user['John_logs'])])

print("Mary", Mary)
print("John", John)

'''
# 將求解組合成字典
Mary = {'logs of day'	     : user['Mary_logs'],
		'start time of day'  : [day for day in Mary_day_list][0],
		'end time of day'    : [day for day in Mary_day_list][-1],
		'total days'         : len(user['Mary_logs']),
		'total logs'         : len(day_list['Mary']),
		'avg log of day'	 : len(day_list['Mary']) / len(user['Mary_logs'])
		}

John = {'logs of day'	     : user['John_logs'],
		'start time of day'  : [day for day in John_day_list][0],
		'end time of day'    : [day for day in John_day_list][-1],
		'total days'		 : len(user['John_logs']),
		'total logs'		 : len(day_list['John']),
		'avg log of day'	 : len(day_list['John']) / len(user['John_logs'])
		}
'''


from keras.models import Sequential
from keras.layers import Convolution2D, MaxPooling2D, Flatten, Dense

classifier = Sequential()

classifier.add(Convolution2D(32, 3, 3, input_shape=(64, 64, 3), activation='relu'))
classifier.add(MaxPooling2D(pool_size=(2, 2)))

classifier.add(Convolution2D(32, 3, 3, activation='relu'))
classifier.add(MaxPooling2D(pool_size=(2, 2)))

classifier.add(Flatten())

classifier.add(Dense(output_dim=128, activation='relu'))
classifier.add(Dense(output_dim=1, activation='sigmoid'))

classifier.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])

from keras.preprocessing.image import ImageDataGenerator

train_datagen = ImageDataGenerator(
        rescale=1./255,
        shear_range=0.1,
        zoom_range=0.1,
        horizontal_flip=True)

test_datagen = ImageDataGenerator(rescale=1./255)

training_set = train_datagen.flow_from_directory('dataset/training_set',
                                                 target_size=(64, 64),
                                                 batch_size=32,
                                                 class_mode='binary')

test_set = test_datagen.flow_from_directory('dataset/test_set',
                                            target_size=(64, 64),
                                            batch_size=32,
                                            class_mode='binary')

classifier.fit_generator(training_set,
                         steps_per_epoch=800,
                         epochs=200,
                         validation_data=test_set,
                         validation_steps=20)

classifier.save('dogcat_bak.h5')

from keras.models import load_model
classifier = load_model('dogcat_bak.h5')

# Making New Predictions
import matplotlib.pyplot as plt
import numpy as np
from keras.preprocessing import image
test_image = image.load_img('dataset/single_prediction/cat_or_dog_2.jpg', target_size=(64,64))
test_image = image.img_to_array(test_image)
test_image = test_image/255
test_image = np.expand_dims(test_image, axis=0)
result = classifier.predict(test_image, steps=1)
if result[:, :] > 0.5:
    value = 'Dog :%1.2f' % (result[0, 0])
    plt.text(20, 62, value, color='red', fontsize=18, bbox=dict(facecolor='white', alpha=0.8))
else:
    value = 'Cat :%1.2f' % (1.0-result[0, 0])
    plt.text(20, 62, value, color='red', fontsize=18, bbox=dict(facecolor='white', alpha=0.8))

plt.imshow(test_image)
plt.show()

# Model Accuracy
x1 = classifier.evaluate_generator(training_set)
x2 = classifier.evaluate_generator(test_set)
print('Training Accuracy  : %1.2f%%     Training loss  : %1.6f' % (x1[1]*100, x1[0]))
print('Validation Accuracy: %1.2f%%     Validation loss: %1.6f' % (x2[1]*100, x2[0]))


from flask import Blueprint, request, make_response
from app.models import Post, db, Comment
from app.forms import PostForm, DeletePostForm, CommentForm
from sqlalchemy import desc

post_routes = Blueprint('posts', __name__)

def validation_errors_to_error_messages(validation_errors):
    """
    Simple function that turns the WTForms validation errors into a simple list
    """
    errorMessages = []
    for field in validation_errors:
        for error in validation_errors[field]:
            errorMessages.append(f'{field.capitalize()} : {error}')
    return errorMessages

@post_routes.route('/', methods=['POST', 'GET'])
def main():
    """
    GET requests return all posts
    POST requests create a new post in the database
    """
    if request.method == 'POST':
        form = PostForm()
        form['csrf_token'].data = request.cookies['csrf_token']

        if form.validate_on_submit():
            content = form.data['content']
            user_id = form.data['user_id']

            new_post = Post(content=content, user_id=user_id)

            db.session.add(new_post)
            db.session.commit()
            return new_post.to_dict()
        elif form.errors:
            print("Form errors: ", form.errors)
            return {'errors': validation_errors_to_error_messages(form.errors)}, 401
    if request.method == 'GET':
        posts = Post.query.order_by(desc(Post.created_at)).all()
        return {"posts": [post.to_dict() for post in posts]}
    return new_post.to_dict()

@post_routes.route('/<int:id>/comments/')
def post_comments(id):
    """
    GET requests return all comments associated with a specific post
    """
    form = CommentForm()
    form['csrf_token'].data = request.cookies['csrf_token']
    if form['csrf_token'].data:
        post_comments = Comment.query.filter(Comment.post_id == id).all()
        return {"comments": [comment.to_dict() for comment in post_comments]}
    if form.errors:
        return form.errors
    return make_response(404)

@post_routes.route('/<int:id>/', methods=['GET', 'PUT'])
def single_post(id):
    """
    GET requests retrieve one post
    PUT requests edit the post
    """
    new_post = Post.query.get(id)
    if request.method == 'PUT':
        form = PostForm()
        form['csrf_token'].data = request.cookies['csrf_token']

        if form.validate_on_submit():
            content = form.data['content']

            new_post.content = content

            db.session.add(new_post)
            db.session.commit()
        elif form.errors:
            return {'errors': validation_errors_to_error_messages(form.errors)}, 401
    if new_post:
        return new_post.to_dict()
    return {"errors": "That post does not exist"}, 404

@post_routes.route('/<int:id>/', methods=['DELETE'])
def delete_post(id):
    """
    Deletes the post matching the id of id
    """
    form = DeletePostForm()
    form['csrf_token'].data = request.cookies['csrf_token']
    if form.validate_on_submit():
        post = Post.query.get(id)
        db.session.delete(post)
        db.session.commit()
        return {}, 200
    return {'errors': validation_errors_to_error_messages(form.errors)}, 401


import argparse

def str2bool(v):
    return v.lower() in ("true", "1")

arg_lists = []
parser = argparse.ArgumentParser()

def add_argument_group(name):
    arg = parser.add_argument_group(name)
    arg_lists.append(arg)
    return arg

# -----------------------------------------------------------------------------
# Network
net_arg = add_argument_group("Network")
net_arg.add_argument(
    "--net_depth", type=int, default=12, help=""
    "number of layers. Default: 12")
net_arg.add_argument(
    "--clusters", type=int, default=500, help=""
    "cluster number in OANet. Default: 500")
net_arg.add_argument(
    "--iter_num", type=int, default=1, help=""
    "iteration number in the iterative network. Default: 1")
net_arg.add_argument(
    "--net_channels", type=int, default=128, help=""
    "number of channels in a layer. Default: 128")
net_arg.add_argument(
    "--use_fundamental", type=str2bool, default=False, help=""
    "train fundamental matrix estimation. Default: False")
net_arg.add_argument(
    "--share", type=str2bool, default=False, help=""
    "share the parameter in iterative network. Default: False")
net_arg.add_argument(
    "--use_ratio", type=int, default=0, help=""
    "use ratio test. 0: not use, 1: use before network, 2: use as side information. Default: 0")
net_arg.add_argument(
    "--use_mutual", type=int, default=0, help=""
    "use matual nearest neighbor check. 0: not use, 1: use before network, 2: use as side information. Default: 0")
net_arg.add_argument(
    "--ratio_test_th", type=float, default=0.8, help=""
    "ratio test threshold. Default: 0.8")

# -----------------------------------------------------------------------------
# Data
data_arg = add_argument_group("Data")
data_arg.add_argument(
    "--data_tr", type=str, default='../data_dump/yfcc-sift-2000-train.hdf5', help=""
    "name of the dataset for train")
data_arg.add_argument(
    "--data_va", type=str, default='../data_dump/yfcc-sift-2000-val.hdf5', help=""
    "name of the dataset for valid")
data_arg.add_argument(
    "--data_te", type=str, default='../data_dump/yfcc-sift-2000-test.hdf5', help=""
    "name of the unseen dataset for test")

# -----------------------------------------------------------------------------
# Objective
obj_arg = add_argument_group("obj")
obj_arg.add_argument(
    "--obj_num_kp", type=int, default=2000, help=""
    "number of keypoints per image")
obj_arg.add_argument(
    "--obj_top_k", type=int, default=-1, help=""
    "number of keypoints above the threshold to use for "
    "essential matrix estimation. put -1 to use all. ")
obj_arg.add_argument(
    "--obj_geod_type", type=str, default="episym",
    choices=["sampson", "episqr", "episym"], help=""
    "type of geodesic distance")
obj_arg.add_argument(
    "--obj_geod_th", type=float, default=1e-4, help=""
    "theshold for the good geodesic distance")

# -----------------------------------------------------------------------------
# Loss
loss_arg = add_argument_group("loss")
loss_arg.add_argument(
    "--weight_decay", type=float, default=0, help=""
    "l2 decay")
loss_arg.add_argument(
    "--momentum", type=float, default=0.9, help=""
    "momentum")
loss_arg.add_argument(
    "--loss_classif", type=float, default=1.0, help=""
    "weight of the classification loss")
loss_arg.add_argument(
    "--loss_essential", type=float, default=0.5, help=""
    "weight of the essential loss")
loss_arg.add_argument(
    "--loss_essential_init_iter", type=int, default=20000, help=""
    "initial iterations to run only the classification loss")
loss_arg.add_argument(
    "--geo_loss_margin", type=float, default=0.1, help=""
    "clamping margin in geometry loss")

# -----------------------------------------------------------------------------
# Training
train_arg = add_argument_group("Train")
train_arg.add_argument(
    "--run_mode", type=str, default="train", help=""
    "run_mode")
train_arg.add_argument(
    "--train_lr", type=float, default=1e-3, help=""
    "learning rate")
train_arg.add_argument(
    "--train_batch_size", type=int, default=32, help=""
    "batch size")
train_arg.add_argument(
    "--gpu_id", type=str, default='0', help='id(s) for CUDA_VISIBLE_DEVICES')
train_arg.add_argument(
    "--num_processor", type=int, default=8, help='numbers of used cpu')
train_arg.add_argument(
    "--train_iter", type=int, default=500000, help=""
    "training iterations to perform")
train_arg.add_argument(
    "--log_base", type=str, default="./log/", help=""
    "save directory name inside results")
train_arg.add_argument(
    "--log_suffix", type=str, default="", help=""
    "suffix of log dir")
train_arg.add_argument(
    "--val_intv", type=int, default=10000, help=""
    "validation interval")
train_arg.add_argument(
    "--save_intv", type=int, default=1000, help=""
    "summary interval")

# -----------------------------------------------------------------------------
# Testing
test_arg = add_argument_group("Test")
test_arg.add_argument(
    "--use_ransac", type=str2bool, default=True, help=""
    "use ransac when testing?")
test_arg.add_argument(
    "--model_path", type=str, default="", help=""
    "saved best model path for test")
test_arg.add_argument(
    "--res_path", type=str, default="", help=""
    "path for saving results")

# -----------------------------------------------------------------------------
# Visualization
vis_arg = add_argument_group('Visualization')
vis_arg.add_argument(
    "--tqdm_width", type=int, default=79, help=""
    "width of the tqdm bar"
)

def get_config():
    config, unparsed = parser.parse_known_args()
    return config, unparsed

def print_usage():
    parser.print_usage()

#
# config.py ends here


import torch
import pickle
import random
import numpy as np
import torch.nn as nn
from tqdm import tqdm
from config import MediSimConfig

SEED = 4
random.seed(SEED)
np.random.seed(SEED)
torch.manual_seed(SEED)
config = MediSimConfig()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
if torch.cuda.is_available():
    torch.cuda.manual_seed_all(SEED)

NUM_TRAINING = 7000
full = pickle.load(open('./data/testDataset.pkl', 'rb'))
dataset = full[:NUM_TRAINING]

import copy
import math
import torch
import torch.nn as nn

def gelu(x):
    return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))

class LayerNorm(nn.Module):
    def __init__(self, hidden_size, eps=1e-12):
        """Construct a layernorm module in the TF style (epsilon inside the square root)."""
        super(LayerNorm, self).__init__()
        self.weight = nn.Parameter(torch.ones(hidden_size))
        self.bias = nn.Parameter(torch.zeros(hidden_size))
        self.variance_epsilon = eps

    def forward(self, x):
        u = x.mean(-1, keepdim=True)
        s = (x - u).pow(2).mean(-1, keepdim=True)
        x = (x - u) / torch.sqrt(s + self.variance_epsilon)
        return self.weight * x + self.bias

class Conv1D(nn.Module):
    def __init__(self, nf, nx):
        super(Conv1D, self).__init__()
        self.nf = nf
        w = torch.empty(nx, nf)
        nn.init.normal_(w, std=0.02)
        self.weight = nn.Parameter(w)
        self.bias = nn.Parameter(torch.zeros(nf))

    def forward(self, x):
        size_out = x.size()[:-1] + (self.nf,)
        x = torch.addmm(self.bias, x.view(-1, x.size(-1)), self.weight)
        x = x.view(*size_out)
        return x

class Attention(nn.Module):
    def __init__(self, nx, n_ctx, config, scale=False):
        super(Attention, self).__init__()
        n_state = nx  # in Attention: n_state=768 (nx=n_embd)
        assert n_state % config.n_head == 0
        self.register_buffer("bias", torch.tril(torch.ones(n_ctx, n_ctx)).view(1, 1, n_ctx, n_ctx))
        self.n_head = config.n_head
        self.split_size = n_state
        self.scale = scale
        self.c_attn = Conv1D(n_state * 3, nx)
        self.c_proj = Conv1D(n_state, nx)

    def _attn(self, q, k, v):
        w = torch.matmul(q, k)
        if self.scale:
            w = w / math.sqrt(v.size(-1))
        nd, ns = w.size(-2), w.size(-1)
        b = self.bias[:, :, ns-nd:ns, :ns]
        w = w * b - 1e10 * (1 - b)
        w = nn.Softmax(dim=-1)(w)
        return torch.matmul(w, v)

    def merge_heads(self, x):
        x = x.permute(0, 2, 1, 3).contiguous()
        new_x_shape = x.size()[:-2] + (x.size(-2) * x.size(-1),)
        return x.view(*new_x_shape)

    def split_heads(self, x, k=False):
        new_x_shape = x.size()[:-1] + (self.n_head, x.size(-1) // self.n_head)
        x = x.view(*new_x_shape)
        if k:
            return x.permute(0, 2, 3, 1)  # (batch, head, head_features, seq_length)
        else:
            return x.permute(0, 2, 1, 3)  # (batch, head, seq_length, head_features)

    def forward(self, x, layer_past=None):
        x = self.c_attn(x)
        query, key, value = x.split(self.split_size, dim=2)
        query = self.split_heads(query)
        key = self.split_heads(key, k=True)
        value = self.split_heads(value)
        if layer_past is not None:
            past_key, past_value = layer_past[0].transpose(-2, -1), layer_past[1]  # transpose back cf below
            key = torch.cat((past_key, key), dim=-1)
            value = torch.cat((past_value, value), dim=-2)
        present = torch.stack((key.transpose(-2, -1), value))  # transpose to have same shapes for stacking
        a = self._attn(query, key, value)
        a = self.merge_heads(a)
        a = self.c_proj(a)
        return a, present

class MLP(nn.Module):
    def __init__(self, n_state, config):  # in MLP: n_state=3072 (4 * n_embd)
        super(MLP, self).__init__()
        nx = config.n_embd
        self.c_fc = Conv1D(n_state, nx)
        self.c_proj = Conv1D(nx, n_state)
        self.act = gelu

    def forward(self, x):
        h = self.act(self.c_fc(x))
        h2 = self.c_proj(h)
        return h2

class Block(nn.Module):
    def __init__(self, n_ctx, config, scale=False):
        super(Block, self).__init__()
        nx = config.n_embd
        self.ln_1 = LayerNorm(nx, eps=config.layer_norm_epsilon)
        self.attn = Attention(nx, n_ctx, config, scale)
        self.ln_2 = LayerNorm(nx, eps=config.layer_norm_epsilon)
        self.mlp = MLP(4 * nx, config)

    def forward(self, x, layer_past=None):
        a, present = self.attn(self.ln_1(x), layer_past=layer_past)
        x = x + a
        m = self.mlp(self.ln_2(x))
        x = x + m
        return x, present

class GPT2Model(nn.Module):
    def __init__(self, config):
        super(GPT2Model, self).__init__()
        self.n_layer = config.n_layer
        self.n_embd = config.n_embd
        self.n_vocab = config.total_vocab_size

        self.vis_embed_mat = nn.Linear(config.total_vocab_size, config.n_embd, bias=False)
        self.pos_embed_mat = nn.Embedding(config.n_positions, config.n_embd)
        block = Block(config.n_ctx, config, scale=True)
        self.h = nn.ModuleList([copy.deepcopy(block) for _ in range(config.n_layer)])
        self.ln_f = LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)

    def forward(self, input_visits, position_ids=None, past=None):
        if past is None:
            past_length = 0
            past = [None] * len(self.h)
        else:
            past_length = past[0][0].size(-2)
        if position_ids is None:
            position_ids = torch.arange(past_length, input_visits.size(1) + past_length, dtype=torch.long,
                                        device=input_visits.device)
            position_ids = position_ids.unsqueeze(0).expand(input_visits.size(0), input_visits.size(1))

        inputs_embeds = self.vis_embed_mat(input_visits)
        position_embeds = self.pos_embed_mat(position_ids)
        hidden_states = inputs_embeds + position_embeds
        presents = []
        for block, layer_past in zip(self.h, past):
            hidden_states, present = block(hidden_states, layer_past)
            presents.append(present)
        hidden_states = self.ln_f(hidden_states)
        return hidden_states, presents

class GPT4EHRHead(nn.Module):
    def __init__(self, model_embeddings_weights, config):
        super(GPT4EHRHead, self).__init__()
        self.n_embd = config.n_embd
        self.set_embeddings_weights(model_embeddings_weights)

    def set_embeddings_weights(self, model_embeddings_weights):
        embed_shape = model_embeddings_weights.shape
        self.decoder = nn.Linear(embed_shape[0], embed_shape[1], bias=False)
        self.decoder.weight = nn.Parameter(model_embeddings_weights.transpose(0, 1))  # Tied weights

    def forward(self, hidden_state):
        code_logits = self.decoder(hidden_state)
        return code_logits

class GPT(nn.Module):
    def __init__(self, config):
        super(GPT, self).__init__()
        self.transformer = GPT2Model(config)
        self.ehr_head = GPT4EHRHead(self.transformer.vis_embed_mat.weight, config)

    def set_tied(self):
        """Make sure we are sharing the embeddings"""
        self.ehr_head.set_embeddings_weights(self.transformer.vis_embed_mat.weight)

    def forward(self, input_visits, position_ids=None, ehr_labels=None, ehr_masks=None, past=None, pos_loss_weight=None):
        hidden_states, presents = self.transformer(input_visits, position_ids, past)
        code_logits = self.ehr_head(hidden_states)
        sig = nn.Sigmoid()
        code_probs = sig(code_logits)
        if ehr_labels is not None:
            shift_probs = code_probs[..., :-1, :].contiguous()
            shift_labels = ehr_labels[..., 1:, :].contiguous()
            loss_weights = None
            if pos_loss_weight is not None:
                loss_weights = torch.ones(shift_probs.shape, device=code_probs.device)
                loss_weights = loss_weights + (pos_loss_weight-1) * shift_labels
            if ehr_masks is not None:
                shift_probs = shift_probs * ehr_masks
                shift_labels = shift_labels * ehr_masks
                if pos_loss_weight is not None:
                    loss_weights = loss_weights * ehr_masks

            bce = nn.BCELoss(weight=loss_weights)
            loss = bce(shift_probs, shift_labels)
            return loss, shift_probs, shift_labels

        return code_probs

    def sample(self, input_visits, random=True, position_ids=None, past=None):
        hidden_states, _ = self.transformer(input_visits, position_ids, past)
        next_logits = self.ehr_head(hidden_states[:,-1:,:])
        sig = nn.Sigmoid()
        next_probs = sig(next_logits)
        if random:
            visit = torch.bernoulli(next_probs)
        else:
            visit = torch.round(next_probs)

        return visit

def get_batch(loc, batch_size, dataset):
    ehr = dataset[loc:loc+batch_size]

    batch_ehr = np.zeros((len(ehr), config.n_ctx, config.total_vocab_size))
    batch_mask = np.zeros((len(ehr), config.n_ctx, 1))
    for i, p in enumerate(ehr):
        visits = p['visits']
        for j, v in enumerate(visits):
            batch_ehr[i,j+1][v] = 1
            batch_mask[i,j+1] = 1
        batch_ehr[i,len(visits),config.code_vocab_size+1] = 1 # Set the final visit to have the end token
        batch_ehr[i,len(visits)+1:,config.code_vocab_size+2] = 1 # Set the rest to the padded visit token

    batch_ehr[:,0,config.code_vocab_size] = 1 # Set the first visits to be the start token
    batch_mask = batch_mask[:,1:,:] # Shift the mask to match the shifted labels and predictions the model will return
    return batch_ehr, batch_mask

model = GPT(config).to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=config.lr)
checkpoint = torch.load('./save/temporal/gpt', map_location=torch.device(device))
model.load_state_dict(checkpoint['model'])
optimizer.load_state_dict(checkpoint['optimizer'])

def convert_ehr(ehrs, index_to_code=None):
    ehr_outputs = []
    for i in range(len(ehrs)):
        ehr = ehrs[i]
        ehr_output = []
        for j in range(1, len(ehr)):
            visit = ehr[j]
            visit_output = []
            indices = np.nonzero(visit)[0]
            end = False
            for idx in indices:
                if idx < config.code_vocab_size:
                    visit_output.append(index_to_code[idx] if index_to_code is not None else idx)
                elif idx == config.code_vocab_size+1:
                    end = True
            if visit_output != []:
                ehr_output.append(visit_output)
            if end:
                break
        ehr_outputs.append({'visits': ehr_output})

    ehr = None
    ehr_output = None
    visit = None
    visit_output = None
    indices = None

    return ehr_outputs

def finish_sequence(model, length, context, batch_size, device='cuda', sample=True):
    with torch.no_grad():
        for _ in range(length-context.size(1)):
            context = torch.cat((context, model.sample(context, sample)), dim=1)

    return context

ehr_dataset = []
with torch.no_grad():
  for i in tqdm(range(0, len(dataset), config.sample_batch_size)):
    batch_ehr, _ = get_batch(i, config.sample_batch_size, dataset)
    batch_ehr = torch.tensor(batch_ehr, dtype=torch.float32).to(device)
    predictions = finish_sequence(model, config.n_ctx, batch_ehr[:,:2,:], batch_size=batch_ehr.size(0), device=device, sample=True)
    batch_converted_ehr = convert_ehr(predictions.detach().cpu().numpy())
    ehr_dataset += batch_converted_ehr

pickle.dump(ehr_dataset, open('./results/datasets_temporal_baselines/extended_gpt.pkl', 'wb'))