import os
import json
import logging
from collections import OrderedDict
from functools import reduce
from pathlib import Path
from typing import Dict, List

#import openpyxl
import pandas as pd
import numpy as np
import xlrd
from xlsxwriter.utility import xl_rowcol_to_cell
import yaml
#import xlsxwriter
#from openpyxl.utils import get_column_letter
from dotenv import load_dotenv

from Vale_validator_check.Vale_validator import Validator_v
from check.check_QD import Check_QD
#from check.check_QD_secondo_livello import Check_QD
from check.check_metodiche import Check_metodiche
from check.check_distretti import Check_distretti
from check.check_priorita import Check_priorita
from check.check_prestazione import Check_prestazione
from flaskr.check.check_canali import Check_canali
from check.check_agende_interne import Check_agende_interne
from flaskr.check.check_inviante import Check_inviante

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
f_handler = logging.FileHandler('generator.log', 'a+', 'utf-8')
c_handler = logging.StreamHandler()
formatter = logging.Formatter(
    '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
f_handler.setFormatter(formatter)
c_handler.setFormatter(formatter)
logger.addHandler(f_handler)
logger.addHandler(c_handler)

load_dotenv()
CAT_NAME = os.getenv("CAT_SISS")
ROOT_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))

class Check_action():

    file_name = ""
    file_data = {}
    catalogo = OrderedDict()
    flag_check_list = []
    error_list = {}
    output_message = ""

    work_sheet = "" #sheet di lavoro di df_mapping
    work_codice_prestazione_siss = ""
    work_descrizione_prestazione_siss = ""
    work_codice_agenda_siss = ""
    work_casi_1_n = ""
    work_abilitazione_esposizione_siss = ""
    work_prenotabile_siss = ""
    work_codici_disciplina_catalogo = ""
    work_descrizione_disciplina_catalogo = ""
    work_codice_QD = ""
    work_descrizione_QD = ""
    work_operatore_logico_QD = ""
    work_codice_metodica = ""
    work_descrizione_metodica = ""
    work_codice_distretto = ""
    work_descrizione_distretto = ""
    work_operatore_logico_distretto = ""
    work_priorita_U = ""
    work_priorita_primo_accesso_D = ""
    work_priorita_primo_accesso_P = ""
    work_priorita_primo_accesso_B = ""
    work_accesso_programmabile_ZP = ""
    work_combinata = ""
    work_codice_agenda_interno = ""
    work_inviante = ""
    work_accesso_farmacia = ""
    work_accesso_CCR = ""
    work_accesso_cittadino = ""
    work_accesso_MMG = ""
    work_accesso_amministrativo = ""
    work_accesso_PAI = ""
    work_gg_preparazione = ""
    work_gg_refertazione = ""
    work_nota_operatore = ""

    work_index_codice_QD = 0
    work_index_codice_SISS_agenda = 0
    work_index_abilitazione_esposizione_SISS = 0
    work_index_codice_prestazione_SISS = 0
    work_index_operatore_logico_distretto = 0
    work_index_codici_disciplina_catalogo = 0
    work_index_operatore_logico_QD = 0

    work_alert_column = ""
    work_delimiter = ""

    def __init__(self):
        self.output_message = ""
        with open("./flaskr/config_validator_Maugeri.yml", "rt", encoding='utf8') as yamlfile:
            data = yaml.load(yamlfile, Loader=yaml.FullLoader)
        logger.debug(data)
        self.work_sheet = data[0]["work_column"]["work_sheet"]
        self.work_codice_prestazione_siss = data[0]["work_column"]["work_codice_prestazione_siss"]
        self.work_descrizione_prestazione_siss = data[0]["work_column"]["work_descrizione_prestazione_siss"]
        self.work_codice_agenda_siss = data[0]["work_column"]["work_codice_agenda_siss"]
        self.work_casi_1_n = data[0]["work_column"]["work_casi_1_n"]
        self.work_abilitazione_esposizione_siss = data[0]["work_column"]["work_abilitazione_esposizione_siss"]
        self.work_prenotabile_siss = data[0]["work_column"]["work_prenotabile_siss"]
        self.work_codici_disciplina_catalogo = data[0]["work_column"]["work_codici_disciplina_catalogo"]
        self.work_descrizione_disciplina_catalogo = data[0]["work_column"]["work_descrizione_disciplina_catalogo"]
        self.work_codice_QD = data[0]["work_column"]["work_codice_QD"]
        self.work_descrizione_QD = data[0]["work_column"]["work_descrizione_QD"]
        self.work_operatore_logico_QD = data[0]["work_column"]["work_operatore_logico_QD"]
        self.work_codice_metodica = data[0]["work_column"]["work_codice_metodica"]
        self.work_descrizione_metodica = data[0]["work_column"]["work_descrizione_metodica"]
        self.work_codice_distretto = data[0]["work_column"]["work_codice_distretto"]
        self.work_descrizione_distretto = data[0]["work_column"]["work_descrizione_distretto"]
        self.work_operatore_logico_distretto = data[0]["work_column"]["work_operatore_logico_distretto"]
        self.work_priorita_U = data[0]["work_column"]["work_priorita_U"]
        self.work_priorita_primo_accesso_D = data[0]["work_column"]["work_priorita_primo_accesso_D"]
        self.work_priorita_primo_accesso_P = data[0]["work_column"]["work_priorita_primo_accesso_P"]
        self.work_priorita_primo_accesso_B = data[0]["work_column"]["work_priorita_primo_accesso_B"]
        self.work_accesso_programmabile_ZP = data[0]["work_column"]["work_accesso_programmabile_ZP"]
        self.work_combinata = data[0]["work_column"]["work_combinata"]
        self.work_codice_agenda_interno = data[0]["work_column"]["work_codice_agenda_interno"]
        self.work_inviante = data[0]["work_column"]["work_inviante"]
        self.work_accesso_farmacia = data[0]["work_column"]["work_accesso_farmacia"]
        self.work_accesso_CCR = data[0]["work_column"]["work_accesso_CCR"]
        self.work_accesso_cittadino = data[0]["work_column"]["work_accesso_cittadino"]
        self.work_accesso_MMG = data[0]["work_column"]["work_accesso_MMG"]
        self.work_accesso_amministrativo = data[0]["work_column"]["work_accesso_amministrativo"]
        self.work_accesso_PAI = data[0]["work_column"]["work_accesso_PAI"]
        self.work_gg_preparazione = data[0]["work_column"]["work_gg_preparazione"]
        self.work_gg_refertazione = data[0]["work_column"]["work_gg_refertazione"]
        self.work_nota_operatore = data[0]["work_column"]["work_nota_operatore"]

        '''self.work_index_sheet = data[1]["work_index"]["work_index_sheet"]
        self.work_index_codice_QD = data[1]["work_index"]["work_index_codice_QD"] -1
        self.work_index_codice_SISS_agenda = data[1]["work_index"]["work_index_codice_SISS_agenda"] -1
        self.work_index_abilitazione_esposizione_SISS = data[1]["work_index"]["work_index_abilitazione_esposizione_SISS"] -1
        self.work_index_codice_prestazione_SISS = data[1]["work_index"]["work_index_codice_prestazione_SISS"] -1
        self.work_index_operatore_logico_distretto = data[1]["work_index"]["work_index_operatore_logico_distretto"] -1
        self.work_index_codici_disciplina_catalogo = data[1]["work_index"]["work_index_codici_disciplina_catalogo"] -1
        self.work_index_operatore_logico_QD = data[1]["work_index"]["work_index_operatore_logico_QD"] - 1 '''

        self.work_alert_column = data[1]["work_index"]["work_alert_column"]
        try:
            self.work_delimiter = data[2]["work_separator"]["work_delimiter"]
        except:
            self.work_delimiter = "," #valore di default

    def import_file(self):
        logging.warning("import excel")

        template_file = input("Enter your mapping file.xlsx: ") ##insert mapping file
        print(template_file)
        template_file = Path(template_file)
        self.file_name = template_file
        self.file_data = template_file
        if template_file.is_file(): #C:\Users\aless\csi-progetti\FaqBot\faqbot-09112020.xlsx
            self.read_exel_file(template_file)
        else:
            print("Il file non esiste, prova a ricaricare il file con la directory corretta.\n")

    def read_exel_file(self, template_file):
        #pd.set_option("display.max_rows", None, "display.max_columns", None)
        df_mapping = pd.read_excel(template_file, sheet_name=self.work_sheet, converters={self.work_codici_disciplina_catalogo: str, self.work_codice_prestazione_siss: str, self.work_codice_metodica: str, self.work_codice_distretto: str}).replace(np.nan, '', regex=True)
        #print ("print JSON")
        #print(sh)

        #catalogo_dir = "c:\\Users\\aless\\exel_validate\\src\\python\\exel_validate\\CCR-BO-CATGP#01_Codifiche attributi catalogo GP++_202007.xls"
        #catalogo_dir = os.path.join(ROOT_DIR, 'CCR-BO-CATGP#01_Codifiche attributi catalogo GP++_110322.xlsx')
        catalogo_dir = os.path.join(ROOT_DIR, CAT_NAME)

        sheet_QD = pd.read_excel(catalogo_dir, sheet_name='QD', converters={"Cod Disciplina": str})
        sheet_Metodiche = pd.read_excel(catalogo_dir, sheet_name='METODICHE', converters={"Codice SISS": str, "Codice Metodica": str})
        sheet_Distretti = pd.read_excel(catalogo_dir, sheet_name='DISTRETTI', converters={"Codice SISS": str, "Codice Distretto": str})

        print("sheet_QD caricato\n")
        #print(sheet_QD)
        print("sheet_Metodiche caricato\n")
        #print(sheet_Metodiche)
        print("sheet_Distretti caricato\n")
        #print(sheet_Distretti)

        self.analizer(df_mapping, sheet_QD, sheet_Metodiche, sheet_Distretti)
        #self.initializer_check_agende_interne()

    def analizer(self, df_mapping, sheet_QD, sheet_Metodiche, sheet_Distretti):

        print("FASE 0: precheck")
        self.check_column_name(df_mapping)

        print('Start analisys:\n', df_mapping)

        print("Fase 1") #FASE 1: CONTROLLO I QUESITI DIAGNOSTICI
        #QD_error = self.check_qd(df_mapping, sheet_QD)
        QD_error = {}
        print("Fase 2") #FASE 2: CONTROLLO LE METODICHE
        #metodiche_error = self.check_metodiche(df_mapping, sheet_Metodiche)
        metodiche_error = {}
        print("Fase 3") #FASE 3: CONTROLLO I DISTRETTI
        #distretti_error = self.check_distretti(df_mapping, sheet_Distretti)
        distretti_error = {}
        print("Fase 4") #FASE 4: CONTROLLO LE PRIORITA'
        #priorita_error = self.check_priorita(df_mapping)
        priorita_error = {}
        print("Fase 5") #FASE 5: CONTROLLO UNIVOCITA' PRESTAZIONI'
        univocita_prestazione_error = self.check_prestazione(df_mapping)
        #univocita_prestazione_error = {}
        print("Fase 6")
        canali_error = self.check_canali(df_mapping)
        #canali_error = {}
        print("Fase 7")
        inviante_error = self.check_inviante(df_mapping)
        #inviante_error = {}
        print("Fase Vale Validator")
        #catalogo_dir = "c:\\Users\\aless\\exel_validate\\src\\python\\exel_validate\\CCR-BO-CATGP#01_Codifiche attributi catalogo GP++_202007.xls"
        #catalogo_dir = os.path.join(ROOT_DIR, 'CCR-BO-CATGP#01_Codifiche attributi catalogo GP++_110322.xls')
        catalogo_dir = os.path.join(ROOT_DIR, CAT_NAME)
        wb = xlrd.open_workbook(catalogo_dir)
        sheet_QD_OW = wb.sheet_by_index(1)
        sheet_Metodiche_OW = wb.sheet_by_index(2)
        sheet_Distretti_OW = wb.sheet_by_index(3)
        QD_validator_error = {}
        metodiche_validator_error = {}
        distretti_validator_error = {}
        #QD_validator_error = Validator_v.ck_QD_description(self, df_mapping, sheet_QD_OW)
        #metodiche_validator_error = Validator_v.ck_metodiche_description(self, df_mapping, sheet_Metodiche_OW)
        #distretti_validator_error = Validator_v.ck_distretti_description(self, df_mapping, sheet_Distretti_OW)

        error_dict = {
            "QD_error": QD_error,
            "metodiche_error": metodiche_error,
            "distretti_error": distretti_error,
            "priorita_error": priorita_error,
            "univocita_prestazione_error": univocita_prestazione_error,
            "QD_validator_error": QD_validator_error,
            "metodiche_validator_error": metodiche_validator_error,
            "distretti_validator_error": distretti_validator_error,
            "canali_error": canali_error,
            "inviante_error": inviante_error
        }

        self._validation(error_dict)

    def check_column_name(self, df_mapping):
        print("check the used column name of the excel file")

    def check_qd(self, df_mapping, sheet_QD):
        print("start checking QD") #Codice Quesito Diagnostico
        #controllo se per ogni Agenda sono inseriti gli stessi QD
        error_dict = {}

        #error_QD_agenda = Check_QD.ck_QD_agenda(self, df_mapping, error_dict)
        error_QD_sintassi = Check_QD.ck_QD_sintassi(self, df_mapping, error_dict)
        error_QD_agenda = Check_QD.ck_QD_agenda(self, df_mapping, error_QD_sintassi)
        error_QD_disciplina_agenda = Check_QD.ck_QD_disciplina_agenda(self, df_mapping, sheet_QD, error_QD_agenda)
        error_QD_descrizione = Check_QD.ck_QD_descrizione(self, df_mapping, sheet_QD, error_QD_disciplina_agenda)
        error_QD_operatori_logici = Check_QD.ck_QD_operatori_logici(self, df_mapping, error_QD_descrizione)

        error_dict = error_QD_operatori_logici
        '''error_list = {
            "error_QD_agenda": error_QD_agenda,
            "error_QD_disciplina_agenda": error_QD_disciplina_agenda,
            "error_QD_sintassi": error_QD_sintassi,
            "error_QD_descrizione": error_QD_descrizione
        }'''

        return error_dict

    def check_metodiche(self, df_mapping, sheet_Metodiche):
        print("start checking Metodiche")
        error_dict = {}

        self.output_message = self.output_message + "\nErrori presenti nelle metodiche e riportate attraverso gli indici:\n"

        error_metodica_sintassi = Check_metodiche.ck_metodica_sintassi(self, df_mapping, error_dict)
        error_metodica_inprestazione = Check_metodiche.ck_metodica_inprestazione(self, df_mapping, sheet_Metodiche, error_metodica_sintassi)
        error_metodica_descrizione = Check_metodiche.ck_metodica_descrizione(self, df_mapping, sheet_Metodiche, error_metodica_inprestazione)

        error_dict = error_metodica_descrizione
        '''error_dict = {
            "error_metodica_inprestazione": error_metodica_inprestazione,
            "error_metodica_separatore": error_metodica_separatore,
            "error_metodica_descrizione": error_metodica_descrizione
        }'''

        #print("error_dict: %s", error_dict)
        return error_dict

    def check_distretti(self, df_mapping, sheet_Distretti):
        print("start checking Distretti")
        error_dict = {}

        error_distretti_sintassi = Check_distretti.ck_distretti_sintassi(self, df_mapping, error_dict)
        error_distretti_inprestazione = Check_distretti.ck_distretti_inprestazione(self, df_mapping, sheet_Distretti, error_distretti_sintassi)
        error_distretti_descrizione = Check_distretti.ck_distretti_descrizione(self, df_mapping, sheet_Distretti, error_distretti_inprestazione)
        error_distretti_operatori_logici = Check_distretti.ck_distretti_operatori_logici(self, df_mapping, error_distretti_descrizione)

        error_dict = error_distretti_operatori_logici
        '''error_dict = {
            "error_distretti_inprestazione": error_distretti_inprestazione,
            "error_distretti_separatore": error_distretti_separatore,
            "error_distretti_descrizione": error_distretti_descrizione
        }'''

        return error_dict

    def check_priorita(self, df_mapping):
        print("start checking priorità e tipologie di accesso")
        error_dict = {}

        error_prime_visite = Check_priorita.ck_prime_visite(self, df_mapping, error_dict)
        error_controlli = Check_priorita.ck_controlli(self, df_mapping, error_prime_visite)
        error_esami_strumentali =Check_priorita.ck_esami_strumentali(self, df_mapping, error_controlli)

        error_dict = error_esami_strumentali
        '''error_list = {
            "error_prime_visite": error_prime_visite,
            "error_controlli": error_controlli,
            "error_esami_strumentali": error_esami_strumentali
        }'''

        return error_dict

    def check_prestazione(self, df_mapping):
        print("start checking univocità delle prestazioni")
        error_dict = {}

        error_casi_1N = Check_prestazione.ck_casi_1n(self, df_mapping, error_dict)
        error_prestazione = Check_prestazione.ck_prestazione(self, df_mapping, error_casi_1N)
        error_prestazione_non_prenotabile = Check_prestazione.ck_prestazione_nonprenotabile(self, df_mapping, error_prestazione)

        error_dict = error_prestazione_non_prenotabile

        return error_dict

    def check_canali(self, df_mapping):
        print("start checking canali di accesso")

        error_dict = {}
        error_canali_vuoti = Check_canali.ck_canali_vuoti(self, df_mapping, error_dict)
        error_canali_PAI = Check_canali.ck_canali_PAI(self, df_mapping, error_canali_vuoti)
        error_canali_abilitati = Check_canali.ck_canali_abilitati(self, df_mapping, error_canali_PAI)

        error_dict = error_canali_abilitati

        return error_dict

    def check_inviante(self, df_mapping):
        print("start checking inviante")

        error_dict = {}

        error_inviante = Check_inviante.ck_inviante(self, df_mapping, error_dict)

        error_dict = error_inviante

        return error_dict

    def _validation(self, error_dict):
        print("questi sono gli errori indivuduati e separati per categoria:\n %s", error_dict)

        '''df = pd.DataFrame(rows_list)
        with pd.ExcelWriter(self.file_name, engine='openpyxl', mode='a') as writer:
            df.to_excel(writer, sheet_name='new_mapping', index=False)'''
        print("\nPer osservare i risultati ottenuti, controllare il file prodotto: check_excel_result.txt")
        file = open("check_excel_result.txt", "w")
        #file.write(json.dumps(error_dict))
        file.write(self.output_message + "\n" + json.dumps(error_dict))
        file.close()

    def findCell(self, sh, searchedValue, start_col):
        result_coord = []
        result_value = []

        for row in range(sh.nrows):
            for col in range(start_col, start_col+1):
                myCell = sh.cell(row, col)
                myValue = sh.cell(row, self.work_index_codice_prestazione_SISS)
                abilita = sh.cell(row, self.work_index_abilitazione_esposizione_SISS)
                if myCell.value == searchedValue and abilita.value == "S":
                    result_coord.append(str(row) + "#" + str(col))
                    result_value.append(myValue.value)
                    #return row, col#xl_rowcol_to_cell(row, col)

        if result_coord == []:
            return -1
        return result_coord#, result_value

    def findCell_agenda(self, sh, searchedValue, start_col):
        result_coord = []
        result_value = []

        for row in range(sh.nrows):
            for col in range(start_col, start_col+1):
                myCell = sh.cell(row, col)
                myValue = sh.cell(row, self.work_index_codice_SISS_agenda) #Codice SISS agenda 15
                #abilita = sh.cell(row, self.work_index_abilitazione_esposizione_SISS-1) #abilitazione esposizione SISS 28
                if myCell.value == searchedValue: # and abilita.value == "S":
                    result_coord.append(str(row) + "|" + str(col))
                    result_value.append(myValue.value)
                    #return row, col#xl_rowcol_to_cell(row, col)

        if result_coord == []:
            return -1
        return result_coord#, result_value

    '''Metodo che aggiunge elemento in una lista esistente o crea la lista nel caso
    non fosse presente'''
    def update_list_in_dict(self, dictio, index, element):
        if index in dictio.keys():
            dictio[index].append(element)
        else:
            dictio[index] = [element]
        return dictio

    '''Metodo per controllare il tipo di una colonna excel'''
    def column_validator(self):
        return ""

    def get_file(self):
        if self.file_name != "":
            return self.file_name
        elif self.file_data != "":
            return self.file_data
        else:
            return None

    def list_duplicates(self, seq):
        seen = set()
        seen_add = seen.add
        # adds all elements it doesn't know yet to seen and all other to seen_twice
        seen_twice = set( x for x in seq if x in seen or seen_add(x) )
        # turn the set into a list (as requested)
        return list( seen_twice )

    def initializer_check_agende_interne(self):
        df_mapping = pd.read_excel(self.file_data, sheet_name=self.work_sheet, converters={self.work_codici_disciplina_catalogo: str, self.work_codice_prestazione_siss: str}).replace(np.nan, '', regex=True)
        error = self.analizer_agende_interne(df_mapping)
        error_dict = {
            "error_Aagende_interne": error
        }
        self._validation(error_dict)

    def analizer_agende_interne(self, df_mapping):
        Agende_interne_error = Check_agende_interne.ck_agende_interne(self, df_mapping, {})

        return Agende_interne_error

k = Check_action()

k.import_file()


import numpy as np
import matplotlib.pyplot as plt

DataTab = np.loadtxt("SFD_residual.dat", skiprows=1)
DataTab = DataTab[DataTab[:, 0].argsort()]
time = DataTab[:,0]
L2 = DataTab[:,2]

delta = input ("input filter width: delta =")
chi = input ("input dissipation control coefficient: chi =")

plt.semilogy(time, L2)
# plt.xlim ([0, 3000])
plt.ylim ([1e-10, 1e-1])
plt.xlabel ("time")
plt.ylabel ("residual")
plt.grid(visible=True, which='major', color='grey', linestyle='-')
plt.grid(visible=True, which='minor', color='grey', linestyle=':', alpha=0.2)
plt.savefig("d"+str(delta)+"x"+str(chi)+".jpg", dpi=600)
plt.show()


import unittest
from utils import build_full_name_variations

class TestBuildFullNameVariations(unittest.TestCase):

    def test_build_full_name_variations(self):
        full_names_and_expectations = [
            ('Stohner', []),
            ('Stohner Jürgen', []),
            ('Stohner, Jürgen', [('Stohner', 'Jürgen')]),
            ('Pothier, Joël F.', [('Pothier', 'Joël F.'), ('Pothier', 'Joël')]),
            ('Pothier, Joël (F.)', [('Pothier', 'Joël (F.)'), ('Pothier', 'Joël')]),
            ('Pehlke-Milde, Jessica', [('Pehlke-Milde', 'Jessica'), ('Pehlke', 'Jessica'), ('Milde', 'Jessica')]),
            ('von Wyl, Agnes', [('von Wyl', 'Agnes'), ('von-Wyl', 'Agnes'), ('von', 'Agnes'), ('Wyl', 'Agnes')]),
            ('Föhn, Martina', [('Föhn', 'Martina')]),
            ('Foo, Bazhong (Frank)', [('Foo', 'Bazhong (Frank)'), ('Foo', 'Bazhong-Frank'), ('Foo', 'Bazhong'), ('Foo', 'Frank')])
        ]

        for fullname_and_expectation in full_names_and_expectations:
            full_name, expectation = fullname_and_expectation
            with self.subTest(full_name=full_name, expectation=expectation):
                self.assertEqual(build_full_name_variations(full_name), expectation, f"Expectation for {full_name} not matched.")

if __name__ == '__main__':
    unittest.main()


import socket
import ssl
import gzip
import tkinter
import tkinter.font

from utils import unchunk, Cache

DEFAULT_ENCODING = 'utf-8'

MAX_REDIRECT = 5

AMP_CHAR_CODES = {
    "quot": '"',
    "amp": "&",
    "lt": "<",
    "gt": ">",
    "nbsp": " ",
    "ndash": "-",
    "copy": "©",
    "#39": "'",
}

WIDTH, HEIGHT = 800, 600
HSTEP, VSTEP = 13, 18

DEFAULT_LEADING = 1.25 # proportional to ascent

BROWSER_MODES = {
    "source": "source",
    "normal": "normal"
}

FONTS = {}

FONT_MODIFIERS = {
    "subscript": "subscript",
    "superscript": "superscript",
}

redirect_counter = 0

cache = Cache()

def get_font(size, weight, slant, family):
    key = (size, weight, slant, family)

    if key not in FONTS:
        font = tkinter.font.Font(
            family=family,
            size=size,
            weight=weight,
            slant=slant,
        )
        FONTS[key] = font
    return FONTS[key]

def request(url, additional_headers = {}, redirect_number = 0):
    full_url = url
    scheme, url = url.split(":", 1)

    assert redirect_number < MAX_REDIRECT

    assert scheme in ["http", "https", "file", "data"], \
        f"Unknown scheme {scheme}"

    if scheme not in ["data"]:
        # all schemes except data are followed by "//"
        host, *path = url[2:].split('/', 1)

        # if path is provided, add it, otherwise default to "/"
        path = "/" + path[0] if path else "/index.html"

    if scheme in ["http", "https"]:
        if cache.has_valid_cache(full_url):
            response_headers, body = cache.retrieve(full_url)
        else:
            s = socket.socket(
                family=socket.AF_INET,
                type=socket.SOCK_STREAM,
                proto=socket.IPPROTO_TCP,
            )

            port = 80 if scheme == "http" else 443

            if ":" in host:
                # if port is provided, overwrite
                host, port = host.split(":", 1)
                port = int(port)

            if scheme == "https":
                ctx = ssl.create_default_context()
                s = ctx.wrap_socket(s, server_hostname=host)

            s.connect((host, port))

            http_request = (
                    b"GET " + bytes(path, 'utf-8') + b" HTTP/1.1\r\n" +
                    b"Host: " + bytes(host, 'utf-8') + b"\r\n" +
                    b"Connection: close\r\n" +
                    b"User-Agent: Manyk\r\n" +
                    b"Accept-Encoding: gzip\r\n"
                    )

            for key, value in additional_headers:
                # Add all additional headers to request
                http_request += bytes(f"{key}: {value}\r\n")

            http_request += b"\r\n" # Add proper HTTP file ending

            s.send(http_request)

            response = s.makefile("rb", newline="\r\n")

            statusline = response.readline().decode()
            version, status, explanation = statusline.split(' ', 2) # No more than 2 to allow explanation to be a sentence

            response_headers = {}
            while True:
                line = response.readline().decode()
                if line == "\r\n": break
                header, value = line.split(':', 1)
                response_headers[header.lower()] = value.strip()

            if 299 < int(status) < 399:
                # redirect
                new_url = f"{scheme}:{host}{response_headers['location']}" if response_headers['location'].startswith('/') else response_headers['location']
                return request(new_url, additional_headers, redirect_number + 1)

            else:
                encoding = DEFAULT_ENCODING
                if "charset" in response_headers.get("content-type"):
                    mime, charset = response_headers["content-type"].split(';')
                    encoding = charset.split('=')[1].strip().lower()

                if "chunked" in response_headers.get('transfer-encoding', {}):
                    body = unchunk(response)
                else:
                    body = response.read()

                if 'content-encoding' in response_headers:
                    assert "gzip" in response_headers.get('content-encoding')
                    body = gzip.decompress(body).decode(encoding=encoding)
                else:
                    body = body.decode(encoding=encoding)

                s.close()
            cache.store(full_url, response_headers, body)

    elif scheme == "file":
        with open(path, "r") as file:
            file = open(path, "r")
            body = file.read()
            response_headers = ""

    elif scheme == "data":
        typeinfo, data = url.split(',', 1)

        # Ignoring media-type, charset and base64 for now
        if not typeinfo:
            mediatype, *mediaoptions = ["text/plain", "charset=US-ASCII"]
        else:
            mediatype, *mediaoptions = typeinfo.split(";")

        response_headers = ""
        body = data

    return response_headers, body

class Text:
    def __init__(self, text, parent):
        self.text = self.transform_amp(text)
        self.children = []
        self.parent = parent

    def __repr__(self):
        return repr(self.text)

    def visualize(self, indent=0):
        print(" " * indent, self.text)

    def transform_amp(self, input_text):
        text = ""
        possible_amp_code_index = 0
        for c in input_text:
            if c == ";":
                possible_amp_code = text[possible_amp_code_index + 1:]
                if possible_amp_code in AMP_CHAR_CODES:
                    # ensure we only replace last portion
                    text = text[:possible_amp_code_index] + AMP_CHAR_CODES[possible_amp_code]
                else:
                    text += c
            else:
                if c == "&":
                    possible_amp_code_index = len(text)
                text += c

        return text

class Element:
    def __init__(self, tag, attributes, parent):
        self.tag = tag
        self.children = []
        self.parent = parent
        self.attributes = attributes

    def __repr__(self):
        return "<" + self.tag + ">"

    def visualize(self, indent=0):
        if self.tag in HTMLParser.SELF_CLOSING_TAGS:
            print(" " * indent, "<" + self.tag + " />")
        elif len(self.children) > 0:
            # Non self-closing with children
            print(" " * indent, "<" + self.tag + ">")
            for child in self.children:
                child.visualize(indent + 2)
            print(" " * indent, "</" + self.tag + ">")
        else:
            # Non self-closing without children
            print(" " * indent, "<" + self.tag + "></" + self.tag + ">")

class HTMLParser:
    SELF_CLOSING_TAGS = [
        "area", "base", "br", "col", "embed", "hr", "img", "input",
        "link", "meta", "param", "source", "track", "wbr",
    ]

    HEAD_TAGS = ["base", "basefont", "bgsound", "noscript", "link", "meta", "title", "style", "script"]

    def __init__(self, body):
        self.body = body
        self.unfinished = []

    def get_attributes(self, text):
        parts = text.split(maxsplit=1)
        tag = parts[0].lower()
        attributes = {}
        for attrpair in parts[1:]:
            if "=" in attrpair:
                key, value = attrpair.split("=", 1)
                if len(value) > 2 and value[0] in ["'", "\""]:
                    value = value[1:-1]
                attributes[key.lower()] = value
            else:
                attributes[attrpair.lower()] = ""

        return tag, attributes

    def add_text(self, text):
        if text.isspace(): return
        self.implicit_tags(None)
        parent = self.unfinished[-1]
        node = Text(text, parent)
        parent.children.append(node)

    def add_tag(self, tag):
        tag, attributes = self.get_attributes(tag)
        if tag.startswith("!"): return

        self.implicit_tags(tag)

        if tag == "p" and repr(self.unfinished[-1]) == "<p>":
            # If '<p>' child of '<p>'
            node = self.unfinished.pop()
            parent = self.unfinished[-1]
            parent.children.append(node)

        if tag.startswith("/"):
            if len(self.unfinished) == 1: return
            node = self.unfinished.pop()
            parent = self.unfinished[-1]
            parent.children.append(node)
        elif tag in HTMLParser.SELF_CLOSING_TAGS:
            parent = self.unfinished[-1]
            node = Element(tag, attributes, parent)
            parent.children.append(node)
        else:
            parent = self.unfinished[-1] if self.unfinished else None
            node = Element(tag, attributes, parent)
            self.unfinished.append(node)

    def implicit_tags(self, tag):
        while True:
            open_tags = [node.tag for node in self.unfinished]

            if open_tags == [] and tag != "html":
                self.add_tag("html")
            elif open_tags == ["html"] and tag not in ["head", "body", "/html"]:
                if tag in self.HEAD_TAGS:
                    self.add_tag("head")
                else:
                    self.add_tag("body")
            else:
                break

    def parse(self):
        text = ""
        in_tag = False
        in_comment = False
        in_script = False
        in_quoted_attribute = False
        current_pattern = ""
        for c in self.body:
            if in_tag and "!--".startswith(current_pattern + c) and not in_comment:
                if current_pattern == "!--":
                    text = ""
                    in_tag = False
                    in_comment = True
                current_pattern += c
            elif current_pattern and not in_script:
                current_pattern = ""

            if in_tag and c == '"':
                in_quoted_attribute = not in_quoted_attribute

            if not in_comment and not current_pattern and not in_script and not in_quoted_attribute:
                if c == "<":
                    in_tag = True
                    if text: self.add_text(text)
                    text = ""
                elif c == ">":
                    in_tag = False
                    if text.startswith("script"):
                        in_script = True
                    self.add_tag(text)
                    text = ""
                else:
                    text += c
            elif in_script:
                if in_tag and "</script>".startswith(current_pattern + c):
                    if current_pattern + c == "</script>":
                        if text: self.add_text(text)
                        self.add_tag("/script")
                        in_script = False
                        in_tag = False
                        text = ""
                        current_pattern = ""
                    else:
                        current_pattern += c
                elif in_tag:
                    text += current_pattern + c
                    current_pattern = ""
                    in_tag = False
                else:
                    if c == "<" and not in_tag:
                        current_pattern += c
                        in_tag = True
                    else:
                        text += c
            elif in_comment:
                text += c
                if "-->" in text:
                    text = ""
                    in_comment = False
            else:
                text += c

        return self.finish()

    def finish(self):
        if len(self.unfinished) == 0:
            self.add_tag("html")
        while len(self.unfinished) > 1:
            node = self.unfinished.pop()
            parent = self.unfinished[-1]
            parent.children.append(node)
        return self.unfinished.pop()

class Layout:
    def __init__(self, tree, font, canvas_width):
        self.display_list = []
        self.cursor_x = HSTEP
        self.cursor_y = VSTEP
        self.canvas_width = canvas_width
        self.line = []

        self.original_font_family = font.actual('family')
        self.original_font_size = font.actual('size')

        self.current_modifier = None

        self.font_weight = font.actual('weight')
        self.font_slant = font.actual('slant')
        self.font_size = font.actual('size')
        self.font_family = font.actual('family')

        self.recurse(tree)
        self.flush()

    def open_tag(self, tag):
        if tag == "i" or tag == "em":
            self.font_slant = "italic"
        elif tag == "b"  or tag == "strong":
            self.font_weight = "bold"
        elif tag == "small":
            self.font_size -= 2
        elif tag == "big":
            self.font_size += 4
        elif tag == "br":
            self.flush()
        elif tag == "code":
            self.font_family = "Monaco"
            self.font_size -= 2
        elif tag == "h1":
            self.font_size = 36
        elif tag == "h2":
            self.font_size = 24
        elif tag == "h3":
            self.font_size = 18
        elif tag == "sup":
            self.current_modifier = FONT_MODIFIERS["superscript"]
        elif tag == "sub":
            self.current_modifier = FONT_MODIFIERS["subscript"]

    def close_tag(self, tag):
        if tag == "i" or tag == "em":
            self.font_slant = "roman"
        elif tag == "b" or tag == "strong":
            self.font_weight = "normal"
        elif tag == "small":
            self.font_size += 2
        elif tag == "big":
            self.font_size -= 4
        elif tag == "p":
            self.flush()
            self.cursor_y += VSTEP
        elif tag == "code":
            self.font_family = self.original_font_family
            self.font_size += 2
        elif tag == "pre":
            self.flush()
            self.cursor_y += VSTEP
        elif tag == "h1":
            self.font_size = self.original_font_size
            self.flush()
            self.cursor_y += VSTEP
        elif tag == "h2":
            self.font_size = self.original_font_size
            self.flush()
            self.cursor_y += VSTEP
        elif tag == "h3":
            self.font_size = self.original_font_size
            self.flush()
        elif tag == "sup":
            self.current_modifier = None
        elif tag == "sub":
            self.current_modifier = None

    def recurse(self, tree):
        if isinstance(tree, Text):
            self.text(tree)
        else:
            self.open_tag(tree.tag)
            for child in tree.children:
                self.recurse(child)
            self.close_tag(tree.tag)

    def text(self, token):
        font = get_font(self.font_size, self.font_weight, self.font_slant, self.font_family)

        lstrip_text = token.text.lstrip(" ")
        rstrip_text = token.text.rstrip(" ")

        pre_whitespace = len(token.text) - len(lstrip_text)
        post_whitespace = len(token.text) - len(rstrip_text)

        split_text = token.text.split()

        for index, word in enumerate(split_text):
            w = font.measure(word)
            if self.cursor_x + w > self.canvas_width - HSTEP:
                self.flush()

            if index == 0:
                # Ensure pre-text whitespaces are added back
                self.cursor_x += pre_whitespace * font.measure(" ")

            self.line.append((self.cursor_x, word, font, self.current_modifier))

            if index + 1 < len(split_text):
                # Add space between all words within text
                self.cursor_x += w + font.measure(" ")
            else:
                # Ensure post-text whitespaces are added back, could be 0
                self.cursor_x += w + font.measure(" ") * post_whitespace

    def flush(self):
        if not self.line: return
        metrics = [font.metrics() for x, word, font, modifier in self.line]
        max_ascent = max(metric["ascent"] for metric in metrics)
        baseline = self.cursor_y + DEFAULT_LEADING * max_ascent

        for x, word, font, modifier in self.line:
            base_ascent = font.metrics('ascent')

            modifier_offset = 0
            if modifier:
                new_font = font.copy()
                new_font.config(size=int(font.actual('size') / 1.5))
                modifier_offset = new_font.metrics('descent')
                if modifier == FONT_MODIFIERS["superscript"]:
                    word_y_offset = baseline - base_ascent - modifier_offset
                    self.display_list.append((x, word_y_offset, word, new_font))
                elif modifier == FONT_MODIFIERS["subscript"]:
                    word_y_offset = baseline - modifier_offset
                    self.display_list.append((x, word_y_offset, word, new_font))
            else:
                word_y_offset = baseline - base_ascent
                self.display_list.append((x, word_y_offset, word, font))

        self.cursor_x = HSTEP
        self.line = []

        max_descent = max(metric["descent"] for metric in metrics)
        self.cursor_y = baseline + DEFAULT_LEADING * max_descent
class Browser:
    SCROLL_STEP = 100

    def __init__(self):
        self.display_list = []
        self.body = ""
        self.body_tokens = []
        self.h_step = HSTEP
        self.v_step= VSTEP

        self.window = tkinter.Tk()
        self.canvas = tkinter.Canvas(
            self.window,
            width=WIDTH,
            height=HEIGHT
        )

        self.document = {
            "height": HEIGHT,
            "width": WIDTH,
        }

        self.canvas.pack(fill='both', expand=True)

        self.font = tkinter.font.Font(
            family="Times",
            size=16,
            weight="normal",
            slant="roman",
        )

        self.window.bind("<Down>", self.scrolldown)
        self.window.bind("<Up>", self.scrollup)
        self.window.bind("<MouseWheel>", self.mouse_scroll)
        self.window.bind("<plus>", self.zoomin)
        self.window.bind("<minus>", self.zoomout)
        self.window.bind("<Configure>", self.resize)

        self.scroll = 0
        self.mode = BROWSER_MODES["normal"]
        self.draw_count = 0

    def mouse_scroll(self, event):
        self.scroll -= event.delta * 3

        if self.scroll < 0:
            self.scroll = 0
        self.draw()

    def resize(self, event):
        if (event.width != self.document["width"] or event.height != self.document["height"]) and self.display_list:
            if event.width != self.document["width"]:
                self.display_list = Layout(self.body_tokens, self.font, event.width).display_list
            self.document = {
                "height": event.height,
                "width": event.width,
            }
            self.draw()

    def scrolldown(self, event):
        self.scroll += Browser.SCROLL_STEP
        self.draw()

    def scrollup(self, event):
        self.scroll -= Browser.SCROLL_STEP
        if self.scroll < 0:
            self.scroll = 0
        self.draw()

    def zoomin(self, event):
        self.font.config(size=int(self.font.actual('size') * 1.2))
        self.display_list = Layout(self.body_tokens, self.font, self.document["width"]).display_list
        self.draw()

    def zoomout(self, event):
        next_font_size = int(self.font.actual('size') / 1.2) if int(self.font.actual('size') / 1.2) > 9 else 9
        self.font.config(size=next_font_size)
        self.display_list = Layout(
            self.body_tokens,
            self.font,
            self.document["width"]
            ).display_list
        self.draw()

    def draw(self):
        self.canvas.delete("all")
        for x, y, c, font in self.display_list:
            if y > self.scroll + self.document["height"]:
                continue # below view
            if y + font.metrics('linespace') < self.scroll:
                continue # above view
            self.canvas.create_text(x, y - self.scroll, text=c, font=font, anchor="nw")

    def load(self, url):
        if url.startswith('view-source:'):
            self.mode = BROWSER_MODES["source"]
            _, url = url.split(":", 1)
            headers, body = request(url)
            body = body.replace("<", "<").replace(">", ">")
            self.body_tokens = HTMLParser(body).parse()
            self.display_list = Layout(self.body_tokens, self.font, self.document["width"]).display_list
            self.draw()
        else:
            self.mode = BROWSER_MODES["normal"]
            headers, body = request(url)
            self.body_tokens = HTMLParser(body).parse()
            input("Press Enter to continue...")
            self.body_tokens.visualize()
            self.display_list = Layout(self.body_tokens, self.font, self.document["width"]).display_list
            self.draw()

if __name__ == "__main__":
    import sys
    Browser().load(sys.argv[1])
    tkinter.mainloop()


# -*- coding: utf-8 -*-
"""Installer for the collective.contact.widget package."""

from setuptools import find_packages
from setuptools import setup

long_description = (
    open('README.rst').read()
    + '\n' +
    'Contributors\n'
    '============\n'
    + '\n' +
    open('CONTRIBUTORS.rst').read()
    + '\n' +
    open('CHANGES.rst').read()
    + '\n')

setup(
    name='collective.contact.widget',
    version='1.14.dev0',
    description="Contact widget",
    long_description=long_description,
    # Get more from http://pypi.python.org/pypi?%3Aaction=list_classifiers
    classifiers=[
        "Development Status :: 5 - Production/Stable",
        "Environment :: Web Environment",
        "Framework :: Plone",
        "Framework :: Plone :: 4.2",
        "Framework :: Plone :: 4.3",
        "License :: OSI Approved :: GNU General Public License v2 (GPLv2)",
        "Operating System :: OS Independent",
        "Programming Language :: Python",
        "Programming Language :: Python :: 2.7",
    ],
    keywords='plone contact widget field',
    author='Vincent Fretin',
    author_email='vincentfretin@ecreall.com',
    url='https://github.com/collective/collective.contact.widget',
    download_url='https://pypi.org/project/collective.contact.widget',
    license='GPL',
    packages=find_packages('src', exclude=['ez_setup']),
    namespace_packages=['collective', 'collective.contact'],
    package_dir={'': 'src'},
    include_package_data=True,
    zip_safe=False,
    install_requires=[
        'five.grok',
        'setuptools',
        'collective.contact.core >= 1.27',
        'future',
        'plone.api',
        'plone.formwidget.contenttree >= 1.0.11',
        'plone.formwidget.autocomplete',
        'z3c.relationfield',
    ],
    extras_require={
        'test': [
            'ecreall.helpers.testing',
            'plone.app.testing',
        ],
    },
    entry_points="""
    """,
)


from datetime import datetime
from pathlib import Path

from PIL import Image, ImageDraw, ImageOps

class CFMImage:

    def __init__(self, file, filename):
        self.filename = filename
        self.file = file
        self.image_data: Image.Image = None
        self.model = None
        self.date = None
        self.width = None
        self.height = None
        self.orientation = None
        self.read_image()

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        if self.image_data is not None:
            self.image_data.close()

    def get_bytes(self):
        if isinstance(self.file, (bytes, str, Path)):
            with open(self.filename) as file:
                return file.read()
        else:
            self.file.seek(0, 0)
            return self.file.read()

    def read_image(self):
        try:
            self.image_data: Image = Image.open(self.file)
            self.get_metadata_with_pil()
            self.rotate_if_necessary()
            self.width, self.height = self.image_data.size
        except Exception:
            self.image_data = None

    def rotate_if_necessary(self):
        self.image_data = ImageOps.exif_transpose(self.image_data)

        # if self.orientation is not None:
        #    if self.orientation == 3:
        #        self.image_data = self.image_data.rotate(180, expand=True)
        #    if self.orientation == 6:
        #        self.image_data = self.image_data.rotate(270, expand=True)
        #    if self.orientation == 8:
        #        self.image_data = self.image_data.rotate(90, expand=True)

    def get_metadata_with_pil(self):
        if self.image_data.getexif() is not None:
            exif = dict(self.image_data.getexif().items())
            if 0x0110 in exif:
                self.model = exif[0x0110].strip("\u0000").strip(" ")
            if 0x9003 in exif:
                try:
                    self.date = datetime.strptime(exif[0x9003], '%Y:%m:%d %H:%M:%S')
                except ValueError:
                    self.date = None
                    # comment récupérer ici l'équivalent de FileModifyDate (voir ExifTool) ?
            if 0x0112 in exif:
                self.orientation = exif[0x0112]

    def get_face(self, face):
        img_copy = self.image_data.copy()
        top, right, bottom, left = face
        return img_copy.crop((left + 1, top + 1, right, bottom))

    def get_image_with_faces(self, face_list) -> Image.Image:
        img_copy = self.image_data.copy()
        draw = ImageDraw.Draw(img_copy)
        for face in face_list:
            top, right, bottom, left = face
            draw.rectangle(((left, top), (right, bottom)), outline=(255, 0, 0), width=4)
        del draw
        return img_copy

    def get_image_with_faces_opencv(self, face_list) -> Image.Image:
        img_copy = self.image_data.copy()
        draw = ImageDraw.Draw(img_copy)
        for face in face_list:
            x, y, width, height = face
            draw.rectangle(((x, y), (x + width, y + height)), outline=(255, 0, 0), width=4)
        del draw
        return img_copy


import torch
import numpy as np
from torch.autograd import Variable
from math import ceil
import random

def prepare_generator_batch(samples, start_letter=0, gpu=False):
    """
    Takes samples (a batch) and returns

    Inputs: samples, start_letter, cuda
        - samples: batch_size x seq_len (Tensor with a sample in each row)

    Returns: inp, target
        - inp: batch_size x seq_len (same as target, but with start_letter prepended)
        - target: batch_size x seq_len (Variable same as samples)
    """

    batch_size, seq_len = samples.size()

    inp = torch.zeros(batch_size, seq_len)
    target = samples
    inp[:, 0] = start_letter
    inp[:, 1:] = target[:, :seq_len-1]

    inp = Variable(inp).type(torch.LongTensor)
    target = Variable(target).type(torch.LongTensor)

    if gpu:
        inp = inp.cuda()
        target = target.cuda()

    return inp, target

def prepare_discriminator_data(pos_samples, neg_samples, gpu=False):
    """
    Takes positive (target) samples, negative (generator) samples and prepares inp and target data for discriminator.

    Inputs: pos_samples, neg_samples
        - pos_samples: pos_size x seq_len
        - neg_samples: neg_size x seq_len

    Returns: inp, target
        - inp: (pos_size + neg_size) x seq_len
        - target: pos_size + neg_size (boolean 1/0)
    """

    inp = torch.cat((pos_samples, neg_samples), 0).type(torch.LongTensor)
    target = torch.ones(pos_samples.size()[0] + neg_samples.size()[0])
    target[pos_samples.size()[0]:] = 0

    # shuffle
    perm = torch.randperm(target.size()[0])
    target = target[perm]
    inp = inp[perm]

    inp = Variable(inp)
    target = Variable(target)

    if gpu:
        inp = inp.cuda()
        target = target.cuda()

    return inp, target

def batchwise_sample(gen, num_samples, batch_size, seq_len):
    """
    Sample num_samples samples batch_size samples at a time from gen.
    Does not require gpu since gen.sample() takes care of that.
    """

    samples = []
    for i in range(int(ceil(num_samples/float(batch_size)))):
        samples.append(gen.sample(batch_size, seq_len))

    return torch.cat(samples, 0)[:num_samples]

def batchwise_oracle_nll(gen, oracle, num_samples, batch_size, max_seq_len, start_letter=0, gpu=False):
    s = batchwise_sample(gen, num_samples, batch_size)
    oracle_nll = 0
    for i in range(0, num_samples, batch_size):
        inp, target = prepare_generator_batch(s[i:i+batch_size], start_letter, gpu)
        oracle_loss = oracle.batchNLLLoss(inp, target) / max_seq_len
        oracle_nll += oracle_loss.data.item()

    return oracle_nll/(num_samples/batch_size)

def load_music_file(file):
    with open(file, 'r') as file:
        text = file.read()

    # get vocabulary set
    words = sorted(tuple(set(text.split())))
    n = len(words)

    # create word-integer encoder/decoder
    word2int = dict(zip(words, list(range(n))))
    int2word = dict(zip(list(range(n)), words))

    # encode all words in dataset into integers
    encoded = np.array([word2int[word] for word in text.split()])

    return n, word2int, int2word, encoded

def batch_music_samples(data, seq_len):
    batches = []
    i=0
    while i+seq_len <= len(data):
        batch = list(data[i:i+seq_len])
        batches.append(batch)
        i += seq_len
    return batches

def train_val_split(data, train_size):
    random.seed(1)
    random.shuffle(data)
    train = torch.LongTensor(data[:train_size])
    val = torch.LongTensor(data[train_size:])
    return train, val

def positive_sample(data, n):
    torch.tensor(data)
    sample_idx = random.sample(range(1, len(data)), n)
    return data[sample_idx]


import asyncio
import typing

import aiofiles.os
from aiogram import types
from aiogram.dispatcher import FSMContext
from aiogram.dispatcher.filters import Text
from aiogram.types import Message, CallbackQuery, InlineKeyboardMarkup, ReplyKeyboardMarkup
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession

import classes.message_delete
from data import workdirs
from functions.f_formattig import formatting_input_schedule_master
from keyboards import generate_markup
from keyboards.default import kb_edit_master, kb_return_main_menu
from keyboards.default.kb_edit_master import get_action_edit_service
from keyboards.inline import ikb_edit_master, ikb_return
from keyboards.inline.ikb_admin import get_week_days_btn
from loader import dp
from misc import state_add_master
from model import database
from model.master import ModelMaster, ModelScheduleMaster

@dp.message_handler(Text(equals="Редактировать мастера"))
async def start_edit_master_handler(m: Message, session: AsyncSession):
    masters = await database.master_barber.MasterBarber.all(session)
    markup = kb_edit_master.get_master_btn(masters)
    markup.add(
        kb_return_main_menu.get_back()
    )
    await m.answer(
        text="Выберите мастера",
        reply_markup=markup
    )
    await state_add_master.StateEditMaster.choice_master.set()

@dp.message_handler(Text(equals="⬅ Назад"), state=state_add_master.StateEditMaster)
async def back_edit_master_handler(m: Message,
                                   session: AsyncSession,
                                   state: FSMContext):
    state_name = await state.get_state()
    state_group = state_name.split(":")[0]
    data = await state.get_data()
    message_ids = data.get("message_ids")

    handlers = {
        state_add_master.StateEditMaster.choice_action.state: {
            'handler': start_edit_master_handler,
            'arguments': [m, session]
        },
        state_add_master.StateEditMaster.schedule_start.state: {
            'handler': choice_edit_master_handler,
            'arguments': [m, session, state, True],
            'message_collection': classes.message_delete.MessageCollection(m.from_user.id, m.bot)
        },
        state_add_master.StateEditMaster.service_edit.state: {
            'handler': edit_service_master_handler,
            'arguments': [m, state],
            'message_collection': classes.message_delete.MessageCollection(m.from_user.id, m.bot)
        },
        state_add_master.StateEditMaster.__name__: {
            'handler': choice_edit_master_handler,
            'arguments': [m, session, state, True]
        },
        state_add_master.StateEditMaster.service_delete.state: {
            'handler': edit_service_master_handler,
            'arguments': [m, state],
            'message_collection': classes.message_delete.MessageCollection(m.from_user.id, m.bot)
        },
    }

    handler = handlers.get(state_name)
    if handler is None:
        handler = handlers.get(state_group)

    handler_func = handler.get('handler')
    handler_arguments: list = handler.get('arguments')
    message_collection: classes.message_delete.MessageCollection = handler.get('message_collection')

    await handler_func(*handler_arguments)

    if message_collection and message_ids:
        message_collection.message_ids = message_ids
        await message_collection.delete()

@dp.message_handler(state=state_add_master.StateEditMaster.choice_master)
async def choice_edit_master_handler(m: Message, session: AsyncSession,
                                     state: FSMContext, back_status: bool = False):
    """Фото, услуги, график работы"""

    data = await state.get_data()
    if not back_status:
        master_barber: database.master_barber.MasterBarber = await database.master_barber.MasterBarber.get_by_name(session, m.text)
        model = ModelMaster().default()
        model.name = master_barber.master
        model.tg_id = master_barber.telegram_id
        data = dict(
            master={
                "id": master_barber.id,
                "name": master_barber.master
            }
        )

        await state.update_data(
            model=model.dict(),
            **data,
        )

    markup = kb_edit_master.get_edit_action_btn()
    markup.add(kb_return_main_menu.get_back())

    await m.answer(
        f"Выберите, что вы хотите отредактировать у мастера - {data.get('master').get('name')} ",
        reply_markup=markup
    )

    await state_add_master.StateEditMaster.choice_action.set()

@dp.message_handler(Text(equals="График работы"), state=state_add_master.StateEditMaster.choice_action)
async def edit_schedule_master_handler(m: Message, session: AsyncSession, state: FSMContext):
    data = await state.get_data()
    master: dict = data.get("master")
    model_master = ModelMaster().load(data.get('model'))
    message_collection = classes.message_delete.MessageCollection(m.from_id, m.bot)
    schedules: typing.List[database.job_barber.JobBarber] = await database.job_barber.JobBarber.all(session=session,
                                                                                master_id=master.get(
                                                                                'id'))
    model_master.schedules = [ModelScheduleMaster().convert(_object) for _object in schedules]
    msg = await m.answer("Вы в разделе График работы",
                         reply_markup=ReplyKeyboardMarkup(resize_keyboard=True).add(kb_return_main_menu.get_back()))

    message_collection.message_ids.append(msg.message_id)

    await state.update_data(model=model_master.dict(),
                            old_model=model_master.dict()

                            )
    msg = await m.answer(formatting_input_schedule_master(model_master.schedules), reply_markup=get_week_days_btn())
    message_collection.message_ids.append(msg.message_id)

    await state.update_data(
        message_ids=message_collection.message_ids,

    )

    await state_add_master.StateEditMaster.schedule_start.set()  # функции в файле schedule_master.py

@dp.callback_query_handler(text='next', state=state_add_master.StateEditMaster.schedule_start)
async def update_schedule_master_handler(c: CallbackQuery,
                                         session: AsyncSession,
                                         state: FSMContext):
    data = await state.get_data()
    model = ModelMaster().load(data.get('model'))
    old_model = ModelMaster().load(data.get('old_model'))
    message_collection = classes.message_delete.MessageCollection(c.from_user.id, c.bot)
    message_collection.message_ids = data.get("message_ids")

    schedules_old_to_dict = {
        _job.day_of_week: {
            'model': _job
        } for _job in old_model.schedules
    }
    schedules = []

    for _job in model.schedules:
        __job_object: dict = schedules_old_to_dict.get(_job.day_of_week)
        if _job != __job_object['model']:
            schedules.append(_job)

    await c.message.edit_text("Идет обновление...")

    for _schedule in schedules:
        await database.job_barber.JobBarber.update(
            session, job_id=_schedule.id, data=_schedule,
        )

    await message_collection.delete()
    markup = kb_edit_master.get_edit_action_btn()
    markup.add(kb_return_main_menu.get_back())
    await c.message.answer(f"График работы успешно изменен у мастера {model.name}", reply_markup=markup)
    await state_add_master.StateEditMaster.choice_action.set()

@dp.message_handler(Text(equals="Фотографии"), state=state_add_master.StateEditMaster.choice_action)
async def edit_photo_master_handler(m: Message):
    await m.answer("Отправьте фотографии работ мастера.",
                   reply_markup=ReplyKeyboardMarkup(resize_keyboard=True).add(kb_return_main_menu.get_back()))

    await state_add_master.StateEditMaster.photo.set()

@dp.message_handler(state=state_add_master.StateEditMaster.photo,
                    content_types=types.ContentTypes.PHOTO)
async def get_photo_master_handler(m: Message, state: FSMContext,
                                   session: AsyncSession, album: typing.List[Message] = None
                                   ):
    if album is None:
        album = [m]

    data = await state.get_data()
    model = ModelMaster().load(data.get("model"))
    master_data: dict = data.get('master')
    msg = await m.answer("Заменяем фотографии...")
    photos: typing.List[
        database.master_works_barber.MasterWorksBarber
    ] = await database.master_works_barber.MasterWorksBarber.all(session=session, master_id=master_data.get('id'))
    await database.master_works_barber.MasterWorksBarber.delete(session, master_id=master_data.get('id'))

    for media in album:
        photo = media.photo[-1]
        filename = f"{photo.file_id}.jpg"
        path_photo = workdirs.WORK_DIRECTORY_PHOTO_MASTER_WORKS / filename
        await media.photo[-1].download(destination_file=path_photo)
        await asyncio.sleep(0.1)
        model.photo_works.append(filename)

    await database.master_works_barber.MasterWorksBarber.save(
        session, master_id=master_data.get('id'),
        master_name=master_data.get('name'), photos=model.photo_works
    )

    markup = kb_edit_master.get_edit_action_btn()
    markup.add(kb_return_main_menu.get_back())

    await msg.delete()
    await m.answer(text="Фотографии успешно заменены", reply_markup=markup)
    await state_add_master.StateEditMaster.choice_master.set()

    for photo in photos:
        await aiofiles.os.remove(workdirs.WORK_DIRECTORY_PHOTO_MASTER_WORKS / photo.photo_path)

@dp.message_handler(Text(equals="Услуги"), state=state_add_master.StateEditMaster.choice_action)
async def edit_service_master_handler(m: Message, state: FSMContext):
    text = "Выберите"
    markup = kb_edit_master.get_action_edit_service()
    markup.add(
        kb_return_main_menu.get_back()
    )

    await state.update_data(
        model=ModelMaster().default().dict()
    )
    await m.answer(
        text=text,
        reply_markup=markup
    )
    await state_add_master.StateEditMaster.choice_action_service.set()

@dp.callback_query_handler(Text(equals='no'), state=state_add_master.StateEditMaster.service_confirmation)
async def add_service_master_handler(c: CallbackQuery, state: FSMContext, session: AsyncSession):
    data = await state.get_data()
    master: dict = data.get('master')
    model_master = ModelMaster().load(data.get('model'))

    await database.service_barber.save_services_master(
        session, master.get('id'), model_master.services
    )
    await c.message.delete()
    markup = kb_edit_master.get_action_edit_service()
    markup.add(
        kb_return_main_menu.get_back()
    )

    await c.message.answer(
        text="Услуги успешно добавлены.",
        reply_markup=markup
    )

    await state_add_master.StateEditMaster.choice_action_service.set()

@dp.message_handler(Text(equals="Удалить услугу"), state=state_add_master.StateEditMaster.choice_action_service)
async def delete_service_master_handler(m: Message, state: FSMContext, session: AsyncSession):
    text = "Выберите услугу, которую хотите удалить.\n" \
           "После нажатия на кнопку - услуга удалится"

    message_collection = classes.message_delete.MessageCollection(m.from_user.id, m.bot)
    data = await state.get_data()
    master = data.get('master')
    ms_id = await m.answer(
        text,
        reply_markup=ReplyKeyboardMarkup(resize_keyboard=True).add(kb_return_main_menu.get_back())
    )
    message_collection.message_ids.append(ms_id.message_id)

    services: typing.List[
        database.service_barber.ServiceBarber
    ] = await database.service_barber.ServiceBarber.all(session=session, master_id=master.get('id'))

    await state.update_data(
        services={
            __service.get('id'): {
                **__service
            } for __service in [_service.json() for _service in services]
        }
    )

    ms_id = await m.answer(
        text=f"Услуги мастера - {master.get('name')}",
        reply_markup=ikb_edit_master.get_service_for_delete_btn(services)
    )
    message_collection.message_ids.append(ms_id.message_id)

    await state.update_data(
        message_ids=message_collection.message_ids
    )

    await state_add_master.StateEditMaster.service_delete.set()

@dp.message_handler(Text(equals='Редактировать услугу'), state=state_add_master.StateEditMaster.choice_action_service)
async def edit_view_service_master_handler(
        m: Message, state: FSMContext, session: AsyncSession
):
    data = await state.get_data()
    master: dict = data.get('master')

    services: typing.List[
        database.service_barber.ServiceBarber
    ] = await database.service_barber.ServiceBarber.all(session, master.get('id'))

    markup = ikb_edit_master.get_service_for_edit_btn(services)
    msg_id_first = await m.answer("Вы в разделе Редактирование услуг", reply_markup=ReplyKeyboardMarkup(
        resize_keyboard=True
    ).add(kb_return_main_menu.get_back()))

    msg_id_second = await m.answer(
        text="Выберите услугу, которую хотите отредактировать",
        reply_markup=markup
    )
    await state.update_data(
        message_ids=[msg_id_first.message_id, msg_id_second.message_id]
    )
    await state_add_master.StateEditMaster.service_edit.set()


from collections import deque

n, k = map(int, input().split())

graph = []
virus_data = []

for i in range(n):
	graph.append(list(map(int, input.split())))
	for j in range(n):
		if graph[i][j] != 0:
			# 바이러스 종류, 초 (해당 초의 상태), X, Y
			virus_data.append((graph[i][j], 0, i, j))

virus_data.sort()
q = deque(virus_data)

target_s, target_x, target_y = map(int, input().split())

dx = [-1, 1, 0, 0]
dy = [0, 0, -1, 1]

while q:
	virus, s, x, y = q.popleft()
	# s가 target_s 일 때는 이미 s초의 상태가 저장되어있는 상태
	# 여기서 break하지 않으면 s초를 넘어간 이후의 상태까지 구하는 것
	if s == target_s:
		break

	for i in range(4):
		nx = x + dx[i]
		ny = y + dy[i]

		if 0 <= nx and nx < n and 0 <= ny and ny < n:
			if graph[nx][ny] == 0:
				graph[nx][ny] = virus
				q.append((virus, s + 1, nx, ny))

print(graph[target_x - 1][target_y - 1])


from datetime import date
import ast
from time import sleep

a = open("dadoscadastro.txt", "a+")
a.close()

def linha(tam=42):
    return '-' * tam

def cabecalho(txt):
    print(linha())
    print(txt.center(42))
    print(linha())

def leiaInt(msg):
    while True:
        try:
            n = int(input(msg))
        except (ValueError, TypeError):
            print('\033[31mERRO: por favor, digite um número inteiro valido.\033[m')
        except KeyboardInterrupt:
            print('\033[31mO usuario preferiu não informar o número.\033[m')
        else:
            return n

def leiastr(msg):
    while True:
        try:
            n = str(input(msg))
        except (ValueError, TypeError):
            print('\033[31mERRO! Digite um Sexo valido, Apenas M ou F\033[m')
        except KeyboardInterrupt:
            print('\033[31mERRO! Digite um Sexo valido, Apenas M ou F\033[m')
        else:
            return n

arquivo = open("dadoscadastro.txt", "r")
linhas = arquivo.readlines()
arquivo.close()
codigo = 0
for linhass in linhas:
    codigo += 1

def Cadastro(nome='<desconhecido>', idade=0, sexo='não informado'):

    arquivo = open("dadoscadastro.txt", "a")
    nome = input('Nome: ').upper()
    nascimento = leiaInt('Ano de nascimento: ')
    atual = date.today().year
    idade = atual - nascimento
    sexo = str(input('Sexo: [M/F] ')).upper()[0]
    while sexo not in 'MF':
        print('\033[31mERRO! Digite um Sexo valido, Apenas M ou F\033[m')
        sexo = str(input('Sexo: [M/F] ')).upper()[0]

    email = str(input('Digite o email: '))
    salario = float(input('Salario do funcionario: R$').replace(',', '').replace('.', ''))

    usuario = {"nome": nome,"data de nascimento": nascimento, "idade": idade, "sexo": sexo, "email": email, "salario": f'{salario:.2f}, "codigo": {codigo}'}
    arquivo.write(str(usuario) + "\n")
    arquivo.close()

    print(linha())
    print('\033[32mCadastrado com sucesso.\033[m')
    print(linha())

def ler_arquivo():
    print(linha())
    print('<FUNCIONARIOS CADASTRADOS>'.center(42))
    print(linha())
    arquivo = open("dadoscadastro.txt", "r")
    linhas = arquivo.readlines()
    arquivo.close()
    indice = 0
    for linhass in linhas:
        print(f'{linhass}')
        indice += 1

    if len(linhas) == 0:
        print('Não Existe Registros aqui.')

def remover_cadastro():
    codigo = leiaInt('Digite o código do funcionário a ser removido: ')
    arquivo = open("dadoscadastro.txt", "r")
    linhas = arquivo.readlines()
    arquivo.close()
    indice = 0
    encontrado = False
    for linha in linhas:
        if f'"codigo": {codigo}' in linha:
            encontrado = True
            break
        indice += 1
    if encontrado:
        del linhas[indice]
        arquivo = open("dadoscadastro.txt", "w")
        arquivo.writelines(linhas)
        arquivo.close()
        print('-'*42)
        print(f'\033[32mFuncionário com código {codigo} removido com sucesso.\033[m')
        print('-'*42)
    else:
        print('-'*42)
        print(f'\033[31mNão foi encontrado nenhum funcionário com código {codigo}.\033[m')
        print('-'*42)

while True:
    print(linha())
    print('CADASTRO DE FUNCIONARIOS DA EMPRESA'.center(42))
    print(linha())

    print('[1] - CADASTRAR NOVA PESSOA\n'
            '[2] - VER LISTA DE FUNCIONARIOS CADASTRADOS\n'
            '[3] - REMOVER CADASTRO\n'
            '[4] - SAIR')

    print(linha())
    opc = leiaInt('Digite sua opção: ')
    print(linha())

    while opc != 1 and opc != 2 and opc != 3 and opc != 4:
        print('\033[31mERRO!Digite uma opção valida!\033[m')
        opc = leiaInt('Digite sua opção: ')
    if opc == 1:
        Cadastro()
    if opc == 2:
        ler_arquivo()
    if opc == 3:
        remover_cadastro()
    if opc == 4:
        print('\033[97mUM MOMENTO...\033[m')
        sleep(3)
        print('\033[33mATÉ LOGO, VOLTE SEMPRE!\033[m')
        break


#!/usr/bin/env python
# coding: utf-8

# In[233]:

get_ipython().run_line_magic('matplotlib', 'inline')
# Necessary Packages
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

# Model and data processing packages
from sklearn.model_selection import train_test_split
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
from sklearn.metrics import roc_curve
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import GridSearchCV

# In[242]:

np.random.seed(234)

# In[243]:

df = pd.read_csv('diabetes.csv')
df.head(5)

# In[244]:

df.shape

# In[245]:

X = df.drop('Outcome', axis=1).values
y = df['Outcome'].values

# In[246]:

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4, random_state=42, stratify=y)

# In[247]:

neighbors = np.arange(1,20)
train_accuracy = np.empty(len(neighbors))
test_accuracy = np.empty(len(neighbors))

# In[248]:

for i,k in enumerate(neighbors):

    knn = KNeighborsClassifier(n_neighbors=k)

    knn.fit(X_train, y_train)

    train_accuracy[i] = knn.score(X_train, y_train)

    test_accuracy[i] = knn.score(X_test, y_test)

# In[249]:

plt.plot(neighbors, test_accuracy, label='Testing Accuracy')
plt.plot(neighbors, train_accuracy, label='Training Accuracy')
plt.legend()
plt.xlabel('Number of Neighbors')
plt.ylabel('Accuracy')
plt.title('KNN Varying Number of Neighbors')
plt.show()

# In[250]:

n_neighbors=18
knn = KNeighborsClassifier(n_neighbors)

# In[251]:

knn.fit(X_train, y_train)

# In[252]:

knn.score(X_test, y_test)

# In[253]:

y_pred = knn.predict(X_test)

# In[254]:

confusion_matrix(y_test, y_pred)

# In[255]:

pd.crosstab(y_test, y_pred, rownames=['True'], colnames=['Predicted'], margins=True)

# In[256]:

print(classification_report(y_test, y_pred))

# In[257]:

y_pred_proba = knn.predict_proba(X_test)[:,1]

# In[258]:

# FPR = False Positive Rate, TPR = True Positive Rate
fpr, tpr, thresholds = roc_curve(y_test, y_pred_proba)

# In[259]:

plt.plot([0,1],[0,1], 'k--')
plt.plot(fpr, tpr, label='KNN')
plt.xlabel('fpr')
plt.ylabel('tpr')
plt.title(f'KNN (n_neighbors={n_neighbors}) ROC Curve')
plt.show()

# In[260]:

roc_auc_score(y_test, y_pred_proba)

# In[261]:

param_grid = {'n_neighbors':np.arange(1,100)}

# In[262]:

knn = KNeighborsClassifier()
knn_cv = GridSearchCV(knn, param_grid, cv=5)
knn_cv.fit(X, y)

# In[263]:

knn_cv.best_score_

# In[264]:

knn_cv.best_params_

# In[265]:

n_neighbors=14
knn = KNeighborsClassifier(n_neighbors)

# In[266]:

knn.fit(X_train, y_train)

# In[267]:

knn.score(X_test, y_test)

# In[268]:

y_pred = knn.predict(X_test)

# In[269]:

confusion_matrix(y_test, y_pred)

# In[270]:

pd.crosstab(y_test, y_pred, rownames=['True'], colnames=['Predicted'], margins=True)

# In[272]:

print(classification_report(y_test, y_pred))

# In[273]:

y_pred_proba = knn.predict_proba(X_test)[:,1]

# In[274]:

fpr, tpr, thresholds = roc_curve(y_test, y_pred_proba)

# In[275]:

plt.plot([0,1],[0,1], 'k--')
plt.plot(fpr, tpr, label='KNN')
plt.xlabel('fpr')
plt.ylabel('tpr')
plt.title(f'KNN (n_neighbors={n_neighbors}) ROC Curve')
plt.show()

# In[276]:

roc_auc_score(y_test, y_pred_proba)

# In[ ]:


#!/usr/bin/python
# -*- coding: utf-8 -*-
import pandas as pd
import numpy as np
from keras import Sequential
from keras.preprocessing.image import ImageDataGenerator
from keras.layers import InputLayer, Dense, Flatten, Conv2D, \
    Activation, MaxPooling2D, Dropout
from keras import backend as K
from keras.models import load_model
from keras.preprocessing.image import load_img
from flask import Flask, render_template, url_for, request, app, jsonify

app = Flask(__name__)

import tensorflow as tf
new_model = tf.keras.models.load_model('saved_model/my_model')

# Check its architecture
# new_model.summary()

@app.route('/')
def home():
    return render_template('home.html')

@app.route('/predict_image', methods=['POST'])
def predict_image():
    file = request.files['file']
    filename = file.filename
    if str(filename).strip():
        image = load_img('dog.jpg', target_size=(256, 256))
        img = np.array(image)
        img = img / 255.0
        img = img.reshape(1, 256, 256, 3)
        label = model.predict(img)
        print ('Predicted Class (0 - Cat , 1- Dog): ',
               round(label[0][0]))
        val = round(label[0][0])
        pred = ('Cat' if val else 'Dog')
        return render_template('home.html', prediction=val,
                               img_name=filename)
    else:
        return render_template('home.html')

if __name__ == '__main__':
    app.run(debug=True)


"""
file containing code for Dummy Books Spider. currently being used to test the various methods rather than utilising a
live site.

"""
from HousingPriceScraper.HousingPriceScraper.spiders.AncestorSpider.Exoskeleton import AncestorSpider
import scrapy

class DummyBooksBaseSpider(AncestorSpider):

    def traverse_site(self, response):

        self.responses.append(response.url)
        pagination_information = self.scrape_to_attribute(response, '//form/strong', 'text')
        number_of_pages = int(pagination_information[0]) / int(pagination_information[2])
        for i in range(1, int(number_of_pages)):
            yield scrapy.Request('http://books.toscrape.com/catalogue/page-{}.html'.format(str(i)),
                                 callback=self.get_items)
            self.requests.append('http://books.toscrape.com/catalogue/page-{}.html'.format(str(i)))

    def get_items(self, response):

        elements = [['name', 'a', 1, 'title'],
                    ['price', 'p', 0, 'text', {'class': 'price_color'}],
                    ['url', 'a', 1, 'href']]
        test = self.scrape_product_box(response, '//article', elements)
        test['page_order'] = list(range(len(test['name'])))
        self.update_urls_config(['http://books.toscrape.com/catalogue/{}'.format(url) for url in test['url']])
        self.validate_save_scraped_data(response.url, test, date_vars=True, attrs=False)
        self.responses.append(response.url)
        if hasattr(self, 'get_attributes'):
            for url in test['url']:
                yield scrapy.Request(url='http://books.toscrape.com/catalogue/{}'.format(url), callback=self.get_attributes)
                self.requests.append('http://books.toscrape.com/catalogue/{}'.format(url))

class DummyBookAttrSpider(AncestorSpider):

    def get_attributes(self, response):

        elements = [('name', '//div/h1', 'text'),
                    ('description', '(//p)[4]', 'text')]
        test = self.scrape_multiple_to_attribute(response, elements)
        attrs = self.scrape_table_to_dict(response, '//th', 'text', '//td', 'text')
        test.update(attrs)
        self.validate_save_scraped_data(response.url, test, date_vars=True, attrs=True)
        self.responses.append(response.url)


from django.test import TestCase
import json
# Create your tests here.

class TestAPI(TestCase):

	# def test_elastic(self):
	# 	response = self.client.get('/elastic/api/elastic/?pathInput=%s&indexInput=%s' %('/home/jk/Jk/python/DjangoProject/New/djangoprojectfinal/MutualFunds.csv','jk'))
	# 	res = (response.content).decode('utf-8')
	# 	res_dict = json.loads(res)
	# 	status = res_dict.get('status')
	# 	self.assertEquals('success',status)

	# def test_deleteInput(self):
	# 	response = self.client.get('/elastic/api/deleteInput/?deleteInput=%s' %('abcd6'))
	# 	res = (response.content).decode('utf-8')
	# 	res_dict = json.loads(res)
	# 	status = res_dict.get('status')
	# 	self.assertEquals('success',status)

	# def test_oneByOneInput(self):
	# 	response = self.client.get('/elastic/api/oneByOneInput/?indexNamePath=%s&jsonObj=%s' %('abcd10','{"mfname":"testMfname","aum":2000,"ret_3yrs":3000,"ret_2yrs":8000,"ret_1yr":2000,"ret_5yrs":2500}'))
	# 	res = (response.content).decode('utf-8')
	# 	res_dict = json.loads(res)
	# 	status = res_dict.get('status')
	# 	self.assertEquals('success',status)

	def test_retrieve(self):
		response = self.client.get('/elastic/api/retrieve/?tablename=%s&indexname=%s' %('MobileDB','jkmob3'))
		res = (response.content).decode('utf-8')
		res_dict = json.loads(res)
		status = res_dict.get('status')
		self.assertEquals('success',status)

Analytics And Visualization Tool/elastic/tests.py


from rptest.clients.rpk import RpkTool
from rptest.clients.types import TopicSpec
from rptest.services.admin import Admin
from rptest.services.cluster import cluster
from rptest.tests.redpanda_test import RedpandaTest
from rptest.services.redpanda import SISettings, get_cloud_storage_type
from rptest.services.kgo_verifier_services import KgoVerifierProducer
from rptest.utils.si_utils import parse_s3_segment_path, quiesce_uploads, BucketView, NTP, NTPR

from ducktape.mark import matrix
from ducktape.utils.util import wait_until

import json
import random
import time

SCRUBBER_LOG_ALLOW_LIST = [
    # Attempts to compute the retention point for the cloud log will fail
    # after a spillover manifest is manually removed.
    r"cloud_storage - .* failed to download manifest {key_not_found}",
    r"cloud_storage - .* failed to download manifest.*cloud_storage::error_outcome:2",
    r"cloud_storage - .* Failed to materialize.*cloud_storage::error_outcome:2",
    r"cloud_storage - .* failed to seek.*cloud_storage::error_outcome:2",
    r"cloud_storage - .* Failed to compute time-based retention",
    r"cloud_storage - .* Failed to compute size-based retention",
    r"cloud_storage - .* Failed to compute retention",
    r"archival - .* Failed to compute archive retention",

    # The test removes a segment from the manifest for the scrubber to detect. A reupload
    # may be attempted somewhere after the base offset of the removed segment. The STM
    # manifest will refuse to apply the command and log the error below.
    r"cloud_storage - .* New replacement segment does not line up with previous segment",
    r"cluster - .* Can't add segment:"
]

class CloudStorageScrubberTest(RedpandaTest):
    scrub_timeout = 90
    partition_count = 3
    message_size = 16 * 1024  # 16KiB
    segment_size = 1024 * 1024  # 1MiB
    to_produce = 100 * 1024 * 1024  # 100MiB per partition
    topics = [TopicSpec(partition_count=partition_count)]

    def __init__(self, test_context):
        self.si_settings = SISettings(
            test_context,
            log_segment_size=self.segment_size,
            cloud_storage_spillover_manifest_max_segments=10,
            cloud_storage_housekeeping_interval_ms=1000 * 10,
            fast_uploads=True)

        super().__init__(
            test_context=test_context,
            extra_rp_conf={
                "cloud_storage_enable_scrubbing": True,
                "cloud_storage_scrubbing_interval_ms": 1000,
                "cloud_storage_scrubbing_interval_jitter_ms": 100,
                # Small quota forces partial scrubs
                "cloud_storage_background_jobs_quota": 30,
                # Disable segment merging since it can reupload
                # the deleted segment and remove the gap
                "cloud_storage_enable_segment_merging": False
            },
            si_settings=self.si_settings)

        self.bucket_name = self.si_settings.cloud_storage_bucket
        self.rpk = RpkTool(self.redpanda)

    def _produce(self):
        # Use a smaller working set for debug builds to keep the test timely
        if self.debug_mode:
            self.to_produce //= 2

        msg_count = self.to_produce * self.partition_count // self.message_size
        KgoVerifierProducer.oneshot(self.test_context,
                                    self.redpanda,
                                    self.topic,
                                    self.message_size,
                                    msg_count,
                                    batch_max_bytes=self.message_size * 8,
                                    timeout_sec=60)

        quiesce_uploads(self.redpanda, [self.topic], timeout_sec=60)

        def all_partitions_spilled():
            bucket = BucketView(self.redpanda)
            for pid in range(self.partition_count):
                spillover_metas = bucket.get_spillover_metadata(
                    NTP(ns="kafka", topic=self.topic, partition=pid))

                if len(spillover_metas) == 0:
                    self.logger.debug(f"{self.topic}/{pid} did not spill yet")
                    return False

            return True

        wait_until(all_partitions_spilled,
                   timeout_sec=60,
                   backoff_sec=10,
                   err_msg="Some or all partitions did not spill")

    def _collect_anomalies(self):
        anomalies_per_ntpr = {}

        admin = Admin(self.redpanda)
        for pid in range(self.partition_count):
            anomalies = admin.get_cloud_storage_anomalies(namespace="kafka",
                                                          topic=self.topic,
                                                          partition=pid)

            ntpr = NTPR(ns=anomalies["ns"],
                        topic=anomalies["topic"],
                        partition=anomalies["partition"],
                        revision=anomalies["revision_id"])

            if ("missing_partition_manifest" not in anomalies
                    and "missing_spillover_manifests" not in anomalies
                    and "missing_segments" not in anomalies
                    and "segment_metadata_anomalies" not in anomalies):
                anomalies = None

            anomalies_per_ntpr[ntpr] = anomalies

        return anomalies_per_ntpr

    def _assert_no_anomalies(self):
        anomalies_per_ntpr = self._collect_anomalies()
        for ntpr, anomalies in anomalies_per_ntpr.items():
            assert anomalies is None, f"{ntpr} reported unexpected anomalies: {anomalies}"

    def _delete_segment_and_await_anomaly(self):
        segment_metas = [
            meta for meta in self.cloud_storage_client.list_objects(
                self.bucket_name) if "log" in meta.key
        ]

        view = BucketView(self.redpanda)
        to_delete = random.choice(segment_metas)
        attempts = 1
        while view.is_segment_part_of_a_manifest(to_delete) == False:
            to_delete = random.choice(segment_metas)
            attempts += 1

            assert attempts < 100, "Too many attempts to find a segment to delete"

        self.logger.info(f"Deleting segment at {to_delete.key}")
        self.cloud_storage_client.delete_object(self.bucket_name,
                                                to_delete.key,
                                                verify=True)

        ntpr = parse_s3_segment_path(to_delete.key).ntpr

        self.logger.info(
            f"Waiting for missing segment anomaly {to_delete.key} to be reported"
        )

        def ntpr_missing_segment():
            anomalies_per_ntpr = self._collect_anomalies()
            self.logger.debug(f"Reported anomalies {anomalies_per_ntpr}")

            if ntpr not in anomalies_per_ntpr:
                return False

            if anomalies_per_ntpr[ntpr] is None:
                return False

            if "missing_segments" not in anomalies_per_ntpr[ntpr]:
                return False

            return to_delete.key in anomalies_per_ntpr[ntpr][
                "missing_segments"]

        wait_until(ntpr_missing_segment,
                   timeout_sec=self.scrub_timeout,
                   backoff_sec=2,
                   err_msg="Missing segment anomaly not reported")

    def _delete_spillover_manifest_and_await_anomaly(self):
        pid = random.randint(0, 2)
        view = BucketView(self.redpanda)
        spillover_metas = view.get_spillover_metadata(
            NTP(ns="kafka", topic=self.topic, partition=pid))
        to_delete = random.choice(spillover_metas)
        ntpr = to_delete.ntpr

        self.logger.info(f"Deleting spillover manifest at {to_delete.path}")
        self.cloud_storage_client.delete_object(self.bucket_name,
                                                to_delete.path,
                                                verify=True)

        self.logger.info(
            f"Waiting for missing spillover anomaly {to_delete.path} to be reported"
        )

        def ntpr_missing_spill_manifest():
            anomalies_per_ntpr = self._collect_anomalies()
            self.logger.debug(f"Reported anomalies {anomalies_per_ntpr}")

            if ntpr not in anomalies_per_ntpr:
                return False

            if anomalies_per_ntpr[ntpr] is None:
                return False

            if "missing_spillover_manifests" not in anomalies_per_ntpr[ntpr]:
                return False

            return to_delete.path in anomalies_per_ntpr[ntpr][
                "missing_spillover_manifests"]

        wait_until(ntpr_missing_spill_manifest,
                   timeout_sec=self.scrub_timeout,
                   backoff_sec=2,
                   err_msg="Missing spillover manifest anomaly not reported")

    def _assert_anomalies_stable_after_leader_shuffle(self):
        self.logger.info("Checking anomalies stay stable after leader shuffle")

        old_anomalies = self._collect_anomalies()
        assert any([
            a is not None for a in old_anomalies.values()
        ]), f"Expected anomalies on at least one ntpr, but got {old_anomalies}"

        admin = Admin(self.redpanda)
        for pid in range(self.partition_count):
            admin.transfer_leadership_to(namespace="kafka",
                                         topic=self.topic,
                                         partition=pid)
            admin.await_stable_leader(namespace='kafka',
                                      topic=self.topic,
                                      partition=pid)

        def anomalies_stable():
            anomalies = self._collect_anomalies()
            self.logger.debug(f"Reported anomalies {anomalies}")

            return anomalies == old_anomalies

        wait_until(
            anomalies_stable,
            timeout_sec=self.scrub_timeout,
            backoff_sec=2,
            err_msg="Reported anomalies changed after leadership shuffle")

    def _assert_anomalies_stable_after_restart(self):
        self.logger.info(
            "Checking anomalies stay stable after full cluster restart")

        old_anomalies = self._collect_anomalies()
        assert any([
            a is not None for a in old_anomalies.values()
        ]), f"Expected anomalies on at least one ntpr, but got {old_anomalies}"

        for node in self.redpanda.nodes:
            self.redpanda.stop_node(node)

        for node in self.redpanda.nodes:
            self.redpanda.start_node(node)

        def anomalies_stable():
            anomalies = self._collect_anomalies()
            self.logger.debug(f"Reported anomalies {anomalies}")

            return anomalies == old_anomalies

        wait_until(anomalies_stable,
                   timeout_sec=self.scrub_timeout,
                   backoff_sec=2,
                   err_msg="Reported anomalies changed after full restart")

    def _assert_segment_metadata_anomalies(self):
        self.logger.info(
            "Fudging manifest and waiting on segment metadata anomalies")

        view = BucketView(self.redpanda)
        manifest = view.manifest_for_ntp(topic=self.topic, partition=0)

        sorted_segments = sorted(manifest['segments'].items(),
                                 key=lambda entry: entry[1]['base_offset'])
        assert len(
            sorted_segments
        ) > 2, f"Not enough segments in manifest: {json.dumps(manifest)}"

        # Remove the metadata for the penultimate segment, thus creating an offset gap
        # for the scrubber to detect
        seg_to_remove_name, seg_to_remove_meta = sorted_segments[-2]
        manifest['segments'].pop(seg_to_remove_name)
        self.logger.info(f"Removing segment with meta {seg_to_remove_meta}")

        detect_at_seg_meta = sorted_segments[-1][1]
        self.logger.info(f"Anomaly should be detected at {detect_at_seg_meta}")

        # Forcefully reset the manifest. Remote writes are disabled first
        # to make this operation safe.
        json_man = json.dumps(manifest)
        self.logger.info(f"Re-setting manifest to:{json_man}")

        self.rpk.alter_topic_config(self.topic, 'redpanda.remote.write',
                                    'false')
        time.sleep(1)

        admin = Admin(self.redpanda)
        admin.unsafe_reset_cloud_metadata(self.topic, 0, manifest)

        self.rpk.alter_topic_config(self.topic, 'redpanda.remote.write',
                                    'true')

        ntpr = NTPR(ns="kafka",
                    topic=self.topic,
                    partition=0,
                    revision=seg_to_remove_meta["ntp_revision"])

        def gap_reported():
            anomalies_per_ntpr = self._collect_anomalies()
            self.logger.debug(f"Reported anomalies {anomalies_per_ntpr}")

            if ntpr not in anomalies_per_ntpr:
                return False

            if anomalies_per_ntpr[ntpr] is None:
                return False

            if "segment_metadata_anomalies" not in anomalies_per_ntpr[ntpr]:
                return False

            seg_meta_anomalies = anomalies_per_ntpr[ntpr][
                "segment_metadata_anomalies"]

            for meta in seg_meta_anomalies:
                if (meta["type"] == "offset_gap"
                        and meta["at_segment"]["base_offset"]
                        == detect_at_seg_meta["base_offset"]):
                    return True

        wait_until(
            gap_reported,
            # A new manifest needs to be uploaded, so the timeout is more generous
            timeout_sec=self.scrub_timeout + 10,
            backoff_sec=2,
            err_msg="Gap not reported")

    @cluster(num_nodes=4, log_allow_list=SCRUBBER_LOG_ALLOW_LIST)
    @matrix(cloud_storage_type=get_cloud_storage_type())
    def test_scrubber(self, cloud_storage_type):
        self._produce()
        self._assert_no_anomalies()
        self._delete_spillover_manifest_and_await_anomaly()
        self._delete_segment_and_await_anomaly()

        self._assert_segment_metadata_anomalies()

        self._assert_anomalies_stable_after_leader_shuffle()
        self._assert_anomalies_stable_after_restart()

        # This test deletes segments, spillover manifests
        # and fudges the manifest. rp-storage-tool also picks
        # up on some of these things.
        self.redpanda.si_settings.set_expected_damage(
            {"missing_segments", "metadata_offset_gaps"})


import sys
import os
import json
import pickle
import configparser

# class imports
from classes import VoskHandler, DataHandler

# Get relevant configs: input_path
config = configparser.ConfigParser()
config.read('config.ini')
input_path = config.get('DATACONFIG','input_path')
output_dir = config.get('DATACONFIG','output_dir') #MAYBE TAKE OUT EVERYWHERE; RELEVANT METHODS IN CLASS ARENT USED ANYMORE
ai_transcripts_path = config.get('DATACONFIG','output_dir')
wd_path = config.get('DATACONFIG','word_dicts_path')
model_path = config.get('MODELCONFIG','model_path')

#*** ----- STEP2: VoskHandler & DataHandler --> Setup VoskModel & specify output path ----- ***#
    # load dict of converted_audio_paths
with open(os.path.join(input_path, 'coversion_dict.pkl'), 'rb') as f:
    converted_audios = pickle.load(f)

    # initialize Vosk-Model
vosky = VoskHandler("vosky", model_path, output_dir)
vosky.initialize_model()

    # Specify output_dir where all files should be stored
DataHandle = DataHandler("data", output_dir)

#*** ----- STEP3: VoskHandler & DataHandler --> Extract & Handle Data ----- ***#
    #VoskHandler takes one file at a time
for file in converted_audios:
    abs_path = converted_audios[file]

    #Transcribe audio to text & get json data dumb in list as result
    res = vosky.transcribe_audio_to_text(abs_path)
    if res is None:
        print(f"[ERROR]Failure during recognizing audio with name {file}.")
        continue

    #convert res into (i) pure_text & (ii) list of word dicts
    data = DataHandle.convert_to_text_and_list_of_word_dicts(res)
    pure_text, word_dicts = data[0], data[1]

    #!!!!! HARD CODED SHIT TAKE OUT
    with open("{}/{}".format(wd_path, file), 'w') as fout:
        json.dump(word_dicts, fout)

    with open("{}/{}.txt".format(ai_transcripts_path, file), 'w') as fout:
            fout.write(pure_text)

print("[INFO] All done.")


from typing import Type
import unittest
from tickets import *
from collections import defaultdict

class TestTicket(unittest.TestCase):

    def test_get_all_tickets(self):
        params = defaultdict(lambda: None)
        tickets = get_tickets(params)
        self.assertGreater(len(tickets), 0)
        test_ticket = Ticket(123, "test.com", "open", "high", "test", "hello world", "2021-11-22T18:29:52Z")
        for ticket in tickets:
            self.assertEqual(type(ticket), type(test_ticket))

    def test_select_ticket(self):
        ticket = get_ticket(5)
        test_ticket = Ticket(123, "test.com", "open", "high", "test", "hello world", "2021-11-22T18:29:52Z")
        self.assertEqual(type(ticket), type(test_ticket))

    def test_invalid_select(self):
        ticket = get_ticket(-239)
        self.assertIsNone(ticket)

    def test_date_comp(self):
        ticket = Ticket(123, "test.com", "open", "high", "test", "hello world", "2021-11-22T18:29:52Z")
        before = "02/21/2013"
        same = "11/22/2021"
        after = "11/23/2021"
        self.assertEqual(ticket.dateCompare(before), 1)
        self.assertEqual(ticket.dateCompare(same), 0)
        self.assertEqual(ticket.dateCompare(after), -1)

class TestParams(unittest.TestCase):

    def test_status(self):
        params = defaultdict(lambda: None)

        params["status"] = "open"
        for ticket in get_tickets(params):
            self.assertEqual(ticket.status, "open")

        params["status"] = "pending"
        for ticket in get_tickets(params):
            self.assertEqual(ticket.status, "pending")

        params["status"] = "closed"
        for ticket in get_tickets(params):
            self.assertEqual(ticket.status, "closed")

    def test_before(self):
        params = defaultdict(lambda: None)

        params["before"] = "11/24/2021"
        for ticket in get_tickets(params):
            self.assertEqual(ticket.dateCompare("11/24/2021"), -1)

    def test_after(self):
        params = defaultdict(lambda: None)

        params["after"] = "11/23/2021"
        for ticket in get_tickets(params):
            self.assertEqual(ticket.dateCompare("11/23/2021"), 1)

    def test_no_matches(self):
        params = defaultdict(lambda: None)
        params["status"] = "open"
        params["before"] = "12/19/1999"
        tickets = get_tickets(params)
        self.assertEqual(len(tickets), 0)

if __name__ == '__main__':
    unittest.main()


from django.conf.urls import url
from . import views

urlpatterns = [
    url(r'^$', views.home, name='home'),
    url(r'^element/$', views.element_list, name='element_list'),
    url(r'^element/(?P<pk>[0-9]+)/$', views.element_detail, name='element_detail'),
    url(r'^element/new/$', views.element_new, name='element_new'),
    url(r'^element/(?P<pk>[0-9]+)/edit/$', views.element_edit, name='element_edit'),
    url(r'^element/(?P<pk>\d+)/remove/$', views.element_remove, name='element_remove'),
    url(r'^order/$', views.order_list, name='order_list'),
    url(r'^order/detail/(?P<pk>[0-9]+)/$', views.order_detail, name='order_detail'),
    url(r'^order/new/$', views.order_new, name='order_new'),
    url(r'^order/new/(?P<pk>[0-9]+)/$', views.order_new_next, name='order_new_next'),
    url(r'^order/(?P<pk>[0-9]+)/edit/$', views.order_edit, name='order_edit'),
    url(r'^order/(?P<pk>\d+)/remove/$', views.order_remove, name='order_remove'),
]