from bs4 import BeautifulSoup
from urllib.request import urlopen
import pandas as pd

url = 'https://movie.naver.com/movie/running/current.naver?order=reserve'
page = urlopen(url)

soup = BeautifulSoup(page, 'lxml')

movie_ul = soup.find('ul', class_='lst_detail_t1').find_all('li')

all_title = []
all_score = []
all_people = []
all_book = []
all_category = []
all_director = []
all_star = []
all_running_time = []
all_release_date = []

for movie in movie_ul:
    all_title.append(movie.find('dt', class_='tit').a.text)
    all_score.append(movie.find('span', class_='num').text)
    all_people.append(movie.find('em').text)
    if movie.find('dl', class_='info_exp'):
        all_book.append(movie.find('dl', class_='info_exp').span.text + '%')
    else:
        all_book.append("예매 정보 없음")

    info = movie.find('dl', class_='info_txt1').find_all('span', class_='link_txt')

    time = movie.find('dl', class_='info_txt1').text
    time = time.replace('\r', '')
    time = time.replace('\t', '')
    time = time.split('\n')

    while '' in time:
        time.remove('')

    running_time = ''
    release_date = ''
    cnt = 0
    for i in range(len(time)):
        if time[i] == '|':
            if cnt == 0:
                if len(time[i+1]) > 5:
                    release_date = time[i+1]
                    running_time = '상영 시간 미공개'
                    break
                running_time = time[i+1]
                cnt += 1
                continue
            if cnt == 1:
                release_date = time[i+1]
                break

    release_date = release_date[0:10]

    all_running_time.append(running_time)
    all_release_date.append(release_date)

    category_info = ''
    for category in info[0].find_all('a'):
        category_info += category.text
        category_info += ','
    category_info = category_info[:-1]

    all_category.append(category_info)

    director_info = ''
    for director in info[1].find_all('a'):
        director_info += director.text
        director_info += ','
    director_info = director_info[:-1]

    all_director.append(director_info)

    star_info = ''
    if len(info) > 2:
        for star in info[2].find_all('a'):
            star_info += star.text
            star_info += ','
        star_info = star_info[:-1]
        all_star.append(star_info)
    else:
        star_info = '출연 배우 없음'
        all_star.append(star_info)

dat_dict = {
    "제목":all_title, "평점":all_score, "참여명수":all_people,
    "예매율":all_book, "개요":all_category, "감독":all_director,
    "배우":all_star , "상영 시간":all_running_time, "개봉일":all_release_date}
dat = pd.DataFrame(dat_dict)
dat.to_csv("네이버영화.csv", index=False)
dat.to_excel("네이버영화.xlsx", index=False)


# -*- coding: utf-8 -*-
# Scipy Sample.
# http://cs231n.github.io/python-numpy-tutorial/
# pip install --upgrade Pillow
"""
Note that the Python Imaging Library (PIL) is not a dependency of SciPy and therefore the pilutil module is not available on systems that don’t have PIL installed.

by http://docs.scipy.org/doc/scipy/reference/misc.html

http://pillow.readthedocs.org/en/latest/installation.html#mac-os-x-installation

pip3 uninstall scipy
pip3 install --upgrade scipy

brew install libtiff libjpeg webp little-cms2
pip3 install --upgrade Pillow
"""
import numpy as np
from scipy.misc import imread, imsave, imresize

# Load an image as an nparray.
img = imread('./dog.png')
print(img.dtype, img.shape)

# Resize.
img_resize = imresize(img, (100, 100))
imsave('./dist/dog_resize.png', img_resize)

# Modify.
img_changed = img * [0, 1, 0, 0.5]
imsave('./dist/dog_green.png', img_changed)

### Distance between points
from scipy.spatial.distance import euclidean, cosine
# Euclidean distance
d = euclidean([1,0], [2,5])
print(d)
# Cosine distance
d = cosine([1,1], [1,5])
print(d)


## Import Environment and FileSystemLoader so we can load in our template.
## Import yaml as our config is yaml.
from jinja2 import Environment, FileSystemLoader
import yaml

def get_template(env):

    ## Open up our yaml file which will map any services depending on the environment.
    with open('service-discovery.yaml', 'r') as file:
        svc_conf = yaml.safe_load(file)

    ## Pulling the values from the service discovery yaml and assigning them to variables.
    monitoringserver = svc_conf[env]['monitoring-server']
    keystore = svc_conf[env]['key-store']
    webservicefrontend = svc_conf[env]['web-service-frontend']
    dbservicebackend = svc_conf[env]['db-service-backend']

    ## Load the templates directory.
    file_loader = FileSystemLoader('templates')
    ## Create our Jinja environment
    jinja_env = Environment(loader=file_loader)
    ## Get our template config file.
    template = jinja_env.get_template("config.yaml")
    ## Create the config file using are variables from the service-discovery yaml.
    outputText = template.render(
        monitoringserver = monitoringserver,
        keystore = keystore,
        webservicefrontend = webservicefrontend,
        dbservicebackend = dbservicebackend
    )

    ## Write the new file
    f = open(f"output/{env}Config.yaml", "w")
    f.write(outputText)
    f.close()


import json

from wfscript.constants.payload import PayloadKey
from wfscript.utils.json import preserialize

from django_wfscript.utils.payload import hash_request_body
from django_wfscript.wfscript_api.models import Payload, Response, Run
from django_wfscript.wfscript_api.views import core_domain

def be_idempotent(payload_hash):
    try:
        payload = Payload.objects.get(md5=payload_hash)
        return dict(payload.response.data, **{PayloadKey.DUPLICATE: True})
    except Payload.DoesNotExist:
        pass

def confirm_resume_info(payload_data, run):
    if not payload_data[PayloadKey.RESUME]:
        raise RuntimeError(f'If {PayloadKey.RUN_ID} is provided, {PayloadKey.RESUME} must be provided as well')
    else:
        provided_method = payload_data[PayloadKey.RESUME].get(PayloadKey.METHOD)
        if run.resume_method != provided_method:
            RuntimeError(f'Next method for Run {run.identity} is {run.resume_method}; '
                         f'Provided: {provided_method}')
        provided_step = payload_data[PayloadKey.RESUME].get(PayloadKey.STEP)
        if run.resume_step != provided_step:
            RuntimeError(f'Next step for Run {run.identity} is {run.resume_step}; '
                         f'Provided: {provided_step}')

def load_run_and_confirm_resume_info(payload_data):
    run = Run.objects.get(identity=payload_data[PayloadKey.RUN_ID])
    confirm_resume_info(payload_data, run)
    return run

def handle_resume_state(run, response_data):
    if response_data[PayloadKey.RESUME]:
        if PayloadKey.STATE in response_data[PayloadKey.RESUME]:
            run.state = response_data[PayloadKey.RESUME].pop(PayloadKey.STATE)
        run.resume_method = response_data[PayloadKey.RESUME][PayloadKey.METHOD]
        run.resume_step = response_data[PayloadKey.RESUME][PayloadKey.STEP]
        run.save()

def api_run_method(request_body):
    payload_hash = hash_request_body(request_body)
    duplicate_results = be_idempotent(payload_hash)
    if duplicate_results:
        return duplicate_results
    else:
        payload_data = json.loads(request_body)
        method_identity = payload_data[PayloadKey.METHOD]
        if payload_data.get(PayloadKey.RUN_ID):
            run = load_run_and_confirm_resume_info(payload_data)
        else:
            run = Run.objects.create(method=method_identity)
        payload = Payload.objects.create(md5=payload_hash, run=run, data=payload_data)
        method_executor = core_domain.get_method_executor(method_identity)
        response_data = preserialize(
            method_executor.run(input_data=payload_data[PayloadKey.INPUT],
                                state=run.state,
                                resume_info=payload_data.get(PayloadKey.RESUME, {})))
        response_data[PayloadKey.RUN_ID] = run.identity
        handle_resume_state(run, response_data)
        Response.objects.create(payload=payload, data=response_data)
        return dict(response_data, **{PayloadKey.DUPLICATE: False})


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Jan 17 14:11:18 2018

@author: will
"""
import matplotlib.pyplot as plt
from tqdm import tqdm
from dendritic_backprop_classes import *

auto_set_optimal_weights = True
train_interneuron_weights = True
train_inter_pyrimidal_weights = True
train_backward_weights = False

hidden_1 = hidden_neuron_layer(30,20,10,10)
output  = output_layer(20,10)
if(auto_set_optimal_weights):
   hidden_1.int_to_pyrA = -hidden_1.above_to_pyrA
   hidden_1.pyr_to_int = output.input_to_pyrB
#test3 = hidden_neuron_layer(20,10,10,10)

vA_store = []
weight_store_int_pyr = []
weight_store_above_pyrA = []
uI_store = []
uP_store = []
soma_store = []

iters  = 10000
for i in tqdm(range(iters)):
   input_layer = np.zeros(30)
   if(i%1000== 0):
      x = np.random.uniform(-1,1,30)
      targets = target_matrix_2.dot(target_matrix_1.dot(x))
      x = non_linearity(x)
   input_layer  = low_pass_filter(input_layer,x,low_pass_filter_timescale_inputs,timestep)
   y = np.ones(10)
   hidden_1.update_membrane_potentials(x,output.uP,1)
   output.update_membrane_potentials(hidden_1.uP,targets)

   if(train_interneuron_weights):
      hidden_1.update_pyr_to_int_weights()
      hidden_1.update_int_to_pyr_weights()

   if(train_inter_pyrimidal_weights):
      hidden_1.update_input_pyr_weights(input_layer)
      output.update_input_pyr_weights(hidden_1.uP)

   if(i%1== 0):
      weight_store_int_pyr.append(np.copy(hidden_1.int_to_pyrA))
      weight_store_above_pyrA .append(np.copy(hidden_1.above_to_pyrA))
      uP_store.append(np.copy(output.uP))
      uI_store.append(np.copy(hidden_1.uI))
      vA_store.append(np.copy(hidden_1.vA))

vA_store = np.array(vA_store)
weight_store_int_pyr= np.array(weight_store_int_pyr)
weight_store_above_pyrA= np.array(weight_store_above_pyrA)
uP_store = np.array(uP_store)
uI_store = np.array(uI_store)
plt.plot(vA_store[:,:,0])
plt.title("Potential of Apical Dendrites")
plt.show()
plt.figure()
plt.plot(np.mean(np.abs(uI_store[:,:,0]- uP_store[:,:,0]),axis=1))
plt.title("Mean abs value of difference between pyr soma rate and interneuron rate")
plt.yscale('log')
plt.show()

diff_vector = []
for i in range(weight_store_int_pyr.shape[0]):
   diff_vector.append(tot_weight_sum(weight_store_int_pyr[i,:,:],weight_store_above_pyrA[i,:,:]))
diff_vector = np.array(diff_vector)
plt.figure()
plt.plot(diff_vector)


"""Gestion des "routes" FLASK et des données pour les films.
Fichier : gestion_films_crud.py
Auteur : OM 2022.04.11
"""
from pathlib import Path

from flask import redirect
from flask import request
from flask import session
from flask import url_for

from APP_PHARMACIE_164.database.database_tools import DBconnection
from APP_PHARMACIE_164.erreurs.exceptions import *
from APP_PHARMACIE_164.clients.gestion_clients_wtf_forms import FormWTFUpdateClient, FormWTFAddClient, FormWTFDeleteClient

"""Ajouter un film grâce au formulaire "client_add_wtf.html"
Auteur : OM 2022.04.11
Définition d'une "route" /film_add

Test : exemple: cliquer sur le menu "Films/Genres" puis cliquer sur le bouton "ADD" d'un "film"

Paramètres : sans

Remarque :  Dans le champ "nom_mail_update_wtf" du formulaire "films/films_update_wtf.html",
            le contrôle de la saisie s'effectue ici en Python dans le fichier ""
            On ne doit pas accepter un champ vide.
"""
@app.route("/clients_afficher/<int:id_client_sel>", methods=['GET', 'POST'])
def clients_afficher(id_client_sel):
    print(" clients_afficher id_client_sel ", id_client_sel)
    if request.method == "GET":
        try:
            with DBconnection() as mc_afficher:
                strsql_client_afficher_data = """SELECT id_client, nom, prenom, date_de_nais,nom_genre,nom_assu FROM t_client
                                            INNER JOIN t_genre ON t_genre.id_genre = t_client.fk_genre
                                            INNER JOIN t_assurance on t_assurance.id_assu = t_client.fk_assu"""
                if id_client_sel == 0:
                    # le paramètre 0 permet d'afficher tous les clients
                    # Sinon le paramètre représente la valeur de l'id du film
                    mc_afficher.execute(strsql_client_afficher_data)
                else:
                    # Constitution d'un dictionnaire pour associer l'id du film sélectionné avec un nom de variable
                    valeur_id_client_selected_dictionnaire = {"value_id_client_selected": id_client_sel}
                    # En MySql l'instruction HAVING fonctionne comme un WHERE... mais doit être associée à un GROUP BY
                    # L'opérateur += permet de concaténer une nouvelle valeur à la valeur de gauche préalablement définie.
                    strsql_client_afficher_data += """ HAVING id_client= %(value_id_client_selected)s"""

                    mc_afficher.execute(strsql_client_afficher_data, valeur_id_client_selected_dictionnaire)

                # Récupère les données de la requête.
                data_client_afficher = mc_afficher.fetchall()
                print("data_client ", data_client_afficher, " Type : ", type(data_client_afficher))

                # Différencier les messages.
                if not data_client_afficher and id_client_sel == 0:
                    flash("""La table "t_client" est vide. !""", "warning")
                elif not data_client_afficher and id_client_sel > 0:
                    # Si l'utilisateur change l'id_mail dans l'URL et qu'il ne correspond à aucun film
                    flash(f"Le client {id_client_sel} demandé n'existe pas !!", "warning")
                else:
                    flash(f"Données client affichés !!", "success")

        except Exception as Exception_client_afficher:
            raise ExceptionClientAfficher(f"fichier : {Path(__file__).name} ; "
                                          f"{clients_afficher.__name__} ;"
                                        f"{Exception_client_afficher}")

    print("clients_afficher  ", data_client_afficher)
    # Envoie la page "HTML" au serveur.
    return render_template("clients/clients_afficher.html", data=data_client_afficher)

@app.route("/client_add_wtf", methods=['GET', 'POST'])
def client_add_wtf():
    # Objet formulaire pour AJOUTER un client
    form = FormWTFAddClient()
    print(" on submit FormWTFAddClient ", form.submit.data, " req ", request.method)
    try:
        if request.method == "POST" and form.submit.data:
            # Récuperer les données du formulaire défini dans APP_PHARMACIE_164/clients/gestion_clients_wtf_forms.py
            nom_client = form.nom_client_wtf.data
            prenom_client = form.prenom_client_wtf.data
            date_nais_client = form.date_nais_client_wtf.data
            genre_selectionne = form.genres_dropdown_wtf.data
            assurance_selectionne = form.assu_dropdown_wtf.data

            valeurs_insertion_dictionnaire = {"value_nom_client": nom_client,
                                              "value_prenom_client": prenom_client,
                                              "value_date_client": date_nais_client,
                                              "value_genre_val_list_dropdown": genre_selectionne,
                                              "value_assu_val_list_dropdown": assurance_selectionne
                                              }
            print("valeurs_insertion_dictionnaire ", valeurs_insertion_dictionnaire)

            strsql_insert_client = """INSERT INTO t_client (id_client,nom,prenom,date_de_nais,fk_genre,fk_assu)
                                    VALUES (NULL,%(value_nom_client)s,%(value_prenom_client)s,%(value_date_client)s,%(value_genre_val_list_dropdown)s,%(value_assu_val_list_dropdown)s) """
            with DBconnection() as mconn_bd:
                mconn_bd.execute(strsql_insert_client, valeurs_insertion_dictionnaire)

            flash(f"Données insérées !!", "success")
            print(f"Données insérées !!")

            # Pour afficher et constater l'insertion du nouveau film (id_client_sel=0 => afficher tous les films)
            return redirect(url_for('clients_afficher', id_client_sel=0))

        if request.method == "GET":
            with DBconnection() as mc_afficher:
                strsql_genres_afficher = """SELECT id_genre, nom_genre FROM t_genre ORDER BY id_genre ASC"""
                mc_afficher.execute(strsql_genres_afficher)

            data_genres = mc_afficher.fetchall()
            print("demo_select_wtf data_genres ", data_genres, " Type : ", type(data_genres))

            """
                Préparer les valeurs pour la liste déroulante de l'objet "FormWTFAddClient"
                la liste déroulante est définie dans le "APP_PHARMACIE_164/clients/gestion_clients_wtf_forms.py"
                le formulaire qui utilise la liste déroulante "APP_PHARMACIE_164/templates/clients/client_add_wtf.html"
            """
            genre_val_list_dropdown = []
            for i in data_genres:
                genre_val_list_dropdown = [(i["id_genre"], i["nom_genre"]) for i in data_genres]

            print("genre_val_list_dropdown ", genre_val_list_dropdown)
            # Les valeurs sont chargées dans la liste déroulante
            form.genres_dropdown_wtf.choices = genre_val_list_dropdown

            with DBconnection() as mc_afficher:
                strsql_assu_afficher = """SELECT id_assu, nom_assu FROM t_assurance ORDER BY id_assu ASC"""
                mc_afficher.execute(strsql_assu_afficher)

            data_assu = mc_afficher.fetchall()
            print("demo_select_wtf data_assu ", data_assu, " Type : ", type(data_assu))

            """
                Préparer les valeurs pour la liste déroulante de l'objet "FormWTFAddClient"
                la liste déroulante est définie dans le "APP_PHARMACIE_164/clients/gestion_clients_wtf_forms.py"
                le formulaire qui utilise la liste déroulante "APP_PHARMACIE_164/templates/clients/client_add_wtf.html"
            """
            assu_val_list_dropdown = []
            for i in data_assu:
                assu_val_list_dropdown = [(i["id_assu"], i["nom_assu"]) for i in data_assu]

            print("assu_dropdown_wtf ", assu_val_list_dropdown)
            # Les valeurs sont chargées dans la liste déroulante
            form.assu_dropdown_wtf.choices = assu_val_list_dropdown

    except Exception as Exception_client_ajouter_wtf:
        raise ExceptionClientsAjouterWtf(f"fichier : {Path(__file__).name}  ;  "
                                        f"{client_add_wtf.__name__} ; "
                                        f"{Exception_client_ajouter_wtf}")

    return render_template("clients/client_add_wtf.html", form=form)

"""Editer(update) un film qui a été sélectionné dans le formulaire "clients_afficher.html"
Auteur : OM 2022.04.11
Définition d'une "route" /film_update

Test : exemple: cliquer sur le menu "Films/Genres" puis cliquer sur le bouton "EDIT" d'un "film"

Paramètres : sans

But : Editer(update) un genre qui a été sélectionné dans le formulaire "genres_afficher.html"

Remarque :  Dans le champ "nom_mail_update_wtf" du formulaire "clients/films_update_wtf.html",
            le contrôle de la saisie s'effectue ici en Python.
            On ne doit pas accepter un champ vide.
"""

@app.route("/client_update_wtf", methods=['GET', 'POST'])
def client_update_wtf():
    # L'utilisateur vient de cliquer sur le bouton "EDIT". Récupère la valeur de "id_client"
    id_client_update = request.values['id_client_btn_edit_html']

    # Objet formulaire pour l'UPDATE
    form_update_client = FormWTFUpdateClient()
    try:

        if request.method == "POST" and form_update_client.submit.data:
            # Récupèrer la valeur du champ depuis "genre_update_wtf.html" après avoir cliqué sur "SUBMIT".
            nom_client_update = form_update_client.nom_client_update_wtf.data
            prenom_client_update = form_update_client.prenom_client_update_wtf.data
            date_nais_client_update = form_update_client.date_nais_client_update_wtf.data
            genre_client_update = form_update_client.genres_dropdown_update_wtf.data
            fk_assu_client_update = form_update_client.assu_dropdown_update_wtf.data

            valeur_update_dictionnaire = {"value_id_client": id_client_update,
                                          "value_nom_client": nom_client_update,
                                          "value_prenom_client": prenom_client_update,
                                          "value_date_client": date_nais_client_update,
                                          "value_genre_update_val_list_dropdown": genre_client_update,
                                          "value_assurance_client": fk_assu_client_update
                                        }
            print("valeur_update_dictionnaire ", valeur_update_dictionnaire)

            str_sql_update_client = """UPDATE t_client SET nom = %(value_nom_client)s,
                                                            prenom = %(value_prenom_client)s,
                                                            date_de_nais = %(value_date_client)s,
                                                            fk_genre = %(value_genre_update_val_list_dropdown)s,
                                                            fk_assu = %(value_assurance_client)s
                                                            WHERE id_client = %(value_id_client)s"""
            with DBconnection() as mconn_bd:
                mconn_bd.execute(str_sql_update_client, valeur_update_dictionnaire)

            flash(f"Donnée mise à jour !!", "success")
            print(f"Donnée mise à jour !!")

            # afficher et constater que la donnée est mise à jour.
            # Afficher seulement le film modifié, "ASC" et l'"id_client_update"
            return redirect(url_for('clients_afficher', id_client_sel=id_client_update))
        elif request.method == "GET":
            # Opération sur la BD pour récupérer "id_mail" et "nom_genre" de la "t_genre"
            str_sql_id_mail = "SELECT * FROM t_client WHERE id_client = %(value_id_client)s"
            valeur_select_dictionnaire = {"value_id_client": id_client_update}
            with DBconnection() as mybd_conn:
                mybd_conn.execute(str_sql_id_mail, valeur_select_dictionnaire)
            # Une seule valeur est suffisante "fetchone()", vu qu'il n'y a qu'un seul champ "nom genre" pour l'UPDATE
            data_client = mybd_conn.fetchone()
            print("data_client ", data_client, " type ", type(data_client), " genre ",
                  data_client["nom"])

            # Afficher la valeur sélectionnée dans le champ du formulaire "client_update_wtf.html"
            form_update_client.nom_client_update_wtf.data = data_client["nom"]
            form_update_client.prenom_client_update_wtf.data = data_client["prenom"]
            form_update_client.date_nais_client_update_wtf.data = data_client["date_de_nais"]
            form_update_client.genres_dropdown_update_wtf.data = data_client["fk_genre"]
            form_update_client.assu_dropdown_update_wtf.data = data_client["fk_assu"]

        if request.method == "GET":
            with DBconnection() as mc_afficher:
                strsql_genres_afficher = """SELECT id_genre, nom_genre FROM t_genre ORDER BY id_genre ASC"""
                mc_afficher.execute(strsql_genres_afficher)

            data_genres = mc_afficher.fetchall()
            print("demo_select_wtf data_genres ", data_genres, " Type : ", type(data_genres))

            """
                Préparer les valeurs pour la liste déroulante de l'objet "FormWTFAddClient"
                la liste déroulante est définie dans le "APP_PHARMACIE_164/clients/gestion_clients_wtf_forms.py"
                le formulaire qui utilise la liste déroulante "APP_PHARMACIE_164/templates/clients/client_add_wtf.html"
            """
            genre_update_val_list_dropdown = []
            for i in data_genres:

                genre_update_val_list_dropdown = [(i["id_genre"], i["nom_genre"]) for i in data_genres]
            print("genre_update_val_list_dropdown ", genre_update_val_list_dropdown)
            # Les valeurs sont chargées dans la liste déroulante
            form_update_client.genres_dropdown_update_wtf.choices = genre_update_val_list_dropdown

            with DBconnection() as mc_afficher:
                strsql_assu_afficher = """SELECT id_assu, nom_assu FROM t_assurance ORDER BY id_assu ASC"""
                mc_afficher.execute(strsql_assu_afficher)

            data_assu = mc_afficher.fetchall()
            print("demo_select_wtf data_assu ", data_assu, " Type : ", type(data_assu))

            """
                Préparer les valeurs pour la liste déroulante de l'objet "FormWTFAddClient"
                la liste déroulante est définie dans le "APP_PHARMACIE_164/clients/gestion_clients_wtf_forms.py"
                le formulaire qui utilise la liste déroulante "APP_PHARMACIE_164/templates/clients/client_add_wtf.html"
            """
            assu_update_val_list_dropdown = []
            for i in data_assu:
                assu_update_val_list_dropdown = [(i["id_assu"], i["nom_assu"]) for i in data_assu]

            print("assu_update_val_list_dropdown ", assu_update_val_list_dropdown)
            # Les valeurs sont chargées dans la liste déroulante
            form_update_client.assu_dropdown_update_wtf.choices = assu_update_val_list_dropdown

    except Exception as ExceptionClient_Update_Wtf:
        raise ExceptionClientUpdateWtf(f"fichier : {Path(__file__).name}  ;  "
                                     f"{client_update_wtf.__name__} ; "
                                     f"{ExceptionClient_Update_Wtf}")

    return render_template("clients/client_update_wtf.html", form_update_client=form_update_client)

"""Effacer(delete) un film qui a été sélectionné dans le formulaire "clients_afficher.html"
Auteur : OM 2022.04.11
Définition d'une "route" /film_delete

Test : ex. cliquer sur le menu "film" puis cliquer sur le bouton "DELETE" d'un "film"

Paramètres : sans

Remarque :  Dans le champ "nom_film_delete_wtf" du formulaire "clients/client_delete_wtf.html"
            On doit simplement cliquer sur "DELETE"
"""

@app.route("/client_delete", methods=['GET', 'POST'])
def client_delete_wtf():
    # Pour afficher ou cacher les boutons "EFFACER"
    data_client_delete = None
    btn_submit_del = None
    # L'utilisateur vient de cliquer sur le bouton "DELETE". Récupère la valeur de "id_mail"
    id_client_delete = request.values['id_client_btn_delete_html']

    # Objet formulaire pour effacer le film sélectionné.
    form_delete_client = FormWTFDeleteClient()
    try:
        # Si on clique sur "ANNULER", afficher tous les films.
        if form_delete_client.submit_btn_annuler.data:
            return redirect(url_for("clients_afficher", id_client_sel=0))

        if form_delete_client.submit_btn_conf_del_film.data:
            # Récupère les données afin d'afficher à nouveau
            # le formulaire "clients/client_delete_wtf.html" lorsque le bouton "Etes-vous sur d'effacer ?" est cliqué.
            data_client_delete = session['data_client_delete']
            print("data_client_delete ", data_client_delete)

            flash(f"Effacer le client de façon définitive de la BD !!!", "danger")
            # L'utilisateur vient de cliquer sur le bouton de confirmation pour effacer...
            # On affiche le bouton "Effacer client" qui va irrémédiablement EFFACER le client
            btn_submit_del = True

        # L'utilisateur a vraiment décidé d'effacer.
        if form_delete_client.submit_btn_del_film.data:
            valeur_delete_dictionnaire = {"value_id_client": id_client_delete}
            print("valeur_delete_dictionnaire ", valeur_delete_dictionnaire)

            str_sql_delete_fk_traitement_client = """DELETE FROM t_pers_traitement WHERE fk_client = %(value_id_client)s"""
            str_sql_delete_fk_telephone_client = """DELETE FROM t_pers_telephone WHERE fk_client = %(value_id_client)s"""
            str_sql_delete_fk_mail_client = """DELETE FROM t_pers_mail WHERE fk_client = %(value_id_client)s"""
            str_sql_delete_fk_adresse_client = """DELETE FROM t_pers_adresse WHERE fk_client = %(value_id_client)s"""
            str_sql_delete_fk_achat_client = """DELETE FROM t_pers_achat_client WHERE fk_client = %(value_id_client)s"""
            str_sql_delete_client = """DELETE FROM t_client WHERE id_client = %(value_id_client)s"""
            # Manière brutale d'effacer d'abord la "fk_film", même si elle n'existe pas dans la "t_pers_mail"
            # Ensuite on peut effacer le film vu qu'il n'est plus "lié" (INNODB) dans la "t_pers_mail"
            with DBconnection() as mconn_bd:
                mconn_bd.execute(str_sql_delete_fk_traitement_client, valeur_delete_dictionnaire)
                mconn_bd.execute(str_sql_delete_fk_telephone_client, valeur_delete_dictionnaire)
                mconn_bd.execute(str_sql_delete_fk_mail_client, valeur_delete_dictionnaire)
                mconn_bd.execute(str_sql_delete_fk_achat_client, valeur_delete_dictionnaire)
                mconn_bd.execute(str_sql_delete_fk_adresse_client, valeur_delete_dictionnaire)
                mconn_bd.execute(str_sql_delete_client, valeur_delete_dictionnaire)

            flash(f"Client définitivement effacé !!", "success")
            print(f"Client définitivement effacé !!")

            # afficher les données
            return redirect(url_for('clients_afficher', id_client_sel=0))
        if request.method == "GET":
            valeur_select_dictionnaire = {"value_id_client": id_client_delete}
            print(id_client_delete, type(id_client_delete))

            # Requête qui affiche le film qui doit être efffacé.
            str_sql_client_delete = """SELECT * FROM t_client WHERE id_client = %(value_id_client)s"""

            with DBconnection() as mydb_conn:
                mydb_conn.execute(str_sql_client_delete, valeur_select_dictionnaire)
                data_client_delete = mydb_conn.fetchall()
                print("data_client_delete...", data_client_delete)

                # Nécessaire pour mémoriser les données afin d'afficher à nouveau
                # le formulaire "clients/client_delete_wtf.html" lorsque le bouton "Etes-vous sur d'effacer ?" est cliqué.
                session['data_client_delete'] = data_client_delete

            # Le bouton pour l'action "DELETE" dans le form. "client_delete_wtf.html" est caché.
            btn_submit_del = False

    except Exception as Exception_client_delete_wtf:
        raise ExceptionClientDeleteWtf(f"fichier : {Path(__file__).name}  ;  "
                                     f"{client_delete_wtf.__name__} ; "
                                     f"{Exception_client_delete_wtf}")

    return render_template("clients/client_delete_wtf.html",
                           form_delete_client=form_delete_client,
                           btn_submit_del=btn_submit_del,
                           data_client_del=data_client_delete
                           )


from datetime import datetime
from models.model_peer import Peer
from typing import Optional

class AddProviderQuery:
    cid: str
    target_peer: Peer
    started_at: datetime
    ended_at: Optional[datetime]
    success: Optional[bool]
    errors: Optional[str]

    def __init__(self, target_peer: Peer, cid: str, started_at: datetime) -> None:
        self.cid = cid
        self.started_at = started_at
        self.target_peer = target_peer
        self.ended_at = None
        self.success = None
        self.errors = None

    def succeeded(self, ended_at: datetime):
        if self.is_done:
            raise Exception(
                f"cid {self.cid} add provider query already done {self.target_peer}")
        self.ended_at = ended_at
        self.success = True

    def failed(self, ended_at: datetime, errors: str):
        if self.is_done:
            raise Exception(
                f"cid {self.cid} add provider query already done {self.target_peer}")
        self.ended_at = ended_at
        self.success = False
        self.errors = errors

    @property
    def is_done(self) -> bool:
        return self.success is not None


# Databricks notebook source
from pyspark.sql import SparkSession

# COMMAND ----------

spark = SparkSession.builder.appName("RS").getOrCreate()

# COMMAND ----------

from pyspark.ml.recommendation import ALS

# COMMAND ----------

from pyspark.ml.evaluation import RegressionEvaluator

# COMMAND ----------

data = spark.read.csv("/FileStore/tables/movielens_ratings.csv", inferSchema=True, header=True)

# COMMAND ----------

data.show()

# COMMAND ----------

data.describe().show()

# COMMAND ----------

training,test = data.randomSplit([0.7,0.3])

# COMMAND ----------

als = ALS(maxIter=5, regParam=0.01, userCol='userId', itemCol='movieId', ratingCol='rating')

# COMMAND ----------

model = als.fit(training)

# COMMAND ----------

predictions = model.transform(test)

# COMMAND ----------

predictions.show()

# COMMAND ----------

evaluator = RegressionEvaluator(metricName="rmse", labelCol="rating", predictionCol="prediction")

# COMMAND ----------

rmse = evaluator.evaluate(predictions)

# COMMAND ----------

print("RSME")
print(rmse)

# COMMAND ----------

single_user = test.filter(test["userId"]==11).select(["movieId", "userId"])

# COMMAND ----------

single_user.show()

# COMMAND ----------

recommendations = model.transform(single_user)

# COMMAND ----------

recommendations.orderBy('prediction', ascending=False).show()

# COMMAND ----------


import requests

url = 'http://10.10.10.139/post.php'

getPara = {
    "wd": "林俊杰"
    }

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36 Edg/93.0.961.47"
}

res = requests.get(url = url,params=getPara,headers=headers)
print(res.text)
print(res.url)
input()


from Tool import app, db
import os
from hashids import Hashids
from flask import Flask, render_template, request, flash, redirect, url_for
from Tool.models import Urls

hashids = Hashids(min_length=4, salt=app.config['SECRET_KEY'])

@app.route('/', methods=['GET', 'POST'])
def index():
    if request.method == 'POST':
        url_data = Urls(og_url=request.form['url'])

        db.session.add(url_data)
        db.session.commit()

        if url_data is None:
            flash('URL is rquired!')
            return redirect(url_for('index'))

        url_id = url_data.id
        hashid = hashids.encode(url_id)
        short_url = request.host_url + hashid

        url_data.short_url = short_url
        db.session.commit()

        return render_template('index.html', short_url=short_url)

    return render_template('index.html')

@app.route('/<id>')
def url_redirect(id):

    original_id = hashids.decode(id)

    if original_id:
        original_id = original_id[0]
        url_data = Urls.query.filter_by(id=original_id).first()
        print(url_data)
        og_url = url_data.og_url
        return redirect(og_url)
    else:
        flash('INVALID URL')
        return redirect(url_for('index'))

if __name__ == '__main__':
    app.run(debug=True)


# -*- coding: utf-8 -*-
# Created by chenlu on May 2nd
import numpy as np
import pandas as pd
import time
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.decomposition import NMF, LatentDirichletAllocation, TruncatedSVD
import gc

def clicks_by_ip(X_total):
    start_time = time.time()
    # Count the number of clicks by ip
    ip_count = X_total.groupby(['ip']).size().rename('clicks_by_ip', inplace=True).reset_index()
    X_total = X_total.merge(ip_count, on='ip', how='left', sort=False)
    X_total['clicks_by_ip'] = X_total['clicks_by_ip'].astype('uint16')

    print(f'[{time.time() - start_time}] Finished to extract clicks_by_ip')
    return X_total

# Aggregation function
def rate_calculation(x):
    """Calculate the attributed rate. Scale by confidence"""
    rate = x.sum() / float(x.count())
    log_group = np.log(100000) # 1000 views -> 60% confidence, 100 views -> 40% confidence
    conf = np.min([1, np.log(x.count()) / log_group])
    return rate * conf

def time_features(df):
    start_time = time.time()
    # Make some new features with click_time column
    df['day'] = df['click_time'].dt.day.astype('uint8')
    df['hour'] = df['click_time'].dt.hour.astype('uint8')
    df['minute'] = df['click_time'].dt.minute.astype('uint8')

    print(f'[{time.time() - start_time}] Finished to extract time_features')
    return df

def confidence_rate_feature(X_total, train):
    start_time = time.time()
    ATTRIBUTION_CATEGORIES = [
        # V1 Features #
        ###############
        ['ip'], ['app'], ['device'], ['os'], ['channel'],

        # V2 Features #
        ###############
        ['app', 'channel'],
        ['app', 'os'],
        ['app', 'device'],

        # V3 Features #
        ###############
        ['channel', 'os'],
        ['channel', 'device'],
        ['os', 'device']
    ]

    # Find frequency of is_attributed for each unique value in column
    freqs = {}
    for cols in ATTRIBUTION_CATEGORIES:

        # New feature name
        new_feature = '_'.join(cols)+'_confRate'

        # Perform the groupby
        group_object = train.groupby(cols)

        # Group sizes
        group_sizes = group_object.size()
        log_group = np.log(100000) # 1000 views -> 60% confidence, 100 views -> 40% confidence
        print(">> Calculating confidence-weighted rate for: {}.\n   Saving to: {}. Group Max /Mean / Median / Min: {} / {} / {} / {}"
              .format(
            cols, new_feature,
            group_sizes.max(),
            np.round(group_sizes.mean(), 2),
            np.round(group_sizes.median(), 2),
            group_sizes.min()
        ))

        # Perform the merge
        train = train.merge(
            group_object['is_attributed']. \
                apply(rate_calculation). \
                reset_index(). \
                rename(
                    index=str,
                    columns={'is_attributed': new_feature}
                )[cols + [new_feature]],
            on=cols, how='left'
        )
    print(f'[{time.time() - start_time}] Finished to extract confidence_rate_feature')
    return train

def group_by_feature(train):
    start_time = time.time()
    # Define all the groupby transformations
    GROUPBY_AGGREGATIONS = [

        # V1 - GroupBy Features #
        #########################
        # Variance in day, for ip-app-channel
        {'groupby': ['ip','app','channel'], 'select': 'day', 'agg': 'var'},
        # Variance in hour, for ip-app-os
        {'groupby': ['ip','app','os'], 'select': 'hour', 'agg': 'var'},
        # Variance in hour, for ip-day-channel
        {'groupby': ['ip','day','channel'], 'select': 'hour', 'agg': 'var'},
        # Count, for ip-day-hour
        {'groupby': ['ip','day','hour'], 'select': 'channel', 'agg': 'count'},
        # Count, for ip-app
        {'groupby': ['ip', 'app'], 'select': 'channel', 'agg': 'count'},
        # Count, for ip-app-os
        {'groupby': ['ip', 'app', 'os'], 'select': 'channel', 'agg': 'count'},
        # Count, for ip-app-day-hour
        {'groupby': ['ip','app','day','hour'], 'select': 'channel', 'agg': 'count'},
        # Mean hour, for ip-app-channel
        {'groupby': ['ip','app','channel'], 'select': 'hour', 'agg': 'mean'},

        # V2 - GroupBy Features #
        #########################
        # Average clicks on app by distinct users; is it an app they return to?
        {'groupby': ['app'],
         'select': 'ip',
         'agg': lambda x: float(len(x)) / len(x.unique()),
         'agg_name': 'AvgViewPerDistinct'
        },
        # How popular is the app or channel?
        {'groupby': ['app'], 'select': 'channel', 'agg': 'count'},
        {'groupby': ['channel'], 'select': 'app', 'agg': 'count'},

        # V3 - GroupBy Features                                              #
        # https://www.kaggle.com/bk0000/non-blending-lightgbm-model-lb-0-977 #
        ######################################################################
        {'groupby': ['ip'], 'select': 'channel', 'agg': 'nunique'},
        {'groupby': ['ip'], 'select': 'app', 'agg': 'nunique'},
        {'groupby': ['ip','day'], 'select': 'hour', 'agg': 'nunique'},
        {'groupby': ['ip','app'], 'select': 'os', 'agg': 'nunique'},
        {'groupby': ['ip'], 'select': 'device', 'agg': 'nunique'},
        {'groupby': ['app'], 'select': 'channel', 'agg': 'nunique'},
        {'groupby': ['ip', 'device', 'os'], 'select': 'app', 'agg': 'nunique'},
        {'groupby': ['ip','device','os'], 'select': 'app', 'agg': 'cumcount'},
        {'groupby': ['ip'], 'select': 'app', 'agg': 'cumcount'},
        {'groupby': ['ip'], 'select': 'os', 'agg': 'cumcount'},
        {'groupby': ['ip','day','channel'], 'select': 'hour', 'agg': 'var'}
    ]

    # Apply all the groupby transformations
    for spec in GROUPBY_AGGREGATIONS:

        # Name of the aggregation we're applying
        agg_name = spec['agg_name'] if 'agg_name' in spec else spec['agg']

        # Name of new feature
        new_feature = '{}_{}_{}'.format('_'.join(spec['groupby']), agg_name, spec['select'])

        # Info
        print("Grouping by {}, and aggregating {} with {}".format(
            spec['groupby'], spec['select'], agg_name
        ))

        # Unique list of features to select
        all_features = list(set(spec['groupby'] + [spec['select']]))

        # Perform the groupby
        gp = train[all_features]. \
            groupby(spec['groupby'])[spec['select']]. \
            agg(spec['agg']). \
            reset_index(). \
            rename(index=str, columns={spec['select']: new_feature})

        # Merge back to X_total
        if 'cumcount' == spec['agg']:
            train[new_feature] = gp[0].values
        else:
            train = train.merge(gp, on=spec['groupby'], how='left')

        # Clear memory
        del gp

    print(f'[{time.time() - start_time}] Finished to extract group_by_feature')
    return train

def next_click_feature(train):
    start_time = time.time()
    GROUP_BY_NEXT_CLICKS = [

        # V1
        {'groupby': ['ip']},
        {'groupby': ['ip', 'app']},
        {'groupby': ['ip', 'channel']},
        {'groupby': ['ip', 'os']},

        # V3
        {'groupby': ['ip', 'app', 'device', 'os', 'channel']},
        {'groupby': ['ip', 'os', 'device']},
        {'groupby': ['ip', 'os', 'device', 'app']}
    ]

    # Calculate the time to next click for each group
    for spec in GROUP_BY_NEXT_CLICKS:

        # Name of new feature
        new_feature = '{}_nextClick'.format('_'.join(spec['groupby']))

        # Unique list of features to select
        all_features = spec['groupby'] + ['click_time']

        # Run calculation
        print(f">> Grouping by {spec['groupby']}, and saving time to next click in: {new_feature}")
        train[new_feature] = train[all_features].groupby(spec['groupby']).click_time.transform(lambda x: x.diff().shift(-1)).dt.seconds

    print(f'[{time.time() - start_time}] Finished to extract next_click_feature')
    return train

def history_click_feature(train):
    start_time = time.time()
    HISTORY_CLICKS = {
        'identical_clicks': ['ip', 'app', 'device', 'os', 'channel'],
        'app_clicks': ['ip', 'app']
    }

    # Go through different group-by combinations
    for fname, fset in HISTORY_CLICKS.items():

        # Clicks in the past
        train['prev_'+fname] = train. \
            groupby(fset). \
            cumcount(). \
            rename('prev_'+fname)

        # Clicks in the future
        train['future_'+fname] = train.iloc[::-1]. \
            groupby(fset). \
            cumcount(). \
            rename('future_'+fname).iloc[::-1]

    # Count cumulative subsequent clicks
    print(f'[{time.time() - start_time}] Finished to extract history_click_feature')
    return train

def topic_feature(X_total):
    """
    categorical feature embedding by using LDA/NMF/LSA(PCA)
    """
    start_time = time.time()
    features = ['ip', 'app', 'device', 'os', 'channel']
    feat_len = len(features)
    n_components = 5

    for i in range(feat_len):
        for j in range(feat_len):
            if i == j:
                continue
            x = features[i]
            y = features[j]
            x_of_y = {}
            print(f">> generate topic embedding by {x}_of_{y}")
            for index, sample in X_total.iterrows():
                x_of_y.setdefault(sample[y], []).append(str(sample[x]))
            ys = list(x_of_y.keys())
            x_as_sentence = [' '.join(x_of_y[y]) for y in ys]
            x_as_matrix = CountVectorizer().fit_transform(x_as_sentence)
            print(f"[{time.time()-start_time}] generate x matrix done")
            # compute LDA topics of ys related to x
            lda = LatentDirichletAllocation(n_components=n_components,
                                            max_iter=3,
                                            learning_method='online',
                                            learning_offset=50.,
                                            random_state=0).fit_transform(x_as_matrix) # topics_of_ys
            lda_array = np.column_stack((np.array(ys), lda))
            print(f"[{time.time()-start_time}] LDA done")
            gc.collect()

            # computer NMF
            nmf = NMF(n_components=n_components,
                      random_state=1,
                      alpha=.1,
                      l1_ratio=.5).fit_transform(x_as_matrix)
            lda_nmf_array = np.column_stack((lda_array, nmf))
            print(f"[{time.time()-start_time}] NMF done")
            gc.collect()

            # compute LSA
            lsa = TruncatedSVD(n_components=n_components,
                               n_iter=3,
                               random_state=42).fit_transform(x_as_matrix)
            lda_nmf_lsa_array = np.column_stack((lda_nmf_array, lsa))
            print(f"[{time.time()-start_time}] LSA done")
            gc.collect()

            # add to X_total
            rename_dict = {0:y}
            t = ['LDA','NMF','LSA']
            for k in range(15):
                rename_dict[k+1] = f'{x}_of_{y}_{t[int(k/5)]}_{k%5+1}'
            temp = pd.DataFrame(data=lda_nmf_lsa_array).rename(index=str, columns=rename_dict)
            X_total = X_total.merge(temp, on=y, how='left', sort=False)
            print(f"[{time.time()-start_time}] merge done")
    X_total['ip'] = X_total['ip'].astype('uint32')
    X_total['app'] = X_total['app'].astype('uint16')
    X_total['device'] = X_total['device'].astype('uint16')
    X_total['os'] = X_total['os'].astype('uint16')
    X_total['channel'] = X_total['channel'].astype('uint16')

    print(f'[{time.time() - start_time}] Finished to extract topic_feature')
    return X_total


# -*- coding: utf-8 -*-
# pylint: disable-msg=C0103,E1101
"""
BURPG Echo
See LICENSE for MIT/X11 license info.
See README.md for all other help.

Note: This class was adapted from Google's Drive API Python Quickstart
https://developers.google.com/drive/v3/web/quickstart/python
"""

#from __future__ import print_function
import os
import httplib2

from apiclient import discovery
from apiclient import http
import oauth2client
from oauth2client import client
from oauth2client import tools

class GoogleDrive:
    """
    GoogleDrive provides a way to connect to a user's Google Drive and upload
    files to it
    """

    SCOPES = 'https://www.googleapis.com/auth/drive.file'
    CLIENT_SECRET_FILE = 'client_secrets.json'
    CLIENT_CREDENTIAL_FILE = 'drive.credentials'
    APPLICATION_NAME = 'BURPG Echo'
    CREDENTIALS = None
    DRIVE_SERVICE = None
    logger = None

    def __init__(self, logger, secret_path='client_secrets.json',
                 credentials_path='drive.credentials', noauth_local_webserver=False):
        """
        Gets valid GDrive user credentials from storage.

        If nothing has been stored, or if the stored credentials are invalid,
        the OAuth2 flow is completed to obtain the new credentials.
        """
        self.CLIENT_SECRET_FILE = secret_path
        self.CLIENT_CREDENTIAL_FILE = credentials_path
        self.logger = logger

        credstore = oauth2client.file.Storage(self.CLIENT_CREDENTIAL_FILE)
        credentials = credstore.get()
        if not credentials or credentials.invalid:
            flow = client.flow_from_clientsecrets(self.CLIENT_SECRET_FILE, self.SCOPES)
            flow.user_agent = self.APPLICATION_NAME

            # Workaround to prevent tools.run_flow from trying to grab our args
            import argparse
            parser = argparse.ArgumentParser(description=__doc__,
                                             formatter_class=argparse.RawDescriptionHelpFormatter,
                                             parents=[tools.argparser])
            dummy_args = []
            if noauth_local_webserver:
                logger.log_verbose("Performing offline auth-flow")
                dummy_args = ["--noauth_local_webserver"]

            credentials = tools.run_flow(flow, credstore, flags=parser.parse_args(dummy_args))
            self.logger.log_verbose('Storing credentials to ' + self.CLIENT_CREDENTIAL_FILE)
        self.CREDENTIALS = credentials
        self.DRIVE_SERVICE = discovery.build(
            'drive', 'v3', http=credentials.authorize(httplib2.Http()))
        self.logger.log_verbose("Google Drive authentication complete")

    def upload_file(self, file_path, parent_folder_id=None):
        """
        Upload a file to Google Drive in a "chunked" manner

        :param file_path: the local path to the file being uploaded
        :param parent_folder_id: the id of the folder where this file should be
        placed
        :returns: the API's response object
        """
        self.logger.log_verbose("Attempting upload:" + file_path +" to " + str(parent_folder_id))
        media = http.MediaFileUpload(file_path, resumable=True)
        file_metadata = {'name': os.path.basename(file_path)}
        if parent_folder_id is not None:
            file_metadata['parents'] = [parent_folder_id]

        request = self.DRIVE_SERVICE.files().create(body=file_metadata, media_body=media)
        response = None
        while response is None:
            status, response = request.next_chunk()
            if status:
                self.logger.log_verbose("Uploaded {:d}%.".format(int(status.progress() * 100)))
        self.logger.log_verbose("Upload complete: " + file_path)
        return response

    def create_folder(self, folder_name):
        """
        Creates a folder in Google Drive and returns the ID

        Note: Google Drive treats folders just like files, except that they have
        the special MIME type "application/vnd.google-apps.folder". To place a
        file in a folder, create the folder using this method and then add the
        ID to the "parents" array property of the "body" argument in
        files().create()

        :param folder_name: the desired name of the created folder
        :returns: the ID of the newly-created folder
        """
        folder_metadata = {
            'name': folder_name,
            'mimeType': 'application/vnd.google-apps.folder'
        }

        folder = self.DRIVE_SERVICE.files().create(body=folder_metadata, fields='id').execute()
        self.logger.log_verbose("Created folder '" + folder_name + "' with ID " + folder.get('id'))
        return folder.get('id')


#!/usr/bin/env python3

# Python 3.7.2

__description__ = '''

Python version of the Cedric's script for the analysis of homoplasy.

'''

def csv2analyse(df, score, metric, minseq, maxseq, mrdelta, norm, corr, maintain, deltaby, combs):
    '''
    Count homoplasy output

    '''

    import collections
    import numpy as np
    import pandas as pd

    # Normalize if required
    if norm:
        if norm == 'lenXnseq':
            df[metric] = df[metric].div(df.nseq, axis = 0).div(df.len, axis = 0)
        elif norm == 'avgLenXnseq':
            df[metric] = df[metric].div(df.nseq, axis = 0).div(df.avgLen, axis = 0)

    # Initialize counts
    namecounts = ["total", "unused", "ndeltaN", "ndeltaP", "ndeltaO", "minScore", "minNumb", "maxScore", "maxNumb"]
    cols = maintain + ["between"] + namecounts
    counts = pd.DataFrame(columns=cols)

    # Group data frame
    index = 0
    g = list(csv.groupby(maintain))
    for element in g:
        names = list(element[0])
        #names = dict(zip(keys,element[0]))  # A dictionary with the names of the combinations. Eg { family: seatoxin, bucket: 50, aligner: CLUSTALO }
        comb  = element[1]  # A dataframe with same keys[0:2]. It has as many columns as the original dataframe.

        # Dictionary of counts
        c = dict(zip( maintain + ["between"],  names  + [deltaby] ))
        c["total"]  = comb.shape[0] * (comb.shape[0]-1)  # Total number of deltas, if original data remained untouched
        comb  = filter_unused(comb, minseq, maxseq, score, metric)
        if comb.shape == 0:
            continue
        delta = compute_delta(comb, score, metric)
        delta = filter_delta(delta, mrdelta)
        c["unused"] = c["total"] - delta.shape[0]  # Number of deltas filtered out
        c["ndeltaN"], c["ndeltaP"], c["ndeltaO"], c["minScore"], c["minNumb"], c["maxScore"], c["maxNumb"] = count_statistics(delta)

        # Merge info to dataframe
        tmp = pd.DataFrame(c, index=[index])
        counts = pd.concat( [counts,tmp] )
        index += 1

    # Reset dtypes
    c = dict(zip(namecounts, ['float64']*len(namecounts)))
    if 'bucket' in maintain:
        c['bucket'] = 'int32'
    counts = counts.astype(c, errors='ignore', copy=False)

    # Set correct correlation: positive or negative
    if corr:
        correlation = corr
    else:
        if counts['ndeltaP'].sum() > counts['ndeltaN'].sum():
            correlation = 'positive'
        else:
            correlation = 'negative'
    if correlation == 'negative':  # Inverse if negative correlation
            tmp = {'minScore':'maxScore', 'maxScore':'minScore', \
                'minNumb' :'maxNumb' , 'maxNumb' :'minNumb'}
            counts = counts.rename(columns=tmp)

    return(counts, correlation)

def filter_unused(df, minseq, maxseq, score, metric):
    ''' Remove rows:
            1. with missing values
            2. nseq < minseq or nseq > maxseq
    '''

    import pandas as pd

    # Remove NA
    df = df.dropna(subset=[score, metric])

    # Remove based on the number of sequences
    cond = (df.nseq < minseq) | (df.nseq > maxseq)
    df = df.drop(df[cond].index)

    df = df.reset_index(drop=True)
    return(df)

def filter_delta(delta, mrdelta):
    ''' Remove rows from delta if:
            1. both abs(delta) < 0.001
            2. 2*abs(m1-m2)/(m1+m2) < mrdelta threshold given by user
    '''

    import pandas as pd

    # Baseline removal
    if mrdelta == -1:
        # Remove if both abs delta == 0
        cond = (abs(delta.delta_score) == 0) & (abs(delta.delta_metric) == 0)
        delta = delta.drop(delta[cond].index)
    else:
        # Remove if both abs delta <= 0.001
        val = 0.001
        cond = (abs(delta.delta_score) <= val) & (abs(delta.delta_metric) <= val)
        delta = delta.drop(delta[cond].index)

        # Remove if metrics are 0
        # These are removes so that we won't have the problem of mrdelta = 0/0 = inf
        cond = (delta.metric1 == delta.metric2) & (delta.metric1 == 0)
        delta = delta.drop(delta[cond].index)

    # Remove if mrdelta < threshold
    cond = (delta.mrdelta < mrdelta)
    delta = delta.drop(delta[cond].index)

    delta = delta.reset_index(drop=True)
    return(delta)

def compute_delta(comb, score, metric):
    ''' Compute delta-score and delta-metric.

    Input
    -----
    comb   : pandas dataframe. The delta values will be computed between each of its rows. Let's say between row1-row2, row1-row3, [...]
    score  : string. tc, sp or col
    metric : string. homo, whomo, whomo2, len, ngap, or ngap2

    Output
    ------
    delta  : pandas dataframe. It has six columns: score1, score2, metric1, metric2, delta_score, delta_metric and mrdelta; and as many rows as deltas.

    '''

    import numpy as np
    import pandas as pd

    # Initialize
    delta = pd.DataFrame(columns=["score1", "score2", "metric1", "metric2"], dtype='float64')

    vals = { "score" : comb[score].tolist(), "metric": comb[metric].tolist() } # Get values from columns 'score' and 'metric'
    for i in range( len(vals["score"]) ):  # For each row

        # Get score1 and score2
        val_score = vals["score"][i]
        mat_score = vals["score"][:]
        del mat_score[i]

        # Get metric1 and metric2
        val_metric = vals["metric"][i]
        mat_metric = vals["metric"][:]
        del mat_metric[i]

        # Merge dataframe
        tmp = pd.DataFrame(np.array([[val_score] * len(mat_score), \
                                     mat_score, \
                                     [val_metric] * len(mat_metric), \
                                     mat_metric]).transpose(), \
                                     columns=["score1", "score2", "metric1", "metric2"])
        delta = pd.concat( [delta, tmp] )

    delta = delta.reset_index(drop=True)

    # Compute deltas
    delta["delta_score"]  = delta.score1  - delta.score2
    delta["delta_metric"] = delta.metric1 - delta.metric2

    # Compute mrdelta
    delta['mrdelta'] = 2 * abs(delta.metric1 - delta.metric2) / (delta.metric1 + delta.metric2)

    return(delta)

def count_statistics(delta):
    '''
    Input
    -----
    delta    : pandas dataframe with the information of the deltas.

    Output
    ------
    ndeltaN  : number of deltas in negative correlation
    ndeltaP  : number of deltas in positive correlation
    ndeltaO  : number of deltas not in negative nor positive correlation
    minScore : sum of score if wrong alignment (according to metric) is always chosen
    minNumb  : counts of wrong alignment considered
    maxScore : sum of score if good alignment (according to metric) is always chosen
    maxNumb  : counts of good alignments considered

    '''

    # Positive correlation between score-metric (the interesting ones)
    # ++, --
    cond = ((delta.delta_score > 0) & (delta.delta_metric > 0)) | ((delta.delta_score < 0) & (delta.delta_metric < 0))
    ndeltaP = delta[cond].shape[0]

    # Negative correlation between score-metric
    # +-, -+
    cond = ((delta.delta_score > 0) & (delta.delta_metric < 0)) | ((delta.delta_score < 0) & (delta.delta_metric > 0))
    ndeltaN = delta[cond].shape[0]

    # Other correlation
    cond = ((delta.delta_score == 0) & (delta.delta_metric != 0)) | ((delta.delta_score != 0) & (delta.delta_metric == 0))
    ndeltaO = delta[cond].shape[0]

    # Score if wrong/good alignment (according to metric) is always chosen
    # Supposing positive correlation (otherwise, they will be changed afterwards)
    minScore, maxScore = 0.0, 0.0
    minNumb, maxNumb = 0, 0
    cond1 = delta.metric1 <  delta.metric2
    cond2 = delta.metric1 >= delta.metric2  # So the equal values will be added as wrong and good alignments (50/50)
    df1 = delta[cond1]
    df2 = delta[cond2]
    # Wrong alignment according to metric
    minScore += df1.score1.sum()
    minScore += df2.score2.sum()
    minNumb += df1.shape[0]
    minNumb += df2.shape[0]
    # Good alignment according to metric
    maxScore += df1.score2.sum()
    maxScore += df2.score1.sum()
    maxNumb += df1.shape[0]
    maxNumb += df2.shape[0]

    return(ndeltaN, ndeltaP, ndeltaO, minScore, minNumb, maxScore, maxNumb)

def calculate_ratio(df, correlation):

    d = {}

    # Calculate
    # Nratio Pratio unusedRatio minacc maxacc deltaacc
    usedNumber = df["total"] - df["unused"]
    d["unusedRatio"] = df["unused"] / df["total"]
    d["minacc"] = df["minScore"] / df["minNumb"]
    d["maxacc"] = df["maxScore"] / df["maxNumb"]
    d["deltaacc"] = d["maxacc"] - d["minacc"]
    if correlation == 'positive':
        d["Ratio"] = df["ndeltaP"] / usedNumber
    else:
        d["Ratio"] = df["ndeltaN"] / usedNumber

    return(d)

def get_ratio(df, correlation, maintain, deltaby):

    import numpy as np
    import pandas as pd

    original = df.copy()

    # Sum by group
    g = df.groupby(maintain)
    lev = [i for i in range(0, len(maintain) )]
    df = g.sum().reset_index(level=lev)

    # Calculate ratio
    d = calculate_ratio(df, correlation)
    tmp1 = df[maintain]
    tmp2 = pd.DataFrame(d)
    df2 = pd.concat( [tmp1, tmp2], axis=1)

    # Determine nfam
    nfam = []
    for element in g:
        comb = element[1]
        cond = comb.unused < comb.total
        if deltaby != 'family':
            nfam.append( len(comb[cond]["family"].unique()) )
        else:
            nfam.append( np.nan )
    df2["nfam"] = nfam
    return(df2)

def get_global_ratio(df, correlation, deltaby):

    import numpy as np
    import pandas as pd

    # Parse df
    namecounts = ["total", "unused", "ndeltaN", "ndeltaP", "ndeltaO", "minScore", "minNumb", "maxScore", "maxNumb"]
    df2 = df[namecounts].sum()

    # Calculate
    d = calculate_ratio(df2, correlation)
    df2 = pd.DataFrame(d, index=[0])

    # Determina nfam
    cond = df.unused < df.total
    if deltaby != 'family':
        nfam = len(df[cond]["family"].unique())
    else:
        nfam = np.nan
    df2['nfam'] = nfam

    return(df2)

def add_columns(df, add_d, maintain, typeCount):

    n = df.shape[0]

    # Add columns
    d = {}
    for k,v in add_d.items():
        d[k] = [v] * n
    tmp = pd.DataFrame(d)
    df = pd.concat( [tmp, df], axis=1 )

    # Column names
    if typeCount == "counts":
        colcounts = ["total", "unused", "ndeltaN", "ndeltaP", "minScore", "minNumb", "maxScore", "maxNumb"]
        cols = ["mrdelta", "score", "metric"] + maintain + ["between"] + colcounts
    else:
        colcounts = ["Ratio", "unusedRatio", "nfam", "minacc", "maxacc", "deltaacc"]
        cols = ["mrdelta", "score", "metric"] + maintain + ["between"] + ["correlation"] + colcounts

    # Reorder columns
    df = df[cols]

    return(df)

if __name__ == '__main__':

    import argparse
    import collections
    import itertools
    import pandas as pd
    import sys

    app = argparse.ArgumentParser(description=__description__)
    app.add_argument("-csv", type=str, required=True, help=".csv filename")
    app.add_argument("-score", type=str, required=True)#, choices=["tc","sp","col"])
    app.add_argument("-metric", type=str, required=True)#, choices=["homo","whomo","whomo2","len","ngap","ngap2"])
    app.add_argument("-norm", choices=['lenXnseq', 'avgLenXnseq'], default=None, \
                    help="Data normalization. Divide metric by (alignment length or average sequence length) and (number of sequences).")
    app.add_argument("-mrdelta", type=float, default=0.0)
    app.add_argument("-minseq", type=int, default=0)
    app.add_argument("-maxseq", type=int, default=100000)
    app.add_argument("-outdir", type=str, required=True, help="Output directory")
    app.add_argument("-decimal", type=int, default=3)
    app.add_argument("-maintain", type=str, nargs='+', default=["bucket","aligner","family"], \
                    help="Combination to be maintained, given in the order that the counts are calculated.")
    app.add_argument("-deltaby", type=str, default="tree", help="Delta calculated considering this variable")
    app.add_argument("-corr", type=str, choices=['negative','positive'], default=None)
    args = app.parse_args()

    # Read homoplasy.csv
    csv = pd.read_csv(args.csv, header=0)

    # unused sequences according to family size
    csv = csv[csv.nseq >= args.minseq]
    csv = csv[csv.nseq <= args.maxseq]
    csv = csv.reset_index(drop=True)

    # Combinations: family, bucket, aligner, tree
    combs = {}
    for element in args.maintain:
        combs[element] = csv[element].unique().tolist()
    combs[args.deltaby] = csv[args.deltaby].unique().tolist()

    # Analyze
    # Get counts
    tmp = args.maintain + [args.deltaby]
    counts, correlation = csv2analyse(csv, args.score, args.metric, args.minseq, args.maxseq, args.mrdelta, args.norm, args.corr, \
                         args.maintain, args.deltaby, combs)   # Original combination: same bucket, aligner and/or family, but different tree
                        #order=tmp, bucket=bucket, aligner=aligner, family=family, tree=tree)   # Original combination: same bucket, aligner and/or family, but different tree

    # Summarize counts
    csvs = {}
    csvs["_".join(args.maintain)] = get_ratio(counts, correlation, args.maintain, args.deltaby)
    d = {'base' : {'mrdelta':args.mrdelta, 'score':args.score, 'metric':args.metric} }
    d["_".join(args.maintain)] = {**d['base'], **{'between': args.deltaby, 'correlation':correlation}}
    for i in range(1, len(args.maintain)):
        iter_comb = itertools.combinations(args.maintain, i)  # All possible combinations to stratify analysis
        for j in iter_comb:
            maint = list(j)
            fixed = [x for x in args.maintain if x not in maint ]
            csvs["_".join(maint)] = get_ratio(counts, correlation, maint, args.deltaby)
            tmp = dict(zip(fixed, ['global']*len(fixed) ))
            d["_".join(maint)] = {**d["_".join(args.maintain)], **tmp}
    csvs["global"] = get_global_ratio(counts, correlation, args.deltaby)
    d["global"] = {**d["_".join(args.maintain)], **dict(zip(args.maintain, ['global']*len(args.maintain) ))}

    # Add mrdelta, score, metric and other columns, reorder & print output
    counts = add_columns(counts, d['base'], args.maintain, "counts")
    counts.round(args.decimal).to_csv(args.outdir + "/" + args.score + "_" + args.metric + "_mrdelta" + str(args.mrdelta) + "_counts.tsv", \
                                        sep="\t", index=False, na_rep='NA')
    for element in csvs:
        csvs[element] = add_columns(csvs[element], d[element], args.maintain, None)
        csvs[element].round(args.decimal).to_csv(args.outdir + "/" + \
            args.score + "_" + args.metric + "_mrdelta" + str(args.mrdelta) + "_" + element + ".tsv",
            sep="\t", index=False, na_rep='NA')


import cv2
import cv2.cv as cv
import numpy as np

global src_pts
global count
count = 0
src_pts = cv2.cv.CreateMat(4,1,cv2.CV_64FC2)

def getPoint(event,x,y,flags,param):
    if event == cv2.EVENT_LBUTTONDOWN:
        global src_pts
        global count
        src_pts[count,0] = (x,y)
        count = count + 1

baseIm = cv2.imread('P1010580.jpg')

cv2.namedWindow('Calibrate')

cv.SetMouseCallback('Calibrate',getPoint)

cv2.imshow('Calibrate',baseIm)

cv2.waitKey(0)

if count==4:
    cv2.destroyAllWindows()

dest_pts = cv2.cv.CreateMat(4,1,cv2.CV_64FC2)

#src_pts[0,0] = (392,144)
#src_pts[1,0] = (592,237)
#src_pts[2,0] = (272,322)
#src_pts[3,0] = (494,436)

dest_pts[0,0] = (0,0)
dest_pts[1,0] = (520,0)
dest_pts[2,0] = (0,520)
dest_pts[3,0] = (520,520)

src_pts = np.array(src_pts)
dest_pts = np.array(dest_pts)

M,mask = cv2.findHomography(src_pts, dest_pts, cv2.RANSAC, 5.0)

#dst = cv2.perspectiveTransform(baseIm,M)

output = cv2.warpPerspective(baseIm,M,(900,900))

cv2.imshow('base',baseIm)
cv2.imshow('output',output)

k = cv2.waitKey(0)


import os
import pytest
import time
from pathlib import Path
from indico.client import IndicoClient
from indico.queries import CreateDataset, CreateModelGroup
from indico.types import ModelGroup, Dataset

PUBLIC_URL = "https://github.com/IndicoDataSolutions/indico-client-python/raw/master/tests/integration/data/"

@pytest.fixture(scope="module")
def airlines_dataset(indico):
    client = IndicoClient()
    dataset_filepath = str(Path(__file__).parents[0]) + "/AirlineComplaints.csv"

    response = client.call(
        CreateDataset(
            name=f"AirlineComplaints-test-{int(time.time())}", files=[dataset_filepath]
        )
    )
    assert response.status == "COMPLETE"
    return response

@pytest.fixture(scope="module")
def too_small_dataset(indico):
    client = IndicoClient()
    dataset_filepath = str(Path(__file__).parents[0]) + "/TooSmall.csv"

    response = client.call(
        CreateDataset(
            name=f"TooSmall-test-{int(time.time())}", files=[dataset_filepath]
        )
    )
    assert response.status == "COMPLETE"
    return response

@pytest.fixture(scope="module")
def airlines_model_group(indico, airlines_dataset: Dataset) -> ModelGroup:
    client = IndicoClient()
    name = f"TestCreateModelGroup-{int(time.time())}"
    mg: ModelGroup = client.call(
        CreateModelGroup(
            name=name,
            dataset_id=airlines_dataset.id,
            source_column_id=airlines_dataset.datacolumn_by_name("Text").id,
            labelset_id=airlines_dataset.labelset_by_name("Target_1").id,
            wait=True,
        )
    )
    return mg

@pytest.fixture(scope="module")
def cats_dogs_image_dataset(indico):
    client = IndicoClient()
    dataset_filepath = str(Path(__file__).parents[0]) + "/dog_vs_cats_small.csv"

    response = client.call(
        CreateDataset(
            name=f"DogsAndCats-test-{int(time.time())}",
            files=dataset_filepath,
            from_local_images=True,
        )
    )
    assert response.status == "COMPLETE"
    return response

@pytest.fixture(scope="module")
def cats_dogs_modelgroup(indico, cats_dogs_image_dataset: Dataset) -> ModelGroup:
    client = IndicoClient()
    name = f"TestCreateObjectDetectionMg-{int(time.time())}"

    model_training_options = {
        "max_iter": 20,
        "lr": 0.1,
        "batch_size": 1,
        "filter_empty": False,
        "test_size": 0.2,
        "use_small_model": True,
    }

    mg: ModelGroup = client.call(
        CreateModelGroup(
            name=name,
            dataset_id=cats_dogs_image_dataset.id,
            source_column_id=cats_dogs_image_dataset.datacolumn_by_name("urls").id,
            labelset_id=cats_dogs_image_dataset.labelset_by_name("label").id,
            model_training_options=model_training_options,
            wait=True,
        )
    )
    return mg

@pytest.fixture(scope="module")
def org_annotate_dataset(indico):
    client = IndicoClient()
    dataset_filepath = str(Path(__file__).parents[0]) + "/org-annotate-labeled.csv"

    response = client.call(
        CreateDataset(
            name=f"OrgAnnotate-test-{int(time.time())}", files=[dataset_filepath]
        )
    )
    assert response.status == "COMPLETE"
    return response

@pytest.fixture(scope="module")
def org_annotate_model_group(indico, org_annotate_dataset: Dataset) -> ModelGroup:
    client = IndicoClient()
    name = f"TestFinetuneModelGroup-{int(time.time())}"
    mg: ModelGroup = client.call(
        CreateModelGroup(
            name=name,
            dataset_id=org_annotate_dataset.id,
            source_column_id=org_annotate_dataset.datacolumn_by_name(
                "News Headlines w/Company Names"
            ).id,
            labelset_id=org_annotate_dataset.labelset_by_name("question_825").id,
            wait=True,
        )
    )
    return mg


import os.path
import scipy.sparse

import numpy as np
import oct2py

from .utils import adjacency_matrix_to_tree
from .reconstruct_tree import ReconstructionMethod
from .raxml_reconstruction import SPECTRALTREE_LIB_PATH

SPECTRALTREE_CHOI_PATH = os.path.join(SPECTRALTREE_LIB_PATH, "ChoilatentTree")

class RG(ReconstructionMethod):
    def __init__(self, distance_metric):
        self.distance_metric = distance_metric

    def __call__(self, observations, taxa_metadata=None):
        oct2py.octave.addpath(SPECTRALTREE_CHOI_PATH)
        oc = oct2py.Oct2Py()
        num_taxa = observations.shape[0]

        D = self.distance_metric(observations)
        #adj_mat = oc.feval("./spectraltree/ChoilatentTree/toolbox/RGb.m",observations+1,0)
        adj_mat = oc.feval(os.path.join(SPECTRALTREE_CHOI_PATH, "toolbox", "RGb.m"),D,1,observations.shape[1], verbose=False)
        adj_mat = np.array(scipy.sparse.csr_matrix.todense(adj_mat))
        tree_RG = adjacency_matrix_to_tree(adj_mat,num_taxa,taxa_metadata)
        return tree_RG

    def __repr__(self):
        return "RG"

class CLRG(ReconstructionMethod):
    def __call__(self, observations, taxa_metadata=None):
        oct2py.octave.addpath('./spectraltree/ChoilatentTree/')
        oc = oct2py.Oct2Py()
        num_taxa = observations.shape[0]
        adj_mat = oc.feval(os.path.join(SPECTRALTREE_CHOI_PATH, "toolbox", "CLRGb.m"),observations+1,0, verbose=False)
        adj_mat = np.array(scipy.sparse.csr_matrix.todense(adj_mat))
        tree_CLRG = adjacency_matrix_to_tree(adj_mat,num_taxa,taxa_metadata)
        return tree_CLRG

    def __repr__(self):
        return "CLRG"


# -*- coding: utf-8 -*-
''' Data structures to describe a finite element mesh.'''

import os
import json
from import_export import mesh_entities as me
from import_export import block_topology_entities as bte

class GroupRecord(object):
    ''' Object defining a grop of entities (nodes, cells, points, or lines).

    :ivar name: group name.
    :ivar nodeIds: identifiers of the nodes.
    :ivar cellIds: identifiers of the cells.
    :ivar pointIds: identifiers of the points.
    :ivar lineIds: identifiers of the lines.
    '''

    def readFromDATFile(self,fName):
        self.name=  os.path.splitext(fName)[0]
        meshDesc= me.MeshData()
        meshDesc.readFromDATFile(fName)
        self.nodeIds= meshDesc.nodes.getTags()
        self.cellIds= meshDesc.cells.getTags()
        self.pointIds= []
        self.lineIds= []

    def setUp(self,name,points,lines):
        self.name= name
        self.nodeIds= []
        self.cellIds= []
        self.pointIds= points
        self.lineIds= lines

    def empty(self):
        return not(self.nodeIds or self.cellIds or self.pointIds or self.lineIds)

    def writeDxfFile(self, dummy):
        '''groups have not representation in dxf files.

        :param dummy: dummy argument (not used).
        '''
        return

    def writeToXCFile(self, xcImportExportData):
        ''' writes the XC commands to define the group in a file.

        :param xcImportExportData:Data used when importing and/or exporting XC models.
        '''
        if(not self.empty()):
            f= xcImportExportData.outputFile
            strCommand= self.name + '= ' + xcImportExportData.setHandlerName + '.defSet("' + self.name +'")'
            f.write(strCommand+'\n')
            for n in self.nodeIds:
                strCommand= self.name + '.getNodes.append(n' + str(n) + ')'
                f.write(strCommand+'\n')
            for e in self.cellIds:
                strCommand= self.name + '.elements.append(e' + str(e) + ')'
                f.write(strCommand+'\n')
            for p in self.pointIds:
                strCommand= self.name + '.getMultiBlockTopology.getPoints.append(' + str(p) + ')'
                f.write(strCommand+'\n')
            for l in self.lineIds:
                strCommand= self.name + '.getMultiBlockTopology.getLines.append(' + str(l) + ')'
                f.write(strCommand+'\n')

    def getDict(self):
        ''' Return the object members in a Python dictionary.'''
        return {'name':self.name, 'nodeIds':self.nodeIds, 'cellIds':self.cellIds, 'pointIds':self.pointIds, 'lineIds':self.lineIds}

    def setFromDict(self, dct):
        ''' Set the data values from the dictionary argument.

        :param dct: dictionary containing the values of the object members.
        '''
        self.name= dct['name']
        self.nodeIds= dct['nodeIds']
        self.cellIds= dct['cellIds']
        self.pointIds= dct['pointIds']
        self.lineIds= dct['lineIds']

class XCImportExportData(object):
    ''' Data used when importing and/or exporting XC
        models.

    :ivar outputFileName: name to use for the output file.
    :ivar problemName: name for the XC problem object.
    :ivar nodeHandlerName: name for the XC node handler.
    :ivar cellHandlerName: name for the XC element handler.
    :ivar setHandlerName: name for the XC group handler.
    :ivar pointHandlerName: name for the XC point handler.
    :ivar lineHandlerName: name for the XC line handler.
    :ivar surfaceHandlerName: name for the XC surface handler.
    :ivar cellConversion: dictionary for cell conversion.
    :ivar outputFile: Python file object used for output.
    :ivar meshDesc: string describing the mesh
    :ivar blockData: block data (points, lines, surfaces, bodies).
    '''

    def __init__(self):
        self.mainDATFile= ""
        self.groupDATFiles= []
        self.dxfLayers= []
        self.outputFileName= 'xc_mesh'
        self.problemName= None
        self.nodeHandlerName= "nodes"
        self.cellHandlerName= "elements"
        self.setHandlerName= "groups"
        self.pointHandlerName= "points"
        self.lineHandlerName= "lines"
        self.surfaceHandlerName= "surfaces"
        self.cellConversion= {}
        self.outputFile= None
        self.meshDesc= None
        self.blockData= None

    def getXCFileName(self):
        return self.outputFileName+'.py'

    def getJSONFileName(self):
        return self.outputFileName+'.json'

    def getDxfFileName(self):
        return self.outputFileName+'.dxf'

    def getBlockHandlerName(self,blockType):
        if(blockType=='line'):
            return 'lines'
        elif(blockType=='face'):
            return 'surfaces'
        elif(blockType=='solid'):
            return 'solids'

    def convertCellType(self,tp):
        if tp in self.cellConversion:
            return self.cellConversion[tp]
        else:
            return None

    def readDATFiles(self):
        self.meshDesc= me.MeshData()
        self.meshDesc.readFromDATFile(self.mainDATFile)
        for fName in self.groupDATFiles:
            grp= GroupRecord()
            grp.name= os.path.splitext(fName)[0]
            grp.readFromDATFile(fName)
            self.meshDesc.groups.append(grp)

    def readDxfFile(self, fName, preprocessor):
        ''' Read from DXF file.

        :param fName: file name.
        :param preprocessor: xc preprocessor for the FE problem.
        '''
        self.blockData= bte.BlockData()
        self.blockData.name= fName
        self.blockData.readFromDxfFile(fName,preprocessor,self.dxfLayers)
        for l in self.dxfLayers:
            grp= GroupRecord()
            grp.setUp(l,self.blockData.pointsInLayer[l],self.blockData.blocksInLayer[l])
            self.blockData.groups.append(grp)

    def readJSONFile(self, fName):
        ''' Read from JSON file.

        :param fName: file name.
        '''
        f= open(fName,'r') # Open JSON file
        dataDict= json.load(f) # returns JSON object as a dictionary.
        self.setFromDict(dataDict)
        f.close()

    def writeDxfFile(self,fileName):
        self.blockData.writeDxfFile(fileName)

    def getXCCommandString(self):
        ''' Return a string with the XC commands that define the model.'''
        strCommand= ''
        if(self.problemName!=None):
            strCommand+= 'problemName= \'' + self.problemName+'\'\n'
            strCommand+= self.problemName + '= xc.FEProblem()'+'\n'
            strCommand+= 'preprocessor= ' + self.problemName + '.getPreprocessor\n'
        strCommand+= self.nodeHandlerName + '= preprocessor.getNodeHandler\n'
        strCommand+= self.cellHandlerName + '= preprocessor.getElementHandler\n'
        strCommand+= self.pointHandlerName + '= preprocessor.getMultiBlockTopology.getPoints\n'
        strCommand+= self.lineHandlerName + '= preprocessor.getMultiBlockTopology.getLines\n'
        strCommand+= self.surfaceHandlerName + '= preprocessor.getMultiBlockTopology.getSurfaces\n'
        #strCommand+= self.lineHandlerName + '= preprocessor.getMultiBlockTopology.getLines\n'
        strCommand+= self.setHandlerName + '= preprocessor.getSets\n'
        if(self.blockData):
            strCommand+= self.blockData.getXCCommandString(self)
        if(self.meshDesc):
            strCommand+= self.meshDesc.getXCCommandString(self)
        return strCommand

    def dumpToXC(self, preprocessor):
        ''' Dump the objects in this container into the preprocessor argument.

        :param preprocessor: XC finite element problem preprocessor.
        '''
        blockTopology= None
        if(self.blockData):
            blockTopology= self.blockData.dumpToXC(preprocessor)
        mesh= None
        if(self.meshDesc):
            mesh= self.meshDesc.dumpToXC(preprocessor)
        return blockTopology, mesh

    def readFromXCSet(self, xcSet):
        ''' Read the objects from the XC set and create the
            corresponding entities here.

        :param preprocessor: XC finite element problem preprocessor.
        '''
        if(len(xcSet.points)>0):
            self.blockData= bte.BlockData()
            self.blockData.readFromXCSet(xcSet)
        if(len(xcSet.nodes)>0):
            self.meshDesc= me.MeshData()
            self.meshDesc.readFromXC(xcSet)

    def writeToXCFile(self):
        ''' Write the model to a XC file.'''
        self.outputFile= open(self.getXCFileName(),"w")
        xcCommandString= self.getXCCommandString()
        self.outputFile.write(xcCommandString)
        self.outputFile.close()

    def writeToJSON(self):
        '''Write the objects that define the mesh.

        :param xcImportExportData: import/export parameters.
        '''
        self.outputFile= open(self.getJSONFileName(),"w")
        blockDict= self.getDict()
        jsonString= json.dumps(blockDict)
        self.outputFile.write(jsonString)
        self.outputFile.close()

    def getDict(self):
        ''' Return the object members in a Python dictionary.'''
        retval= {'mainDATFile':self.mainDATFile, 'groupDATFiles':self.groupDATFiles, 'dxfLayers':self.dxfLayers, 'outputFileName':self.outputFileName, 'problemName':self.problemName, 'nodeHandlerName':self.nodeHandlerName, 'cellHandlerName':self.cellHandlerName, 'setHandlerName':self.setHandlerName, 'pointHandlerName':self.pointHandlerName, 'lineHandlerName':self.lineHandlerName, 'surfaceHandlerName':self.surfaceHandlerName, 'cellConversion':self.cellConversion}
        meshDict= None
        if(self.meshDesc):
            meshDict= self.meshDesc.getDict()
        blockDict= None
        if(self.blockData):
            blockDict= self.blockData.getDict()
        retval.update({'meshDesc':meshDict, 'blockData': blockDict})
        return retval

    def setFromDict(self, dct):
        ''' Set the data values from the dictionary argument.

        :param dct: dictionary containing the values of the object members.
        '''
        self.mainDATFile= dct['mainDATFile']
        self.groupDATFiles= dct['groupDATFiles']
        self.dxfLayers= dct['dxfLayers']
        self.outputFileName= dct['outputFileName']
        self.problemName= dct['problemName']
        self.nodeHandlerName= dct['nodeHandlerName']
        self.cellHandlerName= dct['cellHandlerName']
        self.setHandlerName= dct['setHandlerName']
        self.pointHandlerName= dct['pointHandlerName']
        self.lineHandlerName= dct['lineHandlerName']
        self.surfaceHandlerName= dct['surfaceHandlerName']
        self.cellConversion= dct['cellConversion']
        meshDict= dct['meshDesc']
        if(meshDict):
            self.meshDesc= me.MeshData()
            self.meshDesc.setFromDict(meshDict)
        blockDict= dct['blockData']
        if(blockDict):
            self.blockData= bte.BlockData()
            self.blockData.setFromDict(blockDict)

class MEDMeshData(me.MeshData):
    meshDimension= None
    spaceDimension= None

    def __init__(self,umesh):
        super(MEDMeshData, self).__init__()
        self.meshDimension= umesh.getMeshDimension()
        self.spaceDimension= umesh.getSpaceDimension()
        self.numberOfCells= umesh.getNumberOfCells()
        self.numberOfNodes= umesh.getNumberOfNodes()

        self.nodes.readFromUMesh(umesh)
        self.cells.readFromUMesh(umesh)

    def __str__(self):
        retval= "meshDimension= " + str(self.meshDimension) + '\n'
        retval+= "spaceDimension= " +' '+str(self.spaceDimension) + '\n'
        retval+= super(MEDMeshData, self).__str__()
        return retval

    def getDict(self):
        ''' Return the object members in a Python dictionary.'''
        retval= super().getDict()
        retval.update({'meshDimension':self.meshDimension, 'spaceDimension':self.spaceDimension, 'numberOfCells':self.numberOfCells, 'numberOfNodes':self.numberOfNodes})
        return retval

    def setFromDict(self, dct):
        ''' Set the data values from the dictionary argument.

        :param dct: dictionary containing the values of the object members.
        '''
        super().setFromDict(dct)
        self.meshDimension= dct['meshDimension']
        self.spaceDimension= dct['spaceDimension']
        self.numberOfCells= dct['numberOfCells']
        self.numberOfNodes= dct['numberOfNodes']

import pickle
def dumpMeshes(meshes,fName):
    with open(fName + '.pkl', 'wb') as f:
        pickle.dump(meshes, f, pickle.HIGHEST_PROTOCOL)

def loadMeshes(fName):
    with open(fName + '.pkl', 'r') as f:
        return pickle.load(f)


from flask import Flask, request, abort
import yaml
import json
from functions import *
import datetime

"""
VARIABLE ASSIGNEMENT
"""
# Open the config.yml file and load its contents into the 'config' variable
with open('config.yml', 'r') as file:
    config = yaml.safe_load(file)

# Webex card
with open('ressources/surveyCard.json', 'r') as json_file:
    surveyCard = json.load(json_file)
with open('ressources/resultCard.json', 'r') as json_file:
    resultCard = json.load(json_file)

"""
HANDLE WEBHOOKS
"""
deleteAllWebhooks(config["botToken"])
sendCardWebhookUrl = config["baseWebhookUrl"]+config["sendCardWebhookPath"]
getCardSubmissionUrl = config["baseWebhookUrl"]+config["getCardSubmissionPath"]
getDetailUrl = config["baseWebhookUrl"]+"details"

response = createWebhook("getDetail", getDetailUrl, "attachmentActions", "", config["botToken"])
status_code = 0
while status_code != 200:
    response, status_code = createWebhook("getCardSubmission", getCardSubmissionUrl, "attachmentActions", "", config["botToken"])

initaliseDB(config["botToken"])

print("\nWebhooks creation output: "+str(status_code)+"\n")

app = Flask(__name__)

"""
LISTENING ON /webhook/getcardsubmission RESSOURCE FOR HTTP POST (WEBHOOKS)
"""
@app.route(config["getCardSubmissionPath"], methods=['POST'])
def webhookGet():

    if request.method == 'POST':

        weekNum = getWeekNum()
        data = request.json
        attachement = getAttachement(config["botToken"], data["data"]["id"])

        # READ DATABASE
        with open('data/answers.json', 'r') as openfile:
            previousWeeksAllRooms = json.load(openfile)

        previousWeeks = previousWeeksAllRooms[data["data"]["roomId"]]["roomResults"]
        # REMOVE USERNAME FROM WEEK
        username = getUsernameFromUserid(config["botToken"], attachement["personId"])
        for day in previousWeeks[weekNum-1]["days"]:
            newLine = [s for s in previousWeeks[weekNum-1]["days"][day] if s != username]
            previousWeeks[weekNum-1]["days"][day] = newLine

        # ADD USERNAME IN WEEK
        for day in attachement["inputs"]:
            if attachement["inputs"][day] == "true":
                previousWeeks[weekNum-1]["days"][day].append(username)

        previousWeeksAllRooms[data["data"]["roomId"]]["roomResults"]=previousWeeks

        # UPDATE DATABASE
        with open("data/answers.json", "w") as outfile:
            json.dump(previousWeeksAllRooms, outfile, indent=4)

        # SEND RESULT BACK TO USER
        listDays = []
        listDays = getListDays()
        populateResultCard(resultCard, previousWeeks, weekNum, getRoomSize(config["botToken"], data["data"]["roomId"]), listDays)
        sendCardToPersonId(config["botToken"], data["data"]["personId"], resultCard)

        return 'success', 200

    # ELSE RETURN A HTTP 400 STATUS CODE
    else:
        abort(400)

"""
LISTENING ON LOCALHOST:42101 FOR HTTP POST (WEBHOOKS)
"""
if __name__ == '__main__':
    app.run(host="0.0.0.0", port=42101)


from django.contrib.auth import authenticate, login, get_user_model
from django.http import HttpResponse,JsonResponse
from django.shortcuts import render, redirect

from .forms import ContactForm
from products.models import Product

def home_page(request):
    context = {
        "title":"HELLO WORLD!",
        "content":" Welcome to the homepage",
    }
    if request.user.is_authenticated:
        context["premium_content"] = "YEAHHHHH"
    return render(request, "products/list.html", context)

def about_page(request):
    context = {
        "title": "ABOUT PAGE",
        "content":" Welcome to the aboutpage"
    }
    return render(request, "home_page.html", context)

def contact_page(request):
    contact_form = ContactForm(request.POST or None)
    context = {
         "title": "Contact",
         "content":" Welcome to the contactpage",
         "form": contact_form,

    }
    if contact_form.is_valid():
        print(contact_form.cleaned_data)
        if request.is_ajax():
            return JsonResponse({"message":"Thank you for your submission"})

    if contact_form.errors:
        errors = contact_form.errors.as_json()
        if request.is_ajax():
             return HttpResponse(errors, status=400, content_type='application/json')

    return render(request, "contact/view.html", context)

def home_page_old(request):
    html_ = """
    <!doctype html>

    <meta charset="utf-8">
    <meta name="viewport" content="width=device-width, initial-scale=1">

    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.0-beta1/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-giJF6kkoqNQ00vy+HMDP7azOuL0xtbfIcaT9wjKHr8RbDVddVHyTfAAsrekwKmP1" crossorigin="anonymous">

    <title>Hello, world!</title>

Hello, world!

    """
    return HttpResponse(html_)