from flask import Flask, request
from flask import Response
from flask import jsonify
import json
import threading
import requests
import sys
import queue
import time
import pandas as pd
import utils_balance as ub
from concurrent.futures import ThreadPoolExecutor
from csv import writer
import os
from datetime import datetime
import numpy as np

app = Flask(__name__)
app.debug = True

# Recibir los parametros
arg = sys.argv
# Estados del nodo
id_nodo = int(arg[1]) # ID posicion 1
election = False
puerto_nodo = int(arg[2]) # Puerto del contenedor
ip_container = arg[3] # ip del container
puerto_base = int(arg[5]) # puerto base
total_workers = int(arg[6]) # Total de workers
# puerto_conquer = int(arg[7]) # Puerto del conquer que reunira loa archivos

# Datos del coordinador
puerto_cor = 0
id_cor = 0
without_coordina = False
puertos_kill = []
verificar = False
estatus = True
# Verifica si es el coordinador
if (arg[4] == '1'):
    coordina = True
    id_cor = id_nodo
else:
    coordina = False

pet =0
inicio_global = 0
fin_global = 0

tota = np.power(2,total_workers)-1

# Info of aceptors
class NodeBully:
    def __init__(self,ide,ip_nodo,puerto,total,antesesor,sucesor):
        self.ide = ide
        self.ip_nodo = ip_nodo
        self.puerto = puerto
        self.name = ub.conc(ip_nodo, str(puerto))
        self.id_nodo = ub.hash(self.name,total)
        self.antesesor = antesesor
        self.sucesor = sucesor
        self.sucesor_port = 0
        self.antesesor_port = 0

    def url_request(self):
        url = 'http://'+self.name
        return url

    def set_ide(self,ide):
        self.ide = ide

    def set_ip_nodo(self,ip):
        self.ip_nodo = ip

    def set_puerto(self,puerto):
        self.puerto = puerto

    def set_antesesor(self,ante,port):
        self.antesesor = ante
        self.antesesor_port = port

    def set_sucesor(self,suce,port):
        self.sucesor = suce
        self.sucesor_port = port

# Creamos nodo local
node_local = NodeBully(ide=id_nodo,
                       ip_nodo=ip_container,
                       puerto=puerto_nodo,
                       total=tota,
                       antesesor=0,
                       sucesor=0)

proposer_lider = NodeBully(ide=0,
                       ip_nodo='0',
                       puerto=0,
                       total=tota,
                       antesesor=0,
                       sucesor=0)

nodos_workers = list()

# ------------------------------------------ BULLY  ------------------------------------
# Inicializa otros nodos proposer para conocerlos
# def init_nodos(m):
#     global puerto_base
#     global total_workers
#     global node_local
#     nodos = list()
#     for x in range(total_workers):
#         if (node_local.puerto != (puerto_base+x)):
#             aux = NodeBully(ide=x+1,
#                             ip_nodo=node_local.ip_nodo,
#                             puerto=puerto_base+x)
#             nodos.append(aux)
#     return nodos

# Funcion para obtener puertos
def init_nodos_chord(m):
    global puerto_base
    global node_local
    global total_workers
    nodos = list()
    total = np.power(2,m)-1
    for x in range(total_workers):
        if (x<=total_workers+1):
            if (node_local.puerto != (puerto_base+x)):
                iden = x+1
            else:
                iden=id_nodo
            aux = NodeBully(ide=iden,
                            ip_nodo=ip_container,
                            puerto=(puerto_base+x),
                            total=total,
                            antesesor=0,
                            sucesor=0)
            nodos.append(aux)
        else:
            break
    return nodos

# Recibe los datos del nuevo coordinador
@app.route('/COORDINATOR', methods = ['POST'])
def fun_coordinator():
    global pet
    global election
    global proposer_lider
    global fin_global
    global inicio_global
    global node_local
    # recibimos json
    message = request.get_json()
    # Actualizamos el proposer lider
    proposer_lider.set_ide(message['id'])
    proposer_lider.set_ip_nodo(message['ip'])
    proposer_lider.set_puerto(message['puerto'])
    # Coordinador llegó
    app.logger.info('------ COORDINATOR '+str(proposer_lider.ide)+' ------')
    election = False
    without_coordina = False
    pet =0
    if(inicio_global!=0):
        fin_global = time.time()
        append_list_as_row('./data/tiempos_bully_BaK.csv', [node_local.ide,proposer_lider.ide,(fin_global-inicio_global)])
        inicio_global=0
    # Comienza el proceso de verificacion
    monitor = threading.Thread(target=fun_verificar)
    monitor.start()
    init_chord()
    return jsonify({'response':'RECIBIDO'})

# Hace el proceso de election
@app.route('/ELECTION',methods = ['POST'])
def fun_election():
    global election
    global node_local
    if (election==True):
        # Recibi el post
        message = request.get_json()
        # Id y puerto del nodo que hace peticion de election
        id_vecino = message['id']
        # app.logger.info('------ ELECTION CON EL NODO '+str(id_vecino) + ' ------')
        # Si el nodo actual es mayor
        if (node_local.ide>id_vecino):
            # app.logger.info('------ SOY MAYOR ------')
            return jsonify({'response':'OK'})
        # Si el vecino es mayor
    else:
            # app.logger.info('------ SOY MENOR ------')
        return jsonify({'response':'NO'})

# Funcion de verificacion del coordinator
def fun_verificar():
    global pet
    global election
    global without_coordina
    global proposer_lider
    global node_local
    global inicio_global
    while True:
        try:
            if (election == False):
                # Verificamos si no somos el mismo
                if (proposer_lider.puerto == node_local.puerto):
                    break
                time.sleep(5+(id_nodo))
                url = proposer_lider.ip_nodo+':'+str(proposer_lider.puerto)+'/PRUEBA'
                headers = {'PRIVATE-TOKEN': '<your_access_token>', 'Content-Type':'application/json'}
                req = requests.post('http://'+url, headers=headers)
                # app.logger.info('------ SOY EL NODO No.'+str(id_nodo)+' - '+str(puerto_nodo)+'------')
                pet += 1
                app.logger.info('------('+str(pet)+') ACTIVO LIDER ='+str(proposer_lider.ide)+' - ('+str(node_local.puerto)+')------')
        except requests.exceptions.ConnectionError:
            # Proceso de election
            puertos_kill.append(proposer_lider.puerto)
            # Notifica a todos de election
            time.sleep(10)
            inicio_global = time.time()
            if(election == False and without_coordina == False):
                for proposer in nodos_workers:
                    if (proposer.puerto not in puertos_kill):
                        # requests
                        # app.logger.info('------ ME DI CUENTA ------')
                        app.logger.info('------ ME DI CUENTA ('+str(node_local.ide)+') ------')
                        url = proposer.ip_nodo+':'+str(proposer.puerto)+'/PRE'
                        # url = '148.247.201.221:'+puerto+'/PRE'
                        # Enviar petricion de eleccion
                        headers = {'PRIVATE-TOKEN': '<your_access_token>', 'Content-Type':'application/json'}
                        req = requests.post('http://'+url, headers=headers)
                        response_json = req.json()
                        app.logger.info('------ '+response_json['response']+' ('+str(proposer.ide)+') ------')
                # Inicio proceo de election
                app.logger.info('------ PROCESO ------')
                election = True
                init()
            break

# Verificacion de actividad
@app.route('/PRE',methods = ['POST'])
def pre():
    global election
    election = True
    without_coordina = True
    app.logger.info('------ ELECTION ------')
    return jsonify({'response':'ELECTION'})

# Verificacion de actividad
@app.route('/INICIO_ELECTION',methods = ['POST'])
def inicio_election():
    global election
    # time.sleep(10)
    init()
    return jsonify({'response':'OK'})

# Verificacion de actividad
@app.route('/PRUEBA',methods = ['POST'])
def prueba():
    global proposer_lider
    app.logger.info('------ Verificacion '+str(proposer_lider.ide)+' ------')
    return jsonify({'response':'SI'})

def init():
    global coordina
    global proposer_lider
    global node_local
    # Inicia la election
    cont=0
    for proposer in nodos_workers:
        if (node_local.puerto < proposer.puerto):
            if (proposer.puerto not in puertos_kill):
                # requests
                url = proposer.ip_nodo+':'+str(proposer.puerto)+'/ELECTION'
                # url = '148.247.201.221:'+puerto+'/ELECTION'
                datas = {
                    'id':node_local.ide,
                    'ip':node_local.ip_nodo,
                    'puerto':node_local.puerto
                }
                try:
                    headers = {'PRIVATE-TOKEN': '<your_access_token>', 'Content-Type':'application/json'}
                    req_i = requests.post('http://'+url, data=json.dumps(datas), headers=headers)
                    # Recibir
                    responde_json_ = req_i.json()
                    # Existe uno mayor
                    if (responde_json_['response'] == 'OK'):
                        app.logger.info('------ ('+str(proposer.puerto)+') OK ------')
                        # Se manda notifiacion al primer mayor
                        if(cont==0):
                            mayor_puerto = proposer.puerto
                            mayor_ip = proposer.ip_nodo
                            cont = 1
                except requests.exceptions.ConnectionError:
                    print('')
    # Envia la notificion al siguiente puerto mayor a que haga la election
    if (cont == 1):
        url = mayor_ip+':'+str(mayor_puerto)+'/INICIO_ELECTION'
        # url = '148.247.201.221:'+mayor_puerto+'/INICIO_ELECTION'
        try:
            headers = {'PRIVATE-TOKEN': '<your_access_token>', 'Content-Type':'application/json'}
            req = requests.post('http://'+url, data=json.dumps(datas), headers=headers)
            # Recibir
            responde_json = req.json()
            # Existe uno mayor
            if (responde_json['response'] == 'OK'):
                app.logger.info('------ NODO NUEVO PARA ELECTION '+str(mayor_puerto)+' ------')
                # # Se manda notifiacion al primer mayor
                # if(cont==0):
                #     mayor_puerto = puerto
                #     cont = 1
        except requests.exceptions.ConnectionError:
            print('')
    else:
        # Soy el nuevo coordinador
        proposer_lider = node_local
        coordina = True
        app.logger.info('------ SOY EL NUEVO COORDINATOR No.'+str(node_local.ide)+'------')
        for proposer in nodos_workers:
            if (proposer.puerto not in puertos_kill):
        # for puerto in puertos:
            # if (puerto not in puertos_kill):
                # requests
                url = proposer.ip_nodo+':'+str(proposer.puerto)+'/COORDINATOR'
                # url = '148.247.201.221:'+puerto+'/COORDINATOR'
                datas = {
                    'id':node_local.ide,
                    'ip':node_local.ip_nodo,
                    'puerto':node_local.puerto
                }
                headers = {'PRIVATE-TOKEN': '<your_access_token>', 'Content-Type':'application/json'}
                req = requests.post('http://'+url, data=json.dumps(datas), headers=headers)
                # Recibir
                responde_json = req.json()
                app.logger.info('------ SE NOTIFICO AL NODO '+str(proposer.puerto) + ' - ' + responde_json['response']+' ------')
        init_chord()
        datas = {
        'ip':node_local.ip_nodo,
        'puerto':node_local.puerto
        }
        url = node_local.ip_nodo+':5701/save_workers'
        req = requests.post('http://'+url,data=json.dumps(datas), headers=headers)

# ------------------------------------------ START CONTAINER -----------------------------------

nodos_workers = init_nodos_chord(m=total_workers)

def append_list_as_row(file_name, list_of_elem):
    # Open file in append mode
    with open(file_name, 'a+', newline='\n') as write_obj:
        # Create a writer object from csv module
        csv_writer = writer(write_obj)
        # Add contents of list as last row
        csv_writer.writerow(list_of_elem)

# ------------------------------------------ Balanceo de carga -----------------------------------
nodos_in_m = list()
nodos_online = list()

# Inicializar nodos para chord
def init_chord():
    global nodos_in_m
    global total_workers
    global nodos_workers
    global nodos_online
    # Obtiene los nodos para m = total_workers
    # nodos_in_m = init_nodos_chord(total_workers)
    # Se realiza un request a cada nodos para saber si estan activos
    nodos_online = info_all_nodos(nodos_workers)
    # Se confgura el sucesor y antesesor
    set_sucesor_antesesor(nodos_online)
    # app.logger.error(len(nodos_online))

# Configuramos los nodos antesesores y sucesores
def set_sucesor_antesesor(nodos_online):
    global node_local
    pos = [x.name for x in nodos_online].index(node_local.name)
    app.logger.info('POSICION - ' + str(pos))
    if(pos == 0):
        # Soy el primero
        node_local.set_antesesor(nodos_online[-1].id_nodo, nodos_online[-1].puerto)   # Antesesor
        node_local.set_sucesor(nodos_online[1].id_nodo, nodos_online[1].puerto)        # Sucesor
    elif(pos == len(nodos_online)-1):
        # Soy el ultimo
        node_local.set_antesesor(nodos_online[pos-1].id_nodo, nodos_online[pos-1].puerto)    # Antesesor
        node_local.set_sucesor(nodos_online[0].id_nodo, nodos_online[0].puerto)              # Sucesor
    else:
        # Estoy en el intermedio
        node_local.set_antesesor(nodos_online[pos-1].id_nodo, nodos_online[pos-1].puerto)    # Antesesor
        node_local.set_sucesor(nodos_online[pos+1].id_nodo, nodos_online[pos+1].puerto)      # Sucesor

# Saber que nodos estan activos
def info_all_nodos(nodos):
    global puerto_nodo
    nodos_online_= list()
    #  Convierte lista de json to string
    # json_string = json.dumps([ob.__dict__ for ob in nodos])
    datas = {
        'MSJ': 'INCIO'
    }
    for x in nodos:
        if (x.puerto != node_local.puerto):
            try:
                # app.logger.error('entroooo' + x.url_request())
                req = envio_request_with_datas(url=x.url_request(),
                                           comand='/NODOS',
                                           datas=datas)
                # app.logger.error(req['response'])
                if (req['response']=='OK'): # Guardo los nodos que respondan con un OK
                    # Ultimo nodo añadido es el antesesor
                    nodos_online_.append(x)

            except :
                pass
    app.logger.info('------ RECIBIDOS ------')
    nodos_online_.append(node_local)
    # app.logger.info(nodos_online_)
    nodos_online_ = sorted(nodos_online_, key=lambda chord_node : chord_node.id_nodo)
    return nodos_online_

# Recibe todos los nodos en m
@app.route('/NODOS',methods = ['POST'])
def nodos_recibidos():
    global nodos_in_m
    global estatus
    message = request.get_json()
    aux_nodos = message['MSJ']
    # app.logger.info(estatus)
    if (estatus == True):
        return jsonify({'response':'OK'})
    else:
        return jsonify({'response':'NO'})

# Envio a otro nodos con datos
def envio_request_with_datas(url,comand,datas):
    headers = {'PRIVATE-TOKEN': '<your_access_token>', 'Content-Type':'application/json'}
    req = requests.post(url+comand,data=json.dumps(datas), headers=headers)
    return req.json()

# Envio a otro nodo sin datos
def envio_request_witout_return(url,comand,datas):
    headers = {'PRIVATE-TOKEN': '<your_access_token>', 'Content-Type':'application/json'}
    req = requests.post(url+comand, data=json.dumps(datas), headers=headers)
    return 1

# Retorna nodos vecinos disponibles
def dame_n_nodos(peers):
    global node_local
    global total_workers
    global nodos_online
    if (peers<total_workers+1):
        # peers necesarios son menos o igual a los disponibles
        if(peers==1):
            puertos = list()
            ids = list()
            puertos.append(node_local.sucesor_port)
            ids.append(node_local.sucesor)
            return puertos, ids
        # mis dos vecinos
        elif(peers==2):
            puertos = list()
            ids = list()
            puertos.append(node_local.sucesor_port)
            puertos.append(node_local.antesesor_port)
            ids.append(node_local.sucesor)
            ids.append(node_local.antesesor)
            return puertos, ids
        # si es par
        elif (ub.par(peers)):
            dib = peers/2
            # Se envia al nodo suscesor y antesesor
            puertos_suc = list()
            puertos_ante = list()
            id_suc = list()
            id_ante = list()
            datas = {
                'id_solicitante': node_local.id_nodo,
                'num':dib,
                'ids':id_suc,
                'puertos':puertos_suc,
                'tipo':'S'
            }
            # sucesor
            req = envio_request_with_datas('http://'+node_local.ip_nodo+':'+str(node_local.sucesor_port),'/IP_NODO',datas)
            puertos_suc = req['puertos']
            id_suc=req['ids']
            # antesesor
            datas = {
                'id_solicitante': node_local.id_nodo,
                'num':dib,
                'ids':id_ante,
                'puertos':puertos_ante,
                'tipo':'A'
            }
            req = envio_request_with_datas('http://'+node_local.ip_nodo+':'+str(node_local.antesesor_port),'/IP_NODO',datas)
            puertos_ante = req['puertos']
            id_ante = req['ids']
            return (puertos_suc+puertos_ante),(id_suc+id_ante)
        # Es impar
        elif(ub.par(peers)==False):
            dib = (peers-1)/2
            # Se envia al nodo suscesor y antesesor
            puertos_suc = list()
            puertos_ante = list()
            id_suc = list()
            id_ante = list()
            datas = {
                'id_solicitante': node_local.id_nodo,
                'num':dib+1,
                'ids':id_suc,
                'puertos':puertos_suc,
                'tipo':'S'
            }
            # sucesor
            app.logger.error('********************************************** ' + 'http://'+node_local.ip_nodo+':'+str(node_local.sucesor_port))
            req_suc = envio_request_with_datas('http://'+node_local.ip_nodo+':'+str(node_local.sucesor_port),'/IP_NODO',datas)
            puertos_suc = req_suc['puertos']
            id_suc = req_suc['ids']
            # app.logger.error('Sucesor')
            # app.logger.error(puertos_suc)
            # antesesor
            datas = {
                'id_solicitante': node_local.id_nodo,
                'num':dib,
                'ids':id_ante,
                'puertos':puertos_ante,
                'tipo':'A'
            }
            req_ante = envio_request_with_datas('http://'+node_local.ip_nodo+':'+str(node_local.antesesor_port),'/IP_NODO',datas)
            puertos_ante = req_ante['puertos']
            id_ante = req_ante['ids']
            # app.logger.error('Antesesor')
            # app.logger.error(puertos_ante)
            # app.logger.error((puertos_suc+puertos_ante))
            return (puertos_suc+puertos_ante),(id_suc+id_ante)
    else:
        puertos = list()
        ids = list()
        for x in nodos_online:
            if(node_local.id_nodo!=x.id_nodo):
                puertos.append(x.puerto)
                ids.append(x.id_nodo)
        return puertos,ids

@app.route('/IP_NODO',methods=['POST'])
def ip_port_nodo():
    global node_local
    message = request.get_json()
    id_manager = message['id_solicitante']
    num = message['num']-1
    puertos_dis = message['ids']
    id_dis = message['puertos']
    tipo = message['tipo']

    # Si mi vecino es el utlimo
    if(num <= 0.0):
        # Me añado
        puertos_dis.append(node_local.puerto)
        id_dis.append(node_local.id_nodo)
        # Retorno la lista de id y puertos
        datas = {
            'ids':id_dis,
            'puertos':puertos_dis
        }
        return jsonify(datas)
    else:
    # No soy el ultimo
        datas = {
            'id_solicitante': id_manager,
            'ids':id_dis,
            'puertos':puertos_dis,
            'tipo':tipo,
            'num':num
        }
        if(tipo=='S'):
            # app.logger.error('+++++++++++++++++++++++++++++++++++ http://'+node_local.ip_nodo+':'+str(node_local.sucesor_port))
            req = envio_request_with_datas('http://'+node_local.ip_nodo+':'+str(node_local.sucesor_port),'/IP_NODO',datas)
        elif(tipo=='A'):
            req = envio_request_with_datas('http://'+node_local.ip_nodo+':'+str(node_local.antesesor_port),'/IP_NODO',datas)
        # guardar lo que retorne de listas llenas
        puertos_dis = req['puertos']
        id_dis = req['ids']
        # Me almaceno yo
        puertos_dis.append(node_local.puerto)
        id_dis.append(node_local.id_nodo)
        # Hago json de salida
        datas = {
            'ids':id_dis,
            'puertos':puertos_dis
        }
        # retorno actualizados
        return jsonify(datas)

# Coordinador
if (coordina==True):
    # global node_local
    proposer_lider = node_local
    # Notifico que es el coordinador
    app.logger.info('------ SOY EL COORDINADOR - No.'+str(node_local.ide) + ' ------')
    time.sleep(5)
    app.logger.info(str(nodos_workers)+' lista ------- ')
    for proposer in nodos_workers:
        if (proposer.puerto not in puertos_kill):
            # requests
            # node_local.ip_nodo
            url = proposer.ip_nodo+':'+str(proposer.puerto)+'/COORDINATOR'
            app.logger.info(url)
            # url = '148.247.201.221:'+puerto+'/COORDINATOR'
            datas = {
                'id':node_local.ide,
                'ip':node_local.ip_nodo,
                'puerto':node_local.puerto
            }
            headers = {'PRIVATE-TOKEN': '<your_access_token>', 'Content-Type':'application/json'}
            req = requests.post('http://'+url, data=json.dumps(datas), headers=headers)
            # Recibir
            responde_json = req.json()
            app.logger.info('------ SE NOTIFICO AL NODO '+str(proposer.ide) + ' - ' + responde_json['response']+' ------')

    # Comienza inicio de chord
    init_chord()
    datas = {
        'ip':node_local.ip_nodo,
        'puerto':node_local.puerto
        }
    url = node_local.ip_nodo+':5701/save_workers'
    req = requests.post('http://'+url,data=json.dumps(datas), headers=headers)
    # crate_fingerTable()

# ------------------------------------------ Balanceo de carga -----------------------------------

@app.route('/RECIBIR_BALAANCE',methods = ['POST'])
def k_dividir():
    global node_local
    global total_workers
    global nodos_online
    message = request.get_json()

    # Recibe los datos
    k_ = message['K']
    data_balance = 'D'
    type_balance = 'RR'
    type_cluster = 'Kmeans'
    # need_peers = message['workers']
    need_peers = total_workers
    # tiempo inicial
    inicio = time.time()
    # Atrae lo puertos
    puertos_o,ids = dame_n_nodos(need_peers)
    app.logger.info('--------------------------------------------------'+str(len(puertos_o))+ ' - ' + str(len(ids)))
    # timpo final
    fin = time.time()
    # añadir tiempos a csv
    append_list_as_row('./data/tiempos_busqueda.csv', [need_peers,(fin-inicio)])
    app.logger.error(fin-inicio)

    init_data = ub.init_workres_array(need_peers)
    data_clus = ub.read_CSV(message['name']) # DataPreprocess

    # numero AÑOS DIAS O MESES
    type_balance_str = ub.type_blane_cond(data_balance)
    list_balance = data_clus[type_balance_str].unique()

    if (type_balance=='RR'): # Balanceador Round Robin
        init_data = ub.RaoundRobin(init_data,list_balance,need_peers)
    elif (type_balance=='PS'):# Balanceador PseudoRandom
        init_data = ub.PseudoRandom(init_data,list_balance,need_peers)
    elif (type_balance=='TC'):# Balanceador TwoChoices
        init_data = ub.TwoChoices(init_data,list_balance,need_peers)
    else:# Balanceador Round Robin
        init_data = ub.RaoundRobin(init_data,list_balance,need_peers)

    # app.logger.error(puertos_online)
    peticiones = []
    # inicio=time.time()

    # with ThreadPoolExecutor(max_workers=os.cpu_count()) as executor:

    for x in range(need_peers):
        if (len(init_data[x])>0):
            concac=[]
            for val in init_data[x]:
                # app.logger.info(val)
                concac.append(data_clus[data_clus[type_balance_str]==val])
            data_fin = pd.concat(concac)
            name = "ID_"+str(ids[x])+"_Port_"+str(puertos_o[x])+"_LD="+type_balance+"_DLB="+data_balance
            data_fin.to_csv("./data/"+name+".csv")
            data_clus_n = {"K": k_,
                "name": name,
                "type_cluster":type_cluster,
                "need":need_peers
                }

            url = "http://"+node_local.ip_nodo+":"+str(puertos_o[x])
            req = envio_request_witout_return(url,"/CLUSTERING",data_clus_n)
                # process = Thread(target=envio_request_witout_return,args=(url,"/CLUSTERING",data_clus_n))
                # process.start()
                # peticiones.append(process)
                # peticiones.append(multiprocessing.Process(target=envio_request_witout_return,args=(url,"/CLUSTERING",data_clus_n)))
                # executor.submit(envio_request_witout_return,url,"/CLUSTERING",data_clus_n)
            app.logger.error('-----'+str(fin-inicio))
        # for pet in peticiones:
        #     pet.join()
    return jsonify({'response':'SI'})

# ------------------------------------------ Carga de Trabajo -----------------------------------
def prueba_clus(k_,type_cluster,data_clima,name,wor):
    # app.logger.error(k_)
    data_p =data_clima.iloc[:,[7,8,9,10]]
    for k in k_:
        # KMEANS
        if (type_cluster=="Kmeans"):
            # llamado del clustering
            inicio = time.time()
            k_labels,it = ub.K_means(k,data_p)
            cluster="Kmeans"
            fin = time.time()
            app.logger.error('------ '+str(cluster)+" = "+str(fin-inicio) + ' ------')
        elif (type_cluster=="GM"):
            inicio = time.time()
            k_labels = ub.MixtureModel(k,data_p)
            cluster="GaussianMixture"
            fin = time.time()
            app.logger.error('------ '+str(cluster)+" = "+str(fin-inicio) + ' ------')
        else:
            inicio = time.time()
            k_labels,it = ub.K_means(k,data_p)
            cluster="Kmeans"
            fin = time.time()
            app.logger.error('------ '+str(cluster)+" = "+str(fin-inicio) + ' ------')
        # data send
        append_list_as_row('./data/tiempos_carga.csv', [wor,cluster,(fin-inicio)])
        data_clima['clase']=k_labels
        data_clima.to_csv("./data/results/Clus_"+name+"_DataClust_K="+str(k)+"_"+str(cluster)+".csv")
        return 1

# cLustering
@app.route('/CLUSTERING',methods = ['POST'])
def clustering():
    message = request.get_json()
    # recibir K
    k_ = message['K']
    name = message['name']
    app.logger.info('----------------------------- ' + str(name))
    type_cluster = message['type_cluster']
    wor = message['need']

    data_clima = ub.read_CSV(name)
    # data_p =data_clima.iloc[:,[7,8,9,10]]
    # prueba_clus(k_,type_cluster,data_p,name,wor)
    with ThreadPoolExecutor(max_workers=os.cpu_count()) as executor:
        # app.logger.info('executoooooor')
        executor.submit(prueba_clus,k_,type_cluster,data_clima,name,wor)
        # app.logger.info(executor.result())
    # multi = multiprocessing.Process(target=prueba_clus,args=(k_,type_cluster,data_clima,name))
    # multi.start()
    # for k in k_:
    #     # KMEANS
    #     if (type_cluster=="Kmeans"):
    #         # llamado del clustering
    #         inicio = time.time()
    #         k_labels,it = K_means(k,data_p)
    #         cluster="Kmeans"
    #         fin = time.time()
    #         app.logger.error('------ '+str(cluster)+" = "+str(fin-inicio) + ' - '+str(it) +' ------')
    #     elif (type_cluster=="GM"):
    #         inicio = time.time()
    #         k_labels = MixtureModel(k,data_p)
    #         cluster="GaussianMixture"
    #         fin = time.time()
    #         app.logger.error('------ '+str(cluster)+" = "+str(fin-inicio) + ' ------')
    #     else:
    #         inicio = time.time()
    #         k_labels,it = K_means(k,data_p)
    #         cluster="Kmeans"
    #         fin = time.time()
    #         app.logger.error('------ '+str(cluster)+" = "+str(fin-inicio) + ' - '+str(it) +' ------')
    #     # data send
    #     append_list_as_row('./data/tiempos_carga.csv', [wor,cluster,(fin-inicio)])
    #     data_clima["clase"]=k_labels
    #     data_clima.to_csv("./data/results/Clus_"+name+"_DataClust_K="+str(k)+"_"+str(cluster)+".csv")

    return jsonify({'response':'CLUSTERING TERMINADO'})

if __name__ == '__main__':
    app.run(host= '0.0.0.0',debug=True)


"""Main module."""

# -----------------------------------------------------------------------------
# Imports
# -----------------------------------------------------------------------------
import spikedetekt2
import logging
import imp
import os.path as op

import numpy as np
import tables as tb

from .progressbar import ProgressReporter
from kwiklib.dataio import (BaseRawDataReader, read_raw, excerpt_step,
    to_contiguous, convert_dtype, KwdRawDataReader, ExperimentRawDataReader)
from spikedetekt2.processing import (bandpass_filter, apply_filter, decimate,
    get_threshold, connected_components, extract_waveform,
    compute_pcs, project_pcs, DoubleThreshold)
from kwiklib.utils import (Probe, iterkeys, debug, info, warn, exception,
    display_params, FileLogger, register, unregister)

def _import_module(path):
    module_name = op.basename(path)
    module_name = op.splitext(module_name)[0]
    return imp.load_source(module_name, path)

# -----------------------------------------------------------------------------
# Processing
# -----------------------------------------------------------------------------
def apply_threshold(chunk_fil, threshold=None, **prm):
    # Determine the chunk used for thresholding.
    if prm['detect_spikes'] == 'positive':
        chunk_detect = chunk_fil
    elif prm['detect_spikes'] == 'negative':
        chunk_detect = -chunk_fil
    elif prm['detect_spikes'] == 'both':
        chunk_detect = np.abs(chunk_fil)

    # Perform thresholding.
    # shape: (nsamples, nchannels)
    chunk_threshold = DoubleThreshold(
        strong=chunk_detect > threshold.strong,
        weak=chunk_detect > threshold.weak,
    )
    return chunk_detect, chunk_threshold

def extract_waveforms(chunk_detect=None, threshold=None,
                      chunk_fil=None, chunk_raw=None,
                      probe=None, components=None,
                      **prm):
    # For now, we use the same binary chunk for detection and extraction
    # +/-chunk, or abs(chunk), depending on the parameter 'detect_spikes'.
    chunk_extract = chunk_detect  # shape: (nsamples, nchannels)
    # This is a list of Waveform instances.
    waveforms = []
    for component in components:
        w = extract_waveform(component,
                             chunk_extract=chunk_extract,
                             chunk_fil=chunk_fil,
                             chunk_raw=chunk_raw,
                             threshold_strong=threshold.strong,
                             threshold_weak=threshold.weak,
                             probe=probe,
                             **prm)
        if w is not None:
            waveforms.append(w)

    # Remove skipped waveforms (in overlapping chunk sections).
    # waveforms = [w for w in waveforms if w is not None]
    return waveforms

def add_waveform(experiment, waveform, **prm):
    """Add a Waveform instance to an Experiment."""
    experiment.channel_groups[waveform.channel_group].spikes.add(
        time_samples=waveform.s_offset,
        time_fractional=waveform.s_frac_part,
        recording=waveform.recording,
        waveforms_raw=waveform.raw,
        waveforms_filtered=waveform.fil,
        masks=waveform.masks,
    )

def save_features(experiment, **prm):
    """Compute the features from the waveforms and save them in the experiment
    dataset."""
    nwaveforms_max = prm['pca_nwaveforms_max']
    npcs = prm['nfeatures_per_channel']
    kwik = experiment._files['kwik']

    for chgrp in iterkeys(experiment.channel_groups):
        spikes = experiment.channel_groups[chgrp].spikes
        # Extract a subset of the saveforms.
        nspikes = len(spikes)

        # We convert the extendable features_masks array to a
        # contiguous array.
        if prm.get('features_contiguous', True):
            # Make sure to update the PyTables node after the recreation,
            # to avoid ClosedNodeError.
            spikes.features_masks = to_contiguous(spikes.features_masks, nspikes=nspikes)
        else:
            warn(("The features array has not been converted to a contiguous "
                  "array."))

        # Skip the channel group if there are no spikes.
        if nspikes == 0:
            continue
        nwaveforms = min(nspikes, nwaveforms_max)
        step = excerpt_step(nspikes, nexcerpts=nwaveforms, excerpt_size=1)
        waveforms_subset = spikes.waveforms_filtered[::step]

        # With this option, PCs are directly provided in the PRM file as
        # a NumPy array
        if prm.get('canonical_pcs', None) is not None:
            pcs = prm['canonical_pcs']
            assert isinstance(pcs, np.ndarray)
        else:
            # We take the masks in order to compute the PCs only on
            # the unmasked spikes, for each channel.
            masks = spikes.features_masks[::step,::npcs,1]  # (nspikes, nchannels)
            # Compute the PCs.
            pcs = compute_pcs(waveforms_subset, npcs=npcs, masks=masks)

        # Add PCs to the KWIK file
        kwik.createArray(experiment.channel_groups[chgrp]._node, 'pca_waveforms',
            pcs)

        # Project the waveforms on the PCs and compute the features.
        # WARNING: optimization: we could load and project waveforms by chunks.
        for i, waveform in enumerate(spikes.waveforms_filtered):
            # Convert waveforms from int16 to float32 with scaling
            # before computing PCA so as to avoid getting huge numbers.
            waveform = convert_dtype(waveform, np.float32)
            features = project_pcs(waveform, pcs)
            spikes.features_masks[i,:,0] = features.ravel()

# -----------------------------------------------------------------------------
# File logger
# -----------------------------------------------------------------------------
def create_file_logger(filename):
    # global LOGGER_FILE
    LOGGER_FILE = FileLogger(filename, name='file',
        level=logging.DEBUG)
    register(LOGGER_FILE)
    return LOGGER_FILE

def close_file_logger(LOGGER_FILE):
    unregister(LOGGER_FILE)

# -----------------------------------------------------------------------------
# Main loop
# -----------------------------------------------------------------------------
def run(raw_data=None, experiment=None, prm=None, probe=None,
        _debug=False, convert_only=False):
    """This main function takes raw data (either as a RawReader, or a path
    to a filename, or an array) and executes the main algorithm (filtering,
    spike detection, extraction...)."""
    assert experiment is not None, ("An Experiment instance needs to be "
        "provided in order to write the output.")

    # Create file logger for the experiment.
    LOGGER_FILE = create_file_logger(experiment.gen_filename('log'))

    # Get parameters from the PRM dictionary.
    chunk_size = prm.get('chunk_size', None)
    chunk_overlap = prm.get('chunk_overlap', 0)
    nchannels = prm.get('nchannels', None)

    # Ensure a RawDataReader is instantiated.
    if raw_data is not None:
        if not isinstance(raw_data, BaseRawDataReader):
            raw_data = read_raw(raw_data, nchannels=nchannels)
    else:
        raw_data = read_raw(experiment)

    # Log.
    if convert_only:
        info("Starting file conversion only. Klusta version {1:s}, on {0:s}".format((str(raw_data)), spikedetekt2.__version__))
        info("Running spike detection on a single chunk of spikes only, so as to have some information")
        first_chunk_detected = False # horrible hack - detects spikes on one chunk only so KV doesn't complain
    else:
        info("Starting SpikeDetekt version {1:s} on {0:s}".format((str(raw_data)), spikedetekt2.__version__))
    debug("Parameters: \n" + (display_params(prm)))

    # Get the bandpass filter.
    filter = bandpass_filter(**prm)

    if not (convert_only and first_chunk_detected):
        # Compute the strong threshold across excerpts uniformly scattered across the
        # whole recording.
        threshold = get_threshold(raw_data, filter=filter,
                                  channels=probe.channels, **prm)
        assert not np.isnan(threshold.weak).any()
        assert not np.isnan(threshold.strong).any()
        debug("Threshold: " + str(threshold))

        # Debug module.
        diagnostics_path = prm.get('diagnostics_path', None)
        if diagnostics_path:
            diagnostics_mod = _import_module(diagnostics_path)
            if not hasattr(diagnostics_mod, 'diagnostics'):
                raise ValueError("The diagnostics module must implement a "
                                 "'diagnostics()' function.")
            diagnostics_fun = diagnostics_mod.diagnostics
        else:
            diagnostics_fun = None

    # Progress bar.
    progress_bar = ProgressReporter(period=30.)
    nspikes = 0

    # Loop through all chunks with overlap.
    for chunk in raw_data.chunks(chunk_size=chunk_size,
                                 chunk_overlap=chunk_overlap,):
        # Log.
        debug("Processing chunk {0:s}...".format(chunk))

        nsamples = chunk.nsamples
        rec = chunk.recording
        nrecs = chunk.nrecordings
        s_end = chunk.s_end

        # Filter the (full) chunk.
        chunk_raw = chunk.data_chunk_full  # shape: (nsamples, nchannels)
        chunk_fil = apply_filter(chunk_raw, filter=filter)

        i = chunk.keep_start - chunk.s_start
        j = chunk.keep_end - chunk.s_start

        # Add the data to the KWD files.
        if prm.get('save_raw', False):
            # Do not append the raw data to the .kwd file if we're already reading
            # from the .kwd file.
            if not isinstance(raw_data, (KwdRawDataReader, ExperimentRawDataReader)):
                # Save raw data.
                experiment.recordings[chunk.recording].raw.append(convert_dtype(chunk.data_chunk_keep, np.int16))

        if prm.get('save_high', False):
            # Save high-pass filtered data: need to remove the overlapping
            # sections.
            chunk_fil_keep = chunk_fil[i:j,:]
            experiment.recordings[chunk.recording].high.append(convert_dtype(chunk_fil_keep, np.int16))

        if prm.get('save_low', True):
            # Save LFP.
            chunk_low = decimate(chunk_raw)
            chunk_low_keep = chunk_low[i//16:j//16,:]
            experiment.recordings[chunk.recording].low.append(convert_dtype(chunk_low_keep, np.int16))

        if not (convert_only and first_chunk_detected):
            # Apply thresholds.
            chunk_detect, chunk_threshold = apply_threshold(chunk_fil,
                threshold=threshold, **prm)

            # Remove dead channels.
            dead = np.setdiff1d(np.arange(nchannels), probe.channels)
            chunk_detect[:,dead] = 0
            chunk_threshold.strong[:,dead] = 0
            chunk_threshold.weak[:,dead] = 0

            # Find connected component (strong threshold). Return list of
            # Component instances.
            components = connected_components(
                chunk_strong=chunk_threshold.strong,
                chunk_weak=chunk_threshold.weak,
                probe_adjacency_list=probe.adjacency_list,
                chunk=chunk, **prm)

            # Now we extract the spike in each component.
            waveforms = extract_waveforms(chunk_detect=chunk_detect,
                threshold=threshold, chunk_fil=chunk_fil, chunk_raw=chunk_raw,
                probe=probe, components=components, **prm)

            # DEBUG module.
            # Execute the debug script.
            if diagnostics_fun:
                try:
                    diagnostics_fun(**locals())
                except Exception as e:
                    warn("The diagnostics module failed: " + e.message)

            # Log number of spikes in the chunk.
            nspikes += len(waveforms)

            # We sort waveforms by increasing order of fractional time.
            [add_waveform(experiment, waveform) for waveform in sorted(waveforms)]

            first_chunk_detected = True

        # Update the progress bar.
        progress_bar.update(rec/float(nrecs) + (float(s_end) / (nsamples*nrecs)),
            '%d spikes found.' % (nspikes))

        # DEBUG: keep only the first shank.
        if _debug:
            break

    # Feature extraction.
    save_features(experiment, **prm)

    close_file_logger(LOGGER_FILE)
    progress_bar.finish()


import requests
from datetime import datetime
import json
from config import API_KEY

class WeatherInfo:
    # Constructor for the class
    def __init__(self, cityName, lat = None, lon = None):
        # location info
        self.lat = lat
        self.lon = lon
        self.cityName = cityName

        # weather info
        self.timezone = ""
        self.tempC = ""
        self.tempF = ""
        self.feelsLikeC = ""
        self.feelsLikeF = ""
        self.pressure = ""
        self.humidity = ""
        self.windSpeed = ""
        self.weatherDescription = ""
        self.lastUpdate = None

    def GenerateRequest(self, type) -> str:
        # default setting
        limit = 1
        exclude = "minutely,hourly,daily,alerts"

        # default api call:
        # geocoding api: http://api.openweathermap.org/geo/1.0/direct?q={city name},{state code},{country code}&limit={limit}&appid={API key}
        # weather api: https://api.openweathermap.org/data/3.0/onecall?lat={lat}&lon={lon}&exclude={part}&appid={API key}

        if type == "geo":
            return f"http://api.openweathermap.org/geo/1.0/direct?q={self.cityName}&limit={limit}&appid={API_KEY}"
        else:
            return f"https://api.openweathermap.org/data/3.0/onecall?lat={self.lat}&lon={self.lon}&exclude={exclude}&appid={API_KEY}"

    # Getting weather info
    def GetInfo(self):
        if not self.lat or not self.lon:
            geoRequest = self.GenerateRequest("geo")
            r = requests.get(geoRequest)
            if r.status_code == 200:
                getjson = json.loads(r.text)
                self.lat =  getjson[0]["lat"]
                self.lon = getjson[0]["lon"]

        weatherRequest = self.GenerateRequest("weather")
        r = requests.get(weatherRequest)
        if r.status_code == 200:
            getjson = json.loads(r.text)
            self.timezone = getjson["timezone"]

            tempVal = getjson["current"]["temp"]
            self.tempC = self.TempConvert("c", tempVal)
            self.tempF = self.TempConvert("f", tempVal)

            feelsLikeVal = getjson["current"]["feels_like"]
            self.feelsLikeC = self.TempConvert("c", feelsLikeVal)
            self.feelsLikeF = self.TempConvert("f", feelsLikeVal)

            self.pressure = getjson["current"]["pressure"]
            self.humidity = getjson["current"]["humidity"]
            self.windSpeed = getjson["current"]["wind_speed"]
            self.weatherDescription = getjson["current"]["weather"][0]["description"]

    # This function mostly likely use for testing
    def PrintInfo(self) -> None:
        print(f"Location: {self.cityName}")
        print(f"Timezone: {self.timezone}")
        print("temp: {:0.2f}ºC, feels like: {:0.2f}ºC".format(self.tempC, self.feelsLikeC))
        print(f"Pressure: {self.pressure} hPA")
        print(f"Humidity: {self.humidity}%")
        print(f"Wind Speed: {self.windSpeed} m/s")
        print(f"Weather Description: {self.weatherDescription}")

    def TempConvert(self, type, temp) -> None:
        if type == "c":
            return round(temp - 273.15,2)
        elif type == "f":
            return round((temp - 273.15) * 9/5 + 32,2)


from django.conf import settings
from django.db import models
from django.contrib.auth.models import AbstractUser
from django.utils.translation import ugettext_lazy as _
from utilities.media import write_file
from utilities.media import remove_old_file
from utilities.image_base64 import encode_image, image_exists
from django.db import models

models.options.DEFAULT_NAMES += ('show_in_admin_menu',)

class CustomUser(AbstractUser):
    logo = models.ImageField(
            _('Logo'),
            upload_to='uploads/users/',
            null=True,
            blank=True
        )

    nick = models.CharField(
            _('NICK_LABEL'),
            max_length=255,
            unique=True,
            blank=True
        )

    autor_info = models.TextField(
            _('About You'),
            blank=True
        )
    def __unicode__(self):
        return '%s' % (self.username)

    def __init__(self, *args, **kwargs):
        super(CustomUser, self).__init__(*args, **kwargs)
        if self.logo is not None:
            self._prev_logo = self.logo

    class Meta:
        #swappable = 'AUTH_USER_MODEL'
        verbose_name = _('User')
        verbose_name_plural = _('Users')
        get_latest_by = "created"
        ordering = ('-id',)
        db_table = 'auth_custom_user'
        app_label = 'user'
        show_in_admin_menu = False

    def update_logo(self):
        instance = self
        if instance.logo is not None:
            write_file(instance.logo, instance.id)

            if len(instance._prev_logo.name)> 0:
                if instance._prev_logo.name != instance.logo.name:
                    remove_old_file(instance._prev_logo, instance.id)

            CustomUser.objects.filter(id=instance.id).update(
                    logo=instance.logo
                )

    def get_logo(self, width=38, height=38):
        image = self.logo.name
        logo = settings.STATIC_URL + "images/avatar-%s-%s.png" % (width,height)
        if len(image) and image_exists(image, self.id):
            logo = "data:image/png;base64," + encode_image( image, self.id, width, height)
        return logo

    def get_logo60x60(self,width=60,height=60):
        image = self.logo.name
        logo = settings.STATIC_URL + "img/coment.jpg" % (width,height)
        if len(image) and image_exists(image, self.id):
            logo = "data:image/png;base64," + encode_image( image, self.id, width, height)
        return logo
    def get_logo_comment(self):
        image = self.logo.name
        logo = settings.STATIC_URL + "img/coment.jpg"
        if len(image) and image_exists(image, self.id):
            logo = "data:image/png;base64," + encode_image( image, self.id, width, height)
        return logo


#version: 1.0
#author: Jan Kurpas
#
#
#Tool to create finding charts, with objects marked.
#Start script with:
#
#python fctool.py IMAGE OBJECTS WINDOWSIZE
#
#
#IMAGE: FITS IMAGE showing the sky
#OBJECTS: ASCII File, where the first row gives the plots title, and the following rows give the objects position and name. Write each object in one row, that shall be marked by giving RA DEC STRING
#The Image will be centered on the first object.
#WINDOWSIZE: number, giving the size of the skyarea that will be plotted in arcmin=(1/60 degree)
#FILENAME: string, telling the name of the output file.
#
#Additional options can be defined below
#
#

#Additional Options
cv = 0.99  #Set Cut for percentile
obj_sep = ' ' #Set Seperator in objectlist
#colmap = 'gray' #Set Colormap (black background, white stars: gray; white background, black stars: Greys)
colmap = 'Greys' #Set Colormap (black background, white stars: gray; white background, black stars: Greys)
nr_hdu = 0 # Fits File extension number, where Image is at.
axis_hms = True #Set False if axis labels should be in degree
#arcol = 'white' #Set color of compass
arcol = 'black' #Set color of compass

#-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------

import sys
#Janbrauchte die folgende Zeile, wofuer auch immer
# reload(sys)
# brauchen wir fue Iris' komische Symbole (Ref J Kurpas)
#sys.setdefaultencoding('utf8')
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import colors
from matplotlib.patches import Polygon
from matplotlib.collections import PatchCollection
from astropy.visualization import simple_norm
from astropy.visualization import SqrtStretch
from astropy.visualization import PercentileInterval
from astropy.io import fits
from astropy import wcs
from astropy.nddata import Cutout2D
from astropy.coordinates import Angle

#Function calculates Angular Distance on Sky
def ang_dist(x1,x2):
    return 180*np.arccos(np.sin(x1[1]*np.pi/180)*np.sin(x2[1]*np.pi/180)+np.cos(x1[1]*np.pi/180)*np.cos(x2[1]*np.pi/180)*np.cos((x1[0]-x2[0])*np.pi/180))/np.pi

# Function calculates Percentile value
def perc(Array,lim):
    #2d Array in 1d Array umwandeln und Grenzwert fuer Percentile finden
    Lst = Array.flatten()
    Lst.sort()
    l = len(Lst)
    n = int(round(lim*l))

    #Element, dass bei Grenze
    k = Lst[n]

    #Sicherstellen, dass Normierung durchgefuerht werden kann durch Typanpassung und durchfueren der Normierung, sowie des Ersetzens aller Werte ueber dem Grenzwert durch 1
    Array = np.asarray(Array, dtype=np.float64)
    Array = Array/k
    Array[Array>1] = 1

    #Array zurueckgeben, sowie den Grenzwert an der Perzentile
    return Array,k

#Function checks string for bad characters
def cl_str(x):
    if '\r' in x:
        return x.replace('\r','')
    else:
        return x

#Control whether enough parameters are given
if len(sys.argv) >1:
    inp = sys.argv[1:]
else:
    print('Not enough arguments given.')
    exit()
if len(inp)==1 and (inp[0] == '-h' or inp[0] == '--help'):
    print('Tool to create finding charts, with objects marked.\nStart script with:\n\npython fctool.py IMAGE OBJECTS WINDOWSIZE\n\nIMAGE: FITS image showing the sky\nOBJECTS: ASCII File, where the first row gives the plots title, and the following rows give the objects position and name. Write each object in one row, that shall be marked by giving RA DEC STRING\nThe Image will be centered on the first object.\nWINDOWSIZE: number, giving the size of the skyarea that will be plotted in arcmin=(1/60 degree)\nFILENAME: string, telling the name of the output file.\n\nAdditional options can be defined in the top part of the script.')
    exit()
elif len(inp) ==3 or len(inp) ==2 or (len(inp) ==1 and (inp[0] != '-h' and inp[0] != '--help')):
    print('Not enough arguments given.')
    exit()
    # (M.C.B): I erased "len(inp) ==3".
else:
    #Get parameters
    imlnk = inp[0]
    objfl = inp[1]
    wsz = float(inp[2])
    print('Start fctool\n')
    print('Image: '+imlnk)
    print('Objectfile: '+objfl)
    print('Image Size: '+str(wsz)+' arcmin\n')

    #Get Objectinformation
    info = open(objfl, 'r')
    t = info.read()
    info.close()
    t = t.split('\n')
    title = cl_str(t[0])
    obj = []
    for g in t[1:]:
        tmp = g.split(obj_sep)
        if len(g)>2:
            if (":" in tmp[0]):
                v = tmp[0].split(':')
                RA = Angle(v[0]+'h'+v[1]+'m'+v[2]+'s', unit= 'hourangle')
                DEC = Angle(tmp[1]+'d')
            else:
                RA = Angle(tmp[0]+'d')
                DEC = Angle(tmp[1]+'d')
            s = cl_str(tmp[2])
            obj.append([RA,DEC,s])

    #Get Image
    hdu = fits.open(imlnk)
    hdr = hdu[nr_hdu].header
    dat = hdu[nr_hdu].data
    w = wcs.WCS(hdr)

    print('')
    #Get Central Object Pixelposition:
    print('Central Object Coordinates [deg]: '+str(obj[0][0].deg)+', '+str(obj[0][1].deg))
    objpix_x,objpix_y=w.wcs_world2pix(obj[0][0].deg,obj[0][1].deg,0)
    print('objpix_x,objpix_y:',objpix_x,objpix_y)
    objcoox,obcooy = w.wcs_pix2world(objpix_x,objpix_y,0)

    #Round Object Pixelcoordinates to closest Integer
    print (type(objpix_x))
    xcoord = int(round(float(objpix_x),0))
    ycoord = int(round(float(objpix_y),0))

    #Get subpixel resol.
    spgx = objpix_x-xcoord
    spgy = objpix_y-ycoord

    #print('Central Object Pixel Coordinates: '+str(objpix_x)+', '+str(objpix_y))

    #Initialize Image
    fig = plt.figure()

    #Normierung und Skalierungvornehmen
    naxisx = hdr['NAXIS1']-1
    naxisy = hdr['NAXIS2']-1

    #Determine Pixelsize:
    tpix_x1,ydum1 = w.wcs_pix2world(0,0,0,ra_dec_order = True)
    tpix_x2,ydum2 = w.wcs_pix2world(1,0,0,ra_dec_order = True)
    dx1 = ang_dist([tpix_x1,ydum1],[tpix_x2,ydum2])
    tpix_x1,ydum1 = w.wcs_pix2world(naxisx-1,naxisy,0,ra_dec_order = True)
    tpix_x2,ydum2 = w.wcs_pix2world(naxisx,naxisy,0,ra_dec_order = True)
    dx2 = ang_dist([tpix_x1,ydum1],[tpix_x2,ydum2])
    dx = (dx1+dx2)/2

    xdum1,tpix_y1 = w.wcs_pix2world(0,0,0,ra_dec_order = True)
    xdum2,tpix_y2 = w.wcs_pix2world(0,1,0,ra_dec_order = True)
    dy1 = ang_dist([xdum1,tpix_y1],[xdum2,tpix_y2])
    xdum1,tpix_y1 = w.wcs_pix2world(naxisx,naxisy-1,0,ra_dec_order = True)
    xdum2,tpix_y2 = w.wcs_pix2world(naxisx,naxisy,0,ra_dec_order = True)
    dy2 = ang_dist([xdum1,tpix_y1],[xdum2,tpix_y2])
    dy = (dy1+dy2)/2

    dx = dx #Pixelsize in deg
    dy = dy #Pixelsize in deg
    nxpix = int(round((wsz)/(60*dx),0))
    nypix = int(round(wsz/(60*dy),0))

    if nxpix%2 == 1:
        nrmwx = nxpix/2+1
    else:
        nrmwx = nxpix/2

    if nypix%2 == 1:
        nrmwy = nypix/2+1
    else:
        nrmwy = nypix/2

    gr = [0,0,0,0]  # Array mit Grenzen der Normierung

    #x-Achse
    su = 0
    so = 0
    if xcoord - nrmwx < 0:
        gr[0] = 0
        su = abs(xcoord - nrmwx)
    else :
        gr[0] = xcoord-nrmwx

    gr[1] = xcoord+ nrmwx
    if gr[1] > naxisx:
        so = gr[1]-naxisx
        gr[1] = naxisx
        gr[0] = gr[0]-so
    else:
        gr[1] = gr[1]+su

    #y-Achse
    su = 0
    so = 0
    if ycoord - nrmwy < 0:
        gr[2] = 0
        su = abs(ycoord - nrmwy)
    else :
        gr[2] = ycoord-nrmwy

    gr[3] = ycoord+ nrmwy
    if gr[3] > naxisy:
        so = gr[3]-naxisy
        gr[3] = naxisy
        gr[2] = gr[2]-so
    else:
        gr[3] = gr[3]+su

    #Fall, dass Bild kleiner als eingegebene doppelte Normierungsbreite nrmwx/nrmwy
    if naxisx <= 2*nrmwx:
        gr[0] = 0
        gr[1] = naxisx
    if naxisy <= 2*nrmwy:
        gr[2] = 0
        gr[3] = naxisy

    stretch = SqrtStretch()
    dat = np.asarray(dat, dtype=np.float64)

    #Correct for not a number values, by setting them to -100
    dat[np.isnan(dat)] = -100

    #Trail Cut-Out to get shape of cutout area, in order to normalize right
    cutout = Cutout2D(dat, (xcoord,ycoord), (nypix,nxpix), wcs = w)
    spx = int(cutout.shape[1])/2 #Get x shape of cutout area
    spy = int(cutout.shape[0])/2 #Get y shape of cutout area

    gr[0] = int(gr[0])
    gr[1] = int(gr[1])
    gr[2] = int(gr[2])
    gr[3] = int(gr[3])
    print ('gr[0]:gr[1],gr[2]:gr[3]:',gr[0],gr[1],gr[2],gr[3])
    a,p1 = perc(dat[gr[0]:gr[1],gr[2]:gr[3]],cv)
#    dummyshow = plt.imshow(dat[xcoord-spx:xcoord+spx,ycoord-spy:ycoord+spy], norm = simple_norm(dat[gr[0]:gr[1],gr[2]:gr[3]], stretch = 'sqrt', min_cut=0, max_cut=p1), cmap=colmap)
    plt.clf()
    dat = dat/p1
    dat[dat>1] = 1
    dat = stretch(dat)

    #Bereich ausschneiden
    cutout = Cutout2D(dat, (xcoord,ycoord), (nypix,nxpix), wcs = w)

    #Koordinaten hinzufuegen
    ax = plt.subplot(projection = cutout.wcs)

    #Quellausdehnung berechnen:
    bx = cutout.input_position_cutout[0]+spgx #Get x position of central object in cutout
    by = cutout.input_position_cutout[1]+spgy #Get y position of central object in cutout
    spx = cutout.shape[1] #Get x shape of cutout area
    spy = cutout.shape[0] #Get y shape of cutout area
    mf = 3
    #ccol = '#009ee7'
    ccol = 'green'
    d = spx/80
    if spx/80 < 1:
        d = 1
    r1 = Polygon([[bx+d,by],[bx+mf*d,by]],closed= True, edgecolor = ccol,facecolor = 'none')
    r2 = Polygon([[bx-d,by],[bx-mf*d,by]],closed= True, edgecolor = ccol,facecolor = 'none')
    r3 = Polygon([[bx,by+d],[bx,by+mf*d]],closed= True, edgecolor = ccol,facecolor = 'none')
    r4 = Polygon([[bx,by-d],[bx,by-mf*d]],closed= True, edgecolor = ccol,facecolor = 'none')
    plt.text(bx+d,bx+d,obj[0][2],color =ccol, transform=ax.transData)

    for adob in obj[1:]:
        adobjpix_x,adobjpix_y=w.wcs_world2pix(adob[0].deg,adob[1].deg,0)
        adobjpix_x = adobjpix_x
        adobjpix_y = adobjpix_y
        xdif = adobjpix_x-objpix_x
        ydif = adobjpix_y-objpix_y
        aobx = xdif+ bx
        aoby = ydif+ by
        lwd = 0.8
        col = 'red'
        daob = d
        if aobx>=0 and aoby >= 0 and aobx < spx+5 and aoby < spy+5:
            ax.add_patch(Polygon([[aobx+daob,aoby],[aobx+mf*daob,aoby]],closed= True, edgecolor = col,facecolor = 'none',ls = '-'))#, lw = wsz*lwd))
            ax.add_patch(Polygon([[aobx-daob,aoby],[aobx-mf*daob,aoby]],closed= True, edgecolor = col,facecolor = 'none',ls = '-'))#, lw = wsz*lwd))
            ax.add_patch(Polygon([[aobx,aoby-daob],[aobx,aoby-mf*daob]],closed= True, edgecolor = col,facecolor = 'none',ls = '-'))#, lw = wsz*lwd))
            ax.add_patch(Polygon([[aobx,aoby+daob],[aobx,aoby+mf*daob]],closed= True, edgecolor = col,facecolor = 'none',ls = '-'))#, lw = wsz*lwd))
            plt.text(aobx+daob,aoby+daob,adob[2],color = col, transform=ax.transData)
        print('Plot Star: '+adob[2]+', at '+str(adob[0].deg)+', '+str(adob[1].deg))

    #Koordinateneinstellungen vornehmen
    ax.add_patch(r1)
    ax.add_patch(r2)
    ax.add_patch(r3)
    ax.add_patch(r4)

    if axis_hms == True:
        ax.coords[0].set_major_formatter('hh:mm:ss')
        ax.coords[1].set_major_formatter('dd:mm:ss')
    else:
        ax.coords[0].set_major_formatter('d.ddd')
        ax.coords[1].set_major_formatter('d.ddd')

    ax.coords[0].set_axislabel('Right Ascension')
    ax.coords[1].set_axislabel('Declination')

    #Bild erstellen
    plt.imshow(cutout.data , cmap = colmap, origin = 'lower',vmin = 0, vmax = 1)

    #Weitere anzuzeigende Objekte Plotten
    plt.title(title)

    #Koordinatensystem
    xk = int(round(spx*0.975))
    yk = int(round(spy*0.9))
    txk1 = int(round(spx*0.9))
    txk2 = int(round(spx*0.86))
    tyk1 = int(round(spy*0.925))
    tyk2 = int(round(spy*0.95))

    q = int(round(spx*0.05))
    plt.arrow(xk,yk,(-1)*q,0,head_width=spx/50, head_length=spx/35, fc='white', ec=arcol) #x-Richtung
    plt.text(txk1,txk2,'E',color = arcol)

    plt.arrow(xk,yk,0,q,head_width=spx/50, head_length=spx/35, fc='white', ec=arcol) #Y-Richtung
    plt.text(tyk1,tyk2,'N',color = arcol)

    #Bild speichern (format with duch file extension festgelegt)
    plt.savefig(inp[3], dpi='figure', bbox_inches = 'tight')
    plt.close()
    hdu.close()
    print('Done')
    # (M.C.B): I changed inp[3] to inp[2].


from ..schemas import AudioWordsSchema, AudioImageSchema
from . import audio, MethodView, abort, basepath
from ..models import Word, User
from .. import db
from sqlalchemy.exc import IntegrityError
from flask_jwt_extended import jwt_required, get_jwt_identity
from flask import current_app
import base64, os
from werkzeug.utils import secure_filename

def convert_base64(num):

    # 选择要转换的图片
    selected_image = "record_imgs/" + str(num) + ".png"

    # 打开文件并将其转换为Base64编码
    with open(selected_image, "rb") as image_file:
        encoded_string = base64.b64encode(image_file.read()).decode("utf-8")

    # 将Base64编码添加到data URI字符串中
    data_uri = f"data:image/png;base64,{encoded_string}"

    # 打印Base64编码的data URI字符串
    return data_uri

@audio.route("/word")
class Words(MethodView):
    @audio.arguments(AudioWordsSchema)
    @jwt_required(fresh=True)
    def post(self, word_data):
        user = User.query.filter_by(id=get_jwt_identity()).first()
        if user.role_id != 1:
            return {"status": 403, "message": "权限不足"}, 403
        current_app.logger.info(word_data)

        word = Word(
            stem=word_data["stem"], meaning=word_data["meaning"]
        )

        if "lang" in word_data:
            word.lang = word_data["lang"]
        if "translation" in word_data:
            word.translation = word_data["translation"]
        if "prompt" in word_data:
            word.prompt = word_data["prompt"]

        try:
            db.session.add(word)
            db.session.commit()

        except IntegrityError as e:
            # 数据库出错回滚
            db.session.rollback()
            current_app.logger.error(e)
            abort(422, message="已存在该词汇")

        except Exception as e:
            # 数据库出错回滚
            db.session.rollback()
            current_app.logger.error(e)
            abort(404, message="数据库查询异常")

        return {"status": 201, "message": "录入成功"}, 201

@audio.route("/word/<word_id>/image")
class UploadImage(MethodView):
    @audio.arguments(AudioImageSchema)
    @jwt_required(fresh=True)
    def post(self, image_data, word_id):
        user = User.query.filter_by(id=get_jwt_identity()).first()
        if user.role_id != 1:
            return {"status": 403, "message": "权限不足"}, 403
        try:
            word = Word.query.filter_by(id=word_id).first()
        except Exception as e:
            return {"status": 404, "message": "未找到该词汇"}, 404
        try:
            imageData = base64.b64decode(image_data["image_file"])
            upload_path = os.path.join(
                basepath,
                "record_imgs",
                secure_filename(str(word.id) + ".png"),
            )
            with open(upload_path, "wb") as out_image:
                out_image.write(imageData)
            word.image_path = upload_path
        except Exception as e:
            current_app.logger.error(e)
            return {"status": 400, "message": "未成功上传图片文件"}, 400
        try:
            db.session.add(word)
            db.session.commit()  # SQLAlchemy用
        except Exception as e:
            # 数据库出错回滚
            db.session.rollback()
            current_app.logger.error(e)
            os.remove(upload_path)
            return {"status": 404, "message": "数据库查询异常"}, 404

        return {"status": 201,"message": "ok"}

@audio.route("/word/<word_id>")
class WordById(MethodView):
    def get(self, word_id):
        word = Word.query.filter_by(id=word_id).first()
        return {
            "id": word.id,
            "stem": word.stem,
            "stemtranslation": word.translation,
            # "img": convert_base64(word.id),
            "desc": word.meaning,
            "prompt": word.prompt,
        }


import cobble.command
import cobble.validations
import discord
import databaseManager as dbm
from customcommands.customvalidations import *

import neatTables

class ILBoardCommand(cobble.command.Command):
    def __init__(self, bot: cobble.bot.Bot):
        """
        Parameters:
            bot - The bot object the command will belong to
        """
        super().__init__(bot, "IL Leaderboard", "ilboard", "Show the leaderboard for a given IL", cobble.permissions.EVERYONE)
        self.addArgument(cobble.command.Argument("map", "The IL map", IsMap()))
        self.addArgument(cobble.command.Argument("category", "The IL category", IsILCategory()))
        self.addArgument(cobble.command.Argument("start", "Where to start from", cobble.validations.IsInteger(), True))

    async def execute(self, messageObject: discord.message, argumentValues: dict, attachedFiles: dict) -> str:
        """
        Generate a leaderboard for the given category
        Parameters:
            argumentValues - a dictionary containing values for every argument provided, keyed to the argument name
        """
        if "start" in argumentValues.keys():
            start = int(argumentValues["start"])-1
        else:
            start = 0

        if argumentValues["map"] in dbm.levelNames.values():
            argumentValues["map"] = dbm.getMapFromLevelName(argumentValues["map"])

        lbData = dbm.generateILBoard(argumentValues["map"], argumentValues["category"])[start:start+20]
        tableData = [["Place", "Runner", "Time"]]
        for entry in lbData:
            place = durations.formatLeaderBoardPosition(dbm.fetchILPlace(entry[0], argumentValues["category"]))
            runner = entry[1]
            time = entry[2]
            tableData.append([place, runner, time])

        table = "```"+neatTables.generateTable(tableData, padding=2)+"```"

        return table


# -*- coding: utf-8 -*-

import boto3
import json
import os
import subprocess
import sys
import urllib.parse

from base64 import b64decode
from boto3.dynamodb.conditions import Key, Attr
from datetime import datetime
from datetime import timedelta

ENCRYPTED = os.environ['ENCRYPTED_GOOGLE_API_KEY']
os.environ['GOOGLE_API_KEY'] = boto3.client('kms').decrypt(CiphertextBlob=b64decode(ENCRYPTED))['Plaintext'].decode(
    'utf-8')
ENCRYPTED = os.environ['ENCRYPTED_GOOGLE_APP_ID']
os.environ['GOOGLE_APP_ID'] = boto3.client('kms').decrypt(CiphertextBlob=b64decode(ENCRYPTED))['Plaintext'].decode(
    'utf-8')

def _(cmd):
    return subprocess.check_output(cmd.split())

def lambda_handler(event, context):
    start = datetime.now()

    dynamodb = boto3.resource('dynamodb')
    table = dynamodb.Table(os.environ['TABLE_NAME'])
    title = event['queryStringParameters']['title']

    histories = table.query(
        KeyConditionExpression=Key('title').eq(title),
        ScanIndexForward=False
    )

    if histories['Count'] == 0 or \
            ('cache' in event['queryStringParameters'] and
             event['queryStringParameters']['cache'] == 'no'):
        os.environ['REQUEST_URI'] = 'info_json.sh?title=' \
                                    + urllib.parse.quote_plus(title, safe='+',
                                                              encoding='utf-8')
        json_string = ""
        try:
            json_string = _('bash info_json.sh')
            result = json.loads(json_string)
        except Exception as e:
            print("exception:", e)
            print("string:", json_string)
            raise
    elif histories['Count'] > 0 and 'result' in histories['Items'][0]:
        result = json.loads(histories['Items'][0]['result'])
    else:
        result = {}

    completed = datetime.now()
    timestamp = start.strftime('%Y/%m/%d %H:%M:%S') + '.%03d' % (start.microsecond // 1000)
    history = {
        'title': title,
        'timestamp': timestamp,
        'completed': completed.strftime('%Y/%m/%d %H:%M:%S') + '.%03d' % (completed.microsecond // 1000),
        'event': json.dumps(event),
        'result': json.dumps(result),
        'is_error': 'error' in result,
        'ttl': int((completed + timedelta(days=31)).timestamp())
    }
    table.put_item(Item=history)

    return {
        'isBase64Encoded': False,
        'statusCode': 200 if ('error' not in result) else 403,
        'headers': {'Access-Control-Allow-Origin': '*'},
        'body': json.dumps(result)
    }


import base64
from datetime import datetime
from typing import Optional

import discord
from discord.ext import commands

from configuration import Config

config = Config()

class Encuesta(commands.Cog):
    def __init__(self, bot):
        self.bot = bot

    # FIXME: Here the problem is that with app_commands  I couldn't
    # figure out the option of having "*option" (former implementation),
    # but for slash commands. Tried to look for "multiple optional commands"
    # but I couldn't find anything.
    @commands.hybrid_command(name="encuesta", help="Comando de encuesta", pass_context=True)
    async def encuesta(
        self,
        ctx,
        pregunta: str,
        option_0: Optional[str] = "",
        option_1: Optional[str] = "",
        option_2: Optional[str] = "",
        option_3: Optional[str] = "",
        option_4: Optional[str] = "",
        option_5: Optional[str] = "",
        option_6: Optional[str] = "",
        option_7: Optional[str] = "",
        option_8: Optional[str] = "",
        option_9: Optional[str] = "",
    ):
        def get_number_emoji(n):
            if 0 <= n < 10:
                return f"{n}\N{COMBINING ENCLOSING KEYCAP}"
            else:
                return n

        if not pregunta:
            e = self.get_help_error()
            await ctx.send(embed=e)
            return

        options = [
            option_0,
            option_1,
            option_2,
            option_3,
            option_4,
            option_5,
            option_6,
            option_7,
            option_8,
            option_9,
        ]

        # Log
        with open(str(config.log_file), "a") as f:
            # date;command;message_id;channel;author;message
            message = " ".join(options).encode("utf-8")
            message_enc = base64.b64encode(message)
            line = (
                f'"{datetime.now()}";'
                f'"encuesta";'
                f'"{ctx.message.id}";'
                f'"{ctx.channel.name}";'
                f'"{ctx.message.author.id}";'
                f'"{ctx.message.author}";'
                f'"{message_enc}"\n'
            )
            f.write(line)

        head = f"**Encuesta \N{BAR CHART}**\n" f"Pregunta: **{pregunta}**\n\n"
        foot = (
            "\n\n¡Participa utilizando las reacciones al mensaje \N{HAPPY PERSON RAISING ONE HAND}!"
        )

        # Errors
        n = sum(1 for o in options if o)

        # Only one arg, assumes a Yes/No question
        if n == 0:
            si_emoji = "\N{WHITE HEAVY CHECK MARK}"
            no_emoji = "\N{CROSS MARK}"

            msg = await ctx.send(f"{head}{si_emoji} Sí\n{no_emoji} No{foot}")

            await msg.add_reaction(si_emoji)
            await msg.add_reaction(no_emoji)

        # Error, because one question, with one option makes no sense.
        elif n == 1:
            await ctx.send("\N{NO ENTRY} No es posible tener solo una opción por pregunta.")
            return
        else:
            # Collapse options
            options = [o for o in options if o]

            body = "\n".join(f"{get_number_emoji(i)} {options[i]}" for i in range(n))
            msg = await ctx.send(f"{head}{body}{foot}")

            for i in range(1, n):
                await msg.add_reaction(get_number_emoji(i - 1))

        try:
            await discord.Message.delete(ctx.message)
        except discord.NotFound:
            print("Using slash command, no need to remove original message")

    def get_help_error(self):
        embed = discord.Embed(
            title="\N{NO ENTRY} No se encontró ningún argumento.\n\n",
            description="**¿Cómo realizar encuestas?**",
            colour=0x2B597B,
        )
        embed.add_field(
            name="Para preguntas de Sí y No",
            value='`%encuesta "pregunta"`',
            inline=False,
        )
        embed.add_field(
            name="Para preguntas de varias opciones",
            value='`%encuesta "pregunta" "opción 1" "opción 2" ...`',
            inline=False,
        )
        return embed


from bs4 import BeautifulSoup

with open("index.html") as file:
    html_code = file.read()

soup = BeautifulSoup(html_code, "html.parser")
# Udělá list všech odkazů
all_a = soup.find_all("a")
print(all_a)

for one_a in all_a:
    # nebo místo string napsat gettext
    # print(one_a.string)

    # Printuje jen odkazy
    print(one_a.get("href"))


from typing import NamedTuple

import flax
import jax
import jax.numpy as jnp
import jax.tree_util as jtu
from flax.core.frozen_dict import FrozenDict

import energy
import models

class MetaModule:
    def __init__(self):
        pass

    def __call__(self, meta_state, x, method=None):
        pass

    def reset_hparams(self, rng, sample_input):
        pass

    def reset_params(self, rng, hparams, sample_input):
        pass

class ComplexSynapseMetaParams(NamedTuple):
    omega: FrozenDict
    log_lambda: FrozenDict

class ComplexSynapseParams(NamedTuple):
    base_learner: FrozenDict

class ComplexSynapse(MetaModule):
    def __init__(self, loss_fn_inner, loss_fn_outer, base_learner, l2_reg=None, variant="imaml"):
        self.loss_fn_inner = loss_fn_inner
        self.loss_fn_outer = loss_fn_outer

        self.base_learner = base_learner
        self.l2_reg = l2_reg
        self.variant = variant

        if variant == "complex_synapse":
            assert l2_reg is not None
            self.loss_fn_inner = energy.add(self.loss_fn_inner, energy.complex_synapse(l2_reg))

        elif variant == "imaml":
            assert l2_reg is not None
            self.loss_fn_inner = energy.add(self.loss_fn_inner, energy.imaml(l2_reg))

        elif variant == "init":
            self.loss_fn_inner = self.loss_fn_inner

        elif variant == "l2_reg":
            self.loss_fn_inner = energy.add(self.loss_fn_inner, energy.l2_learned())

        else:
            raise ValueError("Variant \"{}\" not defined.".format(variant))

    def __call__(self, params, hparams, x, method=None):
        return self.base_learner.apply({"params": params.base_learner}, x, method=method)

    def reset_hparams(self, rng, sample_input):
        hparams_omega = self.base_learner.init(rng, sample_input)["params"]

        if self.variant in ["complex_synapse", "l2_reg"]:
            hparams_log_lambda = jtu.tree_map(
                lambda x: jnp.log(self.l2_reg) * jnp.ones_like(x), hparams_omega
            )
        else:
            hparams_log_lambda = FrozenDict()

        return ComplexSynapseMetaParams(hparams_omega, hparams_log_lambda)

    def reset_params(self, rng, hparams, sample_input):
        return ComplexSynapseParams(hparams.omega)

class GainModMetaParams(NamedTuple):
    body: FrozenDict
    head_init: FrozenDict
    bias_init: FrozenDict
    gain_init: FrozenDict

class GainModParams(NamedTuple):
    head: FrozenDict
    bias: FrozenDict
    gain: FrozenDict

class GainMod(MetaModule):
    def __init__(self, loss_fn_inner, loss_fn_outer, hidden_dims, output_dim):
        self.loss_fn_inner = loss_fn_inner
        self.loss_fn_outer = loss_fn_outer

        self.hidden_dims = hidden_dims
        self.output_dim = output_dim

        self.body = models.FeedforwardNetwork(hidden_dims)
        self.head = flax.linen.Dense(output_dim)

    def __call__(self, params, hparams, x):

        features = self.body.apply(
            {"params": hparams.body}, x, params.bias, params.gain,
            method=self.body.forward_modulated
        )

        return self.head.apply({"params": params.head}, features)

    def reset_hparams(self, rng, sample_input):
        rng_body, rng_head = jax.random.split(rng, 2)

        hparams_body = self.body.init(rng_body, sample_input)["params"]
        hparams_head = self.head.init(
            rng_head, jnp.empty((sample_input.shape[0], self.hidden_dims[0]))
        )["params"]

        hparams_bias = FrozenDict({
            l: jnp.zeros(h_dim) for l, h_dim in enumerate(self.hidden_dims)
        })
        hparams_gain = FrozenDict({
            l: jnp.ones(h_dim) for l, h_dim in enumerate(self.hidden_dims)
        })

        return GainModMetaParams(hparams_body, hparams_head, hparams_bias, hparams_gain)

    def reset_params(self, rng, hparams, sample_input):
        params_head = hparams.head_init

        return GainModParams(params_head, hparams.bias_init, hparams.gain_init)

class AlmostNoInnerLoopMetaParams(NamedTuple):
    body: FrozenDict
    head_init: FrozenDict

class AlmostNoInnerLoopParams(NamedTuple):
    head: FrozenDict

class AlmostNoInnerLoop(MetaModule):
    def __init__(self, loss_fn_inner, loss_fn_outer, hidden_dims, output_dim):
        self.loss_fn_inner = loss_fn_inner
        self.loss_fn_outer = loss_fn_outer

        self.hidden_dims = hidden_dims
        self.output_dim = output_dim

        self.body = models.FeedforwardNetwork(hidden_dims)
        self.head = flax.linen.Dense(output_dim)

    def __call__(self, params, hparams, x):

        features = self.body.apply({"params": hparams.body}, x)

        return self.head.apply({"params": params.head}, features)

    def reset_hparams(self, rng, sample_input):
        hparams_body = self.body.init(rng, sample_input)["params"]
        hparams_head = self.head.init(rng, jnp.empty((1, self.hidden_dims[0])))["params"]

        return AlmostNoInnerLoopMetaParams(hparams_body, hparams_head)

    def reset_params(self, rng, hparams, sample_input):
        params_head = hparams.head_init

        return AlmostNoInnerLoopParams(params_head)


from PySide2.QtCore import Qt, QPointF, QMimeData, QAbstractListModel, QModelIndex
from PySide2.QtWidgets import QListView
from sortedcontainers import SortedDict

from nfb_studio.serial import mime, hooks

class Toolbox(QAbstractListModel):
    """A list of scheme nodes that can be dragged to the scheme.
    """

    DragMimeType = "application/x-toolbox-drag"
    """Toolbox assumes that all its items have the same MIME type."""

    def __init__(self, parent=None):
        super().__init__(parent)

        self._items = SortedDict()

    def getView(self):
        """Get a new QListView suitable for displaying the toolbox."""
        v = QListView()
        v.setModel(self)

        v.setSelectionMode(v.SingleSelection)
        v.setDragEnabled(True)
        v.setDragDropMode(v.DragOnly)

        return v

    def rowCount(self, parent=QModelIndex()):
        return len(self._items)

    def data(self, index: QModelIndex, role=Qt.DisplayRole):
        i = index.row()

        if role == Qt.DisplayRole:
            return self._items.peekitem(i)[0]
        return None

    def headerData(self, section, orientation: Qt.Orientation, role=Qt.DisplayRole):
        if role == Qt.DisplayRole:
            if orientation == Qt.Vertical:
                return section
            return "Node toolbox"
        return None

    def addItem(self, name, item):
        i = self._items.bisect_right(name)

        self.beginInsertRows(QModelIndex(), i, i)
        self._items[name] = item
        self.endInsertRows()

    def removeItem(self, name):
        """Remove an item with a specified name from a toolbox.
        If an item with such a name does not exist, does nothing.
        """
        if name not in self._items:
            return None

        i = self._items.index(name)

        self.beginRemoveRows(QModelIndex(), i, i)
        item = self._items.pop(name)
        self.endRemoveRows()

        return item

    def flags(self, index: QModelIndex):
        default_flags = super().flags(index)

        if index.isValid():
            return default_flags | Qt.ItemIsDragEnabled
        return default_flags

    def mimeTypes(self):
        return [self.DragMimeType]

    def mimeData(self, indexes):
        assert len(indexes) == 1
        i = indexes[0].row()

        package = QMimeData()
        mime.dump(self._items.peekitem(i)[1], package, self.DragMimeType, hooks=hooks.qt)

        return package

    def schemeDropEvent(self, scheme, event):
        """Event to be executed when an item from this toolbox gets dropped into a scheme.
        This function can be added as a custom drop event for this toolbox's MIME type, which is usually done by the
        SchemeEditor.
        """
        package = event.mimeData()
        node = mime.load(package, self.DragMimeType, hooks=hooks.qt)

        pos = event.scenePos() - QPointF(
            node.boundingRect().size().width()/2,
            node.boundingRect().size().height()/2
        )
        node.setPos(pos)
        scheme.addItem(node)

    # Serialization ====================================================================================================
    def serialize(self) -> dict:
        return self._items

    @classmethod
    def deserialize(cls, data: dict):
        obj = cls()
        obj._items = data

        return obj


import pytest

from broker.tasks.cloudfront import (
    create_distribution,
    disable_distribution,
    wait_for_distribution_disabled,
    delete_distribution,
)
from broker.extensions import db
from broker.models import CDNServiceInstance, Operation

from tests.lib import factories

@pytest.fixture
def service_instance():
    service_instance = factories.CDNServiceInstanceFactory.create(
        id="1234",
        domain_names=["example.com", "foo.com"],
        domain_internal="fake1234.cloudfront.net",
        route53_alias_hosted_zone="Z2FDTNDATAQYW2",
        cloudfront_distribution_id="FakeDistributionId",
        cloudfront_origin_hostname="origin_hostname",
        cloudfront_origin_path="origin_path",
        origin_protocol_policy="https-only",
        forwarded_headers=["HOST"],
    )
    new_cert = factories.CertificateFactory.create(
        service_instance=service_instance,
        private_key_pem="SOMEPRIVATEKEY",
        iam_server_certificate_id="certificate_id",
        id=1002,
    )
    current_cert = factories.CertificateFactory.create(
        service_instance=service_instance,
        private_key_pem="SOMEPRIVATEKEY",
        iam_server_certificate_id="certificate_id",
        id=1001,
    )
    factories.ChallengeFactory.create(
        domain="example.com",
        validation_contents="example txt",
        certificate_id=1001,
        answered=True,
    )
    factories.ChallengeFactory.create(
        domain="foo.com",
        validation_contents="foo txt",
        certificate_id=1001,
        answered=True,
    )
    factories.ChallengeFactory.create(
        domain="example.com",
        validation_contents="example txt",
        certificate_id=1002,
        answered=False,
    )
    factories.ChallengeFactory.create(
        domain="foo.com",
        validation_contents="foo txt",
        certificate_id=1002,
        answered=False,
    )
    service_instance.current_certificate = current_cert
    service_instance.new_certificate = new_cert
    db.session.add(service_instance)
    db.session.add(current_cert)
    db.session.add(new_cert)
    db.session.commit()
    db.session.expunge_all()
    return service_instance

@pytest.fixture
def provision_operation(service_instance):
    operation = factories.OperationFactory.create(
        id=4321, service_instance=service_instance
    )
    return operation

def test_create_distribution_existing_distribution(
    clean_db, service_instance, cloudfront, provision_operation
):
    cloudfront.expect_get_distribution(
        caller_reference="asdf",
        domains=service_instance.domain_names,
        certificate_id=service_instance.new_certificate.iam_server_certificate_id,
        origin_hostname=service_instance.cloudfront_origin_hostname,
        origin_path=service_instance.cloudfront_origin_path,
        distribution_id=service_instance.cloudfront_distribution_id,
        status="InProcess",
    )
    create_distribution.call_local(provision_operation.id)

def test_create_distribution_referencing_nonexistent_distribution(
    clean_db, service_instance, cloudfront, provision_operation
):
    cloudfront.expect_get_distribution_returning_no_such_distribution(
        distribution_id=service_instance.cloudfront_distribution_id
    )
    cloudfront.expect_create_distribution(
        caller_reference=service_instance.id,
        domains=service_instance.domain_names,
        certificate_id=service_instance.new_certificate.iam_server_certificate_id,
        origin_hostname=service_instance.cloudfront_origin_hostname,
        origin_path=service_instance.cloudfront_origin_path,
        distribution_id="FakeDistributionId",
        distribution_hostname="fake1234.cloudfront.net",
        bucket_prefix="1234/",
    )

    create_distribution.call_local(provision_operation.id)

    db.session.expunge_all()
    service_instance = CDNServiceInstance.query.get("1234")

    assert service_instance.cloudfront_distribution_arn
    assert service_instance.cloudfront_distribution_arn.startswith("arn:aws:cloudfront")
    assert service_instance.cloudfront_distribution_arn.endswith("FakeDistributionId")
    assert service_instance.cloudfront_distribution_id == "FakeDistributionId"
    assert service_instance.domain_internal == "fake1234.cloudfront.net"

def test_deprovision_skips_disabling_when_no_cloudfront_distribution_present(
    clean_db, cloudfront
):
    service_instance = factories.CDNServiceInstanceFactory.create(
        id="1233",
        domain_names=["example.com", "foo.com"],
        domain_internal="fake1233.cloudfront.net",
        route53_alias_hosted_zone="Z2FDTNDATAQYW2",
    )
    operation = factories.OperationFactory.create(
        id="5677",
        state=Operation.States.IN_PROGRESS.value,
        action=Operation.Actions.DEPROVISION.value,
    )
    db.session.commit()
    db.session.expunge_all()

    # try these things - it'll blow up if we try to call cloudfront because we've stubbed boto
    disable_distribution.call_local("5677")
    wait_for_distribution_disabled.call_local("5677")
    delete_distribution.call_local("5677")


import sys
import numpy as np
import json

import pandas as pd
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

def existsOR( data, keys ):
  ret = False
  for k in keys:
    ret = ret or (k in data)
  return ret

def existsAND( data, keys ):
  ret = True
  for k in keys:
    ret = ret and (k in data)
  return ret

def sub( series, i ):
  return [ v[i] for v in series.values[:-1] ]

kIsCartesian = False
lbl_xyz = ['x','y','z']
lbl_quat = ['qx','qy','qz', 'qw']
ln_color = ['r', 'g', 'b', 'k']

plt.ion()

argc = len( sys.argv )
if argc < 2:
  raise RuntimeError( "Expects an argument: path to json file to plot.")

fp = sys.argv[1]
print( "Will plot {}".format( fp ) )

f = open( fp )
f_str = f.read()
f_str = f_str.replace( "nan", "\"NaN\"" )
json_data = json.loads( f_str )

data = pd.DataFrame.from_dict( json_data['data'] )

kIsCartesian = False
if 'type' in json_data:
  kIsCartesian = ( json_data['type'].lower().startswith('cart') )
if 'x' in data and 'dx' in data:
  kIsCartesian = False
  if len( data['x'].values[0] ) == 7 and len( data['dx'].values[0] ) == 6:
    kIsCartesian = True

print( "{}".format( "CARTESIAN" if kIsCartesian else "VECTOR" ) )

t = data['time'].values[:-1]

if existsOR( data, ['s', 'ds', 'dds'] ):
  f, axarr = plt.subplots(3)
  if 's' in data:
    axarr[0].plot( t, data['s'].values[:-1] )
    axarr[0].set_title( "s" )
  if 'ds' in data:
    axarr[1].plot( t, data['ds'].values[:-1] )
    axarr[1].set_title( "ds" )
  if 'dds' in data:
    axarr[2].plot( t, data['dds'].values[:-1] )
    axarr[2].set_title( "dds" )

if existsOR( data, ['seg_idx',] ):
  f, axarr = plt.subplots(1)
  axarr.plot( t, data['seg_idx'].values[:-1] )
  axarr.set_title( "segment index" )

if kIsCartesian:
  suffixes = []
  if existsOR( data, ['x', 'dx', 'ddx'] ):
    suffixes.append( ('','-') )
  if existsOR( data, ['x_target', 'dx_target', 'ddx_target'] ):
    suffixes.append( ('_target',':') )
  ft, axarrt = plt.subplots(1,2)
  f, axarr = plt.subplots(2,2)
  for (suffix, ln) in suffixes:
    ax_i = 0
    ax_j = 0
    for i in range( 3 ):
      axarrt[ax_i+0].plot(t, sub( data['x'+suffix], i ), ln+ln_color[i], label="dof={}".format( lbl_xyz[i]+suffix ) )
    axarrt[ax_i+0].set_title( "position" )
    axarrt[ax_i+0].legend()
    for i in range( 4 ):
      axarrt[ax_i+1].plot(t, sub( data['x'+suffix], 3+i ), ln+ln_color[i], label="dof={}".format( lbl_quat[i]+suffix ) )
    axarrt[ax_i+1].set_title( "orientation" )
    axarrt[ax_i+1].legend()
    for (vec, name) in [( data['dx'+suffix], 'velocity' ), ( data['ddx'+suffix], 'acceleration' ) ]:
      for i in range( 3 ):
        axarr[ax_i+0, ax_j].plot(t, sub( vec, i ), ln+ln_color[i], label="dof={}".format( lbl_xyz[i]+suffix ) )
      axarr[ax_i+0, ax_j].set_title( "linear " + name )
      axarr[ax_i+0, ax_j].legend()
      for i in range( 3 ):
        axarr[ax_i+1, ax_j].plot(t, sub( vec, 3+i ), ln+ln_color[i], label="dof={}".format( lbl_xyz[i]+suffix ) )
      axarr[ax_i+1, ax_j].set_title( "angular " + name )
      axarr[ax_i+1, ax_j].legend()
      ax_j += 1

    # 3D plot
    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    ax.plot( sub( data['x'], 0 ), sub( data['x'], 1 ), sub( data['x'], 2 ) )
    ax.set_xlabel('X')
    ax.set_ylabel('Y')
    ax.set_zlabel('Z')
    ax.set_title('3D position')

else: # NOT cartesian
  suffixes = []
  if existsOR( data, ['x', 'dx', 'ddx'] ):
    suffixes.append( ('','-') )
  if existsOR( data, ['x_target', 'dx_target', 'ddx_target'] ):
    suffixes.append( ('_target',':') )
  f, axarr = plt.subplots(1,3)
  for (suffix, ln) in suffixes:
    ax_j = 0
    for (vec, name) in [( data['x'+suffix], 'position' ), ( data['dx'+suffix], 'velocity' ), ( data['ddx'+suffix], 'acceleration' ) ]:
      for i in range( len(vec.values[0]) ):
        axarr[ax_j].plot(t, sub( vec, i ), label="dof={}".format( lbl_xyz[i] ) )
      axarr[ax_j].set_title( name )
      axarr[ax_j].legend()
      ax_j += 1

plt.show()

try:
  input("Enter to quit.")
except:
  pass


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from utils.base_solver import BaseSolver

import os, logging, importlib, re, copy, random, tqdm, argparse
import os.path as osp
import cPickle as pickle
import numpy as np
from collections import defaultdict

import tensorflow as tf
import tensorflow.contrib.slim as slim
import torch

from utils import config as cfg
from utils import dataset, utils
from utils.evaluator import Multi_Evaluator

from run_symnet_multi import make_parser
from test_symnet import SolverWrapper as BaseSolverWrapper

def main():
    logger = logging.getLogger('MAIN')

    parser = make_parser()
    args = parser.parse_args()
    args.network = 'symnet_multi'
    utils.display_args(args, logger)
    logger.info("Loading dataset")

    test_dataloader = dataset.get_dataloader(args.data, 'test',
        batchsize=args.test_bz, args=args)

    logger.info("Loading network and solver")
    network = importlib.import_module('network.'+args.network)
    net = network.Network(test_dataloader, args)

    with utils.create_session() as sess:
        sw = SolverWrapper(net, test_dataloader, args)
        sw.trainval_model(sess, args.epoch)

class SolverWrapper(BaseSolverWrapper):

    def trainval_model(self, sess, max_epoch):
        logger = self.logger('test_model')
        logger.info('Begin testing')

        score_op, train_summary_op = self.construct_graph(sess)
        self.initialize(sess)
        sess.graph.finalize()

        evaluator = Multi_Evaluator()
        all_attr = []
        all_pred = defaultdict(list)

        for image_ind, batch in tqdm.tqdm(enumerate(self.test_dataloader), total=len(self.test_dataloader), postfix='test'):

            predictions = self.network.test_step(sess, batch, score_op)
            all_attr.append(torch.from_numpy(batch[1]))
            for key in score_op.keys():
                all_pred[key].append(predictions[key])

        for name, pred in all_pred.items():
            mAP, mAUC = evaluator(
                np.concatenate(pred,0),
                np.concatenate(all_attr,0)
            )
            report_dict = {
                'mAP':          mAP,
                'mAUC':         mAUC,
                'name':         self.args.name,
                'epoch':        self.args.epoch,
            }

            print("%s: "%(name) + utils.formated_multi_result(report_dict))
        logger.info('Finished.')

if __name__=="__main__":
    main()


import matplotlib.pyplot as plt
import numpy as np

x = np.arange(0, 1,0.00001)
y = np.sqrt(1 - x**2)

plt.plot(x, y,'y')
plt.plot(x, -y,'y')
plt.gca().set_aspect('equal')
plt.show()


#!/usr/bin/env python3

import os
import logging
import numpy as np

def sigmoid(x, derivative=False):
    return x*(1-x) if derivative else 1/(1+np.exp(-x))

def _get_pipe_model_name(pipe, name):
    for n, m in pipe.steps:
        if n == name:
            return m
    return None

def _get_pipe_model_idx(pipe, idx):
    return pipe.steps[idx][1]

def get_pipe_model(pipe, indicator):
    return _get_pipe_model_idx(pipe, indicator) if isinstance(indicator, int) else \
           _get_pipe_model_name(pipe, indicator)

def get_pipe_transform(pipe, x):
    for name, model in pipe.steps[:-1]:
        x = model.transform(x)
    return x

def print_pipe_model(model):
    if model is None:
        return None

    str = ''
    for step in model.steps:
        str += "%s: %s\n" % step;
    return str;

def which_set(sets, k):
    nset = len(sets)
    for i in range(nset):
        if k in sets[i]:
            return i
    return nset

def split_dict(d, *key_sets):
    ret = [{} for _ in range(len(key_sets) + 1)]

    for k, v in d.items():
        ret[which_set(key_sets, k)][k] = v

    return ret

# search file with its path matching @path_prefix
# if more than one candicate, throw exception
# return:
#    path, suffix
def search_path(path_prefix):
    if os.path.exists(path_prefix):
        return path_prefix, ''

    dir = os.path.dirname(path_prefix)
    prefix = os.path.basename(path_prefix)

    candicates = [f for f in os.listdir(dir) if f.startswith(prefix) ]
    if len(candicates) > 1:
        raise Exception("more than one file under '%s' with filename prefix '%s'" % (dir, prefix))
    elif len(candicates) == 0:
        return None, None
    else:
        path = os.path.join(dir, candicates[0])
        suffix = candicates[0][len(prefix):]
        return path, suffix

# return matched range [begin, end)
def find_seq(seq, subseq, is_eql=None, to_skip=None, begin=0, end=None):
    if to_skip is None:
        to_skip = lambda item: False

    if is_eql is None:
        is_eql = lambda x1, x2: x1 == x2

    if end is None:
        end = len(seq)

    len_subseq = len(subseq)

    for idx in range(begin, end):
        p, q = idx, 0
        while p < end and q < len_subseq:
            if p != idx and to_skip(seq[p]):  #skip stopword if not the frist.
                p += 1
            elif not is_eql(seq[p], subseq[q]):
                break
            else:
                p, q = p+1, q+1

        if q == len_subseq:
            return idx, p

    return -1, -1;

def rfind_seq(seq, subseq, is_eql=None, to_skip=None, begin=0, end=None):
    if to_skip is None:
        to_skip = lambda item: False

    if is_eql is None:
        is_eql = lambda x1, x2: x1 == x2

    if end is None:
        end = len(seq)

    len_subseq = len(subseq)

    for idx in range(end-1, begin-1, -1):
        p, q = idx, len_subseq -1
        while p >= 0 and q >= 0:
            if p != idx and to_skip(seq[p]):  #skip stopword if not the frist.
                p -= 1
            elif not is_eql(seq[p], subseq[q]):
                break
            else:
                p, q = p-1, q-1

        if q == -1:
            return p+1, idx+1

    return -1, -1;

def set_logging(verbose=0):
    log_level = logging.DEBUG   if verbose >= 2 else \
                logging.INFO    if verbose == 1 else \
                logging.WARNING if verbose == 0 else \
                logging.ERROR
    logging.basicConfig(level=log_level,
            format="%(asctime)s.%(msecs)03d [%(levelname)s] [%(module)s] %(message)s", datefmt="%H:%M:%S")

def assert_eq(expected, val):
    if expected != val:
        raise RuntimeError("Assert equal error: Expected '{}' is not equal to '{}'".format(expected, val))

def test_search_path():
    def to_test(path_prefix):
        path, suffix = search_path(path_prefix)
        print("path_prefix: %s ->  path: %s, suffix: %s" % (path_prefix, path, suffix))

    to_test('.model/knn')
    to_test('.model/knn/AssertFail')
    to_test('.model/knn/BadPrefix')
    to_test('/etc/app')

def test_find_seq():
    seq = 'Buying food on the street is nothing new but in the UK this idea is really taking off Its a great way of sampling freshly cooked dishes from around the world Rob and Neil discuss the subject and hear from an expert who explains the popularity in this type of food plus you can learn nothing new vocabulary along the way'.split()

    def to_skip(w):
        return w in {'on', 'the', 'is', 'in', 'a', 'an', 'of', 'from', 'and'}

    assert_eq((1, 7), find_seq(seq, 'food street nothing'.split(), None, to_skip))
    assert_eq((1, 7), find_seq(seq, 'food on the street is nothing'.split()))
    assert_eq((38, 44), find_seq(seq, 'hear expert who explains'.split(), None, to_skip))

    assert_eq((1, 7), rfind_seq(seq, 'food street nothing'.split(), None, to_skip))
    assert_eq((1, 7), rfind_seq(seq, 'food on the street is nothing'.split()))
    assert_eq((38, 44), rfind_seq(seq, 'hear expert who explains'.split(), None, to_skip))

    assert_eq((6, 8), find_seq(seq, 'nothing new'.split(), None, to_skip))
    assert_eq((6, 8), find_seq(seq, 'nothing new'.split(), None, to_skip, 6))
    assert_eq((55, 57), find_seq(seq, 'nothing new'.split(), None, to_skip, 7))

    assert_eq((55, 57), rfind_seq(seq, 'nothing new'.split(), None, to_skip))
    assert_eq((6, 8), rfind_seq(seq, 'nothing new'.split(), None, to_skip, 56))

if __name__  == '__main__':
     test_find_seq()


# "x.py" as in "execute"
import os
import sys
import subprocess
import requests
from datetime import datetime, timedelta
from threading import Thread, Event

token = "{{ token }}"
logs_url = "{{ logs_url }}"
heartbeat_url = "{{ heartbeat_url }}"

class HeartbeatThread(Thread):
    def __init__(self, event):
        Thread.__init__(self)
        self.stopped = event

    def run(self):
        while not self.stopped.wait(10):
            requests.get(heartbeat_url, headers={"Authorization": token})

stop_flag = Event()
thread = HeartbeatThread(stop_flag)
thread.start()

process = subprocess.Popen(["sh", "wrapper.sh"], stdout=subprocess.PIPE)

last_post = datetime.utcnow()
five_seconds = timedelta(seconds=5)
log_lines = b""

for line in iter(process.stdout.readline, b""):

    log_lines += line

    if datetime.utcnow() - last_post > five_seconds:
        last_post = datetime.utcnow()
        requests.post(logs_url, headers={"Authorization": token}, data=log_lines)
        log_lines = b""

requests.post(logs_url, headers={"Authorization": token}, data=log_lines)
stop_flag.set()