import numpy as np
import matplotlib
import matplotlib.pyplot as plt
from keras.models import Sequential
from keras.optimizers import Adam
from keras.callbacks import ModelCheckpoint
from keras.models import load_model
from keras.layers import Lambda,Conv2D,MaxPooling2D,Dropout,Dense,Flatten,Activation

from helper import get_class_names,get_train_data,get_test_data,plot_images,plot_model

matplotlib.style.use('ggplot')

#import class names
class_names = get_class_names()
#print(class_names)
num_classes = len(class_names)
#print(num_classes)

#height and width of the images
IMAGE_SIZE=32
#3 channels RGB
CHANNELS = 3

#Fetch and decode Data
#Load the training dataset, labels are integers whereas class is one-hot vectors

images_train, labels_train, class_train = get_train_data()
#print('Labels_Train: ',labels_train)
#print('Class_Train: ',class_train)

images_test, labels_test, class_test = get_test_data()
#print('Training set size:\t',len(images_train))
#print('Testing set size:\t',len(images_test))

def cnn_model():
    model = Sequential()

    model.add(Conv2D(32,(3,3), activation='relu', padding='same', input_shape=(IMAGE_SIZE,IMAGE_SIZE,CHANNELS)))
    model.add(Conv2D(32,(3,3), activation='relu'))
    model.add(MaxPooling2D(pool_size=(2,2)))
    model.add(Dropout(0.25))

    model.add(Conv2D(64,(3,3), activation='relu', padding='same'))
    model.add(Conv2D(64,(3,3), activation='relu'))
    model.add(MaxPooling2D(pool_size=(2,2)))
    model.add(Dropout(0.25))

    model.add(Flatten())

    model.add(Dense(512, activation='relu'))
    model.add(Dropout(0.5))

    model.add(Dense(num_classes, activation='softmax'))
    model.save('cifar10_cnn_model')

    model.summary()

    return model

model = cnn_model()
#print(model)

checkpoint = ModelCheckpoint('best_model_simple.h5', #model filename
                            monitor='val_loss', #quantity to monitor
                            verbose=0,#verbosity = 0 or 1
                            save_best_only=True,#The latest best model will not be overwritten
                            mode='auto')#The decision to overwrite model is made)
                                    #automatically depending on the quantity to monitor

model.compile(loss='categorical_crossentropy',#Better loss function for neural networs
            optimizer=Adam(lr=1.0e-4),#Adam optimizer with 1.0e-4 learning rate
            metrics=['accuracy'])#Metrics to be evaluted by model

model_details = model.fit(images_train,class_train,
                          batch_size=128,#number of sample per gradient update
                          epochs=10,#number of iterations
                          validation_data=(images_test,class_test),
                          callbacks=[checkpoint],
                          verbose=1)
#print(model_details)

scores = model.evaluate(images_test,class_test,verbose=0)
#print('Accuracy: %.2f%%'%(scores[1]*100))

#plot_model(model_details)

class_pred = model.predict(images_test,batch_size=32)
#print(class_pred[0])

labels_pred = np.argmax(class_pred,axis=1)
#print(labels_pred)

correct = (labels_pred==labels_test)
#print(correct)
#print('Number of currect prediction: %d'%sum(correct))

num_images = len(correct)
#print('Accurecy: %.2f%%'%((sum(currect)*100)/num_images))

incorrect = (correct == False)
#Images of the test-set that have been incorrectly classified.
images_error = images_test[incorrect]
#Get predicted classes for those images
labels_error = labels_pred[incorrect]
#Get true classes for those images.
labels_true = labels_test[incorrect]

prediction = model.predict(images_test)

def plot_images(i,prediction_array,true_label,img):
	prediction_array, true_label, img = prediction_array[i], true_label[i], img[i]
	plt.xticks([])
	plt.yticks([])
	plt.imshow(img, cmap = plt.cm.binary)
	predicted_label = np.argmax(prediction_array)
	if predicted_label == true_label:
		color = 'blue'
	else:
		color = 'red'

	plt.xlabel('{} {:2.0f}% ({})'.format(class_names[predicted_label],
				    100*np.max(prediction_array),
				    class_names[true_label],
                                    color = color))

def plot_value_array(i,prediction_array,true_label):
	prediction_array, true_label = prediction_array[i], true_label[i]
	plt.xticks([])
	plt.yticks([])
	thisplot = plt.bar(range(10), prediction_array, color = '#777777' )
	plt.ylim([0,1])
	predicted_label = np.argmax(prediction_array)
	thisplot[predicted_label].set_color('red')
	thisplot[true_label].set_color('blue')

num_rows = 5
num_cols = 3
num_images = num_rows * num_cols
plt.figure(figsize=(2*2*num_cols,2*num_rows))
for i in range(num_images):
	plt.subplot(num_rows, num_cols*2, 2*i+1)
	plot_images(i,prediction,labels_test, images_test)
	plt.subplot(num_rows,num_cols*2, 2*i+2)
	plot_value_array(i,prediction,labels_test)
plt.show()

'''plot_images(images=images_error[0:9],
            labels_true=labels_true[0:9],
            class_names=class_names,
            labels_pred=labels_error[0:9])'''


import numpy as np
from collections import Counter

###################### IMPACT AND LEADING PERSISTENT USERS ######################
def temporal_highimpact_impacts( edgelists, num_highimpact_users, target='target', weight='weight'):
    """Return an array with the leading impact vector over time as well as the array of leading users.

    Inputs:
        - edgelists: array of edgelists for each time with the number of retweets from user i to user j.
    """

    temporal_highimpact_impact_array = []
    temporal_highimpact_users_array = []
    for edgelist in edgelists:

        leading_impact_vector = get_leading_impact_vector( edgelist, num_highimpact_users, target=target, weight=weight )

        temporal_highimpact_impact_array.append( leading_impact_vector )
        temporal_highimpact_users_array.append( set( leading_impact_vector.index ) )

    return temporal_highimpact_impact_array, temporal_highimpact_users_array

def temporal_leading_impacts(edgelists, num_highimpact_users, num_persistent_users, target='target', weight='weight'):
    """ Return impact vector for leading (N) persistent (M) users in edgelist.

    If num_highimpact_users:float in [0,1], it is treated as a percentage. If num_leading_users:int > 0, it is treated as absolute number of users.

    Inputs:
        - edgelists: array of edgelists for each time with the number of retweets from user i to user j.
        - num_highimpact_users: number of high-impact users (N in the paper).
        - num_persistent_users: number of leading users considered as persistent (M in the paper).
    Outputs:
        - persistent_impact_vec: array of impact vectors of the leading persistent users
        - persistent_users_vec: array of the leading persistent users
        - users_persistence: persistence of the leading users
    """

    leading_impact_vec, leading_users_vec = temporal_highimpact_impacts( edgelists, num_highimpact_users, target=target, weight=weight )

    users_persistence = get_users_persistence( leading_users_vec )
    users_persistence = users_persistence.most_common( num_persistent_users )

    persistent_users = get_users_from_users_persistence_dict( users_persistence )

    persistent_impact_vec = []
    persistent_users_vec  = []
    for leading_impact in leading_impact_vec:
        persistent_impacts = leading_impact[ leading_impact.index.isin( persistent_users ) ]

        persistent_impact_vec.append( persistent_impacts  )
        persistent_users_vec.append( set( persistent_impacts.index )  )

    return persistent_impact_vec, persistent_users_vec, users_persistence

def get_impact_vector(edgelist, target='target', weight='weight'):
    """ Returns impact vector for all users in edgelist.
    """
    return edgelist.groupby( target ).sum().sort_values( weight )

# In the article, N = 50 leading weekly users, M = 50 of persistent users
def get_leading_impact_vector(edgelist, num_leading_users, target='target', weight='weight'):
    """ Return impact vector for leading (N) persistent (M) users in edgelist.
    If num_leading_users:float in [0,1], it is treated as a percentage. If num_leading_users:int > 0, it is treated as absolute number of users.
    """

    impact_vec = get_impact_vector( edgelist, target=target, weight=weight )

    if 0 < num_leading_users < 1:

        ## Option 1: based on percentage of retweets
        impact_cum_dist = impact_vec[ weight ].cumsum()
        impact_cum_dist /= impact_cum_dist.iloc[-1]

        return impact_vec[ impact_cum_dist > num_leading_users]

        ## Option 2: based on percentage of population
        # population_cum_dist = pd.Series(range(1 , len(impact_vec)+1), index=impact_vec.index)
        # population_cum_dist /= population_cum_dist.iloc[-1]

        # return impact_vec[ population_cum_dist > num_leading_users]

    elif (type(num_leading_users) == int) & (num_leading_users > 0):

        return impact_vec.iloc[ -num_leading_users: ]

    else:
        print('Some error for {}'.format(num_leading_users))

def get_users_persistence(users_array):#, num_users=None):
    '''Return a dictionary of all the users in `influential_users:list[set]` as well as their persistence.
    '''

    users_persistence = np.array([])

    # append all users into a single array
    for users in users_array:

        users_persistence = np.append(users_persistence, np.array(list(users)))

    users_persistence = Counter( users_persistence )
    users_persistence = {k: v for k, v in sorted(users_persistence.items(), key=lambda item: item[1], reverse=True)}

    return Counter(users_persistence)

def get_users_from_users_persistence_dict( users_persistence ):
    return [ users for (users, persistence) in users_persistence ]

# TODO: audiences:array[dict], chambers:array[dict], array of overlap matrices, chambers/audiences networks

###################### CHAMBERS & AUDIENCES ######################

def get_audience( user, edgelist, source='source', target='target' ):
    '''Returns the audience:set of the user given an edgelist.
    '''
    return set( edgelist[ edgelist[ target ] == user ][ source ].unique() )

def get_chamber( user, edgelist, users_excluded=False, source='source', target='target'):
    """Returns the chamber:set of the user given an edgelist.

    users_excluded can be either False or a list of users to exclude from chamber.
    """

    audience = get_audience( user, edgelist, source=source, target=target )
    audience_out_ego_network = edgelist[ edgelist[ source ].isin( audience ) ]
    # remove the leading user from the audience ego network
    audience_out_ego_network = audience_out_ego_network[ audience_out_ego_network[ target ] != user ]

    # exclude specific users from chamber.
    if users_excluded != False:
        audience_out_ego_network = audience_out_ego_network[ ~audience_out_ego_network[ target ].isin( users_excluded ) ]

    chamber = set( audience_out_ego_network[ target ].unique() )
    return chamber

def get_chamber_from_audience(user, edgelist, audience, users_excluded=False, source='source', target='target'):
    """Returns the chamber:set of the user given an edgelist and a given audience.

    users_excluded can be either False or a list of users to exclude from chamber.
    """

    audience_out_ego_network = edgelist[ edgelist[ source ].isin( audience ) ]
    # remove the leading user from the audience ego network
    audience_out_ego_network = audience_out_ego_network[ audience_out_ego_network[ target ] != user ]

    # exclude specific users from chamber.
    if users_excluded != False:
        audience_out_ego_network = audience_out_ego_network[ ~audience_out_ego_network[ target ].isin( users_excluded ) ]

    chamber = set( audience_out_ego_network[ target ].unique() )
    return chamber

def get_audience_edgelist( user, edgelist, source='source', target='target' ):
    '''Builds audience network of leading user `user`.
    '''

    audience = get_audience( user, edgelist, source, target )

    audience_edgelist = get_edgelist_from_audience( audience, edgelist, source, target )
    return audience_edgelist

def get_chamber_edgelist( user, edgelist, users_excluded=False, source='source', target='target'):
    '''Builds chamber network of leading user `user`.
    '''

    chamber = get_chamber( user, edgelist, users_excluded, source, target )

    chamber_edgelist = get_edgelist_from_chamber( chamber, edgelist, source, target )
    return chamber_edgelist

# helper
def get_edgelist_from_chamber(chamber, edgelist, source='source', target='target'):
    chamber_edgelist = edgelist[ (edgelist[ source ].isin( chamber )) & (edgelist[ target ].isin( chamber )) ]
    return chamber_edgelist

def get_edgelist_from_audience(audience, edgelist, source='source', target='target'):
    audience_edgelist = edgelist[ (edgelist[ source ].isin( audience )) & (edgelist[ target ].isin( audience )) ]
    return audience_edgelist

def get_chambers_of_users( users, edgelist, users_excluded=False, source='source', target='target', return_network=False ):
    """Get the chamber of all the users in `users`.

    users_excluded can be {False, list:str (list of global excluded users), list:list:str (list of excluded users per week)}
    """

    chambers_dict = {}
    chamber_networks_dict = {}
    for i, user in enumerate(users):

        chamber = get_chamber( user, edgelist, users_excluded=users_excluded, source=source, target=target )
        chambers_dict[user] = chamber

        if return_network:
            chamber_edgelist = get_edgelist_from_chamber( chamber, edgelist, source, target )
            chamber_networks_dict[user] = chamber_edgelist

    if return_network:
        return chambers_dict, chamber_networks_dict
    else:
        return chambers_dict

def get_audiences_of_users( users, edgelist, source='source', target='target', return_network=False ):
    """Get the audience of all the users in `users`.

    users_excluded can be {False, list:str (list of global excluded users), list:list:str (list of excluded users per week)}
    """

    audiences_dict = {}
    audience_networks_dict = {}
    for i, user in enumerate(users):

        audience = get_audience( user, edgelist, source=source, target=target )
        audiences_dict[user] = audience

        if return_network:
            audience_edgelist = get_edgelist_from_audience( audience, edgelist, source, target )
            audience_networks_dict[user] = audience_edgelist

    if return_network:
        return audiences_dict, audience_networks_dict
    else:
        return audiences_dict

def temporal_chambers(users, edgelists, users_excluded=False, source='source', target='target', return_networks=False):
    """Get the chambers of all the users in users for every edgelist in edgelists. It is assumed that edgelists are ordered temporally.
    """

    # preallocation
    temporal_chambers_vec = [{}] * len(edgelists)
    temporal_chamber_networks_vec = [{}] * len(edgelists)

    # check if there is a list of excluded users. If there is, check if it's a temporal (leading) or a static (persistent) list
    list_of_lists = False
    if users_excluded != False:
        list_of_lists = (type(users_excluded[0]) == list) | (type(users_excluded[0]) == set)

    for (t,edgelist) in enumerate(edgelists):

        if return_networks:
            if list_of_lists:
                chamber_per_user_dict, chamber_network_per_user_dict = get_chambers_of_users( users[t], edgelist, users_excluded[t], source, target, return_networks )
            else:
                chamber_per_user_dict, chamber_network_per_user_dict = get_chambers_of_users( users[t], edgelist, users_excluded, source, target, return_networks )

            # append chamber networks for time t
            temporal_chamber_networks_vec[t] = chamber_network_per_user_dict

        else:
            if list_of_lists:
                chamber_per_user_dict = get_chambers_of_users( users[t], edgelist, users_excluded[t], source, target, return_networks )
            else:
                chamber_per_user_dict = get_chambers_of_users( users[t], edgelist, users_excluded, source, target, return_networks )

        # append chambers for time t
        temporal_chambers_vec[t] = chamber_per_user_dict

    if return_networks:
        return temporal_chambers_vec, temporal_chamber_networks_vec
    else:
        return temporal_chambers_vec

def temporal_audiences( users, edgelists, source='source', target='target', return_networks=False ):
    """Get the audiences of all the users in users for every edgelist in edgelists. It is assumed that edgelists are ordered temporally.
    """

    # preallocation
    temporal_audiences_vec = [{}] * len(edgelists)
    temporal_audience_networks_vec = [{}] * len(edgelists)

    for (t,edgelist) in enumerate(edgelists):

        if return_networks:
            audience_per_user_dict, audience_network_per_user_dict = get_audiences_of_users( users[t], edgelist, source, target, return_networks )
            # append audience networks for time t
            temporal_audience_networks_vec[t] = audience_network_per_user_dict

        else:
            audience_per_user_dict = get_audiences_of_users( users[t], edgelist, source, target, return_networks )

        # append audiences for time t
        temporal_audiences_vec[t] = audience_per_user_dict

    if return_networks:
        return temporal_audiences_vec, temporal_audience_networks_vec
    else:
        return temporal_audiences_vec

# helper
def get_chamber_from_audience(user, edgelist, audience, users_excluded=False, source='source', target='target'):
    """Returns the chamber:set of the user given an edgelist and a given audience.

    users_excluded can be either False or a list of users to exclude from chamber.
    """

    audience_out_ego_network = edgelist[ edgelist[ source ].isin( audience ) ]
    # remove the leading user from the audience ego network
    audience_out_ego_network = audience_out_ego_network[ audience_out_ego_network[ target ] != user ]

    # exclude specific users from chamber.
    if users_excluded != False:
        audience_out_ego_network = audience_out_ego_network[ ~audience_out_ego_network[ target ].isin( users_excluded ) ]

    chamber = set( audience_out_ego_network[ target ].unique() )
    return chamber


from flask import Blueprint, render_template, session, render_template_string, redirect, url_for
from app.models import UserSimple
from app.models import User, Role

from app import db

from flask_security import Security, SQLAlchemyUserDatastore, auth_required, hash_password, login_required

user_datastore = SQLAlchemyUserDatastore(db, User, Role)

from flask_security.models import fsqla_v3 as fsqla

from flask_login import current_user

users_bp = Blueprint('users_bp', __name__)

@users_bp.route('/app/users/getall')
def getallusers():
    users = UserSimple.get_users()
    return render_template('users_simple.html', users=users)

@users_bp.route('/app/users/setsession')
def setsession():
    userid = session['_user_id']
    all_ret = User.get_user(userid)
    email_ret = all_ret.email
    session['useremail'] = all_ret.email
    return render_template_string('hello {{ what }}', what=email_ret)

@users_bp.route('/app/users/getusername')
def getusername():
    userid = session['_user_id']
    username_full = User.get_username(userid)
    return render_template_string('hello {{ what }}', what=userid)

@users_bp.route('/app/user/removeself', methods=['GET', 'POST'])
@login_required
def remove():
    userid = session['_user_id']
    delete_user = User.query.filter_by(fs_uniquifier=userid).first()
    db.session.delete(delete_user)
    db.session.commit()
    return redirect(url_for('home_bp.home'))


from django.conf.urls import url

from FeeApp import views

urlpatterns = [
    url(r'^fee_list/', views.fee_list, name='fee_list'),
    url(r'^fee_modi/', views.fee_modi, name='fee_modi'),
    url(r'^fee_detail/', views.fee_detail, name='fee_detail'),
    url(r'^fee_add/', views.fee_add, name='fee_add'),
    url(r'^get_cost_list/', views.AllCostView.as_view(), name='get_cost_list'),
    url(r'^get_cost/', views.CostView.as_view(), name='get_cost'),
    url(r'^add_cost/', views.add_cost, name='add_cost'),
    url(r'^update_to_cost_status/', views.update_to_cost_status, name='update_to_cost_status'),
    url(r'^update_to_cost/', views.update_to_cost, name='update_to_cost'),
    url(r'^delete_to_cost/', views.delete_to_cost, name='delete_to_cost'),
]


import os

''' TF_CPP_MIN_LOG_LEVEL
0 = all messages are logged (default behavior)
1 = INFO messages are not printed
2 = INFO and WARNING messages are not printed
3 = INFO, WARNING, and ERROR messages are not printed
'''
'''
print('backend :', keras.backend.backend())
print('keras version :', keras.__version__)
'''
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

import io
import pandas as pd
import numpy
import scipy
import shutil
import datetime
import pxpLibreriaDataFrame as pd_pxp
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import OneHotEncoder
from keras.models import Sequential
from keras.layers import Dense
from contextlib import redirect_stdout
from pickle import dump

logPrograma = []
logPrograma += [pd_pxp.registroMomento("*) Iniciando programa de entrenamiento", True)]

##################################################################
# Parametrizo
archivoGuardaModelo = "Diabetes_Modelo.json"
archivoGuardaPesos = "Diabetes_Peso.h5"
archivoGuardaTransformaciones = "Diabetes_Transformaciones.pkl"
archivoInfoEntrenamiento = "Diabetes_InfoEntrenamiento.txt"
archivoDeEntrenamiento = "DiabetesColor_ParaEntrenamientoCompleto.csv"  # "DiabetesColor_ParaEntrenamiento.csv"
archivoPrefijoPlot = "Diabetes_graph_"
columnaObjetivo='aparicion'
mostrarGraficos = False
mostrarMejoresCols = False
mostrarGraficoMejoresCols = False
kantidadColumnasBuscadas = 1
split_dataframe_para_test = 0.20
epocas = 20
lote = 5
split_validacion_fit_size = 0.20
##################################################################
# Parametrizo columnas
column_trans = ColumnTransformer(
    [('Colo', OneHotEncoder(dtype='int', handle_unknown='ignore'), ['COLOR'])],
    remainder='passthrough')

##################################################################
# Levanto los datos
logPrograma += [pd_pxp.registroMomento("*) Levantando datos de " + archivoDeEntrenamiento, True)]
df_leida = pd.read_csv(archivoDeEntrenamiento, sep=';')

# df = pd.read_csv("pima-indians-diabetesHEADANEXO.csv", sep=';')
if mostrarGraficoMejoresCols:
    pd_pxp.ExplorarDatosPrevios(df_leida)

##################################################################
logPrograma += [pd_pxp.registroMomento("*) Transformando Columnas ", True)]
df_leida.rename(columns={columnaObjetivo:'objetivo'}, inplace=True)

coltra = column_trans.fit(df_leida)
transformacion = column_trans.transform(df_leida)
# La transformacion puede devolver distintas cosas... no uso pd.DataFrame.sparse.from_spmatrix por que funciona pero adentro queda sparce igual
if type(transformacion) == scipy.sparse.csr.csr_matrix:
    transformacion=transformacion.toarray()
df_transformada = pd.DataFrame(transformacion, columns=coltra.get_feature_names())

# # # salta columnas # # #n
# df_transformada=df_leida.drop(["COLOR"], axis=1)
##############################################################
logPrograma += [pd_pxp.registroMomento("*) Iniciando Split de dataframes ", True)]
train, test = train_test_split(df_transformada, test_size=split_dataframe_para_test, random_state=42, shuffle=True)

# PREPARO ENTRENAMIENTO
train_entrada = pd.DataFrame(train).drop(["objetivo"], axis=1)  # Todos menos lo que voy a predecir
train_salida = pd.DataFrame(train['objetivo'])  # Lo que voy a predecir

# PREPARO TEST
test_entrada = pd.DataFrame(test).drop(["objetivo"], axis=1)  # Todos menos lo que voy a predecir
test_salida = pd.DataFrame(test['objetivo'])  # Lo que voy a predecir

# Mostrar Mejores Columnas
if mostrarMejoresCols:
    pd_pxp.MostrarMejoresColumnas(df_leida, train_entrada, train_salida, kantidadColumnasBuscadas)
    cero = input("\nPresionar 0 para terminar")
    if cero == '0':
        print("Chau!")
        quit()

# Verifico Variabilidad
analisisVariabilidad = ''
sets = (('train_salida', train_salida), ('test_salida', test_salida))
for set_name, set_data in sets:
    analisisVariabilidad = analisisVariabilidad + '-' * 10 + ' ' + set_name + ' ' + '-' * 10 + "\n"
    analisisVariabilidad = analisisVariabilidad + set_data['objetivo'].value_counts().to_string() + "\n"
print(analisisVariabilidad)

# Entrenar y Fit
numpy.random.seed(7)
q_dim_entradas = train_entrada.columns.size

# crea el modelo
model = Sequential()
model.add(Dense(12, input_dim=q_dim_entradas, activation='relu'))
# model.add(Dropout(0.4))
model.add(Dense(8, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
# sigmoid  capa de salida, 1 neurona, Espero 1 salida: cero o uno. Recibe todas las salidas de la capa anterior
# por eso es densa. Pero debe devolver un numero entre cero y uno.

logPrograma += [pd_pxp.registroMomento("*) Compilando Modelo ", True)]
# Compila el modelo: Internamente armo el grafo.
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
# adam es la mas completa de las funciones de ajustar pesos (adam: variante de pesos por gradiente)
# funcion de perdida: binary_crossentropy ideal para salidas entre cero y uno.
# metrics: lo que va imprimiendo de salida.

logPrograma += [pd_pxp.registroMomento("*) Creando Modelo ", True)]
# Ajusta el modelo
history = model.fit(x=train_entrada, y=train_salida, epochs=epocas, validation_split=split_validacion_fit_size,
                    batch_size=lote,
                    shuffle=True)  # 72.64 #72.80

# batch_size - cada cuanto reajusto los pesos?
# epochs - cuantas veces recorro los datos ajustando pesos?

logPrograma += [pd_pxp.registroMomento("*) Evaluando Modelo ", True)]
####################################################################
#  EVALUACION : Incluye evaluate y .predict con identico dataset TEST
scoresTrain = model.evaluate(test_entrada, test_salida)
test_predecido = model.predict(test_entrada)
confundida = confusion_matrix(test_salida, test_predecido.round())
confundidaPxp = pd_pxp.matrizConfusionPxp(confundida)

logPrograma += [pd_pxp.registroMomento("*) Guardando Modelo ", True)]
#####################################################################
# Guardar resultado del entrenamiento
model_json = model.to_json()
with open(archivoGuardaModelo, "w") as json_file:
    json_file.write(model_json)
# serializar los pesos a HDF5
model.save_weights(archivoGuardaPesos)

#SACAR DESDE ACA
# guardo las transformaciones
dump(coltra, open(archivoGuardaTransformaciones, "wb"))
#SACARE HASTA ACA

##################################################################################
logPrograma += [pd_pxp.registroMomento("*) Armando resultados finales ", True)]
resultadoContraTest = "\nEvaluacion contra grupo de test :\n" + \
                      "     {metrica} : {valormetrica:.2f}% \n" + \
                      "     {metrica0} : {valormetrica0:.2f}%\n"
resultadoContraTest = resultadoContraTest.format(metrica=model.metrics_names[1],
                                                 valormetrica=scoresTrain[1] * 100,
                                                 metrica0=model.metrics_names[0],
                                                 valormetrica0=scoresTrain[0] * 100)

resultadoEvaluacion = confundidaPxp.obtenerResultadoEvaluacion()

fileMem = io.StringIO()
with redirect_stdout(fileMem):
    print(" Archivos de entrada: ")
    print(' -' + archivoDeEntrenamiento)
    print(
        " Archivos de salida : ")
    print(' -' + archivoGuardaModelo)
    print(' -' + archivoGuardaPesos)
    print(' -' + archivoInfoEntrenamiento)
    print(' -' + archivoGuardaTransformaciones)
    print(resultadoEvaluacion)
    print(confundida)
    print(resultadoContraTest)
    print('#' * 10, "MODEL SUMMARY", '#' * 10)
    model.summary()
    print('Epochs: ' + str(epocas))
    print('batch_size: ' + str(lote))
    print('split_validacion_fit_size: ' + str(split_validacion_fit_size))
    print('split_dataframe_para_test: ' + str(split_dataframe_para_test))
    print('#' * 10, "SHAPE ENTRENAMIENTO", '#' * 10)
    print(train_entrada.shape)
    print(train_entrada.columns.tolist())
    print('#' * 10, "SHAPE TEST", '#' * 10)
    print(test_entrada.shape)
    print(train_entrada.columns.tolist())
    print('#' * 10, "Analisis de variabilidad", '#' * 10)
    print(analisisVariabilidad)
    print('#' * 10, "Log de performance", '#' * 10)
    logPrograma += [pd_pxp.registroMomento("*) FIN DEL PROCESO ", True)]
    for mom in logPrograma:
        print(mom)

# print(fileMem.getvalue())
with open(archivoInfoEntrenamiento, 'w') as fileDisk:
    fileDisk.write(fileMem.getvalue())  # Lo abro con w entonces lo pisa

# Imprimo
pd_pxp.mostrarPerformanceEntrenamiento(history, archivoPrefijoPlot, mostrarGraficos)
pd_pxp.mostrarMatriz(confundida, archivoPrefijoPlot, mostrarGraficos)

prefijo = datetime.datetime.now().strftime("%m_%d_%Y %H_%M_%S")+' '+' '#+str(tp)+' '
destino = 'Historia/'+prefijo+archivoInfoEntrenamiento
print('* Realizando copia en: '+destino)
# Copia el archivo desde la ubicación actual a Historia
shutil.copy(archivoInfoEntrenamiento, destino)
####################################### SALIDA FINAL
print("Modelo Guardado! Informacion en : " + archivoInfoEntrenamiento)
print('(' * 15, "SALIDA FINAL", ')' * 15)
print(resultadoEvaluacion)
print('Precision tp/(tp+fp): {mcPrecision:,.2%}'.format(mcPrecision=confundidaPxp.mcPrecision))
print('Recall    tp/(tp+fn): {mcRecall:,.2%}'.format(mcRecall=confundidaPxp.mcRecall))
print('Acurracy         : {mcAccuracy:,.2%}'.format(mcAccuracy=confundidaPxp.mcAccuracy))


import requests
import whisper
import torch
from pydub import AudioSegment
from io import BytesIO
import io
import queue
import threading
import datetime
import os
import gradio as gr

import matplotlib as mpl
font_cache_path = mpl.get_cachedir() + '/fontList.cache'
print(font_cache_path)
#%rm $font_cache_path

# Step 1: Get the radio stream
stream_url = "http://stream.zeno.fm/y9y2bhvzs4zuv"  # Your stream URL

# Set inference device
device = "cuda" if torch.cuda.is_available() else "cpu"

# Set language
language_whisper = "english"

model_fp32 = whisper.load_model(
    name="small",
    device=device)

# Create a queue to store the audio files
audio_queue = queue.Queue()

# Create placeholders for the audio player, log text area, and transcript text area
log_text_area = gr.outputs.Textbox(label="Log")
transcript_text_area = gr.outputs.Textbox(label="Transcript")
audio_player = gr.outputs.Audio(type='filepath', label="Audio")

# Function to transcribe audio files
def transcribe_audio():
    while True:
        # Get the next audio file from the queue
        audio_file = audio_queue.get()
        print(audio_file)
        print("start transcribe")

        # Log the start of transcription
        gr.set_output(log_text_area="transcription start")

        # Transcribe the audio
        result = model_fp32.transcribe(audio_file)

        # Update the audio player and the text box
        audio_player.audio(audio_file)
        gr.set_output(transcript_text_area=result["text"])
        gr.set_output(log_text_area="transcription complete")

        # Log the end of transcription
        gr.set_output(log_text_area="transcription complete of " + audio_file)
        # Mark the task as done
        audio_queue.task_done()

# Start the transcription thread
transcription_thread = threading.Thread(target=transcribe_audio)
transcription_thread.start()

# Download the stream
response = requests.get(stream_url, stream=True)

# Create a buffer for the incoming audio data
buffer = BytesIO()

gr.set_output(log_text_area="Starting stream processing")

# Process the stream in chunks of 10 seconds
for block in response.iter_content(1024):
    # Write the audio data to our buffer
    buffer.write(block)
    # If we have 10 seconds of audio data
    if buffer.tell() >= 10 * 1024 * 100:
        gr.set_output(log_text_area="Saving audio file")
        # Get the current datetime
        now = datetime.datetime.now()

        # Create a directory for the audio files
        os.makedirs('audio', exist_ok=True)

        # Create a filename with the datetime and directory path
        filename = f'audio/stream_{now.strftime("%Y%m%d_%H%M%S")}.wav'

        # Save the buffer to a file
        with open(filename, 'wb') as f:
            f.write(buffer.getvalue())

        # Add the filename to the queue
        audio_queue.put(filename)

        # Log the creation of the audio file
        gr.set_output(log_text_area="Created audio file " + filename)

        # Clear the buffer
        buffer.seek(0)
        buffer.truncate()


from dataclasses import dataclass
from typing import List, Optional, Tuple

import numpy as np
import numpy.typing as npt
from shapely.geometry import CAP_STYLE, LineString, Polygon

from nuplan.common.actor_state.ego_state import EgoState
from nuplan.common.actor_state.scene_object import SceneObject
from nuplan.common.actor_state.state_representation import Point2D, StateSE2
from nuplan.common.maps.abstract_map import AbstractMap
from nuplan.common.maps.abstract_map_objects import Lane, StopLine
from nuplan.common.maps.maps_datatypes import SemanticMapLayer
from nuplan.planning.metrics.evaluation_metrics.base.violation_metric_base import ViolationMetricBase
from nuplan.planning.metrics.metric_result import MetricStatistics, MetricStatisticsType, Statistic, TimeSeries
from nuplan.planning.scenario_builder.abstract_scenario import AbstractScenario
from nuplan.planning.simulation.history.simulation_history import SimulationHistory, SimulationHistorySample
from nuplan.planning.simulation.observation.idm.utils import create_path_from_se2, path_to_linestring
from nuplan.planning.simulation.observation.observation_type import DetectionsTracks, Observation
from nuplan.planning.simulation.occupancy_map.strtree_occupancy_map import STRTreeOccupancyMapFactory
from nuplan.planning.simulation.path.utils import trim_path_up_to_progress

@dataclass
class VelocityRecord:
    """
    Class to track velocity and distance to stop line at a timestamp.
    """

    velocity: float  # [m/s^2], Velocity at the current timestamp
    timestamp: int  # timestamp
    distance_to_stop_line: float  # [m], Distance to the stop line

@dataclass
class VelocityData:
    """
    Class to track VelocityRecord over the simulation history.
    """

    data: List[VelocityRecord]

    def add_data(self, velocity: float, timestamp: int, distance_to_stop_line: float) -> None:
        """
        Add new data to the list
        :param velocity: [m/s^2], Velocity at the current timestamp
        :param timestamp: Timestamp
        :param distance_to_stop_line: [m], Distance to the stop line.
        """
        if self.data is None:
            self.data = []

        self.data.append(
            VelocityRecord(velocity=velocity, timestamp=timestamp, distance_to_stop_line=distance_to_stop_line)
        )

    @property
    def velocity_np(self) -> npt.NDArray[np.float32]:
        """
        Velocity in numpy representation.
        """
        return np.asarray([data.velocity for data in self.data])

    @property
    def timestamp_np(self) -> npt.NDArray[np.int32]:
        """
        Timestamp in numpy representation.
        """
        return np.asarray([data.timestamp for data in self.data])

    @property
    def distance_to_stop_line_np(self) -> npt.NDArray[np.float32]:
        """
        Distance to stop line in numpy representation.
        """
        return np.asarray([data.distance_to_stop_line for data in self.data])

    @property
    def min_distance_stop_line_record(self) -> VelocityRecord:
        """
        Return velocity record of minimum distance stop line
        :return A velocity record.
        """
        distance_to_stop_line = self.distance_to_stop_line_np
        index = np.argmin(distance_to_stop_line)
        return self.data[int(index)]

    @property
    def min_velocity_record(self) -> VelocityRecord:
        """
        Return minimum velocity record
        :return A velocity record.
        """
        index = np.argmin(self.velocity_np)
        return self.data[int(index)]

class EgoStopAtStopLineStatistics(ViolationMetricBase):
    """
    Ego stopped at stop line metric.
    """

    def __init__(
        self,
        name: str,
        category: str,
        max_violation_threshold: int,
        distance_threshold: float,
        velocity_threshold: float,
    ) -> None:
        """
        Initializes the EgoProgressAlongExpertRouteStatistics class
        Rule formulation: 1. Get the nearest stop polygon (less than the distance threshold).
                          2. Check if the stop polygon is in any lanes.
                          3. Check if front corners of ego cross the stop polygon.
                          4. Check if no any leading agents.
                          5. Get min_velocity(distance_stop_line) until the ego leaves the stop polygon.
        :param name: Metric name
        :param category: Metric category
        :param max_violation_threshold: Maximum threshold for the violation when computing the score
        :param distance_threshold: Distances between ego front side and stop line lower than this threshold
        assumed to be the first vehicle before the stop line
        :param velocity_threshold: Velocity threshold to consider an ego stopped.
        """
        super().__init__(name=name, category=category, max_violation_threshold=max_violation_threshold)
        self._distance_threshold = distance_threshold
        self._velocity_threshold = velocity_threshold
        self._stopping_velocity_data: List[VelocityData] = []
        self._previous_stop_polygon_fid: Optional[str] = None

    @staticmethod
    def get_nearest_stop_line(map_api: AbstractMap, ego_pose_front: LineString) -> Optional[Tuple[str, Polygon]]:
        """
        Retrieve the nearest stop polygon
        :param map_api: AbstractMap map api
        :param ego_pose_front: Ego pose front corner line
        :return Nearest stop polygon fid if distance is less than the threshold.
        """
        center_x, center_y = ego_pose_front.centroid.xy
        center = Point2D(center_x[0], center_y[0])

        # If ego is not in lane, then we do not need to proceed.
        if not map_api.is_in_layer(center, layer=SemanticMapLayer.LANE):
            return None

        stop_line_fid, distance = map_api.get_distance_to_nearest_map_object(center, SemanticMapLayer.STOP_LINE)

        if stop_line_fid is None:
            return None

        stop_line: StopLine = map_api.get_map_object(stop_line_fid, SemanticMapLayer.STOP_LINE)
        lane: Optional[Lane] = map_api.get_one_map_object(center, SemanticMapLayer.LANE)

        # Check if the stop polygon intersects with the ego lane
        if lane is not None:
            return stop_line_fid, stop_line.polygon if stop_line.polygon.intersects(lane.polygon) else None

        return None

    @staticmethod
    def check_for_leading_agents(detections: Observation, ego_state: EgoState, map_api: AbstractMap) -> bool:
        """
        Get the nearest leading agent
        :param detections: Detection class
        :param ego_state: Ego in oriented box representation
        :param map_api: AbstractMap api
        :return True if there is a leading agent, False otherwise
        """
        if isinstance(detections, DetectionsTracks):

            if len(detections.tracked_objects.tracked_objects) == 0:
                return False

            ego_agent = ego_state.agent

            # Check if any missing instance token
            for index, box in enumerate(detections.tracked_objects):
                if box.token is None:
                    box.token = str(index + 1)
            scene_objects: List[SceneObject] = [ego_agent]
            scene_objects.extend([scene_object for scene_object in detections.tracked_objects])

            occupancy_map = STRTreeOccupancyMapFactory.get_from_boxes(scene_objects)
            agent_states = {
                scene_object.token: StateSE2(
                    x=scene_object.center.x, y=scene_object.center.y, heading=scene_object.center.heading
                )
                for scene_object in scene_objects
            }
            ego_pose: StateSE2 = agent_states['ego']
            lane = map_api.get_one_map_object(ego_pose, SemanticMapLayer.LANE)

            # Construct ego's path to go
            ego_baseline = lane.baseline_path
            ego_progress = ego_baseline.get_nearest_arc_length_from_position(ego_pose)
            progress_path = create_path_from_se2(ego_baseline.discrete_path)
            ego_path_to_go = trim_path_up_to_progress(progress_path, ego_progress)

            # Check for intersection between the path and the any other agents
            ego_path_to_go = path_to_linestring(ego_path_to_go)
            intersecting_agents = occupancy_map.intersects(
                ego_path_to_go.buffer((scene_objects[0].box.width / 2), cap_style=CAP_STYLE.flat)
            )

            # If there are other agents ahead of the ego in the same lane there should be at least two agents
            if intersecting_agents.size > 1:
                return True

        return False

    def _compute_velocity_statistics(self, scenario: AbstractScenario) -> List[MetricStatistics]:
        """
        Compute statistics in each stop line
        :param scenario: Scenario running this metric
        :return A list of metric statistics.
        """
        if not self._stopping_velocity_data:
            return []

        mean_ego_min_distance_to_stop_line = []
        mean_ego_min_velocity_before_stop_line = []
        aggregated_timestamp_velocity = []
        aggregated_timestamps = []
        ego_stop_status = []

        for velocity_data in self._stopping_velocity_data:

            # Stop at stop line success rate, we check the velocity of minimum distance to the stop line in each record
            min_distance_velocity_record = velocity_data.min_distance_stop_line_record
            mean_ego_min_distance_to_stop_line.append(min_distance_velocity_record.distance_to_stop_line)
            mean_ego_min_velocity_before_stop_line.append(min_distance_velocity_record.velocity)

            if (
                min_distance_velocity_record.distance_to_stop_line < self._distance_threshold
                and min_distance_velocity_record.velocity < self._velocity_threshold
            ):
                stop_status = True
            else:
                stop_status = False

            ego_stop_status.append(stop_status)
            aggregated_timestamp_velocity.append(velocity_data.velocity_np)
            aggregated_timestamps.append(velocity_data.timestamp_np)

        # Aggregate
        statistics = [
            Statistic(
                name='number_of_ego_stop_before_stop_line',
                unit=MetricStatisticsType.COUNT.unit,
                value=sum(ego_stop_status),
                type=MetricStatisticsType.COUNT,
            ),
            Statistic(
                name='number_of_ego_before_stop_line',
                unit=MetricStatisticsType.COUNT.unit,
                value=len(ego_stop_status),
                type=MetricStatisticsType.COUNT,
            ),
            Statistic(
                name='mean_ego_min_distance_to_stop_line',
                unit='meters',
                value=float(np.mean(mean_ego_min_distance_to_stop_line)),
                type=MetricStatisticsType.VALUE,
            ),
            Statistic(
                name='mean_ego_min_velocity_before_stop_line',
                unit='meters_per_second_squared',
                value=float(np.mean(mean_ego_min_velocity_before_stop_line)),
                type=MetricStatisticsType.VALUE,
            ),
        ]

        aggregated_timestamp_velocity = np.hstack(aggregated_timestamp_velocity)  # type: ignore
        aggregated_timestamps = np.hstack(aggregated_timestamps)  # type: ignore
        velocity_time_series = TimeSeries(
            unit='meters_per_second_squared',
            time_stamps=list(aggregated_timestamps),
            values=list(aggregated_timestamp_velocity),
        )

        results = self._construct_metric_results(
            metric_statistics=statistics, time_series=velocity_time_series, scenario=scenario
        )
        return results  # type: ignore

    def _save_stopping_velocity(
        self,
        current_stop_polygon_fid: str,
        history_data: SimulationHistorySample,
        stop_polygon_in_lane: Polygon,
        ego_pose_front: LineString,
    ) -> None:
        """
        Save velocity, timestamp and distance to a stop line if the ego is stopping
        :param current_stop_polygon_fid: Current stop polygon fid
        :param history_data: History sample data at current timestamp
        :param stop_polygon_in_lane: The stop polygon where the ego is in
        :param ego_pose_front: Front line string (front right corner and left corner) of the ego.
        """
        # Distance between the front stop line and ego's front footprint
        stop_line: LineString = LineString(stop_polygon_in_lane.exterior.coords[:2])
        distance_ego_front_stop_line = stop_line.distance(ego_pose_front)
        current_velocity = history_data.ego_state.dynamic_car_state.speed
        current_timestamp = history_data.ego_state.time_point.time_us

        # If it is in the same stop polygon
        if current_stop_polygon_fid == self._previous_stop_polygon_fid:
            self._stopping_velocity_data[-1].add_data(
                velocity=current_velocity,
                timestamp=current_timestamp,
                distance_to_stop_line=distance_ego_front_stop_line,
            )
        else:
            # If it is a new stop polygon
            self._previous_stop_polygon_fid = current_stop_polygon_fid
            velocity_data = VelocityData([])
            velocity_data.add_data(
                velocity=current_velocity,
                timestamp=current_timestamp,
                distance_to_stop_line=distance_ego_front_stop_line,
            )
            self._stopping_velocity_data.append(velocity_data)

    def compute(self, history: SimulationHistory, scenario: AbstractScenario) -> List[MetricStatistics]:
        """
        Returns the ego stopped at stop line metric
        :param history: History from a simulation engine
        :param scenario: Scenario running this metric
        :return the estimated ego stopped at stop line metric.
        """
        # Extract ego pose in box3d.
        ego_states: List[EgoState] = history.extract_ego_state

        # Get egos' front footprint
        ego_pose_fronts: List[LineString] = [
            LineString(
                [
                    state.car_footprint.oriented_box.geometry.exterior.coords[0],
                    state.car_footprint.oriented_box.geometry.exterior.coords[3],
                ]
            )
            for state in ego_states
        ]

        # Get stop polygons
        scenario_map: AbstractMap = history.map_api

        # Get the nearest stop polygon.
        for ego_pose_front, ego_state, history_data in zip(ego_pose_fronts, ego_states, history.data):

            # 1) Get the nearest stop polygon if it is in the same lane as ego.
            stop_polygon_info: Optional[Tuple[str, Polygon]] = self.get_nearest_stop_line(
                map_api=scenario_map, ego_pose_front=ego_pose_front
            )

            if stop_polygon_info is None:
                continue

            fid, stop_polygon_in_lane = stop_polygon_info

            # 2) Check if front corners cross the stop polygon
            ego_pose_front_stop_polygon_distance: float = ego_pose_front.distance(stop_polygon_in_lane)

            # The front corners cross the stop polygon if distance is zero
            if ego_pose_front_stop_polygon_distance != 0:
                continue

            # 3) Check if no any leading agents.
            detections: Observation = history_data.observation
            has_leading_agent = self.check_for_leading_agents(
                detections=detections, ego_state=ego_state, map_api=scenario_map
            )

            # Skip if there is any leading agent in the stop polygon (means the ego is not the first vehicle)
            if has_leading_agent:
                continue

            # 4) Accumulate velocity
            self._save_stopping_velocity(
                current_stop_polygon_fid=fid,
                history_data=history_data,
                stop_polygon_in_lane=stop_polygon_in_lane,
                ego_pose_front=ego_pose_front,
            )

        results = self._compute_velocity_statistics(scenario=scenario)
        return results


import random
import numpy as np
import librosa
import nlpaug.augmenter.audio as naa

def loudness(
    sig: np.ndarray,
    augment_times: int,
    factor_low: int,
    factor_high: int
) -> list:
    """Changes the loudness of the signal.

    Args:
        sig: Input signal.
        augment_times: Number of times to augment original signal.
        factor_low: Factor to change the lower loudness of the signal.
        factor_high: Factor to change the upper loudness of the signal.

    Returns:
        loud_sig: Returns the loudness changed signal.
    """

    loud_sig = []
    for i in range(augment_times):
        factor = tuple(random.uniform(factor_low, factor_high)
                       for i in range(2))
        aug = naa.LoudnessAug(factor=factor)
        loud_sig.append(np.array(aug.augment(sig)).flatten())
    return loud_sig

def time_stretch(
    signal: np.ndarray,
    augment_times: int,
    factor: float
) -> list:
    """Changes the speed of the signal.

    Args:
        signal: Input signal.
        augment_times: Number of times to augment original signal.
        factor: Factor to change the speed of the signal.

    Returns:
        stretched_sig: Returns the speed changed signal.
    """

    stretched_signal = []
    for i in range(augment_times):
        stretched_signal.append(librosa.effects.time_stretch(signal, factor))
        factor = np.random.uniform(0.1, 1)
    return stretched_signal

def pitch_shift(
        signal: np.ndarray,
        augment_times: int,
        sr: int,
        factor: tuple,
        zone: tuple = (0, 1),
        coverage: int = 1):
    """Changes the pitch of the signal.

    Args:
        signal: Input signal.
        augment_times: Number of times to augment original signal.
        sr: Sampling rate of the input signal.
        factor: Factor to change the pitch of the signal.
        zone: Range of pitch shift. Default value is (0, 1).
        coverage: Percentage of audio to apply pitch shift. Default value is 1.

    Returns:
        pitched_sig: Returns the pitch changed signal.
    """

    pitched_signal = []
    for i in range(augment_times):
        pitched_signal.append(naa.PitchAug(
            sampling_rate=sr, zone=zone, coverage=coverage,
            factor=factor).augment(signal)[0])
        factor = np.random.uniform(0, 50, size=2)
        factor = np.sort(factor)
    return pitched_signal


# -*- coding: utf-8 -*-

"""
@Author: Yacan Man
@Email: myxc@live.cn
@Website: https://www.manyacan.com
@software: PyCharm
@file: main.py
@time: 2023/6/26 21:04
"""
import os
import time
import hues
import torch
import torchvision
from torch.utils.data import DataLoader
from torch.utils.tensorboard.writer import SummaryWriter
from torch.nn import Conv2d, MaxPool2d, Flatten, Linear, CrossEntropyLoss

# 数据路径
DATA_DIR = './data/'

# ToTensor
transformer = torchvision.transforms.Compose([
    torchvision.transforms.ToTensor()
])

# Train data and test data
train_data = torchvision.datasets.CIFAR10(
    root=DATA_DIR,
    transform=transformer,
    train=True,
    download=True
)

test_data = torchvision.datasets.CIFAR10(
    root=DATA_DIR,
    transform=transformer,
    train=False,
    download=True
)

# Data loader
train_loader = DataLoader(
    dataset=train_data,
    batch_size=64,
    shuffle=True,
    num_workers=2,
    drop_last=True
)

test_loader = DataLoader(
    dataset=test_data,
    batch_size=64,
    shuffle=True,
    num_workers=2,
    drop_last=True
)

# VGG Model
vgg_pretrained = torchvision.models.vgg16(weights=torchvision.models.VGG16_Weights.IMAGENET1K_V1)

# Training function
def train(dataloader, model, loss_fn, optimizer, lr_scheduler=None, verbose=True):
    size = len(dataloader.dataset)
    num_batches = len(dataloader)  # dataloader中的批量样本数
    model.train()
    train_loss, correct = 0, 0
    for batch, (X, y) in enumerate(dataloader):
        X, y = X.to(device), y.to(device)

        # Compute prediction error
        y_pred = model(X)
        loss = loss_fn(y_pred, y)
        train_loss += loss

        # Backpropagation
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        if lr_scheduler:
            lr_scheduler.step()

        if verbose:
            if batch % 50 == 0:
                loss, current = loss.item(), batch * len(X)
                hues.info(f"Loss: {loss:>7f}  [{current:>5d}/{size:>5d}]")

        correct += (y_pred.argmax(1) == y).type(torch.float).sum().item()
    train_loss /= num_batches
    correct /= size
    return train_loss, correct

# Training function
def test(dataloader, model, loss_fn, verbose=True):
    size = len(dataloader.dataset)  # 样本总数
    num_batches = len(dataloader)  # dataloader中的批量样本数
    model.eval()
    test_loss, correct = 0, 0
    with torch.no_grad():
        for X, y in dataloader:
            X, y = X.to(device), y.to(device)
            y_pred = model(X)
            test_loss += loss_fn(y_pred, y).item()
            correct += (y_pred.argmax(1) == y).type(torch.float).sum().item()
    test_loss /= num_batches
    correct /= size
    if verbose:
        hues.info(f"Test Error: Accuracy: {(100 * correct):>0.1f}%, Avg loss: {test_loss:>8f} \n")
    return test_loss, correct  # For reporting tuning results/ early stopping

if __name__ == '__main__':
    hues.info(f'The length of train data is {len(train_data)}, the length of test data is {len(test_data)}.')

    # Finding device
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

    # Building model
    model = vgg_pretrained
    model.classifier.add_module('Linear', torch.nn.Linear(1000, 10))
    model.to(device)
    hues.info(model)

    # Loss function
    loss_fun = CrossEntropyLoss()
    loss_fun.to(device)

    # Optimizer
    learning_rate = 0.0001
    optimizer = torch.optim.SGD(vgg_pretrained.parameters(), lr=learning_rate)
    # 学习率的衰减, 每经过5轮更新梯度，学习率衰减为原来的0.9
    scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=.9)

    # Using tensorboard to visualize network structure
    LOG_DIR = 'logs'
    if not os.path.exists(LOG_DIR):
        os.mkdir(LOG_DIR)
    writer = SummaryWriter(LOG_DIR)

    # Running model
    epochs = 300
    for i in range(epochs):
        print('+' * 25, f'Epoch {i}'.center(10), '+' * 25)
        train_loss, train_cor = train(train_loader, model, loss_fun, optimizer)
        scheduler.step()
        test_loss, test_cor = test(test_loader, model, loss_fun)
        writer.add_scalar('Loss/Train', train_loss, i)
        writer.add_scalar('Loss/Test', test_loss, i)
        writer.add_scalar('Accuracy/Train', train_cor, i)
        writer.add_scalar('Accuracy/Test', test_cor, i)
        writer.add_scalar('Learning Rate', optimizer.state_dict()["param_groups"][0]["lr"], i)

    hues.success("Done!")

    # Saving model
    timer = time.strftime('%Y-%m-%d_%H-%M-%S', time.localtime())
    MODEL_DIR = 'model'
    if not os.path.exists(MODEL_DIR):
        os.mkdir(MODEL_DIR)
    torch.save(model, f'{MODEL_DIR}/model_{timer}.pth')

    # Saving the structure of model
    demo_input = torch.ones(64, 3, 32, 32).to(device)  # batch_size=6, in_channels=3, height=28, weight=28.
    writer.add_graph(model, demo_input)

    writer.close()  # Closing writer
    # tensorboard --logdir='logs'


import fitz
import os
import sys
import tabulate

PDF_PATH = "MSC_Nastran_2022.4_Quick_Reference_Guide.pdf"
dims = os.get_terminal_size()
COLS = dims.columns

def reformat_format(table: list) -> list:
    # Make left adjust with 8 character spacing
    for ind, row in enumerate(table):
        table[ind] = [field.ljust(8) for field in row]
        if ind > 1:
            table[ind] = [''] + table[ind]
    return table

def reformat_describer(table: list, remarks: int) -> list:
    # Remove duplicate headers (if spanning multiple pages)
    while ['Describer', 'Meaning'] in table[1:]:
        ind = table.index(['Describer', 'Meaning'], 2)
        del table[ind]
        remarks -= 1
    # Combine entries and calculate maximum column widths
    maxcols = [len(table[0][0]), len(table[0][1])]
    for ind, row in enumerate(table):
        table[ind] = [row[0]] + [''.join(row[1:])]
        maxcols[0] = max(maxcols[0], len(table[ind][0]))
        maxcols[1] = max(maxcols[1], len(table[ind][1]))
    while ['v','v'] in table:
        ind = table.index(['v','v'])
        del table[ind]
        remarks -= 1
    while ['v', ''] in table:
        ind = table.index(['v',''])
        del table[ind]
        remarks -= 1
    if maxcols[1] > COLS:
        maxcols[1] = COLS - maxcols[0] - 7
    return table, remarks, maxcols

def main(card):
    with fitz.open(PDF_PATH) as doc:
        toc = doc.get_toc()
        for i, section in enumerate(toc):
            if section[1] == card:
                break
        pages = []
        for page in doc.pages(section[2]-1, toc[i+1][2]-1, 1):
            pix = page.get_pixmap()
            imgname = "page-%i.png" % page.number
            pix.save(imgname)
            # os.startfile(imgname)
            text = page.get_text("dict")
            text['blocks'] = sorted(text['blocks'], key=lambda d: d['bbox'][1])
            bbox = []
            img = mpimg.imread(imgname)
            imgplot = plt.imshow(img)
            ax = plt.gca()
            for val in text['blocks']:
                rect = Rectangle(
                    val['bbox'][:2],
                    val['bbox'][2]-val['bbox'][0],
                    val['bbox'][3]-val['bbox'][1],
                    linewidth=1,
                    edgecolor='r',
                    facecolor='none',
                    )
                ax.add_patch(rect)
                bbox.append(val['bbox'])
            pages.append(text['blocks'][1:-3])
            # text['blocks'][0]['lines'][0]['spans'][0]['text']
            # plt.show()

        text = []
        for sections in pages:
            for section in sections:
                tmp = []
                for line in section['lines']:
                    for span in line['spans']:
                        tmp.append(span['text'])
                text.append(tmp)
        # Find Example index
        i = 0
        while 'Example' not in text[i][0]:
            i+=1
        example = i
        # Find Describer index
        while 'Describer' not in text[i][0]:
            i+=1
        describer = i
        # Find Remarks index
        while 'Remarks' not in text[i][0]:
            i+=1
        remarks = i
        # Print card and short description
        print('\n'+':  '.join(text[0])+'\n')
        # Print long description
        print(''.join(text[1])+'\n')
        # Print card format
        print(text[2][0])
        text[3:example] = reformat_format(text[3:example])
        out = tabulate.tabulate(
            text[3:example],
            headers='firstrow',
            tablefmt='fancy_grid',
        )
        print(out+'\n')
        # Print describer table
        text[describer:remarks], remarks, maxcols = reformat_describer(text[describer:remarks], remarks)
        out = tabulate.tabulate(
            text[describer:remarks],
            headers='firstrow',
            tablefmt='fancy_grid',
            maxcolwidths=maxcols,
        )
        print(out+'\n')

if __name__ == "__main__":
    from bulk_data import BULK_DATA
    entry = BULK_DATA(sys.argv[1])
    print(entry)


import torch
import torch.nn.functional as F
from dataclasses import dataclass
import torch.nn as nn
from .sageconv import SAGEConv

class L2NormalizationLayer(torch.nn.Module):
    def __init__(self, dim=1, eps=1e-12):
        super(L2NormalizationLayer, self).__init__()
        self.dim = dim
        self.eps = eps

    def forward(self, x):
        return F.normalize(x, p=2, dim=self.dim, eps=self.eps)

class Swish(nn.Module):
    def forward(self, x):
        return x * torch.sigmoid(x)

class TransformerBlock(nn.Module):
    def __init__(self,dim=256, num_heads=4, expand=2, attn_dropout=0.1, drop_rate=0.1, activation=Swish()):
        super().__init__()
        self.norm1 = nn.LayerNorm(dim)
        self.attn = nn.MultiheadAttention(dim, num_heads, dropout=attn_dropout,batch_first=True)
        self.dropout1 = nn.Dropout(drop_rate)
        self.norm2 = nn.LayerNorm(dim)
        self.linear1 = nn.Linear(dim, dim*expand, bias=False)
        self.linear2 = nn.Linear(dim*expand, dim, bias=False)
        self.dropout2 = nn.Dropout(drop_rate)
        self.activation = activation

    def forward(self, inputs):
        x = self.norm1(inputs)
        x, _ = self.attn(x, x, x)
        x = self.dropout1(x)
        x = x + inputs
        attn_out = x

        x = self.norm2(x)
        x = self.activation(self.linear1(x))
        x = self.linear2(x)
        x = self.dropout2(x)
        x = x + attn_out
        return x

@dataclass
class GraphModelArugments:
    num_opcodes: int = 120
    opcode_dim: int = 64
    node_features: int = 112+opcode_dim
    node_feature_dim:int = 192
    config_features: int = 14
    conf_feature_dim: int = 64

    feature_dropout: float=0.0
    feature_expand: int = 1

    graphsage_in: int = node_feature_dim+conf_feature_dim
    graphsage_hidden: int = graphsage_in

    graphsage_layers: int = 2
    conv_aggregation_type: str = "max"
    mlp_layers: int = 2

    final_dropout: float = 0.0
    embedding_dropout: float = 0.0

    attention_blocks: int = 0
    drop_rate: float = 0.1
    attention_dropout: float = 0.1
    num_heads: int = 4

    is_pair_modeling: bool = False

import torch.nn as nn
import torch.nn.functional as F

class MLP(nn.Module):
    def __init__(self, input_dim, dims, alpha=0.2,use_bias=True):
        super(MLP, self).__init__()

        layers = []
        dims = [input_dim] + dims  # Add input dimension to the beginning
        for i in range(1, len(dims)):
            layers.append(nn.Linear(dims[i-1], dims[i], bias=use_bias))
            if i < len(dims) - 1:  # Do not append activation for the last layer
                layers.append(nn.LeakyReLU(alpha))
        self.net = nn.Sequential(*layers)

    def forward(self, x):
        return self.net(x)

class GraphSage(torch.nn.Module):
    def __init__(self,arguments: GraphModelArugments):
        super().__init__()
        self.sageconvs = torch.nn.ModuleList([SAGEConv(in_channels=arguments.graphsage_hidden,out_channels=arguments.graphsage_hidden,aggr=arguments.conv_aggregation_type) for _ in range(arguments.graphsage_layers)])
        self.mlps = torch.nn.ModuleList([MLP(input_dim=arguments.graphsage_hidden,dims=[arguments.node_feature_dim]*arguments.mlp_layers) for _ in range(arguments.graphsage_layers)])
        self.prenet = MLP(input_dim=arguments.graphsage_in,dims=[arguments.node_feature_dim]*arguments.mlp_layers)

    def forward(self,node_features, config_features, edges, *args,**kwargs):
        x = node_features
        x = torch.concat([config_features, x],dim=1)
        x = self.prenet(x)
        x = F.leaky_relu(x,negative_slope=0.2)

        for layer,mlps in zip(self.sageconvs,self.mlps):
            y = x
            y = torch.concat([config_features, y], dim=1)
            y = F.leaky_relu(mlps(layer(y,edges)),negative_slope=0.2)
            x += y
        return x

class LayoutGraphModel(torch.nn.Module):
    def __init__(self,arguments: GraphModelArugments):
        super().__init__()
        self.arguments = arguments
        self.opcode_embeddings = torch.nn.Embedding(arguments.num_opcodes,arguments.opcode_dim)
        if arguments.graphsage_layers>0:
            print("Skipping graph layer")
            self.graph_encoder = GraphSage(arguments=arguments)
        else:
            self.graph_encoder = None

        self.node_features_mlp = torch.nn.Sequential(
            torch.nn.Linear(arguments.node_features,arguments.node_features*arguments.feature_expand),
            torch.nn.Dropout(arguments.feature_dropout),
            torch.nn.LeakyReLU(inplace=True),
            torch.nn.Linear(arguments.node_features*arguments.feature_expand,arguments.node_feature_dim),
            torch.nn.LeakyReLU(inplace=True),
        )

        self.config_features_mlp = torch.nn.Sequential(
            torch.nn.Linear(arguments.config_features,arguments.config_features*arguments.feature_expand),
            torch.nn.Dropout(arguments.feature_dropout),
            torch.nn.LeakyReLU(inplace=True),
            torch.nn.Linear(arguments.config_features*arguments.feature_expand,arguments.conf_feature_dim),
            torch.nn.LeakyReLU(inplace=True),
        )

        if arguments.attention_blocks>0:
            self.aggregation_norm = torch.nn.LayerNorm(arguments.graphsage_hidden)

            self.attention_module = torch.nn.Sequential(
                *[
                    TransformerBlock(arguments.graphsage_hidden,arguments.num_heads,attn_dropout=arguments.attention_dropout,drop_rate=arguments.drop_rate) for _ in range(arguments.attention_blocks)
                ]
            )
        else:
            self.attention_module = None
        self.embed_drop = torch.nn.Dropout(arguments.embedding_dropout)
        if not self.arguments.is_pair_modeling:
            self.final_classifier = torch.nn.Sequential(
                torch.nn.Dropout(arguments.final_dropout),
                torch.nn.Linear(arguments.node_feature_dim,1,bias=False)
            )

    def forward(self,node_features, node_config_features,node_separation, node_ops, edges, batches,*args,**kwargs):
        opcode_embed = self.embed_drop(self.opcode_embeddings(node_ops))
        x = torch.cat([node_features,opcode_embed],dim=1)
        x = self.node_features_mlp(x)
        node_config_features = self.config_features_mlp(node_config_features)
        if self.graph_encoder is not None:
            x = self.graph_encoder(x,node_config_features,edges)
        start_idx = 0
        aggregated = [[] for _ in torch.unique(batches)]
        for ns,b in zip(node_separation,batches):
            end_idx = ns
            aggregated[b].append(torch.sum(x[start_idx:end_idx], dim=0))
            start_idx = end_idx
        aggregated = torch.stack([torch.stack(x) for x in aggregated])

        if self.arguments.is_pair_modeling:
            aggregated = self.aggregation_norm(aggregated)
            aggregated_fc = self.attention_module(aggregated)
            return torch.tanh(torch.bmm(aggregated,aggregated_fc.transpose(1,2)))   # This is like calculating the cosine between two vectors. We add 1 to make the value in range [0,2]
        elif self.attention_module is not None:
            aggregated = self.aggregation_norm(aggregated)
            aggregated = self.attention_module(aggregated)

        aggregated = torch.squeeze(self.final_classifier(aggregated),2)
        return aggregated


# -*- coding: utf-8 -*-

# Python 3.6
# author: EJ Jang
# date: 2018.06.14

# Options - Batch or Backfill (bkfil: False, True)
add_lback_kwarg = dict(add_lback_yr=34, add_lback_qtr=48)
fileName = 'SustG'
mapping = 'worldscope'
# option = 'batch'
# freq = 'W'
import sys
option = sys.argv[1]
freq = sys.argv[2]
print('# Starting Factor - {}_{} ({})'.format(mapping, fileName, freq))

import numpy as np
import pandas as pd
import datetime as dt
import time
import re
from functools import reduce

from batch_utils.utils_dateSeq import batch_sequence
from batch_utils.utils_mapping import get_Mapping, getUnique_TMSRS_CD
from batch_utils.utils_mapping_orig import get_Mapping_orig
from batch_utils.ItemInfo import Item_lst, RatioItem_lst, CurrItem_lst
from batch_utils.common import chunker, chunker_count, chg_to_type, add_mapped_tick
from batch_utils.common import firstCheck_duplicates, secondCheck_columns
from batch_utils.common import check_mapping_df, save_batch
from batch_utils.common import batch_monitor_msg, batch_finish_msg

from batch_utils import WS_retrieve_custom, WS_year, WS_qtr_currToHist, WS_resample
from batch_utils import simple_add, simple_mult, simple_div, simple_subtract
from batch_utils import align_add, align_mult, align_div, align_subtract
from batch_utils import substitute_Value
from batch_utils.WS.WS_histChg import get_HistAvg, get_HistChgAvg

# Date Sequence to be made by this batch
bkfil, rtvStart, seq_DT = batch_sequence(option, freq)

# Getting the Universe in TMSRS_CD ~ Code Map
allSec = getUnique_TMSRS_CD()
codeMap = get_Mapping_orig(mapping)
trim_codeMap = codeMap[codeMap['TMSRS_CD'].isin(allSec)].copy()
trim_codeMap_uniq = trim_codeMap['Code'].unique()

print(trim_codeMap.iloc[:2])
print('\n>>> Total Mapping Securities #: {}'.format(trim_codeMap.shape[0]))
print('>>> Total Performing Securities #: {}'.format(trim_codeMap_uniq.shape[0]))

# Checking Level of Duplicates in codeMap
chk_codeMap = check_mapping_df(trim_codeMap)

DF_ROE_qtr, DF_ROE_yr = pd.DataFrame(), pd.DataFrame()
DF_Payout_qtr, DF_Payout_yr = pd.DataFrame(), pd.DataFrame()
DF_epsg_qtr, DF_epsg_yr = pd.DataFrame(), pd.DataFrame()

# Transform Security List into Batch Chunks
tot_n = chunker_count(trim_codeMap_uniq, 50)
Code_lst_tot = chunker(trim_codeMap_uniq.tolist(), 50)

# Bring needed items
re_item = RatioItem_lst.loc['ROE_ws']
ni_item = Item_lst.loc['NetIncome_EPS']
po_item = RatioItem_lst.loc['Payout_ws']
poc_item = CurrItem_lst.loc['Payout_curr']

st_time = time.time()
for i, Code_lst in enumerate(Code_lst_tot):
    # Get Data
    roe_yr, roe_qtr = WS_retrieve_custom(
        re_item['Item'], re_item['Table'], Code_lst, None, bkfil, **add_lback_kwarg)
    eps_yr, eps_qtr = WS_retrieve_custom(
        ni_item['Item'], ni_item['Table'], Code_lst, 'sum', bkfil, **add_lback_kwarg)

    payout_yr = WS_year(po_item['Item'], po_item['Table'], Code_lst, bkfil, add_lback=34)
    payout_qtr = WS_qtr_currToHist(poc_item['Item'], poc_item['Table'], Code_lst, bkfil, add_lback=48)

    # Qtr
    roe_qtr_ = get_HistAvg(roe_qtr, k=8)
    payout_qtr_ = get_HistAvg(payout_qtr, k=8)
    epsg_qtr = get_HistChgAvg(eps_qtr, k=5, k2=4, growth=True)

    fill_cols = ['FiscalPrd', 'FiscalPrd2', 'CalPrdEndDate', 'Value_']
    roe_qtr_samp = WS_resample(seq_DT, roe_qtr_, fill_cols=fill_cols + ['FILLyr'])
    payout_qtr_samp = WS_resample(seq_DT, payout_qtr_, fill_cols=fill_cols)
    epsg_qtr_samp = WS_resample(seq_DT, epsg_qtr, fill_cols=fill_cols + ['Ratio'])

    DF_ROE_qtr = DF_ROE_qtr.append(roe_qtr_samp)
    DF_Payout_qtr = DF_Payout_qtr.append(payout_qtr_samp)
    DF_epsg_qtr = DF_epsg_qtr.append(epsg_qtr_samp)

    # Year
    roe_yr_ = get_HistAvg(roe_yr, k=2)
    payout_yr_ = get_HistAvg(payout_yr, k=2)
    epsg_yr = get_HistChgAvg(eps_yr, k=2, k2=1, growth=True)

    fill_cols = ['FiscalPrd', 'CalPrdEndDate', 'Value_']
    roe_yr_samp = WS_resample(seq_DT, roe_yr, fill_cols=fill_cols + ['FILLyr'])
    payout_yr_samp = WS_resample(seq_DT, payout_yr, fill_cols=fill_cols + ['FILLyr'])
    epsg_yr_samp = WS_resample(seq_DT, epsg_yr, fill_cols=fill_cols + ['Ratio'])

    DF_ROE_yr = DF_ROE_yr.append(roe_yr_samp, sort=False)
    DF_Payout_yr = DF_Payout_yr.append(payout_yr_samp, sort=False)
    DF_epsg_yr = DF_epsg_yr.append(epsg_yr_samp, sort=False)

    batch_monitor_msg(i, tot_n, st_time)
batch_finish_msg(tot_n, st_time)

DF_ROE_trail = substitute_Value(DF_ROE_yr, DF_ROE_qtr)
DF_Payout_trail = substitute_Value(DF_Payout_yr, DF_Payout_qtr)
DF_epsg_trail = substitute_Value(DF_epsg_yr, DF_epsg_qtr)

DF_ROE_trail = DF_ROE_trail[
    DF_ROE_trail['Value_'].notnull()
    ].sort_values(['Code', 'BASE_DT']).copy()
DF_ROE_trail.loc[
    DF_ROE_trail['Value_'].abs() > 100, 'Value_'] = DF_ROE_trail.loc[
    DF_ROE_trail['Value_'].abs() > 100, 'Value_'].apply(lambda x: 100 * np.sign(x))

DF_Payout_trail.loc[DF_Payout_trail['Value_'] < 0, 'Value_'] = np.nan
DF_Payout_trail = DF_Payout_trail[
    DF_Payout_trail['Value_'].notnull()
    ].sort_values(['Code', 'BASE_DT']).copy()
DF_Payout_trail.loc[DF_Payout_trail['Value_'] > 100, 'Value_'] = 100

DF_epsg_trail = DF_epsg_trail[
    DF_epsg_trail['Value_'].notnull()
    ].sort_values(['Code', 'BASE_DT']).copy()

DF_ROE_trail.reset_index(drop=True, inplace=True)
DF_Payout_trail.reset_index(drop=True, inplace=True)
DF_epsg_trail.reset_index(drop=True, inplace=True)

cols = ['BASE_DT', 'TMSRS_CD', 'Code', 'Value_yr', 'Value_qtr', 'Value_',
        'RGN_TP_CD', 'freq', 'ref']
DF_ROE_trail = add_mapped_tick(DF_ROE_trail, trim_codeMap)[cols]
DF_Payout_trail = add_mapped_tick(DF_Payout_trail, trim_codeMap)[cols]
DF_epsg_trail = add_mapped_tick(DF_epsg_trail, trim_codeMap)[cols]

DF_ROE_trail['StyleName'] = 'ROE_l2yr'
DF_Payout_trail['StyleName'] = 'Payout_l2yr'
DF_epsg_trail['StyleName'] = 'EPSg_l2yr'

#*------------
secondCheck_columns(DF_ROE_trail, DF_Payout_trail, DF_epsg_trail)
#*------------

DF = reduce(
    lambda x, y: pd.DataFrame.append(x, y, sort=False),
    [DF_ROE_trail, DF_Payout_trail])
DF.reset_index(drop=True, inplace=True)

# Save!
save_batch(bkfil, DF, mapping, fileName + '_ROEnPayout')

# Save!
save_batch(bkfil, DF_epsg_trail, mapping, fileName + '_EPSg')

from batch_utils.fill_monitor import print_fillReport
out = print_fillReport(bkfil, freq, DF)
out = print_fillReport(bkfil, freq, DF_epsg_trail)

# <<Sustainable Growth Script>>
dir_ = 'save_total' if bkfil else 'save_batch'
DF_IBES_Payout = pd.read_pickle('{}/IBES_refIBES_Payout.p'.format(dir_))
DF_IBES_ROE = pd.read_pickle('{}/IBES_refIBES_ROE.p'.format(dir_))

# Merge IBES with Worldscope for (normalized) Payout & ROE (comb. average)
TMP = DF_IBES_Payout[
    ['BASE_DT', 'TMSRS_CD', 'Payout_avg']
    ].rename(columns={'Payout_avg': 'Value_1'}).copy()
DF_Payout_norm = pd.merge(
    DF_Payout_trail, TMP, on=['BASE_DT', 'TMSRS_CD'], how='left')
DF_Payout_norm['Value_norm'] = DF_Payout_norm.apply(
    lambda x: (x['Value_'] + x['Value_1']) / 2 if pd.notnull(x['Value_1'])
    else x['Value_'], axis=1)

TMP = DF_IBES_ROE[
    ['BASE_DT', 'TMSRS_CD', 'ROE_avg']
    ].rename(columns={'ROE_avg': 'Value_1'}).copy()
DF_ROE_norm = pd.merge(
    DF_ROE_trail, TMP, on=['BASE_DT', 'TMSRS_CD'], how='left')
DF_ROE_norm['Value_norm'] = DF_ROE_norm.apply(
    lambda x: (x['Value_'] + x['Value_1']) / 2 if pd.notnull(x['Value_1'])
    else x['Value_'], axis=1)

# Calculate Sustainable Growth Factor
TMP0 = DF_ROE_norm[
    ['BASE_DT', 'TMSRS_CD', 'Value_norm']].rename(
    columns={'Value_norm': 'ROE_norm'})
TMP1 = DF_Payout_norm[
    ['BASE_DT', 'TMSRS_CD', 'Value_norm']].rename(
    columns={'Value_norm': 'Payout_norm'})

DF_SustG_tot = pd.merge(TMP0, TMP1, on=['BASE_DT', 'TMSRS_CD'])
DF_SustG_tot['Value_'] = DF_SustG_tot['ROE_norm'] * (
    1 - DF_SustG_tot['Payout_norm'] / 100)
DF_SustG_tot['StyleName'], DF_SustG_tot['freq'] = 'SustG', None
DF_SustG_tot['Code'], DF_SustG_tot['RGN_TP_CD'] = None, None
#*------------
secondCheck_columns(DF_SustG_tot, DF)
#*------------

# Save!
save_batch(bkfil, DF_SustG_tot, 'comb', fileName)
out = print_fillReport(bkfil, freq, DF_SustG_tot, chk_col='Value_')


from typing import Dict, Optional, Sequence, Tuple, Union

import hydra
import omegaconf
import torch
from torch import nn as nn
from torch.nn import functional as F

from cmrl.models.util import gaussian_nll
from cmrl.models.layers import ParallelEnsembleLinearLayer, truncated_normal_init
from cmrl.models.networks.mlp import EnsembleMLP
from cmrl.models.util import to_tensor

class TransitionConditionalMutualInformationTest(EnsembleMLP):
    _MODEL_FILENAME = "conditional_mutual_information_test.pth"

    def __init__(
        self,
        # transition info
        obs_size: int,
        action_size: int,
        # algorithm parameters
        ensemble_num: int = 7,
        elite_num: int = 5,
        residual: bool = True,
        learn_logvar_bounds: bool = False,
        # network parameters
        num_layers: int = 4,
        hid_size: int = 200,
        activation_fn_cfg: Optional[Union[Dict, omegaconf.DictConfig]] = None,
        # others
        device: Union[str, torch.device] = "cpu",
    ):
        super().__init__(
            ensemble_num=ensemble_num,
            elite_num=elite_num,
            device=device,
        )
        self.obs_size = obs_size
        self.action_size = action_size
        self.residual = residual
        self.learn_logvar_bounds = learn_logvar_bounds

        self.num_layers = num_layers
        self.hid_size = hid_size

        self.parallel_num = self.obs_size + self.action_size + 1

        self._input_mask = 1 - torch.eye(self.parallel_num, self.obs_size + self.action_size).to(self.device)

        def create_activation():
            if activation_fn_cfg is None:
                return nn.ReLU()
            else:
                return hydra.utils.instantiate(activation_fn_cfg)

        hidden_layers = [
            nn.Sequential(
                self.create_linear_layer(obs_size + action_size, hid_size),
                create_activation(),
            )
        ]
        for i in range(num_layers - 1):
            hidden_layers.append(
                nn.Sequential(
                    self.create_linear_layer(hid_size, hid_size),
                    create_activation(),
                )
            )
        self.hidden_layers = nn.Sequential(*hidden_layers)

        self.mean_and_logvar = self.create_linear_layer(hid_size, 2 * self.obs_size)
        self.min_logvar = nn.Parameter(
            -10 * torch.ones(self.parallel_num, 1, 1, self.obs_size), requires_grad=learn_logvar_bounds
        )
        self.max_logvar = nn.Parameter(
            0.5 * torch.ones(self.parallel_num, 1, 1, self.obs_size), requires_grad=learn_logvar_bounds
        )

        self.apply(truncated_normal_init)
        self.to(self.device)

    def create_linear_layer(self, l_in, l_out):
        return ParallelEnsembleLinearLayer(l_in, l_out, parallel_num=self.parallel_num, ensemble_num=self.ensemble_num)

    @property
    def input_mask(self):
        return self._input_mask

    def mask_input(self, x: torch.Tensor) -> torch.Tensor:
        assert x.ndim == 4
        assert self._input_mask.ndim == 2
        input_mask = self._input_mask[:, None, None, :]
        return x * input_mask

    def forward(
        self,
        batch_obs: torch.Tensor,  # shape: (parallel_num, )ensemble_num, batch_size, obs_size
        batch_action: torch.Tensor,  # shape: ensemble_num, batch_size, action_size
    ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
        assert len(batch_action.shape) == 3 and batch_action.shape[-1] == self.action_size

        batch_action = batch_action.repeat((self.parallel_num, 1, 1, 1))
        if len(batch_obs.shape) == 3:  # non-repeat or first repeat
            batch_obs = batch_obs.repeat((self.parallel_num, 1, 1, 1))

        batch_input = torch.concat([batch_obs, batch_action], dim=-1)

        masked_input = self.mask_input(batch_input)
        hidden = self.hidden_layers(masked_input)
        mean_and_logvar = self.mean_and_logvar(hidden)

        mean = mean_and_logvar[..., : self.obs_size]
        logvar = mean_and_logvar[..., self.obs_size :]
        logvar = self.max_logvar - F.softplus(self.max_logvar - logvar)
        logvar = self.min_logvar + F.softplus(logvar - self.min_logvar)

        if self.residual:
            mean += batch_obs

        return mean, logvar

    def get_nll_loss(self, model_in: Dict[(str, torch.Tensor)], target: torch.Tensor) -> torch.Tensor:
        pred_mean, pred_logvar = self.forward(**model_in)
        target = target.repeat((self.parallel_num, 1, 1, 1))

        nll_loss = gaussian_nll(pred_mean, pred_logvar, target, reduce=False)
        nll_loss += 0.01 * (self.max_logvar.sum() - self.min_logvar.sum())
        return nll_loss


"""
TODO:
    [ ] Interact with API
        [ ] Investigate Facebook API
        [ ] Investigate Discord API
    [ ] create "calendar-to-post" pipeline
"""

from __future__ import print_function

import datetime
import os.path

# = Google API =

from google.auth.transport.requests import Request
from google.oauth2.credentials import Credentials
from google_auth_oauthlib.flow import InstalledAppFlow
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError

# = Discord API =

import discord

# = Facebook API =

import requests

# Important variables that are private
from var import *
# `var` file contains the following variables:
#   BKB_CAL_ID [string] -> Identifying string for a particular calendar

# If modifying these scopes, delete the file token.json.

def Get_2w_google_api():
    """Shows basic usage of the Google Calendar API.
    Prints the start and name of the next 10 events on the user's calendar.
    """

    SCOPES = ['https://www.googleapis.com/auth/calendar.readonly']
    creds = None
    # The file token.json stores the user's access and refresh tokens, and is
    # created automatically when the authorization flow completes for the first
    # time.
    if os.path.exists('./tolken/token.json'):
        creds = Credentials.from_authorized_user_file('./tolken/token.json', SCOPES)
    # If there are no (valid) credentials available, let the user log in.
    if not creds or not creds.valid:
        if creds and creds.expired and creds.refresh_token:
            creds.refresh(Request())
        else:
            flow = InstalledAppFlow.from_client_secrets_file(
                './tolken/credentials.json', SCOPES)
            creds = flow.run_local_server(port=0)
        # Save the credentials for the next run
        with open('./tolken/token.json', 'w') as token:
            token.write(creds.to_json())

    try:
        service = build('calendar', 'v3', credentials=creds)

        # Call the Calendar API
        now_datetime = datetime.datetime.utcnow()
        now = now_datetime.isoformat() + 'Z'  # 'Z' indicates UTC time
        print('Upcoming events for the next 2 weeks')
        events_result = service.events().list(calendarId= BKB_CAL_ID , timeMin=now,
                                              timeMax= (now_datetime + datetime.timedelta(weeks=2)).isoformat() + 'Z', singleEvents=True,
                                              orderBy='startTime').execute()
        events = events_result.get('items', [])

        if not events:
            print('No upcoming events found.')
            return

        # Prints the start and name of the next 10 events
        return events

    except HttpError as error:
        print('An error occurred: %s' % error)
        return None

def send_to_facebook(events: list[dict]) -> Exception | None:

    page_id_1 = BKB_FACEBOOK_PAGE_ID
    facebook_access_token_1 = BKB_FACEBOOK_PAGE_TOKEN

    msg: list[str] = message_generator(events=events)

    post_url = 'https://graph.facebook.com/{}/feed'.format(page_id_1) # Figure out if correct link.
    payload = {
    'message': msg,
    'access_token': facebook_access_token_1
    }
    r = requests.post(post_url, data=payload)
    print(r.text)

    return None

async def send_to_discord(events: list[dict]) -> list[Exception]:
    """
        events er en dictonary med alle relevante envents for den kommende uke[^1]

        [^1] kommende uke er 7 dager etter den dagen hvor programmet hentet eventer.
    """

    if "discord_announcment_client" not in globals():
        print("Client has not been inisiated")
        return NameError("func 'send_to_discord': 'discord_announcment_client' does not exist.")
    # lim inn kode for å sende en str til "annonser", pass på formatering og at det er på riktig kanal
    # IKKE RAISE ERROR! Programmet må kjøre kontinuelig heletiden. Heller returner en Error isteded.
    ret_messages = list()
    for event in message_generator(events):
        try:
            await discord_announcment_client.send(event)
        except Exception as err:
            ret_messages.append(err)

    return ret_messages

def main():
    pass

def message_generator(events: dict) -> list[str]:
    """
        Generates a string meant to be sendt to end-clients.
        Fokuserer på 3 nøkkelord i root:
            - item["description"] -> str
            - item["start"]       -> dict
            - item["end"]         -> dict
        NOTE:
            Det er mange nøkkelord vi kan bruke... Dette krever lengere undersøkelse etter vi går live.
    """
    messages = list()
    for item in events: # is going to be replaced by a "standard" function so we can have fledxible text generation
        start = datetime.datetime.fromisoformat(item["start"])
        end   = datetime.datetime.fromisoformat(item["end"])
        top    = "date: {}\nStart: {} -> End: {}".format(start.date(),":".join(str(start.time()).split(":")[:2]),":".join(str(end.time()).split(":")[:2]))
        middel = item["description"]
        end    = ""

        messages.append("--------------".join([top,middel,end]))

    return messages

def create_contact() -> Exception | None:
    """
        Creates the nessesary connections to services.
        services:
            - Google calendar API (håndter et annet sted, se "Get_2w_google_api()" )
            - Facebook Graph API (muligens trenger ikke?)
            - Discord API
    """
    global discord_announcment_client, facebook_client

    discord_announcment_client = discord.Client().get_guild(BKB_SERVER_ID).get_channel(BKB_ANNOUNCMENT_CHANNEL_ID)
    facebook_client = ""

    return None

def time_diff(date1: datetime.datetime, date2: datetime.datetime) -> Exception | datetime.timedelta:
    diff = date2 - date1

    if datetime.timedelta() < diff < datetime.timedelta(hour = 24):
        return diff
    elif diff < datetime.timedelta():
        return datetime.timedelta()
    elif diff > datetime.timedelta(days=1):
        return datetime.timedelta(days=1)
    else:
        return ValueError("The difference is not messurable.")

if __name__ == "__main__":
    create_contact()
    send_to_discord({})


from typing import List

class Solution:
    def getWinner(self, arr: List[int], k: int) -> int:
        if k == 1:
            return max(arr[0], arr[1])
        if k >= len(arr):
            return max(arr)

        current_winner = arr[0]
        consecutive_wins = 0

        for i in range(1, len(arr)):
            if current_winner > arr[i]:
                consecutive_wins += 1
            else:
                current_winner = arr[i]
                consecutive_wins = 1

            if consecutive_wins == k:
                return current_winner

        return current_winner

if __name__ == '__main__':
    solution = Solution()

    # Test Case 1
    arr1 = [2, 1, 3, 5, 4, 6, 7]
    k1 = 2
    result1 = solution.getWinner(arr1, k1)
    print("Result 1:", result1)

    # Test Case 2
    arr2 = [3, 2, 1]
    k2 = 10
    result2 = solution.getWinner(arr2, k2)
    print("Result 2:", result2)