import json
from types import SimpleNamespace
import os.path
import pprint
from absl import logging
import numpy as np
import tensorflow.compat.v1 as tf
import tensorflow_addons as tfa
import kws_streaming.data.input_data as input_data
from kws_streaming.models import models
from kws_streaming.models import utils

import math

from transformers import AdamWeightDecay

def train(flags):
  """Model training."""

  flags.training = True

  # Set the verbosity based on flags (default is INFO, so we see all messages)
  logging.set_verbosity(flags.verbosity)

  # Start a new TensorFlow session.
  tf.reset_default_graph()
  config = tf.ConfigProto()
  config.gpu_options.allow_growth = True
  sess = tf.Session(config=config)
  tf.keras.backend.set_session(sess)

  audio_processor = input_data.AudioProcessor(flags)

  time_shift_samples = int((flags.time_shift_ms * flags.sample_rate) / 1000)

  # Figure out the learning rates for each training phase. Since it's often
  # effective to have high learning rates at the start of training, followed by
  # lower levels towards the end, the number of steps and learning rates can be
  # specified as comma-separated lists to define the rate at each stage. For
  # example --how_many_training_steps=10000,3000 --learning_rate=0.001,0.0001
  # will run 13,000 training loops in total, with a rate of 0.001 for the first
  # 10,000, and 0.0001 for the final 3,000.
  training_steps_list = list(map(int, flags.how_many_training_steps.split(',')))
  learning_rates_list = list(map(float, flags.learning_rate.split(',')))
  if len(training_steps_list) != len(learning_rates_list):
    raise Exception(
        '--how_many_training_steps and --learning_rate must be equal length '
        'lists, but are %d and %d long instead' % (len(training_steps_list),
                                                   len(learning_rates_list)))
  logging.info(flags)

  model = models.MODELS[flags.model_name](flags)
  if flags.distill_teacher_json:
    with open(flags.distill_teacher_json, 'r') as f:
      teacher_flags = json.load(f, object_hook=lambda d: SimpleNamespace(
        **{ k: v for k, v in flags.__dict__.items() if not k in d },
        **d))
    teacher_base = models.MODELS[teacher_flags.model_name](teacher_flags)
    hard_labels = tf.keras.layers.Lambda(lambda logits: tf.one_hot(tf.math.argmax(logits, axis=-1), depth=flags.label_count))
    teacher = tf.keras.models.Sequential([teacher_base, hard_labels])
    teacher_base.trainable = False
    teacher.trainable = False
  else:
    teacher = None
    teacher_flags = None

  base_model = model

  logging.info(model.summary())

  # save model summary
  utils.save_model_summary(model, flags.train_dir)

  # save model and data flags
  with open(os.path.join(flags.train_dir, 'flags.txt'), 'wt') as f:
    pprint.pprint(flags, stream=f)

  loss = tf.keras.losses.CategoricalCrossentropy(from_logits=True, label_smoothing=flags.label_smoothing)
  metrics = ['accuracy']

  if flags.optimizer == 'adam':
    optimizer = tf.keras.optimizers.Adam(epsilon=flags.optimizer_epsilon)
  elif flags.optimizer == 'momentum':
    optimizer = tf.keras.optimizers.SGD(momentum=0.9)
  elif flags.optimizer == 'novograd':
    optimizer = tfa.optimizers.NovoGrad(
        lr=0.05,
        beta_1=flags.novograd_beta_1,
        beta_2=flags.novograd_beta_2,
        weight_decay=flags.novograd_weight_decay,
        grad_averaging=bool(flags.novograd_grad_averaging))
  elif flags.optimizer == 'adamw':
    # Exclude some layers for weight decay
    exclude = ["pos_emb", "class_emb", "layer_normalization", "bias"]
    optimizer = AdamWeightDecay(learning_rate=0.05, weight_decay_rate=flags.l2_weight_decay, exclude_from_weight_decay=exclude)
  else:
    raise ValueError('Unsupported optimizer:%s' % flags.optimizer)

  loss_weights = [ 0.5, 0.5, 0.0 ] if teacher else [ 1. ] # equally weight losses form label and teacher, ignore ensemble output
  model.compile(optimizer=optimizer, loss=loss, loss_weights=loss_weights, metrics=metrics)

  train_writer = tf.summary.FileWriter(flags.summaries_dir + '/train',
                                       sess.graph)
  validation_writer = tf.summary.FileWriter(flags.summaries_dir + '/validation')

  sess.run(tf.global_variables_initializer())

  if flags.start_checkpoint:
    model.load_weights(flags.start_checkpoint).expect_partial()
    logging.info('Weights loaded from %s', flags.start_checkpoint)

  if teacher_flags and teacher_flags.start_checkpoint:
    # Load weights into teacher base as this is the actual model that was saved, teacher includes hard label head
    teacher_base.load_weights(teacher_flags.start_checkpoint).assert_existing_objects_matched()
    logging.info('Distillation teacher weights loaded from %s', teacher_flags.start_checkpoint)

  start_step = 0

  logging.info('Training from step: %d ', start_step)

  # Save graph.pbtxt.
  tf.train.write_graph(sess.graph_def, flags.train_dir, 'graph.pbtxt')

  # Save list of words.
  with tf.io.gfile.GFile(os.path.join(flags.train_dir, 'labels.txt'), 'w') as f:
    f.write('\n'.join(audio_processor.words_list))

  best_accuracy = 0.0

  # prepare parameters for exp learning rate decay
  training_steps_max = np.sum(training_steps_list)
  lr_init = learning_rates_list[0]
  exp_rate = -np.log(learning_rates_list[-1] / lr_init)/training_steps_max
  mode = 'training'

  if flags.lr_schedule == 'cosine':
    # Currently, no restarts are performed, so it is just a cosine decay over the entire
    # training process. I think this is how DeiT does it.
    lr_init = lr_init * flags.batch_size / 512
    num_train = audio_processor.set_size(mode)
    warmup_steps = int((num_train / flags.batch_size) * flags.warmup_epochs)
    first_decay_steps=training_steps_max

  # Training loop.
  for training_step in range(start_step, training_steps_max + 1):
    if training_step > 0:
      offset = (training_step -
                1) * flags.batch_size if flags.pick_deterministically else 0

      # Pull the audio samples we'll use for training.
      train_fingerprints, train_ground_truth = audio_processor.get_data(
          flags.batch_size, offset, flags, flags.background_frequency,
          flags.background_volume, time_shift_samples, mode,
          flags.resample, flags.volume_resample, sess)

      if flags.lr_schedule == 'exp':
        learning_rate_value = lr_init * np.exp(-exp_rate * training_step)
      elif flags.lr_schedule == 'linear':
        # Figure out what the current learning rate is.
        training_steps_sum = 0
        for i in range(len(training_steps_list)):
          training_steps_sum += training_steps_list[i]
          if training_step <= training_steps_sum:
            learning_rate_value = learning_rates_list[i]
            break
      elif flags.lr_schedule == 'cosine':
        learning_rate_value = lr_init * min(1, float(training_step) / max(1, warmup_steps)) * (math.cos(math.pi * training_step / training_steps_max) + 1) / 2.
      else:
        raise ValueError('Wrong lr_schedule: %s' % flags.lr_schedule)

      tf.keras.backend.set_value(model.optimizer.learning_rate, learning_rate_value)

      one_hot_labels = tf.keras.utils.to_categorical(train_ground_truth, num_classes=flags.label_count)

      if teacher:
        teacher_labels = teacher.predict_on_batch(train_fingerprints)
        one_hot_labels = [ one_hot_labels, teacher_labels, one_hot_labels ] # third is for the ensemble output, gradient is unused

      result = model.train_on_batch(train_fingerprints, one_hot_labels)

      if teacher:
        loss_total, loss_label, loss_teacher, loss_average, acc_label, acc_teacher, acc_ensemble = result
        differences = (teacher_labels != one_hot_labels).astype(dtype=int).sum()
        logging.info(
            'Step #%d: rate %f, accuracy %.2f%%, cross entropy %f, teacher acc %.2f%% (%d diff), teacher cross entropy %f, ensemble acc %.2f%%',
            *(training_step, learning_rate_value, acc_label * 100, loss_total, acc_teacher * 100, differences, loss_teacher, acc_ensemble * 100))
        summary = tf.Summary(value=[
            tf.Summary.Value(tag='accuracy', simple_value=acc_label),
            tf.Summary.Value(tag='teacher_accuracy', simple_value=acc_teacher),
            tf.Summary.Value(tag='ensemble_accuracy', simple_value=acc_ensemble),
        ])
      else:
        loss_label, acc_label = result
        logging.info(
            'Step #%d: rate %f, accuracy %.2f%%, cross entropy %f',
            *(training_step, learning_rate_value, acc_label * 100, loss_label))
        summary = tf.Summary(value=[
            tf.Summary.Value(tag='accuracy', simple_value=acc_label),
        ])

      train_writer.add_summary(summary, training_step)

    is_last_step = (training_step == training_steps_max)
    if (training_step % flags.eval_step_interval) == 0 or is_last_step:
      set_size = audio_processor.set_size('validation')
      set_size = int(set_size / flags.batch_size) * flags.batch_size
      total_accuracy = 0.0
      count = 0.0
      for i in range(0, set_size, flags.batch_size):
        validation_fingerprints, validation_ground_truth = audio_processor.get_data(
            flags.batch_size, i, flags, 0.0,
            0.0, 0, 'validation',
            0.0, 0.0, sess)

        one_hot_labels = tf.keras.utils.to_categorical(validation_ground_truth, num_classes=flags.label_count)
        if teacher:
          one_hot_labels = [ one_hot_labels, one_hot_labels, one_hot_labels ]
        # Run a validation step and capture training summaries for TensorBoard
        # with the `merged` op.
        result = model.test_on_batch(validation_fingerprints,
                                     one_hot_labels)

        if teacher:
          loss_total, loss_label, loss_teacher, loss_average, acc_label, acc_teacher, acc_ensemble = result
          summary = tf.Summary(value=[
          tf.Summary.Value(tag='accuracy', simple_value=acc_ensemble),
          tf.Summary.Value(tag='label_head_accuracy', simple_value=acc_label),
          tf.Summary.Value(tag='distill_head_accuracy', simple_value=acc_teacher),
          ])
          accuracy = acc_ensemble
        else:
          loss_label, acc_label = result
          summary = tf.Summary(value=[
              tf.Summary.Value(tag='accuracy', simple_value=acc_label),])
          accuracy = acc_label

        validation_writer.add_summary(summary, training_step)

        total_accuracy += accuracy
        count = count + 1.0

      total_accuracy = total_accuracy / count
      logging.info('Step %d: Validation accuracy = %.2f%% (N=%d)',
                   *(training_step, total_accuracy * 100, set_size))

      # Save the model checkpoint when validation accuracy improves
      if total_accuracy >= best_accuracy:
        best_accuracy = total_accuracy
        # overwrite the best model weights
        model.save_weights(flags.train_dir + 'best_weights')
      logging.info('So far the best validation accuracy is %.2f%%',
                   (best_accuracy * 100))

  tf.keras.backend.set_learning_phase(0)
  set_size = audio_processor.set_size('testing')
  set_size = int(set_size / flags.batch_size) * flags.batch_size
  logging.info('set_size=%d', set_size)
  total_accuracy = 0.0
  count = 0.0

  for i in range(0, set_size, flags.batch_size):
    test_fingerprints, test_ground_truth = audio_processor.get_data(
        flags.batch_size, i, flags, 0.0, 0.0, 0, 'testing', 0.0, 0.0, sess)

    one_hot_labels = tf.keras.utils.to_categorical(test_ground_truth, num_classes=flags.label_count)
    if teacher:
      one_hot_labels = [ one_hot_labels, one_hot_labels, one_hot_labels ]
    result = model.test_on_batch(test_fingerprints, one_hot_labels)

    total_accuracy += result[-1] if teacher else result[1]
    count = count + 1.0
  total_accuracy = total_accuracy / count

  logging.info('Final test accuracy = %.2f%% (N=%d)',
               *(total_accuracy * 100, set_size))
  with open(os.path.join(flags.train_dir, 'accuracy_last.txt'), 'wt') as fd:
    fd.write(str(total_accuracy * 100))
  model.save_weights(flags.train_dir + 'last_weights')


from multiprocessing import Queue, Process
import requests
from datetime import datetime, timedelta
import pandas as pd
import sqlite3
from random import randint
import numpy as np
import requests_cache
from time import sleep
import io
import sys
import time
import random

class announcement():
    def __init__(self, df, today):

        if len(sys.argv) == 0 or not today:
            #print('setting up cache')
            requests_cache.install_cache('estimize_cache')
        self.conn = sqlite3.connect('estimize_data.sqlite', timeout=30)

        self.df = df
        #print(self.df)

        self.get_yahoo()
        self.add_day_if_amc()
        self.get_change()
        self.get_day_change()
        #self.get_rev_surprise()
        self.get_eps_growth_rate()
        self.get_revs_growth_rate()
        self.get_avg_eps_beat_rate()
        self.get_average_ROI()
        self.get_industry()
        self.get_sue()

        self.df = pd.DataFrame(self.df).transpose()
        #print(self.df)

    def add_day_if_amc(self):
        if self.df['Time'] == "AMC":
            date = datetime.strptime(self.df['Date'], "%Y-%m-%d")
            date + timedelta(days=1)
            while date.isoweekday()>=6:
                date + timedelta(days=1)
            self.df['Date'] = str(date).split(" ")[0]

    def get_yahoo(self):
        try:
            url = "http://chart.finance.yahoo.com/table.csv?s={0}&a=12&b=01&c=2010&d=12&e=01&f=2018&g=d&ignore=.csv".format(self.df['Symbol'])
            data = io.StringIO(requests.get(url).text)
            self.yahoo_df = pd.read_csv(data)
        except Exception as e:
            self.yahoo_df = None

    def get_eps_growth_rate(self):
        # (q315-q314)/q315
        self.df['EPS_Growth'] = None
        try:
            prev_qtr = self.df['Qtr'][:2]+str(int(self.df['Qtr'][2:])-1)

            sql = "select `EPS` from estimize_processed where Symbol='%s' and Qtr = '%s'" % (self.df['Symbol'], prev_qtr)
            df = pd.read_sql(sql, self.conn)
            if not df.empty:
                eps_growth_rate = (float(self.df['EPS']) - float(df['EPS'])) / float(df['EPS'])

                self.df['EPS_Growth'] = eps_growth_rate

        except Exception as e:
            print(e)

    def get_revs_growth_rate(self):
        # (q315-q314)/q315
        self.df['Rev_Growth'] = None
        try:
            prev_qtr = self.df['Qtr'][:2]+str(int(self.df['Qtr'][2:])-1)

            sql = "select `Revs` from estimize_processed where Symbol='%s' and Qtr = '%s'" % (self.df['Symbol'], prev_qtr)
            df = pd.read_sql(sql, self.conn)
            if not df.empty:
                eps_growth_rate = (float(self.df['Revs']) - float(df['Revs'])) / float(df['Revs'])

                self.df['Rev_Growth'] = eps_growth_rate
        except Exception as e:
            print("here", e)

    def get_avg_eps_beat_rate(self):
        # average percent beat for last 4 quarters
        self.df['Avg_Beat'] = None
        self.df['Avg_Estimize_Beat'] = None
        try:
            sql = "select `Percent_Beat` from estimize_processed where Symbol='%s' order by Date desc limit 4" % (self.df['Symbol'])
            df = pd.read_sql(sql, self.conn)

            if len(df)>=3:
                avg_beat = sum(df['Percent_Beat'].astype(float))/len(df['Percent_Beat'])

                self.df['Avg_Beat'] = avg_beat

        except Exception as e:
            #print("here 2", e)
            pass

        try:
            sql = "select `Estimize_Percent_Beat` from estimize_processed where Symbol='%s' order by Date desc limit 4" % (self.df['Symbol'])
            df = pd.read_sql(sql, self.conn)

            if len(df)>=3:
                avg_beat = sum(df['Estimize_Percent_Beat'].astype(float))/len(df['Estimize_Percent_Beat'])

                self.df['Avg_Estimize_Beat'] = avg_beat

        except Exception as e:
            #print("here 3", e)
            pass

    def get_average_ROI(self):
        # average return of the last four quarters
        self.df['Avg_ROI'] = None
        try:
            sql = "select `ROI` from estimize_processed where Symbol='%s' order by Date desc limit 4" % (self.df['Symbol'])
            df = pd.read_sql(sql, self.conn)

            if len(df)>=3:

                avg_roi = sum(df['ROI'])/len(df['ROI'])

                self.df['Avg_ROI'] =avg_roi

        except Exception as e:
            #print("here 4", e)
            pass

    def get_industry(self):
        self.df['Industry'] = None
        try:
            html_text = requests.get("http://finviz.com/quote.ashx?t="+self.df['Symbol']).text
            df = pd.read_html(io.StringIO(html_text))[2]

            industry = df[5][6].split(" | ")[0]
            industry_num = None

            if industry=="Basic Materials":
                industry_num = 0
            elif industry=="Conglomerates":
                industry_num = 1
            elif industry=="Consumer Goods":
                industry_num = 2
            elif industry=="Financial":
                industry_num = 3
            elif industry=="Healthcare":
                industry_num = 4
            elif industry=="Industrial Goods":
                industry_num = 5
            elif industry=="Services":
                industry_num = 6
            elif industry=="Technology":
                industry_num = 7
            elif industry=="Utilities":
                industry_num = 8

            self.df['Industry'] = industry_num
        except Exception as e:
            #print("here 5", e)
            pass

    def get_sue(self):
        self.df['SUE'] = None
        self.df['Estimize_SUE'] = None
        try:
            sql = "select Surprise as SUE from estimize_processed where Symbol == '%s' order by Date desc limit 4;" % (self.df['Symbol'])

            df = pd.read_sql(sql, self.conn)

            if len(df)>=3 and sue != 0:
                sue = np.std(df['SUE'].astype('float'));
                sue = (float(self.df['Surprise'])) / sue

                self.df['SUE'] = sue

        except Exception as e:
            #print("here 6", e)
            pass

        try:
            sql = "select Estimize_Surprise as SUE from estimize_processed where Symbol == '%s' order by Date desc limit 4;" % (self.df['Symbol'])

            df = pd.read_sql(sql, self.conn)

            if len(df)>=3:
                sue = np.std(df['SUE'].astype('float'));
                sue = (float(self.df['Estimize_Surprise'])) / sue

                self.df['Estimize_SUE'] = sue

        except Exception as e:
            #print("here 7", e)
            pass

    def get_day_change(self):
        self.df['Before_ROI'] = None

        try:

            start_date = str(self.df['Date']).split(" ")[0]

            start_index = self.yahoo_df[self.yahoo_df['Date']==start_date].index

            open = self.yahoo_df.iloc[start_index]['Open'].values[0]
            close = self.yahoo_df.iloc[start_index+3]['Close'].values[0]

            overnight_roi = (open-close)/close

            self.df['Before_ROI'] = (overnight_roi * 100)
        except Exception as e:
            #print("here 8", e)
            pass

    def get_change(self):
        self.df['ROI'] = None
        self.df['Open'] = None
        self.df['Close'] = None
        try:
            start_date = str(self.df['Date']).split(" ")[0]

            change_df = self.yahoo_df[self.yahoo_df['Date']>=start_date].tail()
            change_df = change_df.reset_index(drop=True)

            open = change_df.ix[4,'Open']
            close = change_df.ix[0,'Close']

            roi = (close - open)/open
            if roi<2 and roi>-2:
                self.df['ROI'] = (roi * 100)
                self.df['Open'] = open
                self.df['Close'] = close
        except Exception as e:
            print("get change exception", e)


import customtkinter
from tkinter import *

customtkinter.set_appearance_mode('dark')
customtkinter.set_default_color_theme('blue')

root = customtkinter.CTk()
root.geometry('500x500')

vertical = Scale(root, from_=0, to=400, sliderlength=10,length=200)
vertical.pack(padx=10,pady=10)

def slide():
    lbl = customtkinter.CTkLabel(root, text=horizontal.get()).pack()
    root.geometry(str(horizontal.get()) + 'x' + str(vertical.get()))

horizontal = Scale(root, from_=0, to=400, orient=HORIZONTAL,length=300)
horizontal.pack()

lbl = customtkinter.CTkLabel(root, text=horizontal.get()).pack()

btn = customtkinter.CTkButton(root,text='Click Me',command=slide).pack(padx=10,pady=10)
root.mainloop()


"""The handler for training and evaluation."""

import os
from argparse import ArgumentParser

import torch
import wandb
# os.environ["WANDB_MODE"] = "offline"
# wandb.init()
from pytorch_lightning import Trainer
from pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint
from pytorch_lightning.callbacks.progress import TQDMProgressBar
from pytorch_lightning.loggers import WandbLogger
from pytorch_lightning.strategies.ddp import DDPStrategy
from pytorch_lightning.utilities import rank_zero_info

from pl_tsp_model import TSPModel
from pl_mis_model import MISModel

torch.cuda.amp.autocast(enabled=True)
torch.cuda.empty_cache()

def arg_parser():
  parser = ArgumentParser(description='Train a Pytorch-Lightning diffusion model on a TSP dataset.')
  parser.add_argument('--task', type=str, required=True)
  parser.add_argument('--storage_path', type=str, required=True)
  parser.add_argument('--validation_split', type=str, default='data/tsp/tsp50_val_concorde.txt')
  parser.add_argument('--test_split', type=str, default='data/tsp/tsp50_test_concorde.txt')
  parser.add_argument('--validation_examples', type=int, default=64)

  parser.add_argument('--num_workers', type=int, default=16)
  parser.add_argument('--fp16', action='store_true')
  parser.add_argument('--use_activation_checkpoint', action='store_true')

  parser.add_argument('--diffusion_schedule', type=str, default='linear')
  parser.add_argument('--diffusion_steps', type=int, default=1000)
  parser.add_argument('--inference_diffusion_steps', type=int, default=1000)
  parser.add_argument('--inference_schedule', type=str, default='linear')
  parser.add_argument('--inference_trick', type=str, default="ddim")
  parser.add_argument('--sequential_sampling', type=int, default=1)
  parser.add_argument('--parallel_sampling', type=int, default=1)

  parser.add_argument('--n_layers', type=int, default=12)
  parser.add_argument('--hidden_dim', type=int, default=256)
  parser.add_argument('--sparse_factor', type=int, default=-1)
  parser.add_argument('--aggregation', type=str, default='sum')
  parser.add_argument('--two_opt_iterations', type=int, default=1000)
  parser.add_argument('--save_numpy_heatmap', action='store_true')

  parser.add_argument('--project_name', type=str, default='tsp_diffusion')
  parser.add_argument('--wandb_entity', type=str, default=None)
  parser.add_argument('--wandb_logger_name', type=str, default=None)
  parser.add_argument("--resume_id", type=str, default=None, help="Resume training on wandb.")
  parser.add_argument('--ckpt_path', type=str, default=None)
  parser.add_argument('--resume_weight_only', action='store_true')

  parser.add_argument('--do_valid_only', action='store_true')
  parser.add_argument('--do_test_only', action='store_true')

  parser.add_argument('--rewrite_ratio', type=float, default=0.25)
  parser.add_argument('--norm', action='store_true')
  parser.add_argument('--rewrite', action='store_true')
  parser.add_argument('--rewrite_steps', type=int, default=3)
  parser.add_argument('--inference_steps', type=int, default=5)

  args = parser.parse_args()
  return args

def main(args):
    args.wandb_logger_name += "_norm" if args.norm else ""
    args.wandb_logger_name += "_{}".format(args.rewrite_ratio)
    args.wandb_logger_name += "_{}".format(args.parallel_sampling)
    args.wandb_logger_name += "_{}".format(args.inference_diffusion_steps)

    project_name = args.project_name

    if args.task == 'tsp':
      model_class = TSPModel
      saving_mode = 'min'
    elif args.task == 'mis':
      model_class = MISModel
      saving_mode = 'max'
    else:
        raise NotImplementedError

    model = model_class(param_args=args)
    os.makedirs(os.path.join(args.storage_path, f'models'), exist_ok=True)
    wandb_id = os.getenv("WANDB_RUN_ID") or wandb.util.generate_id()
    wandb_logger = WandbLogger(
      name=args.wandb_logger_name,
      project=project_name,
      entity=args.wandb_entity,
      save_dir=os.path.join(args.storage_path, f'models'),
      id=args.resume_id or wandb_id,
    )
    rank_zero_info(f"Logging to {wandb_logger.save_dir}/{wandb_logger.name}/{wandb_logger.version}")

    checkpoint_callback = ModelCheckpoint(
      monitor='val/solved_cost', mode=saving_mode,
      save_top_k=3, save_last=True,
      dirpath=os.path.join(wandb_logger.save_dir,
                            args.wandb_logger_name,
                            wandb_logger._id,
                            'checkpoints'),
    )
    lr_callback = LearningRateMonitor(logging_interval='step')

    trainer = Trainer(
      accelerator="auto",
      devices=torch.cuda.device_count() if torch.cuda.is_available() else None,
      # devices=None,
      callbacks=[TQDMProgressBar(refresh_rate=20), checkpoint_callback, lr_callback],
      logger=wandb_logger,
      check_val_every_n_epoch=1,
      strategy=DDPStrategy(static_graph=True),
      precision=16 if args.fp16 else 32,
      inference_mode=False
    )

    rank_zero_info(
      f"{'-' * 100}\n"
      f"{str(model.model)}\n"
      f"{'-' * 100}\n"
    )

    ckpt_path = args.ckpt_path

    if args.do_valid_only:
      trainer.validate(model, ckpt_path=ckpt_path)
    elif args.do_test_only:
      trainer.test(model, ckpt_path=ckpt_path)
    else:
      trainer.validate(model, ckpt_path=ckpt_path)
      trainer.test(model, ckpt_path=ckpt_path)
    trainer.logger.finalize("success")

if __name__ == '__main__':
    args = arg_parser()
    main(args)


"""
This module takes a bundle of datapackages and flattens them.

Because we have enabled the generation of multiple data packages as a part of a
data package "bundle", we need to squish the multiple data packages together in
order to put all of the pudl data into one data package. This is especailly
useful for converting the data package to a SQLite database or any other format.

The module does two main things:
 - squish the csv's together
 - squish the metadata (datapackage.json) files together

The CSV squishing is pretty simple and is all being done in
`flatten_data_packages_csvs`. We are assuming and enforcing that if two data
packages include the same dataset, that dataset has the same ETL parameters
(years, tables, states, etc.). The metadata is slightly more complicated to
compile because each element of the metadata is structured differently. Most of
that work is being done in `flatten_data_package_metadata`.
"""

import json
import logging
import os
import pathlib
import shutil

import pudl

logger = logging.getLogger(__name__)

##############################################################################
# Flattening PUDL Data Packages
##############################################################################

def flatten_data_packages_csvs(pkg_bundle_dir, pkg_name='pudl-all'):
    """
    Copy the CSVs into a new data package directory.

    Args:
        pkg_bundle_dir (path-like): the subdirectory where the bundle of data
            packages live
        pkg_name (str): the name you choose for the flattened data package.

    """
    # set where the flattened datapackage is going to live
    all_dir = pathlib.Path(pkg_bundle_dir, pkg_name)
    # delete the subdirectory if it exists
    if os.path.exists(all_dir):
        shutil.rmtree(all_dir)
    # make the subdirectory..
    os.mkdir(all_dir)
    # we also need the sub-subdirectory for the data
    all_data_dir = pathlib.Path(pkg_bundle_dir, pkg_name, 'data')
    os.mkdir(all_data_dir)
    # for each of the package directories, copy over the csv's
    for pkg_dir in pkg_bundle_dir.iterdir():
        # copy all the csv's except not from all_dir - would make duplicates or
        # from the epacems package (because it has CEMS and EIA files).
        if pkg_dir != all_dir:  # and 'epacems' in pkg_dir.name:
            for csv in pathlib.Path(pkg_dir, 'data').iterdir():
                # if the csv already exists, shutil.copy will overrite. this is
                # fine because we've already checked if the parameters are the
                # same
                shutil.copy(csv, all_data_dir)
        # for the CEMS pacakge, only pull the actual CEMS tables.
        # elif 'epacems' in pkg_dir.name:
        #    for csv in pathlib.Path(pkg_dir, 'data').iterdir():
        #        shutil.copy(csv, all_data_dir)
                # if 'hourly_emissions_epacems' in csv.name:
                #    shutil.copy(csv, all_data_dir)

def compile_data_packages_metadata(pkg_bundle_dir,
                                   pkg_name='pudl-all'):
    """
    Grab the metadata from each of your dp's.

    Args:
        pkg_bundle_dir (path-like): the subdirectory where the bundle of data
            packages live
        pkg_name (str): the name you choose for the flattened data package.

    Returns:
        dict: pkg_descriptor_elements

    """
    pkg_descriptor_elements = {}
    for pkg_dir in pkg_bundle_dir.iterdir():
        if pkg_dir.name != pkg_name:
            with open(pathlib.Path(pkg_dir, "datapackage.json")) as md:
                metadata = json.load(md)
            for thing in ['id', 'licenses', 'homepage', 'profile',
                          'created', 'sources', 'contributors', 'resources',
                          'autoincrement']:
                try:
                    pkg_descriptor_elements[thing].append(metadata[thing])
                except KeyError:
                    pkg_descriptor_elements[thing] = [metadata[thing]]
    return(pkg_descriptor_elements)

def flatten_data_package_metadata(pkg_bundle_dir,
                                  pkg_name='pudl-all'):
    """
    Convert a bundle of PULD data package metadata into one file.

    Args:
        pkg_bundle_dir (path-like): the subdirectory where the bundle of data
            packages live
        pkg_name (str): the name you choose for the flattened data package.

    Returns:
        dict: pkg_descriptor

    """
    # grab the peices of metadata from each of the data packages
    pkg_descriptor_elements = compile_data_packages_metadata(pkg_bundle_dir,
                                                             pkg_name=pkg_name)
    # the beginning of the data package descriptor
    pkg_descriptor = {
        'name': pkg_name,
        'title': 'flattened bundle of pudl data packages',
    }
    # the uuid for the individual data packages should be exactly the same
    if not len(set(pkg_descriptor_elements['id'])) == 1:
        raise AssertionError(
            'too many ids found in data packages metadata')
    # for these pkg_descriptor items, they should all be the same, so we are
    # just going to grab the first item for the flattened metadata
    for item in ['id', 'licenses', 'homepage']:
        pkg_descriptor[item] = pkg_descriptor_elements[item][0]
    # we're gonna grab the first 'created' timestap (each dp generates it's own
    # timestamp, which are slightly different)
    pkg_descriptor['created'] = min(pkg_descriptor_elements['created'])
    # these elements are dictionaries that have different items inside of them
    # we want to grab all of the elements and have no duplicates
    for item in ['sources', 'contributors', 'resources']:
        # flatten the list of dictionaries
        list_of_dicts = \
            [item for sublist in pkg_descriptor_elements[item]
                for item in sublist]
        if item == 'contributors':
            # turn the dict elements into tuple/sets to de-dupliate it
            pkg_descriptor[item] = [dict(y) for y in set(
                tuple(x.items()) for x in list_of_dicts)]
        elif item == 'sources' or 'resources':
            item_list = []
            # pull only one of each dataset creating a dictionary with the
            # dataset as the key
            for i in dict([(item_dict['title'], item_dict)
                           for item_dict in list_of_dicts]).values():
                item_list.append(i)
            pkg_descriptor[item] = item_list
    pkg_descriptor['autoincrement'] = pudl.helpers.merge_dicts(
        pkg_descriptor_elements['autoincrement'])
    return(pkg_descriptor)

def get_all_sources(pkg_descriptor_elements):
    """Grab list of all of the datasets in a data package bundle."""
    titles = set()
    for sources in pkg_descriptor_elements['sources']:
        for source in sources:
            titles.add(source['title'])
    return(titles)

def get_same_source_meta(pkg_descriptor_elements, title):
    """Grab the the source metadata of the same dataset from all datapackages."""
    samezies = []
    for sources in pkg_descriptor_elements['sources']:
        for source in sources:
            if source['title'] == title:
                samezies.append(source)
    return(samezies)

def check_for_matching_parameters(pkg_bundle_dir, pkg_name):
    """
    Check to see if the ETL parameters for datasets are the same across dp's.

    Args:
        pkg_bundle_dir (path-like): the subdirectory where the bundle of data
            packages live
        pkg_name (str): the name you choose for the flattened data package.
    """
    logger.info('checking for matching etl parameters across datapackages')
    # grab all of the metadata components
    pkg_descriptor_elements = compile_data_packages_metadata(pkg_bundle_dir,
                                                             pkg_name=pkg_name)
    # grab all of the "titles" (read sources)
    titles = get_all_sources(pkg_descriptor_elements)
    # check if
    for title in titles:
        samezies = get_same_source_meta(pkg_descriptor_elements, title)
        # for each of the source dictionaries, check if they are the same
        for source_dict in samezies:
            if not samezies[0] == source_dict:
                raise AssertionError(f'parameters do not match for {title}')

def flatten_pudl_datapackages(pudl_settings,
                              pkg_bundle_name,
                              pkg_name='pudl-all'):
    """
    Combines a collection of PUDL data packages into one.

    Args:
        pkg_bundle_name (str): the name of the subdirectory where the bundle of
            data packages live. Normally, this name will have been generated in
            `generate_data_packages`.
        pudl_settings (dict) : a dictionary filled with settings that mostly
            describe paths to various resources and outputs.
        pkg_name (str): the name you choose for the flattened data package.

    Returns:
        dict: a dictionary of the data package validation report.

    """
    # determine the subdirectory for the package bundles...
    pkg_bundle_dir = pathlib.Path(pudl_settings['datapackage_dir'],
                                  pkg_bundle_name)
    if not os.path.exists(pkg_bundle_dir):
        raise AssertionError(
            "The datapackage bundle directory does not exist. ")

    # check that data packages that have the same sources have the same parameters
    check_for_matching_parameters(pkg_bundle_dir, pkg_name)

    # copy the csv's into a new data package directory
    flatten_data_packages_csvs(pkg_bundle_dir,
                               pkg_name=pkg_name)
    # generate a flattened dp metadata descriptor
    pkg_descriptor = flatten_data_package_metadata(pkg_bundle_dir,
                                                   pkg_name=pkg_name)
    # using the pkg_descriptor, validate and save the data package metadata
    report = pudl.load.metadata.validate_save_pkg(
        pkg_descriptor,
        pkg_dir=pathlib.Path(pkg_bundle_dir, pkg_name))
    return(report)


# db_logger.py

import logging
import os

LOG_FILE = 'database_operations.log'

if not os.path.exists('logs'):
    os.makedirs('logs')

logging.basicConfig(
    filename=os.path.join('logs', LOG_FILE),
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(message)s'
)


"""
Created on Wed Mar 30 22:02:43 2022

@author: luis_
"""
from tkinter import simpledialog
import nltk
import matplotlib

carpeta_nombre = "C:\\Users\\luis_\\OneDrive\\Documentos\\PLN\\"
archivo_nombre = "texto1.txt"

with open(carpeta_nombre+archivo_nombre,"r",encoding="utf8") as archivo:
    texto = archivo.read()

tokens = nltk.word_tokenize(texto, "spanish")

text = nltk.Text(tokens)

text.collocations()

print("---------------------------------------------------")

text.similar("google")

print("---------------------------------------------------")

text.dispersion_plot(["google", "datos"])


import pandas as pd
import requests

filename = 'images_url.csv'
path_image = 'images/'

links = pd.read_csv(filename)
for i, url in enumerate(links.values):
    r = requests.get(url[0])
    with open(f'{path_image}/image-{i}.jpg', 'wb') as f:
        f.write(r.content)


from winregistry import WinRegistry as Reg
import os
reg = Reg()
os.system('cls')
path = r'HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\IDConfigDB\Hardware Profiles\0001'
print('-Main Menu-\n[1] Dump Current HWID\n[2] Replace Current HWID [!]\n[3] Exit')
choice = input('HWID-Tool> ')
if choice == '1':
	os.system('cls')
	print('\nCurrent HWID : ' + str(reg.read_value(path, 'HwProfileGuid')).split("'")[7])
	exit()
elif choice == '2':
	os.system('cls')
	print('\n\n[WARNING] Replacing your current HWID can cause driver errors,\ninvalidate licenses with other programs\nor cause other compatibility issues.\nUse caution before proceeding!')
	choice = input('Do you really want to replace your HWID? [Y/N] : ')
	if choice == 'N':
		exit()
	elif choice == 'Y':
		os.system('cls')
		newHWID = '{' + input('Alright, enter your new HWID : ') + '}'
		os.system('cls')
		print('Are you sure you want to change your HWID to\n' + newHWID )
		choice2 = input('[Y/N] : ')
		if choice2 == 'N':
			exit()
		elif choice2 == 'Y':
			print('OK, Trying to write new HWID.')
			try:
				reg.write_value(path, 'HwProfileGuid', r'' + newHWID, 'REG_SZ')
				print('New HWID Saved!')
			except:
				print('Error! Failed to write new HWID, did you run this as admin?')
				exit()
			exit()
		else:
			print('Invalid Choice')
			exit()
	else:
		print('Invalid Choice')
		exit()
elif choice == '2':
	print('Exited.')
	exit()
else:
	print('Invalid Choice')
	exit()


import numpy as np
import matplotlib.pyplot as plt

data_cnt = 1000
data_test_mean = 10.4
data_test_std = 0.5
decimal = 2
bin_cnt = 5

def gen_data(mean, std, size, decimal):
    return np.around(np.random.default_rng().normal(mean, std, size), decimals=decimal)

def gen_plot(arr, bin_cnt, mu, sigma, save=False):
    plt.figure()
    count, bins, ignored = plt.hist(arr, bin_cnt, density=True)
    plt.plot(bins, 1/(sigma*np.sqrt(2*np.pi))*np.exp(-(bins-mu)**2/(2*sigma**2)), linewidth=2, color='r')
    if save == False:
        plt.show()
    else:
        plt.savefig(save)
    plt.clf()

def gen_stat(arr):
    print("Mean: ", np.mean(arr))
    print("Standard Deviation: ", np.std(arr))
    print("Variance: ", np.var(arr))

if __name__ == "__main__":
    data_test = gen_data(data_test_mean, data_test_std, data_cnt, decimal)
    print(data_test)
    np.savetxt("./dataset/data_1.csv", data_test, fmt="%."+str(decimal)+"f")
    print("File written!")
    gen_plot(data_test, bin_cnt, data_test_mean, data_test_std, save="./output/data_1.png")
    print("Plot saved!")
    gen_stat(data_test)
    print("Statistics generated!")


import streamlit as st
from streamlit_option_menu import option_menu
st.set_page_config(layout="wide")
from PIL import Image
from source.Reg import Regression
from source.Class import Classification
from source.Cluster import Clustering
import source.Cluster as clt

# st.set_page_config(layout="wide")

with open('style/final.css') as f:
        st.markdown(f"<style>{f.read()}</style>",unsafe_allow_html=True)
imcol1, imcol2, imcol3 = st.columns((2,5,3))
with imcol1:
    st.write("")
with imcol2:
    st.image('image/Logo_final.png')
    st.markdown("")
with imcol3:
    st.write("")
#---------Side bar-------#

with st.sidebar:
    selected = st.selectbox("",
                     ['Select Application',"Classification","Regression","Clustering"],key='text')
    Library = st.selectbox("",
                     ["Library Used","Streamlit","Image","Pandas","matplotlib","scikit-learn"],key='text1')

    # Gcp_cloud = st.selectbox("",
    #                  ["GCP Services Used","VM Instance","Computer Engine","Cloud Storage"],key='text2')
    # GPT_TOOL =  st.selectbox(" ",('Models Used','GPT3 - Davinci','GPT-3.5 Turbo Model'),key='text3')
    st.markdown("## ")
    href = """<form action="#">
            <input type="submit" value="Clear/Reset" />
            </form>"""
    st.sidebar.markdown(href, unsafe_allow_html=True)
#--------------function calling-----------#
if __name__ == "__main__":
    try:
        if selected == "Select Application":
            pass
            st.markdown("<hr style=height:2.5px;background-color:gray>",unsafe_allow_html=True)
        if selected == "Regression":
            Regression()
        if selected == "Classification":
            Classification()
        if selected == "Clustering":
            Clustering()
    except BaseException as error:
        st.error(error)


import requests
import os
import pickle

if __name__ == "__main__":
    os.environ['NO_PROXY'] = '127.0.0.1'
    response = requests.get("http://localhost:5000/ping")
    print(response.text)
    print(response.content)
    print(response.json()[0])
    responses = pickle.loads(response.content)
    print(responses)

    # url = "http://127.0.0.1:5000/evaluate-model"
    # data = {'model_name': 'test', 'data': 'here'}
    #
    # response = requests.post(url, data)
    # print(response.text)


import json
from typing import Callable, Dict, List, Union

from ....utils import parse_readable_size
from ...web import web_api, MarsServiceWebAPIHandler, MarsWebAPIClientMixin
from ..core import SessionInfo
from .core import AbstractSessionAPI

def _encode_size(size: Union[str, Dict[str, List[int]]]) -> str:
    if not isinstance(size, dict):
        return size
    else:
        return ",".join(f"{k}={v}" for k, v in size.items())

def _decode_size(encoded: str) -> Union[int, str, Dict[str, Union[int, List[int]]]]:
    if not encoded:
        return 0
    if "," not in encoded and "=" not in encoded:
        try:
            return int(encoded)
        except ValueError:
            return int(parse_readable_size(encoded)[0])
    else:
        ret = dict()
        for kv in encoded.split(","):
            k, v = kv.split("=", 1)
            ret[k] = int(parse_readable_size(v)[0])
        return ret

class SessionWebAPIBaseHandler(MarsServiceWebAPIHandler):
    async def _get_oscar_session_api(self):
        from .oscar import SessionAPI

        return await self._get_api_by_key(SessionAPI, "Session", with_key_arg=False)

class SessionWebAPIHandler(SessionWebAPIBaseHandler):
    @classmethod
    def get_root_pattern(cls):
        return "/api/session(?:/(?P<sub_path>[^/]*)$|$)"

    @web_api("(?P<session_id>[^/]+)", method="put")
    async def create_session(self, session_id: str):
        oscar_api = await self._get_oscar_session_api()
        addr = await oscar_api.create_session(session_id)
        self.write(addr)

    @web_api("(?P<session_id>[^/]+)", method="delete")
    async def delete_session(self, session_id: str):
        oscar_api = await self._get_oscar_session_api()
        await oscar_api.delete_session(session_id)

    @web_api("", method="delete")
    async def delete_all_sessions(self):
        oscar_api = await self._get_oscar_session_api()
        await oscar_api.delete_all_sessions()

    @web_api(
        "(?P<session_id>[^/]+)", method="get", arg_filter={"action": "check_exist"}
    )
    async def has_session(self, session_id: str):
        oscar_api = await self._get_oscar_session_api()
        res = await oscar_api.has_session(session_id)
        self.write("1" if res else "0")

    @web_api(
        "(?P<session_id>[^/]*)",
        method="get",
        arg_filter={"action": "get_last_idle_time"},
    )
    async def get_last_idle_time(self, session_id: str):
        session_id = session_id or None
        oscar_api = await self._get_oscar_session_api()
        res = await oscar_api.get_last_idle_time(session_id)
        self.write(str(res) if res else "")

    @web_api("", method="get")
    async def get_sessions(self):
        oscar_api = await self._get_oscar_session_api()
        res = await oscar_api.get_sessions()
        self.write(
            json.dumps({"sessions": [{"session_id": info.session_id} for info in res]})
        )

class SessionWebLogAPIHandler(SessionWebAPIBaseHandler):
    _root_pattern = "/api/session/(?P<session_id>[^/]+)/op/(?P<op_key>[^/]+)/log"

    @web_api("", method="get")
    async def fetch_tileable_op_logs(self, session_id: str, op_key: str):
        oscar_api = await self._get_oscar_session_api()
        offsets = _decode_size(self.get_argument("offsets", None))
        sizes = _decode_size(self.get_argument("sizes", None))
        log_result = await oscar_api.fetch_tileable_op_logs(
            session_id, op_key, offsets, sizes
        )
        self.write(json.dumps(log_result))

web_handlers = {
    SessionWebAPIHandler.get_root_pattern(): SessionWebAPIHandler,
    SessionWebLogAPIHandler.get_root_pattern(): SessionWebLogAPIHandler,
}

class WebSessionAPI(AbstractSessionAPI, MarsWebAPIClientMixin):
    def __init__(self, address: str, request_rewriter: Callable = None):
        self._address = address.rstrip("/")
        self.request_rewriter = request_rewriter

    async def get_sessions(self) -> List[SessionInfo]:
        addr = f"{self._address}/api/session"
        res = await self._request_url("GET", addr)
        res_obj = json.loads(res.body.decode())
        return [SessionInfo(**kw) for kw in res_obj["sessions"]]

    async def create_session(self, session_id: str) -> str:
        addr = f"{self._address}/api/session/{session_id}"
        res = await self._request_url(path=addr, method="PUT", data=b"")
        return res.body.decode()

    async def delete_session(self, session_id: str):
        addr = f"{self._address}/api/session/{session_id}"
        await self._request_url(path=addr, method="DELETE")

    async def delete_all_sessions(self):
        addr = f"{self._address}/api/session"
        await self._request_url(path=addr, method="DELETE")

    async def has_session(self, session_id: str):
        addr = f"{self._address}/api/session/{session_id}"
        params = dict(action="check_exist")
        res = await self._request_url("GET", addr, params=params)
        return bool(int(res.body.decode()))

    async def get_last_idle_time(
        self, session_id: Union[str, None] = None
    ) -> Union[float, None]:
        session_id = session_id or ""
        addr = f"{self._address}/api/session/{session_id}"
        params = dict(action="get_last_idle_time")
        res = await self._request_url("GET", addr, params=params)
        content = res.body.decode()
        return float(content) if content else None

    async def fetch_tileable_op_logs(
        self,
        session_id: str,
        tileable_op_key: str,
        chunk_op_key_to_offsets: Dict[str, List[int]],
        chunk_op_key_to_sizes: Dict[str, List[int]],
    ) -> Dict:
        addr = f"{self._address}/api/session/{session_id}/op/{tileable_op_key}/log"
        params = dict(
            offsets=_encode_size(chunk_op_key_to_offsets),
            sizes=_encode_size(chunk_op_key_to_sizes),
        )
        res = await self._request_url("GET", addr, params=params)
        return json.loads(res.body.decode())


# -*- coding: utf-8 -*-

import argparse
import os
import pickle
import logging
import timeit
import warnings

from methods import Reader
from methods.feature_engineer2 import FeatureEngineer
from methods.baby_cry_predictor import BabyCryPredictor

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--load_path_data',
                        default='')
    parser.add_argument('--load_path_model',
                        default='output/model/')
    parser.add_argument('--save_path',
                        default='output/prediction/')
    parser.add_argument('--file_name', default='recording.wav')
    parser.add_argument('--log_path',
                        default='logs/')

    # Arguments
    args = parser.parse_args()
    load_path_data = args.load_path_data
    load_path_model = args.load_path_model
    file_name = args.file_name
    save_path = args.save_path
    log_path = args.log_path

    ####################################################################################################################
    # Set up logging
    ####################################################################################################################

    logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s',
                        datefmt='%Y-%m-%d %I:%M:%S %p',
                        filename=os.path.join(log_path, 'logs_pc_methods_test_model.log'),
                        filemode='w',
                        level=logging.INFO)

    ####################################################################################################################
    # READ RAW SIGNAL
    ####################################################################################################################

    logging.info('Reading {0}'.format(file_name))
    start = timeit.default_timer()

    # Read signal (first 5 sec)
    file_reader = Reader(os.path.join(load_path_data, file_name))
    signal, _ = file_reader.read_audio_file()

    stop = timeit.default_timer()
    logging.info('Time taken for reading file: {0}'.format(stop - start))

    ####################################################################################################################
    # FEATURE ENGINEERING
    ####################################################################################################################

    logging.info('Starting feature engineering')
    start = timeit.default_timer()

    # Feature extraction
    engineer = FeatureEngineer()

    processed_signal = engineer.feature_engineer(signal)
    # processed_signal.drop('label', axis=1, inplace=True)
    stop = timeit.default_timer()
    logging.info('Time taken for feature engineering: {0}'.format(stop - start))

    ####################################################################################################################
    # MAKE PREDICTION
    ####################################################################################################################

    logging.info('Predicting...')
    start = timeit.default_timer()

    # https://stackoverflow.com/questions/41146759/check-sklearn-version-before-loading-model-using-joblib
    with warnings.catch_warnings():
      warnings.simplefilter("ignore", category=UserWarning)

      with open((os.path.join(load_path_model, 'model.pkl')), 'rb') as fp:
          model = pickle.load(fp)

    predictor = BabyCryPredictor(model)

    prediction = predictor.classify(processed_signal)

    stop = timeit.default_timer()
    logging.info('Time taken for prediction: {0}. Is it a baby cry?? {1}'.format(stop - start, prediction))

    ####################################################################################################################
    # SAVE
    ####################################################################################################################

    logging.info('Saving prediction...')

    # Save prediction result
    with open(os.path.join(save_path, 'prediction.txt'), 'w') as text_file:
        text_file.write("{0}".format(prediction))

    logging.info('Saved! {0}'.format(os.path.join(save_path, 'prediction.txt')))

    return prediction

if __name__ == '__main__':
    print(main())


import requests

from src.utils.environment import Environment
from src.utils.logger import get_logger
from src.utils.errors import MrBotNotSupportedGitRepository, MrBotFailedCreateMergeRequest

logger = get_logger(__name__)

class Bot():
    """ MR Bot to create in version control apps. """
    def create_merge_request(self,
                             tool: str,
                             repo: str,
                             project_id: int,
                             repo_name: str,
                             owner: str,
                             src_branch: str,
                             dest_branch: str,
                             old_version: str,
                             new_version: str,
                             notes: str,
                             delete_src_branch: bool) -> str:
        """
        Call propper MR creationg method in class based on repo url

        Parameters:
            tool(str): Name of tool which version was checked
            repo(str): Local repository git url
            project_id(int): Gitlab project ID (Empty if not Gitlab)
            repo_name(str): Repository name (Empty if Gitlab)
            owner(str): Organization owner (Empty if Gitlab)
            src_branch(str): Generated branch where version update was pushed
            dest_branch(str): Main repository branch (ususally: master/main)
            old_version(str): Old version which was detected in local repository
            new_version(str): New version which was scrapped from OSS project
            notes(str): Release notes from OSS project
            delete_src_branch(bool): if True src branch checkbox would be enabled, else disabled
        Returns:
            str: Generated Merge Request url after webhook
        Raises:
            MrBotNotSupportedGitRepository: Raises when non supported version control
        """
        mr_title = f'{tool.capitalize()} version upgrade: {old_version} => {new_version}'

        if 'gitlab' in repo:
            return self.create_gitlab_mr(
                project_id=project_id,
                src_branch=src_branch,
                dest_branch=dest_branch,
                mr_title=mr_title,
                notes=notes,
                delete_src_branch=delete_src_branch
            )
        elif 'github' in repo:
            return self.create_github_pr(
                repo_name=repo_name,
                owner=owner,
                src_branch=src_branch,
                dest_branch=dest_branch,
                mr_title=mr_title,
                notes=notes
            )
        elif 'bitbucket' in repo:
            return self.create_bitbucket_pr(
                owner=owner,
                repo_name=repo_name,
                mr_title=mr_title,
                src_branch=src_branch,
                dest_branch=dest_branch,
                notes=notes,
                delete_src_branch=delete_src_branch
            )
        else:
            logger.error('%s auto MR/PR not supported', repo)
            raise MrBotNotSupportedGitRepository

    def create_gitlab_mr(self,
                         project_id: int,
                         dest_branch: str,
                         src_branch: str,
                         mr_title: str,
                         notes: str,
                         delete_src_branch: bool) -> str:
        """
        Method to create Merge Request in Gitlab

        Parameters:
            project_id(int): Gitlab project id
            dest_branch(str): Destination branch where to merge
            src_branch(str): Source branch which to merge
            mr_title(str): Merge Request title
            notes(str): Release notes from OSS project
            delete_src_branch(bool): if True src branch checkbox would be enabled, else disabled
        Returns:
            str: Generated Merge Request url after webhook
        Raises:
            MrBotFailedCreateMergeRequest: Raises when POST response NOT OK (200/201)
        """
        url = f'https://gitlab.com/api/v4/projects/{project_id}/merge_requests'

        headers = {'PRIVATE-TOKEN': f'{Environment.git_access_token}'}

        payload = {
            'source_branch': src_branch,
            'target_branch': dest_branch,
            'title': mr_title,
            'description': notes,
            'remove_source_branch': delete_src_branch
        }

        response = requests.post(url=url, json=payload, headers=headers, timeout=60)
        if not response.ok:
            logger.error('Failed to create MR in GitLab')
            logger.error(response.json())
            raise MrBotFailedCreateMergeRequest(repo=str(project_id))

        logger.info('Merge request in gitlab was created: %s', mr_title)
        return response.json()['web_url']

    def create_github_pr(self,
                         owner: str,
                         repo_name: str,
                         dest_branch: str,
                         src_branch: str,
                         mr_title: str,
                         notes: str) -> str:
        """
        Method to create Pull Request in Github

        Parameters:
            owner(str): Repository owner (Organization name)
            repo_name(int): Repository name in Github
            dest_branch(str): Destination branch where to merge
            src_branch(str): Source branch which to merge
            mr_title(str): Pull Request title
            notes(str): Release notes from OSS project
        Returns:
            str: Generated Pull Request url after webhook
        Raises:
            MrBotFailedCreateMergeRequest: Raises when POST response NOT OK (200/201)
        """
        url = f'https://api.github.com/repos/{owner}/{repo_name}/pulls'

        headers = {
            'Accept': 'application/vnd.github+json',
            'Authorization': f'Bearer {Environment.git_access_token}',
            'X-GitHub-Api-Version': '2022-11-28'
        }

        payload = {
            'title': 'mr_title',
            'owner': owner,
            'repo': repo_name,
            'body': notes,
            'base': src_branch,
            'head': dest_branch
        }

        response = requests.post(url=url, json=payload, headers=headers, timeout=60)
        if not response.ok:
            logger.error('Failed to create PR in Github')
            logger.error(response.json())
            raise MrBotFailedCreateMergeRequest(repo=repo_name)

        logger.info('Pull request in github was created: %s', mr_title)
        return response.json()['html_url']

    def create_bitbucket_pr(self,
                         owner: str,
                         repo_name: str,
                         dest_branch: str,
                         src_branch: str,
                         mr_title: str,
                         notes: str,
                         delete_src_branch: bool) -> str:
        """
        Method to create Pull Request in Bitbucket

        Parameters:
            owner(str): Repository owner (Organization name)
            repo_name(int): Repository name in Bitbucket
            dest_branch(str): Destination branch where to merge
            src_branch(str): Source branch which to merge
            mr_title(str): Pull Request title
            notes(str): Release notes from OSS project
            delete_src_branch(bool): if True src branch checkbox would be enabled, else disabled
        Returns:
            str: Generated Pull Request url after webhook
        Raises:
            MrBotFailedCreateMergeRequest: Raises when POST response NOT OK (200/201)
        """
        url = f'https://api.bitbucket.org/2.0/repositories/{owner}/{repo_name}/pullrequests'

        headers = {'Content-Type': 'application/json'}

        payload = {
            'title': mr_title,
            'description': notes,
            'close_source_branch': delete_src_branch,
            'destination': {
                'branch': {
                    'name': dest_branch
                }
            },
            'source': {
                'branch': {
                    'name': src_branch
                }
            }
        }

        response = requests.post(url=url, json=payload, headers=headers, auth=(Environment.git_username, Environment.git_access_token), timeout=60)
        if not response.ok:
            logger.error('Failed to create PR in BitBucket')
            logger.error(response.json())
            raise MrBotFailedCreateMergeRequest(repo=repo_name)

        logger.info('Pull request in bitbucket was created: %s', mr_title)
        return response.json()


# basic packages #
import os
import numpy as np
import pickle

# gen_PSTH_log #
from .PSTH_gen_PSTH_log import gen_PSTH_log

# for ANOVA  #
#from scipy import stats
import pandas as pd
from statsmodels.formula.api import ols
from statsmodels.stats.anova import anova_lm

# plot #
from matplotlib import pyplot as plt

def neuron_period_activity_analysis(hp,
                                    log,
                                    trial_list,
                                    model_dir,
                                    rule,
                                    seltive_epoch,
                                    analy_epoch,
                                    n_types=('exh_neurons','mix_neurons'),
                                    norm = True,
                                    PSTH_log = None,
                                    last_step = True,
                                    bin_wid=0.5,):

    print("Start neuron period activity analysis")
    print("\trule: "+rule+" selective epoch: "+seltive_epoch+" analysis epoch: ", analy_epoch)

    with open(model_dir+'/task_info.pkl','rb') as tinf:
        task_info = pickle.load(tinf)

    if PSTH_log is None:
        PSTH_log = gen_PSTH_log(hp,trial_list,model_dir,rule,seltive_epoch,n_types=n_types,norm=norm)

    if isinstance(analy_epoch, str):
        start = task_info[rule]['epoch_info'][analy_epoch][0]
        end = task_info[rule]['epoch_info'][analy_epoch][1]
    elif isinstance(analy_epoch, (tuple,list)):
        start = int(analy_epoch[0]/hp["dt"])
        end = int(analy_epoch[1]/hp["dt"])
    else:
        raise ValueError('Wrong analy_epoch format!')

    is_dict = False
    is_list = False
    if isinstance(trial_list, dict):
        temp_list = list()
        is_dict = True
        for value in trial_list[rule].values():
            temp_list += value
        temp_list = sorted(set(temp_list))
    elif isinstance(trial_list, list):
        temp_list = trial_list
        is_list = True

    trial_sort_by_matur = dict()
    fire_rate_dict = dict()

    for trial_num in temp_list:
        growth = log['perf_'+rule][trial_num//log['trials'][1]]
        if (is_list and growth > hp['mid_target_perf']) or (is_dict and trial_num in trial_list[rule]['mature']):
            mature = 'mature'
        elif (is_list and growth > hp['early_target_perf']) or (is_dict and trial_num in trial_list[rule]['mid']):
            mature = 'mid'
        elif is_list or (is_dict and trial_num in trial_list[rule]['early']):
            mature = 'early'

        if mature not in trial_sort_by_matur:
            trial_sort_by_matur[mature] = list()
            fire_rate_dict[mature] = list()
        trial_sort_by_matur[mature].append(trial_num)

    if last_step:
        for key, value in trial_sort_by_matur.items():
            trial_sort_by_matur[key] = value[-1:]

    for mature_key, sub_trial_list in trial_sort_by_matur.items():
        for trial_num in sub_trial_list:
            fire_rate_dict[mature_key] += list(PSTH_log[trial_num][:,start:end].mean(axis = 1))

    #ANOVA#
    #f,p = stats.f_oneway(*list(fire_rate_dict.values()))

    dict_melt = dict()
    dict_melt['Maturation'] = list()
    dict_melt['Fire_rate'] = list()
    for key,value in fire_rate_dict.items():
        dict_melt['Maturation'] += [key for i in range(len(value))]
        dict_melt['Fire_rate'] += list(value)

    df_melt = pd.DataFrame(dict_melt)
    model = ols('Fire_rate~C(Maturation)',data=df_melt).fit()
    anova_table = anova_lm(model, typ = 2)

    p = anova_table['PR(>F)'][0]
    df_g = anova_table['df'][0]
    df_res = anova_table['df'][1]

    #print("\tP value:",anova_table['PR(>F)'][0])

    # plot #
    colors = {'early':'green','mid':'blue','mature':'red'}
    save_path = 'figure/figure_'+model_dir.rstrip('/').split('/')[-1]+'/'+rule+'/'+seltive_epoch+'/'+'_'.join(n_types)+'/'
    fig,axes = plt.subplots(2,1,figsize=(12,15))
    for mature,fire_rate in fire_rate_dict.items():
        axes[0].hist(fire_rate,bins=int(max(fire_rate)/bin_wid)+1,histtype="step",alpha=0.6,\
            color=colors[mature],label=mature+' mean:%.3f'%(np.mean(fire_rate)),density=1)
    axes[0].legend()
    axes[0].set_xlabel("activity")

    m_keys = list(fire_rate_dict.keys())
    axes[1].boxplot([fire_rate_dict[m_key] for m_key in m_keys], labels = m_keys)
    axes[1].set_ylabel("activity")

    fig.suptitle("rule: "+rule+" selective epoch: "+seltive_epoch+" analysis epoch: "+str(analy_epoch)+\
        "\n p value: %.3e"%(p)+" group df: %.1f"%(df_g)+" residual df: %.1f"%(df_res))

    if isinstance(analy_epoch, str):
        plt.savefig(save_path+rule+'_'+analy_epoch+'_activity_oneway_anova_analysis.png')
        plt.savefig(save_path+rule+'_'+analy_epoch+'_activity_oneway_anova_analysis.pdf')
    elif isinstance(analy_epoch, (tuple,list)):
        plt.savefig(save_path+rule+'_'+str(analy_epoch[0])+'_'+str(analy_epoch[1])+'_activity_oneway_anova_analysis.png')
        plt.savefig(save_path+rule+'_'+str(analy_epoch[0])+'_'+str(analy_epoch[1])+'_activity_oneway_anova_analysis.pdf')


import pickle
import numpy as np
import skimage.transform
import skimage.util
import skimage.io
import skimage.color
import sklearn
import warnings
from pathlib import Path
import os
from collections import Counter
import matplotlib.pyplot as plt
import pandas as pd

def rescale_channel(batch):
	"""
	Rescale all channels of an image to (-1, 1) independently
	param: batch: a 4D array of immages with shape [index, height, width, depth]
	return: rescaled images with data type np.float32
	"""
	batch = batch.astype(np.float32)
	min = np.min(batch, axis=(1,2), keepdims=True)		# not sure this is a good idea, might cause color balance issues
	max = np.max(batch, axis=(1,2), keepdims=True)
	mu = 0.5*(max + min)
	rg = 0.5*(max - min + 1.0e-6)
	batch = (batch - mu)/rg
	return batch

def rescale_flat(batch):
	"""
	Rescale all channels of an image to (-1, 1) simultaneously
	param: batch: a 4D array of immages with shape [index, height, width, depth]
	return: rescaled images with data type np.float32
	"""
	batch = batch.astype(np.float32)
	min = np.min(batch, axis=(1,2,3), keepdims=True)
	max = np.max(batch, axis=(1,2,3), keepdims=True)
	mu = 0.5*(max + min)
	rg = 0.5*(max - min + 1.0e-6)
	batch = (batch - mu)/rg
	return batch

class Data(object):
	"""
	A class for self-contained handling of datasets
	"""
	def __init__(self, feature, label, scaler=None):
		"""
		Initialize
		param: feature: data features, here assumed to be 4d numpy array
		param: label: data labels, assumed to be 1d numpy array
		param: scaler: the scaling function to be applied to generated batches (default = rescale_channel)
		"""
		assert len(feature) == len(label)
		self.feature = feature
		self.label = label
		self.length = len(self.feature)
		self.__data = None
		self.__scaler = scaler
		if scaler is None:
			self.__scaler = rescale_channel

	def classes(self):
		return set(self.label)

	def size_classes(self):
		return len(self.classes())

	def input_shape(self):
		return self.feature[0].shape	# not sure if this will work

	def scale_function(self):
		return self.__scaler

	def shuffle(self):
		self.__data = (self.feature, self.label)
		self.feature, self.label = sklearn.utils.shuffle(self.feature, self.label)

	def reset(self):
		if self.__data:
			self.feature = self.__data[0]
			self.label = self.__data[1]
			self.__data = None

	def batch_count(self, size_batch):
		n = self.length // size_batch
		if self.length % size_batch:
			n += 1
		return n

	def make_batches(self, size_batch):
		for b in range(0, self.length, size_batch):
			x = self.feature[b:b+size_batch]
			y = self.label[b:b+size_batch]
			yield self.__scaler(x), y.astype(np.int32), len(x)

	def make_batch(self):
		return self.__scaler(self.feature), self.label.astype(np.int32), self.length

	def subset(self, n):
		return Data(self.feature[:n], self.label[:n], scaler=self.__scaler)

def load_data(scaler=None, train=True):
	file_train = "./data/train.p"		# These are the pickle files
	file_valid = "./data/valid.p"
	file_test = "./data/test.p"
	if train:
		with open(file_train, mode="rb") as f:
			data_train = pickle.load(f)
		with open(file_valid, mode="rb") as f:
			data_valid = pickle.load(f)
		dataset_train = Data(data_train["features"], data_train["labels"], scaler=scaler)
		dataset_valid = Data(data_valid["features"], data_valid["labels"], scaler=scaler)
		return dataset_train, dataset_valid
	else:
		with open(file_test, mode="rb") as f:
			data_test = pickle.load(f)
		dataset_test = Data(data_test["features"], data_test["labels"], scaler=scaler)
		return dataset_test

def load_augment(filename):
	path = Path(filename)
	if path.exists():
		with open(filename, mode="rb") as f:
			data_augment = pickle.load(f)
		return data_augment
	else:
		return None

def save_augment(filename, dict):
	path = Path(filename)
	if path.exists():
		path.unlink()
	with open(filename, mode="wb") as f:
		pickle.dump(dict, f)

def jitter_image_float(source, xn=0, yn=0):
	"""
	Generate an image shifted by xn, yn pixels
	param: source: input source image
	param: xn: vertical shift in pixels
	param: yn: horizontal shift in pixels
	return: shifted image
	"""
	if xn == 0 and yn == 0:
		return source
	x_pad = [(xn, 0)] if xn > 0 else [(0, -xn)]
	y_pad = [(yn, 0)] if yn > 0 else [(0, -yn)]
	y_pad.append((0,0))
	source_p = skimage.util.pad(source, pad_width=x_pad + y_pad, mode="edge")
	x_crp = [(0, xn)] if xn > 0 else [(-xn, 0)]
	y_crp = [(0, yn)] if yn > 0 else [(-yn, 0)]
	y_crp.append((0,0))
	source_c = skimage.util.crop(source_p, crop_width=x_crp + y_crp)
	return source_c

def transform_image(source, angle=0.0, refine=False, xn=0, yn=0):
	"""
	Generate a slightly modified image from a source image
	param: source: the source image
	param: angle: the rotation angle (optional)
	param: refine: a refinement switch
	param: xn: the desired vertical jitter
	param: yn: the desired horizontal jitter
	return: modified image with shape identical to source
	"""
	if angle == 0.0 and xn == 0 and yn == 0 and not refine:
		return source
	source_f = skimage.util.img_as_float(source)
	if angle != 0.0:
		source_r = skimage.transform.rotate(source_f, angle, order=1, mode="edge")
	else:
		source_r = source_f
	if refine:
		source_u = skimage.transform.pyramid_expand(source_r, order=3)
		source_r = skimage.transform.pyramid_reduce(source_u)
	if xn != 0 or yn != 0:
		source_r = jitter_image_float(source_r, xn=xn, yn=yn)
	with warnings.catch_warnings():
		warnings.simplefilter("ignore")
		return skimage.util.img_as_ubyte(source_r)

def generate_batch(source_image, source_label):
	"""
	Generate a series of images based on stochastic transforms of source images
	param: source_image: the source images assumed to be a 4-d array with axis=0 as the image selection axis
	param: source_label: the source image labels assumed to be a 1-d array
	return: tuple (target_image, target_label)
	note: the labels are not modified in any way, just copied to the output
	"""
	assert len(source_image) == len(source_label)
	bump = np.array([-2, -1, 0, 0, 1, 2], dtype=np.int32)
	generated = np.zeros_like(source_image)
	for i in range(len(source_image)):
		xn = np.random.choice(bump)
		yn = np.random.choice(bump)
		angle = 35.0*np.random.uniform(low=-1.0)
		use_refine = (np.random.uniform() > 0.5)
		if abs(angle) < 2.0:
			angle = 0.0
			use_refine = True
		generated[i,:,:,:] = transform_image(source_image[i], angle, refine=use_refine, xn=xn, yn=yn)
	return generated, source_label

def augment_data(base_data, filename=None):
	"""
	Generate an augmented data set from the input source data
	param: base_data: the input source data set
	param: filename: a list of filenames
	from which previously generated batches will be read,
	or if the files do not exist, to which the generated batches will be written
	return: a dataset containing the union of base data and all generated batches
	"""
	full_data, full_label = None, None
	if filename:
		full_data = base_data.feature
		full_label = base_data.label
		for fn in filename:
			from_file = load_augment(fn)
			if from_file is not None:
				fake_data = from_file["features"]
				fake_label = from_file["labels"]
			else:
				fake_data, fake_label = generate_batch(base_data.feature, base_data.label)
			if from_file is None:
				save_dict = {"features": fake_data, "labels": fake_label}
				save_augment(fn, save_dict)
			full_data = np.concatenate([full_data, fake_data])
			full_label = np.concatenate([full_label, fake_label])
	else:
		fake_data, fake_label = generate_batch(base_data.feature, base_data.label)
		full_data = np.concatenate([base_data.feature, fake_data])
		full_label = np.concatenate([base_data.label, fake_label])
	return Data(full_data, full_label, scaler=base_data.scale_function())

def data_summary(train, valid=None, test=None):
	"""
	Print summary statistics of data sets
	param: train: training data set
	param: valid: validation data set (optional)
	param: test: testing data set (optional)
	return: nothing
	"""
	print("Number of training examples = {}".format(train.length))
	if valid: print("Number of validation examples = {}".format(valid.length))
	if test: print("Number of testing examples = {}".format(test.length))
	print("Image data shape = {}".format(train.input_shape()))
	print("Number of unique classes = {}".format(train.size_classes()))

def data_statistics(datasets):
	"""
	Calculate counts of labels in data sets
	param: datasets: A list of data sets
	return: tuple(classes, list(counts))
	"""
	classes = []
	for ds in datasets:
		classes += ds.classes()
	classes = list(set(classes))
	stats = []
	for ds in datasets:
		counter = Counter(ds.label)
		counts = np.zeros((len(classes)))
		for cls in classes:
			if cls in counter:
				counts[cls] = counter[cls]
		stats.append(counts)
	return (classes, stats)

def plot_statistics(datasets, title=None, setlabels=None):
	"""
	Try to coerce a vertical plot of the frequency counts of data sets
	param: datasets: the data sets for which the plot should be made
	param: title: a plot title (optional)
	param: setlabels: a set of labels for classes (optional)
	return: nothing
	"""
	colors = ["r", "g", "b"]
	classes, stats = data_statistics(datasets)
	n_groups = len(classes)
	fig, axes = plt.subplots(figsize=(20,16))
	index = np.array(classes, dtype=np.float32)
	bar_width = 0.25
	has_legend = bool((setlabels is not None) and len(setlabels) == len(stats))
	idx = 0
	for sts in stats:
		total = np.sum(sts)
		if total > 0.0: sts /= total
		lbl = setlabels[idx] if has_legend else None
		clr = colors[idx % 3]
		plt.bar(index, sts, bar_width, label=lbl, color=clr)
		index += bar_width
		idx += 1
	if title is not None: plt.title(title, fontsize=30)
	plt.xlabel("Traffic sign", fontsize=20)
	plt.ylabel("Fraction of data set", fontsize=20)
	plt.legend()
	plt.tight_layout()
	plt.show()

def plot_statistics_y(datasets, title=None, setlabels=None, figsize=(20,16)):
	"""
	Try to coerce a horizontal plot of the frequency counts of data sets
	param: datasets: the data sets for which the plot should be made
	param: title: a plot title (optional)
	param: setlabels: a set of labels for classes (optional)
	return: nothing
	"""
	colors = ["r", "g", "b"]
	classes, stats = data_statistics(datasets)
	n_groups = len(classes)
	fig, axes = plt.subplots(figsize=figsize)
	index = np.array(classes, dtype=np.float32)
	bar_width = 0.25
	has_legend = bool((setlabels is not None) and len(setlabels) == len(stats))
	idx = 0
	for sts in stats:
		total = np.sum(sts)
		if total > 0.0: sts /= total
		lbl = setlabels[idx] if has_legend else None
		clr = colors[idx % 3]
		plt.barh(index, sts, bar_width, label=lbl, color=clr)
		index += bar_width
		idx += 1
	if title is not None: plt.title(title, fontsize=30)
	names = load_names()
	index -= 2*bar_width
	axes.set_yticks(index)
	axes.set_yticklabels(names)
	axes.invert_yaxis()
	plt.xlabel("Fraction of data set", fontsize=20)
	plt.ylabel("Traffic sign", fontsize=20)
	plt.legend()
	plt.tight_layout()
	plt.show()

def load_names():
	"""
	Read the label names from a csv file
	return: list of strings
	"""
	file_names = "./data/signnames.csv"
	data = pd.read_csv(file_names)
	cols = data.columns
	return data[cols[1]].values

def sample_images(images, labels):
	"""
	Select a sample image from images for each unique label in labels
	param: images: the source image set
	labels: the labels corresponding to the images
	"""
	label_set = list(set(labels))
	selection = []
	for lbl in label_set:
		available = [i for i,lb in enumerate(labels) if lb == lbl]
		k = np.random.choice(available)
		selection.append(images[k])
	return selection

def display_image_samples(dataset):
	"""
	Display a sample of images from a dataset
	param: dataset: the dataset from which to sample
	return: nothing
	"""
	samples = sample_images(dataset.feature, dataset.label)
	subtitles = load_names()
	display_image_grid_alt(samples, title="Image samples", subtitle=subtitles)

def display_image_row(image_set, title=None, subtitle=None):
	"""
	Display a row of images
	Note: untested
	"""
	n = len(image_set)
	fig, axes = plt.subplots(ncols=n)
	fig.tight_layout()
	if title:
		fig.suptitle(title)
	has_subtitles = bool((subtitle is not None) and len(subtitle) == n)
	for j in range(n):
		axes[j].imshow(image_set[j])
		axes[j].set_axis_off()
		if has_subtitles:
			axes[j].set_title(subtitle[j])
	plt.show()

def display_image_grid(image_set, title=None, subtitle=None, ncol=4, figsize=(12,20)):
	"""
	Display a grid of images
	param: image_set: the images to be displayed
	param: title: Title for the plot (optional)
	param: subtitle: Subtitles for each subplot (optional)
	param: ncol: the number of columns to arrange (optional, default = 4)
	param: figsize: the size of the figure (optional)
	return: nothing
	"""
	n = len(image_set)
	nrow = n//ncol
	if n % ncol != 0: nrow += 1
	fig, axes = plt.subplots(nrows=nrow, ncols=ncol, figsize=figsize)
	fig.tight_layout()
	if title:
		fig.suptitle(title, fontsize=20, y=1.0)
	has_subtitles = bool((subtitle is not None) and len(subtitle) == n)
	idx = 0
	for i in range(nrow):
		for j in range(ncol):
			axes[i][j].set_axis_off()
			if idx >= n: continue
			axes[i][j].imshow(image_set[idx])
			if has_subtitles:
				axes[i][j].set_title(subtitle[idx][:20] + "...[" + str(i) + "]")
			idx += 1
	plt.show()

def display_image_grid_alt(image_set, title=None, subtitle=None, ncol=4, figsize=(12,20)):
    """
    Display a grid of images
    param: image_set: the images to be displayed
    param: title: Title for the plot (optional)
    param: subtitle: Subtitles for each subplot (optional)
    param: ncol: the number of columns to arrange (optional, default = 4)
    param: figsize: the size of the figure (optional)
    return: nothing
    """
    # this was written due to problems with the above function in notebooks
    n = len(image_set)
    nrow = n//ncol
    tcks = np.array([])
    if n % ncol != 0: nrow += 1
    fig = plt.figure(figsize=figsize)
    if title:
        fig.suptitle(title, fontsize=20, y=0.92)
    has_subtitles = bool((subtitle is not None) and len(subtitle) == n)
    for i in range(n):
        axis = fig.add_subplot(nrow, ncol, i+1)
        plt.xticks(tcks)
        plt.yticks(tcks)
        axis.imshow(image_set[i])
        if has_subtitles:
            axis.set_xlabel(subtitle[i][:20] + "...[" + str(i) + "]")
        else:
            axis.set_xlabel(i)
    plt.show()

def display_predictions(features, actual, predict, figsize=(16, 18)):
	"""
	Display a set of predictions as generated by a trained model
	param: features: the features used to generate the predictions
	param: actual: the actual ground-truth labels
	param: predict: the predictions generated by the model
	return: nothing
	"""
	label_names = load_names()
	n_class = len(label_names)
	n_featr = len(features)
	fig, axes = plt.subplots(nrows=n_featr, ncols=2, figsize=figsize)
	fig.tight_layout()
	fig.suptitle("Softmax predictions", fontsize=20, y=1.1)
	n_predict = len(predict.indices[0])
	bar_index = np.arange(n_predict)
	margin = 0.05
	width = (1.0 - 2.0*margin)/n_predict
	for i, (f, act, pred_idx, pred_val) in enumerate(zip(features, actual, predict.indices, predict.values)):
		name_pred = [label_names[pi] for pi in pred_idx]
		name_actl = label_names[act]
		axes[i][0].imshow(f)
		axes[i][0].set_title(name_actl)
		axes[i][0].set_axis_off()
		axes[i][1].barh(bar_index + margin, pred_val[::-1], width)
		axes[i][1].set_yticks(bar_index + margin)
		axes[i][1].set_yticklabels(name_pred[::-1])
		axes[i][1].set_xticks([0, 0.5, 1.0])
	plt.show()

def prediction_accuracy(actual, predict):
	n = len(actual)
	if n == 0: return 0, 1.0
	correct = 0.0
	for act, prd in zip(actual, predict.indices):
		if act == prd[0]:
			correct += 1.0
	return n, correct/float(n)

def compute_class_scores(confusion):
	"""
	Calculate per-class measures from the the confusion matrix
	param: confusion: the confusion matrix as generated by a model
	return: dictionary of {name: (precision, recall, f-score)}
	"""
	names = load_names()
	n = len(names)
	assert n == confusion.shape[0]
	scores = {}
	for i in range(n):
		nm = names[i]
		true_pos = confusion[i,i]
		flse_pos = np.sum(confusion[:,i]) - true_pos
		flse_neg = np.sum(confusion[i,:]) - true_pos
		prec = true_pos/(true_pos + flse_pos)
		recl = true_pos/(true_pos + flse_neg)
		fscr = 2.0*prec*recl/(prec + recl)
		scores[nm] = (prec, recl, fscr)
	return scores

def read_test_images():
	"""
	Read a set of images from disc and save as pickle file
	Alpha channel information is removed if present
	Images read are assumed to be size 32x32
	return: nothing
	"""
	path = "./data/web/proc"
	files = [f for f in os.listdir(path) if f.startswith("img")]
	imgs_read = np.zeros((len(files), 32, 32, 3), dtype=np.uint8)
	for i in range(imgs_read.shape[0]):
		pth = os.path.join(path, files[i])
		img = skimage.io.imread(pth)
		if img.shape[2] == 4:
			img = skimage.color.rgba2rgb(img)
			with warnings.catch_warnings():
				warnings.simplefilter("ignore")
				img = skimage.util.img_as_ubyte(img)
		imgs_read[i,:,:,:] = img
	labels = np.array([14, 14, 40, 13, 20, 18, 1, 2, 2, 27, 23, 33, 25, 14], dtype=np.int8)
	web_test = Data(imgs_read, labels)
	save_dict = {"features": imgs_read, "labels": labels}
	save_augment("./data/web_test.p", save_dict)


from django.urls import path
from AdminDash import views

urlpatterns = [
    path('adminHome/', views.adminHome,name='admin_home'),
    path('adminLoginProcess/', views.adminLoginProcess, name='admin_login_process'),
    path('adminLogoutProcess/', views.adminLogoutProcess, name='admin_logout_process'),

]


from django.urls import path

from . import views
from . import rel

urlpatterns = [
    path('login/', views.login, name='login'),
    path('logoff/', views.logoff, name='logoff'),
    path('signup/', views.signup, name='signup'),
    path('signout/', views.signout, name='signout'),

    path('sobrevivente/', views.sobrevivente, name='sobrevivente'),
    path('posicao/', views.posicao, name='posicao'),
    path('relato/<int:relatado>/', views.relato, name='relato'),
    path('compra/<int:oferta>/', views.compra, name='compra'),

    path('rel/infeccoes/', rel.infeccoes, name='infeccoes'),
    path('rel/recursos/', rel.recursos, name='recursos'),
    path('rel/recursos-perdidos/', rel.recursos_perdidos, name='recursos_perdidos'),
    path('rel/sob-relatores/', rel.sob_relatores, name='sob_relatores'),
    path('rel/sob-distancia/', rel.sob_distancia, name='sob_sistancia'),
    path('rel/sob-pontos/', rel.sob_pontos, name='sob_pontos'),

    path('sobreviventes/', views.sobreviventes, name='sobreviventes'),
    path('itens/', views.itens, name='itens'),
    path('relatos/', views.relatos, name='relatos'),
    path('ofertas/', views.ofertas, name='ofertas'),
]


import os
import json
import sqlite3
from dotenv import load_dotenv
from driver import Driver
from datetime import datetime, timedelta

start=datetime.now()
# Load environment variables and webdriver
load_dotenv()  # take environment variables from .env.
driver = Driver()

# Initialize database
con = sqlite3.connect("linkedin.db")
cur = con.cursor()
cur.execute("""
            CREATE TABLE IF NOT EXISTS post(
            id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(4))) || '-' || lower(hex(randomblob(2))) || '-4' || substr(lower(hex(randomblob(2))),2) || '-a' || substr(lower(hex(randomblob(2))),2) || '-%' || lower(hex(randomblob(6)))),
            title VARCHAR(255) NOT NULL
            );
        """
)

# Sign in to LinkedIn Account defined in .env file
driver.get("https://www.linkedin.com/")
driver.find_element_by_id("session_key").send_keys(os.getenv("LINKEDIN_USER"))
driver.find_element_by_id("session_password").send_keys(os.getenv("LINKEDIN_PASS"))
driver.find_element_by_class_name("sign-in-form__submit-btn--full-width").click()

# Load all posts to list
# Scroll to last post
# Find new posts
# Add new posts to the list

input()
posts = driver.find_elements_by_class_name("update-components-text")

for post in posts:
    print(post.text)
    cur.execute("INSERT INTO post(title) VALUES("
                f"""'{post.text.replace("'", "").replace('"', "")}');"""
    )
    con.commit()
while True:
    new_posts = driver.find_elements_by_class_name("update-components-text")
    for new_post in new_posts:
        if new_post not in posts:
            print(new_post.text)
            cur.execute(
                "INSERT INTO post(title) VALUES("
                f"""'{new_post.text.replace("'", "").replace('"', "")}');"""
            )
            con.commit()
            posts.append(new_post)
    if datetime.now()-start > timedelta(minutes=5) :
        break
    driver.scroll_to_end_of_page()

# Serializing json
json_object = json.dumps([{"post":post.text}for post in posts], indent=4)

driver.quit()


import numpy as np
from scipy.io import loadmat
from collections import deque
from scipy.io import savemat
import os

class Env(object):
    """
    The wireless video streaming environment
    """

    def __init__(self, env_params):
        # Copying the env params
        self.road_distances = env_params['road_distances']
        self.acc_std = env_params['acc_std']
        self.speed_min, self.speed_max = env_params['speed_bound']
        self.cell_radius = env_params['cell_radius']
        self.Nt = env_params['Nt']
        self.delta_T = env_params['delta_T']  # Duration of each TF (s)

        self.noise = env_params['noise'] - 30  # dBm to dB
        self.bandwidth = env_params['bandwidth']
        self.power = env_params['power']
        self.max_rate = env_params['max_rate']

        self.num_seg = env_params['num_seg']
        self.seg_TF = env_params['seg_TF']  # Number of TF for each segment
        self.vbr_mean = env_params['vbr_mean']
        self.vbr_std = env_params['vbr_std']
        self.vbr_max = env_params['vbr_max']
        self.playback_start = env_params['playback_start']
        self.video_length = self.seg_TF * self.num_seg

        self.stop_pos = env_params['stop_pos']
        self.stop_time_range = env_params['stop_time_range']

        # Default status
        self.speed = None
        self.speed_before_stop = None
        self.seg_size = None  # Segment size vector
        self.pos = None  # Position of t
        self.TF_cnt = None  # Number of TF that the current segment to be played in the current time step, [0, seg_TF]
        self.seg_cnt = None  # Index of the segment that is going to be played in the current time step, [0, num_seg]
        self.buffer = None
        self.cum_buffer = None
        self.total_size = None
        self.progress = None
        self.ext_trajectory = None  # external defined user pos
        self.road = None
        self.stop_time = None
        self.stop_cnt = None

        # Reset state
        self.state = self.reset()
        self.state_dim = len(self.state)
        self.action_dim = 1

        # Params for computing the polynomial approximation of exp(-E1^(-1)(x))/E1^(-1)(x) from MATLAB
        self.p = loadmat('matlab/func_fitting/p.mat')['p'].astype(np.float32)

    def reset(self, ext_seg_size=None, ext_trajectory=None, ext_road_choice=None, reset_to_playback_start=True):

        self.TF_cnt = 1  # Start from 1 because the video is played (i.e. episode starts) after receiving the first seg
        self.seg_cnt = 0

        if ext_seg_size is None:
            self.seg_size = np.minimum(np.random.normal(self.vbr_mean, self.vbr_std, self.num_seg),
                                       self.vbr_max) * self.delta_T * self.seg_TF
        else:
            self.seg_size = ext_seg_size
        self.total_size = np.sum(self.seg_size)

        if ext_road_choice is None:
            self.road = np.random.choice(len(self.road_distances))
        else:
            self.road = ext_road_choice

        if ext_trajectory is None:
            self.ext_trajectory = None
            self.speed = np.random.uniform(self.speed_min, self.speed_max)
            pos_0 = np.random.uniform(0, 2 * self.cell_radius)
            self.pos = [pos_0 for _ in range(self.Nt)]  # duplicated padding
            self.stop_time = np.random.uniform(self.stop_time_range[0], self.stop_time_range[1])
            self.stop_cnt = 0
        else:
            self.ext_trajectory = deque(ext_trajectory)
            pos_0 = self.ext_trajectory.popleft()
            self.pos = [pos_0 for _ in range(self.Nt)]

        self.buffer = self.seg_size[0]
        self.cum_buffer = self.buffer
        self.progress = self.cum_buffer / self.total_size

        snr = self.compute_snr()
        self.state = np.r_[self.progress, self.TF_cnt, self.seg_size[self.seg_cnt], self.buffer, snr]

        if reset_to_playback_start:
            [self.step() for _ in range(self.playback_start)]

        return self.state

    def step(self, rate=None):
        energy, shortage, done, cut_off = 0, 0, 0, 0

        if rate is not None:
            rate = np.squeeze(rate)
            max_rate = (self.total_size - self.cum_buffer) / self.delta_T

            # buffer evolution
            if self.TF_cnt == self.seg_TF:  # the current segment is finishing
                self.TF_cnt = 0
                self.buffer = self.buffer + rate * self.delta_T - self.seg_size[self.seg_cnt]
                self.seg_cnt += 1
            else:
                self.buffer = self.buffer + rate * self.delta_T

            self.cum_buffer = self.cum_buffer + rate * self.delta_T
            self.progress = self.cum_buffer / self.total_size  # total video downloading progress

            # stalling or not
            if self.seg_size[self.seg_cnt] <= self.buffer:
                self.TF_cnt += 1
            else:
                shortage = self.seg_size[self.seg_cnt] - self.buffer

            rate = np.minimum(rate, max_rate)
            energy = self.power_rate(rate, self.state[4]) * self.delta_T  # compute the energy before pos changes
            if self.cum_buffer >= self.total_size:
                done = 1

        # Position evolution
        self.pos[1:] = self.pos[:-1]
        if self.ext_trajectory is None:
            if self.pos[0] > self.stop_pos and self.stop_cnt * self.delta_T < self.stop_time:
                self.stop_cnt += 1
                if self.speed > 0:
                    self.speed_before_stop = self.speed
                self.speed = 0
            else:
                if self.speed == 0:
                    self.speed = self.speed_before_stop
                self.pos[0] = self.pos[0] + self.speed * self.delta_T
                self.speed += np.random.normal(scale=self.acc_std) * self.delta_T
                self.speed = np.clip(self.speed, self.speed_min, self.speed_max)

        else:
            self.pos[0] = self.ext_trajectory.popleft()
            if len(self.ext_trajectory) == 0:
                cut_off = 1

        # SNR
        snr_db = self.compute_snr()
        self.state = np.r_[self.progress, self.TF_cnt, self.seg_size[self.seg_cnt], self.buffer, snr_db]
        return self.state, energy, shortage, done, cut_off

    def compute_snr(self):
        """
        Compute the receive SNRs (unit transmit power) between the user and its 1st and 2nd nearest BSs
        based on the user's current position and road
        """
        snr_db = np.zeros(self.Nt * 2)
        road_dist = self.road_distances[self.road]
        for i in range(self.Nt):
            x = np.mod(self.pos[i], 2 * self.cell_radius)
            if x < self.cell_radius:
                dist_1 = np.sqrt(x ** 2 + road_dist ** 2)
                dist_2 = np.sqrt((2 * self.cell_radius - x) ** 2 + road_dist ** 2)
            else:
                dist_2 = np.sqrt(x ** 2 + road_dist ** 2)
                dist_1 = np.sqrt((2 * self.cell_radius - x) ** 2 + road_dist ** 2)

            snr_db[2 * i] = -35.3 - 37.6 * np.log10(dist_1) - self.noise  # SNR nearest BS
            snr_db[2 * i + 1] = -35.3 - 37.6 * np.log10(dist_2) - self.noise  # second nearest BS
        return snr_db

    def power_rate(self, rate, snr_dB):
        # Compute the average transmit power based on the average data rate
        snr = np.power(10, (snr_dB / 10))
        x = rate / self.bandwidth * np.log(2)
        y = np.matmul(np.power(x, np.arange(0, len(self.p))), self.p)[0]
        power = 1 / snr * (np.maximum(y, 0) - x)
        return power

    def get_next_seg_size(self):
        # Get the size of the segment to be played in the next TF
        if self.TF_cnt == self.seg_TF:
            cnt = self.seg_cnt + 1
        else:
            cnt = self.seg_cnt
        return [self.seg_size[cnt]]

    def gen_traces(self, num, length, data_dir=None, name=None, verbose=True, seed=None):
        """
        Generate traces including user position, SNR, segment size, and road choice. Save the traces to data_dir.
        :param num: Number of traces to be generated
        :param length: Length of each trace
        :param data_dir: Directory to save the trace data
        :param name: Name of the data
        :param verbose: Print the information or not
        :param seed: Random seed
        :return: SNR and user position traces
        """
        if seed is not None:
            np.random.seed(seed)
        pos_data = []
        snr_data = []
        seg_size_data = []
        road_data = []
        v_data = []
        for i in range(num):
            if verbose:
                print('Generating trace: {}'.format(i))

            self.reset(reset_to_playback_start=False)
            seg_size_data.append(self.seg_size)  # Generate seg size
            road_data.append(self.road)  # Generate road choice

            # Generate pos
            pos_trace = [self.pos[0]]
            snr_trace = [self.state[4]]
            v_trace = [self.speed]
            for t in range(length - 1):
                self.step()
                pos_trace.append(self.pos[0])
                snr_trace.append(self.state[4])
                v_trace.append(self.speed)
            pos_data.append(pos_trace)
            snr_data.append(snr_trace)
            v_data.append(v_trace)

        if data_dir is not None:
            if not os.path.exists(data_dir):
                os.makedirs(data_dir)
            savemat(data_dir + '/' + name + '.mat', {'pos_data': pos_data,
                                                     'seg_size_data': seg_size_data,
                                                     'road_data': road_data,
                                                     'snr_data': snr_data,
                                                     'v_data': v_data})
        return snr_data, pos_data, v_data


from django.urls import path

from . import views

urlpatterns = [
    path("", views.index, name="index"),
    path("wiki/create",views.create, name ="create"),
    path("wiki/search",views.search, name ="search"),
    path("wiki/<str:entry>", views.page, name="page"),
    path("wiki/<str:title>/edit", views.edit, name="edit"),
    path("wiki/<str:title>/delete", views.delete, name="delete"),
    path("wiki/<str:title>/save", views.save, name="save"),
    path("wiki", views.random_page, name="random_page"),

]


import logging
from logging import Logger

from danswer.configs.app_configs import LOG_LEVEL

def get_log_level_from_str(log_level_str: str = LOG_LEVEL) -> int:
    log_level_dict = {
        "CRITICAL": logging.CRITICAL,
        "ERROR": logging.ERROR,
        "WARNING": logging.WARNING,
        "INFO": logging.INFO,
        "DEBUG": logging.DEBUG,
        "NOTSET": logging.NOTSET,
    }

    return log_level_dict.get(log_level_str.upper(), logging.INFO)

def setup_logger(
    name: str = __name__, log_level: int = get_log_level_from_str()
) -> Logger:
    logger = logging.getLogger(name)

    # If the logger already has handlers, assume it was already configured and return it.
    if logger.handlers:
        return logger

    logger.setLevel(log_level)

    formatter = logging.Formatter(
        "%(asctime)s %(filename)20s%(lineno)4s : %(message)s",
        datefmt="%m/%d/%Y %I:%M:%S %p",
    )

    handler = logging.StreamHandler()
    handler.setLevel(log_level)
    handler.setFormatter(formatter)

    logger.addHandler(handler)

    return logger