import tensorflow as tf
tf.enable_eager_execution()

import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
from sklearn.model_selection import train_test_split

import random
import unicodedata
import re
import numpy as np
import os
import io
import time
import tqdm

TRAIN_FILE = 'athena_train.txt'
TEST_FILE = 'athena_test.txt'

# Read, then decode for py2 compat.
data = open(TRAIN_FILE, 'rb').read().decode(encoding='utf-8')
data = data.replace('(', ' ( ').replace(')', ' ) ').replace('[', ' [ ').replace(']', ' ] ').replace('{', ' { ').replace('}', ' } ')

train = []
test = []
para = ''
train_flag = True
start = True
for line in data.splitlines():
    if line == '':
        train_flag = (random.random() > 0.05)
        if not start:
            if train_flag:
                train.append(para)
            else:
                test.append(para)
            para = ''
    elif train_flag:
        para += line + '\n'
    else:
        para += line + '\n'
    start = False

train_x = ['<start> '  + item.splitlines()[0] + ' <end>' for item in train]
train_y = ['<start> ' + ' '.join((' '.join(item.splitlines()[1:])).split()) + ' <end>' for item in train]
test_x = ['<start> '  + item.splitlines()[0] + ' <end>' for item in test]
test_y = ['<start> ' + ' '.join((' '.join(item.splitlines()[1:])).split()) + ' <end>' for item in test]

train_x = [' '.join(i for i in line.split() if not (i.isalpha() and len(i)==1)) for line in train_x]
train_y = [' '.join(i for i in line.split() if not (i.isalpha() and len(i)==1)) for line in train_y]
print(train_y[0])

data = ' '.join(i for i in data.split() if not (i.isalpha() and len(i)==1))

lang_tokenizer = tf.keras.preprocessing.text.Tokenizer(filters=' \n\t\r\n', lower = False)
lang_tokenizer.fit_on_texts(['<start> ' + data + ' <end>'])

train_x = lang_tokenizer.texts_to_sequences(train_x)

train_y = lang_tokenizer.texts_to_sequences(train_y)
test_x = lang_tokenizer.texts_to_sequences(test_x)
test_y = lang_tokenizer.texts_to_sequences(test_y)

train_x = tf.keras.preprocessing.sequence.pad_sequences(train_x, padding='post')
train_y = tf.keras.preprocessing.sequence.pad_sequences(train_y, padding='post')[:, :50]
test_x = tf.keras.preprocessing.sequence.pad_sequences(test_x, padding='post')
test_y = tf.keras.preprocessing.sequence.pad_sequences(test_y, padding='post')[:, :50]

BUFFER_SIZE = len(train_x)
BATCH_SIZE = 5
steps_per_epoch = int(len(train_x)//BATCH_SIZE)
test_steps_per_epoch = int(len(test_x)//BATCH_SIZE)
embedding_dim = 256
units = 1024
vocab_size = len(lang_tokenizer.word_index)+1

train_dataset = tf.data.Dataset.from_tensor_slices((train_x, train_y)).shuffle(BUFFER_SIZE)
train_dataset = train_dataset.batch(BATCH_SIZE, drop_remainder=True)
test_dataset = tf.data.Dataset.from_tensor_slices((test_x, test_y)).shuffle(BUFFER_SIZE)
test_dataset = test_dataset.batch(BATCH_SIZE, drop_remainder=True)

example_input_batch, example_target_batch = next(iter(train_dataset))
print(example_input_batch.shape, example_target_batch.shape)

class Encoder(tf.keras.Model):
  def __init__(self, vocab_size, embedding_dim, enc_units, batch_sz):
    super(Encoder, self).__init__()
    self.batch_sz = batch_sz
    self.enc_units = enc_units
    self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)
    self.gru = tf.keras.layers.GRU(self.enc_units,
                                   return_sequences=True,
                                   return_state=True,
                                   recurrent_initializer='glorot_uniform')

  def call(self, x, hidden):
    x = self.embedding(x)
    output, state = self.gru(x, initial_state = hidden)
    return output, state

  def initialize_hidden_state(self):
    return tf.zeros((self.batch_sz, self.enc_units))

encoder = Encoder(vocab_size, embedding_dim, units, BATCH_SIZE)

class BahdanauAttention(tf.keras.layers.Layer):
  def __init__(self, units):
    super(BahdanauAttention, self).__init__()
    self.W1 = tf.keras.layers.Dense(units)
    self.W2 = tf.keras.layers.Dense(units)
    self.V = tf.keras.layers.Dense(1)

  def call(self, query, values):
    # query hidden state shape == (batch_size, hidden size)
    # query_with_time_axis shape == (batch_size, 1, hidden size)
    # values shape == (batch_size, max_len, hidden size)
    # we are doing this to broadcast addition along the time axis to calculate the score
    query_with_time_axis = tf.expand_dims(query, 1)

    # score shape == (batch_size, max_length, 1)
    # we get 1 at the last axis because we are applying score to self.V
    # the shape of the tensor before applying self.V is (batch_size, max_length, units)
    score = self.V(tf.nn.tanh(
        self.W1(query_with_time_axis) + self.W2(values)))

    # attention_weights shape == (batch_size, max_length, 1)
    attention_weights = tf.nn.softmax(score, axis=1)

    # context_vector shape after sum == (batch_size, hidden_size)
    context_vector = attention_weights * values
    context_vector = tf.reduce_sum(context_vector, axis=1)

    return context_vector, attention_weights

attention_layer = BahdanauAttention(10)

class Decoder(tf.keras.Model):
  def __init__(self, vocab_size, embedding_dim, dec_units, batch_sz):
    super(Decoder, self).__init__()
    self.batch_sz = batch_sz
    self.dec_units = dec_units
    self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)
    self.gru = tf.keras.layers.GRU(self.dec_units,
                                   return_sequences=True,
                                   return_state=True,
                                   recurrent_initializer='glorot_uniform')
    self.fc = tf.keras.layers.Dense(vocab_size)

    # used for attention
    self.attention = BahdanauAttention(self.dec_units)

  def call(self, x, hidden, enc_output):
    # enc_output shape == (batch_size, max_length, hidden_size)
    context_vector, attention_weights = self.attention(hidden, enc_output)

    # x shape after passing through embedding == (batch_size, 1, embedding_dim)
    x = self.embedding(x)

    # x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size)
    x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)

    # passing the concatenated vector to the GRU
    output, state = self.gru(x)

    # output shape == (batch_size * 1, hidden_size)
    output = tf.reshape(output, (-1, output.shape[2]))

    # output shape == (batch_size, vocab)
    x = self.fc(output)

    return x, state, attention_weights

decoder = Decoder(vocab_size, embedding_dim, units, BATCH_SIZE)

optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=0.001)

def loss_function(real, pred):
  mask = tf.math.logical_not(tf.math.equal(real, 0))
  loss_ = tf.keras.losses.sparse_categorical_crossentropy(real, pred, from_logits = True)

  mask = tf.cast(mask, dtype=loss_.dtype)
  loss_ *= mask

  return tf.reduce_mean(loss_)

def train_step(inp, targ, enc_hidden):
  loss = 0

  with tf.GradientTape() as tape:
    enc_output, enc_hidden = encoder(inp, enc_hidden)

    dec_hidden = enc_hidden

    dec_input = tf.expand_dims([lang_tokenizer.word_index['<start>']] * BATCH_SIZE, 1)

    # Teacher forcing - feeding the target as the next input
    for t in range(1, targ.shape[1]):
      # passing enc_output to the decoder
      predictions, dec_hidden, _ = decoder(dec_input, dec_hidden, enc_output)

      loss += loss_function(targ[:, t], predictions)

      # using teacher forcing
      dec_input = tf.expand_dims(targ[:, t], 1)

  batch_loss = (loss / int(targ.shape[1]))

  variables = encoder.trainable_variables + decoder.trainable_variables

  gradients = tape.gradient(loss, variables)

  optimizer.apply_gradients(zip(gradients, variables))

  return batch_loss

EPOCHS = 30

for epoch in range(EPOCHS):
  start = time.time()

  enc_hidden = encoder.initialize_hidden_state()
  total_loss = 0

  pbar = tqdm.tqdm(enumerate(train_dataset.take(steps_per_epoch)), total = steps_per_epoch)
  for (batch, (inp, targ)) in pbar:
    batch_loss = train_step(inp, targ, enc_hidden)
    total_loss += batch_loss
    pbar.set_description('LOSS : %f' % batch_loss)

  test_loss = 0
  for (batch, (inp, targ)) in tqdm.tqdm(enumerate(test_dataset.take(test_steps_per_epoch)), total = test_steps_per_epoch):
    loss = 0
    enc_output, enc_hidden = encoder(inp, enc_hidden)

    dec_hidden = enc_hidden

    dec_input = tf.expand_dims([lang_tokenizer.word_index['<start>']] * BATCH_SIZE, 1)

    # Teacher forcing - feeding the target as the next input
    for t in range(1, targ.shape[1]):
      # passing enc_output to the decoder
      predictions, dec_hidden, _ = decoder(dec_input, dec_hidden, enc_output)

      loss += loss_function(targ[:, t], predictions)

      # using teacher forcing
      dec_input = tf.expand_dims(targ[:, t], 1)

    batch_loss = (loss / int(targ.shape[1]))
    test_loss += batch_loss

  # saving (checkpoint) the model every 2 epochs
  encoder.save_weights('struct_encoder_' + str(epoch))
  decoder.save_weights('struct_decoder_' + str(epoch))

  print('Epoch {} Loss {:.4f} Test Loss {:.4f}'.format(epoch + 1,
                                      total_loss / steps_per_epoch, test_loss/test_steps_per_epoch))
  print('Time taken for 1 epoch {} sec\n'.format(time.time() - start))


import os

from tqdm import tqdm
import pandas as pd
import numpy as np
from typing import Union

import torch
from torch.optim import AdamW

from metalearner.data.rawdata import load_raw_data
from metalearner.data.ast_dataloader import load_ast_dataset
from metalearner.model.recur_net import Model_Recursive_LSTM_v2
from metalearner.learner.base_learner import BaseLearner
from utils.util import warn_once, notify, read_yes, TrainTestPair
from metalearner.learn_utils import CMOutput

from os import environ
environ['train_device'] = 'cuda:0' # training device: 'cpu' or 'cuda:X'
environ['store_device'] = 'cuda:0' # Data storing device:  'cpu' or 'cuda:X'
train_device= torch.device(environ.get('train_device', "cuda"))
store_device= torch.device(environ.get('store_device', "cuda"))

def mape_criterion(outputs, labels):
    eps = 1e-5
    return 100 * torch.mean(torch.abs((labels - outputs)/(labels + eps)))

def mse_criterion(outputs, labels):
    return torch.nn.MSELoss()(labels, outputs)

class RecurLSTMLearner(BaseLearner):
    def __init__(self, disable_norm, loss_func_name,
            residual=False, input_size=164, **kwargs):
        super(RecurLSTMLearner, self).__init__(**kwargs)

        if loss_func_name.lower() == "mape":
            self.loss_func = mape_criterion
        elif loss_func_name.lower() == "mse":
            self.loss_func = mse_criterion
        else:
            raise ValueError()
        self.loss_func_name = loss_func_name

        self.model = Model_Recursive_LSTM_v2(input_size, disable_norm, residual=residual, drops=[0.112, 0.112, 0.112, 0.112])
        print(f"Model parameter count: {self.model.para_cnt()}")
        self.init_device()

        self.lr_scheduler = 'one_cycle'

    def init_optimizer(self):
        self.optimizer = AdamW(self.model.parameters(), weight_decay=0.375e-2)

    def _inference(self, inputs) -> CMOutput:
        outputs = self.model(inputs)
        # inputs = (inputs[0], inputs[1].to(original_device))
        return CMOutput(outputs)

    def _loss(self, outputs: CMOutput, labels):
        loss = self.loss_func(outputs.preds, labels)
        return loss

    def add_monotir_summary(self, outputs: CMOutput, x, y, loss):
        if self.monitor.is_summary:
            self.monitor.summary([
                ("scalar", "Loss/Train", loss),
            ])

    def init_device(self):
        self.normed_zero_output = None
        if self.data_meta_info is not None:
            if self.use_clip:
                self.normed_zero_output = torch.Tensor(
                        [self.data_meta_info.standardize_output(0)])
            self.output_avg = self.data_meta_info.output_avg
            self.output_std = self.data_meta_info.output_std
        else:
            self.output_avg = None
            self.output_std = None

        if self.train_device:
            self.model.to(self.train_device)
            if self.normed_zero_output is not None:
                self.normed_zero_output = self.normed_zero_output.to(self.train_device)

    def data_to_train_device(self, inputs, labels):
        if self.train_device and labels.device != self.train_device:
            return (inputs[0], inputs[1].to(train_device)), labels.to(train_device)
        else:
            return inputs, labels

    def prepare_test_pair(self, val_data, verbose=True):
        val_xs, val_ys, val_dis = zip(*val_data)
        return val_xs, val_ys, val_dis

    def train_one_batch(self, inputs, labels):
        self.model.train()
        # original_device = labels.device
        inputs, labels = self.data_to_train_device(inputs, labels)
        self.cached_data["train"] = (inputs, labels)
        # zero the parameter gradients
        self.optimizer.zero_grad()
        ### Forward
        # track history if only in train
        with torch.set_grad_enabled(True):
            outputs = self._inference(inputs)

            assert outputs.preds.shape == labels.shape
            loss = self._loss(outputs, labels)
            self.add_monotir_summary(outputs, inputs, labels, loss)
            if self.loss_func_name != "mape":
                mape = mape_criterion(outputs.preds, labels)
            else:
                mape = loss
            ### Backward + optimize only if in training phase
            loss.backward()
            self.optimizer.step()

        for hook in self.batch_hooks:
            hook()
        # print(self.monitor.epoch_cnt, self.monitor.train_step, loss)
        ### Summary, check and cache
        self.monitor.step()
        if self.monitor.is_cache:
            if self.monitor.epoch_cnt > 2:
                self.check_exit()
        # labels = labels.to(original_device)

        # ### debug
        # if self.monitor.train_step > 3:
        #     print(self.monitor.train_step)
        #     self.stop_training()

        return loss

    def train_model(self, dataloader_dict, num_epochs=100, log_every=5):
        self.stop_checker.max_epoch = num_epochs
        self.model.register_hooks_for_grads_weights(self.monitor)
        self.train(TrainTestPair(dataloader_dict["train"], dataloader_dict["val"]))

    def loss(self, outputs: CMOutput, dataset_or_x, y):
        return self._loss(outputs, y)

    def predict(self, *args, **kwargs):
        self.model.eval()
        return super().predict(*args, **kwargs)

    def forward_compute_metrics(self, input_data, element_wise_test=False):
        self.model.eval()
        all_preds = []
        all_labels = []
        if isinstance(input_data, torch.utils.data.DataLoader):
            for batch_x, batch_y, _ in input_data:
                for idx in range(len(batch_x)):
                    _x, _y = batch_x[idx], batch_y[idx]
                    _x, _y = self.data_to_train_device(_x, _y)
                    outputs = self._inference(_x)
                    all_preds.append(outputs)
                    all_labels.append(_y)
        elif isinstance(input_data, tuple):
            X, Y = input_data
            # TODO: Naive approach to adapt to the tiramisu
            # case for X Y didn't have the same shape
            if len(X) != len(Y):
                _x, _y = X, Y
                _x, _y = self.data_to_train_device(_x, _y)
                outputs = self._inference(_x)
                all_preds.append(outputs)
                all_labels.append(_y)
            else:
                for idx in range(len(Y)):
                    _x, _y = X[idx], Y[idx]
                    _x, _y = self.data_to_train_device(_x, _y)
                    outputs = self._inference(_x)
                    all_preds.append(outputs)
                    all_labels.append(_y)
        # tiramisu, x,y value paris
        elif isinstance(input_data, list):
            for idx, (_x, _y) in enumerate(input_data):
                _x, _y = self.data_to_train_device(_x, _y)
                outputs = self._inference(_x)
                all_preds.append(outputs)
                all_labels.append(_y)
        else:
            raise ValueError()
        outputs = CMOutput.concat(all_preds, 0)
        labels = torch.cat(all_labels, 0)
        metrics = self.compute_metrics(outputs, labels, element_wise_test=element_wise_test)
        metrics["loss"] = float(self.loss(outputs, None, labels).data.cpu().numpy())
        return outputs, metrics, labels

    def get_results_df(self, batches_list, log=False):
        df = pd.DataFrame()
        self.model.eval()
        torch.set_grad_enabled(False)
        all_outputs=[]
        all_labels=[]

        for k, (inputs, labels, di) in tqdm(list(enumerate(batches_list))):
            original_device = labels.device
            inputs, labels = self.data_to_train_device(inputs, labels)
            outputs = self.model(inputs)
            assert outputs.shape == labels.shape
            all_outputs.append(outputs)
            all_labels.append(labels)
            inputs = (inputs[0], inputs[1].to(original_device))
            labels = labels.to(original_device)
        preds = torch.cat(all_outputs).cpu().detach().numpy().reshape((-1,))
        labels = torch.cat(all_labels).cpu().detach().numpy().reshape((-1,))
        preds = np.around(np.abs(preds), decimals=6)
        labels = np.around(labels, decimals=6)

        ### Denormalization
        preds = self.data_meta_info.de_standardize_output(preds)
        labels = self.data_meta_info.de_standardize_output(labels)

        assert preds.shape == labels.shape
        df['prediction'] = np.array(preds)
        df['labels'] = np.array(labels)
        df['abs_diff'] = np.abs(preds - labels)
        df['RMSE'] = np.sqrt(np.abs(np.power(preds-labels, 2)))
        df['MAPE'] = np.abs(df.labels - df.prediction) / df.labels * 100

        describe = df.describe()
        # print(df)
        print(describe)

        return df

    def save(self, path=None):
        _path = path if path is not None else self.cache_path
        if _path is None:
            return
        super(RecurLSTMLearner, self).save(_path)
        torch.save(self.model, os.path.join(_path, "Model_Recursive_LSTM_v2.torch"))

    def load(self, path=None):
        _path = path if path is not None else self.cache_path
        super(RecurLSTMLearner, self).load(_path)
        self.model = torch.load(os.path.join(_path, "Model_Recursive_LSTM_v2.torch"))
        self.init_device()

    def _init_loss_func(self):
        warn_once(f"{self.__class__.__name__} init loss function when the object is created")

def test_tiramisu(files_or_dir, learning_params):
    if isinstance(files_or_dir, str):
        root_path, _, files = list(os.walk(files_or_dir))[0]
        files = [os.path.join(root_path, f) for f in files if f.endswith(".npy")]
    elif isinstance(files_or_dir, list):
        files = files_or_dir
    elif isinstance(files_or_dir, dict): # fix: tiramisu
        files = files_or_dir['tasks'][1]
    else:
        raise
    raw_data = load_raw_data(files, learning_params, verbose=True, force=True)
    print(f"Raw data size (before pre-processing): {raw_data.size}")
    raw_data.preprocess(time_lb=learning_params["ave_lb"], verbose=True)
    print(f"Raw data size (after pre-processing): {raw_data.size}")
    if learning_params["disable_norm"]:
        warn_once("Disable normalization")
        raw_data.metainfo.to_norm_input = False
        raw_data.metainfo.to_norm_output = False
    else:
        raw_data.metainfo.tsfm_hub.parse_y_norm_method("std", None, {
            "avg": raw_data.metainfo.output_avg, "std": raw_data.metainfo.output_std})
        raw_data.metainfo.tsfm_hub.parse_x_norm_method("min-max", None, {
            "min": raw_data.metainfo.input_min, "max": raw_data.metainfo.input_max})
        raw_data.metainfo.tsfm_hub.print()

    dataset, val_batches_list, val_batches_indices, train_batches_list, train_batches_indices = load_ast_dataset(raw_data, train_device, store_device)

    ### debug
    # avg, std, flops, ast_features, node_ids, serialized_tree = xydata[0]
    # ast = AST.deserialize_tree(serialized_tree)
    # print(node_ids)
    # print(serialized_tree)
    # print(ast)

    bl_dict = {'train': train_batches_list, 'val': val_batches_list}
    # here, please careful that LSTM learning rate could be different from the previous. Change input config if necessary.
    learner = RecurLSTMLearner(
        learning_params["disable_norm"], learning_params["loss_func"],
        residual=learning_params["residual"],
        data_meta_info = raw_data.metainfo,
        cache_path=learning_params["cache_dir"],
        debug=learning_params["debug"],
        tb_log_dir=learning_params["tb_logdir"],
        )

    if learning_params["load_cache"]:
        if not read_yes(f"Load cost model at {learner.cache_path}"):
                exit(0)
        learner.load()

    learner.train_model(dataloader_dict=bl_dict, num_epochs=10000, log_every=1)
    # Basic results on the test and validation set
    val_df = learner.get_results_df(val_batches_list)


import os
import collections
import json
import pickle
import time
from datetime import datetime, timedelta, date
from concurrent.futures import ThreadPoolExecutor
import click
from appdirs import user_cache_dir

import calendar

import math

try:
    import numpy as np
except:
    np = None

def np_exception(function):
    def wrapper(*args, **kwargs):
        if not np:
            raise ModuleNotFoundError("Please install pandas and numpy using \n pip install pandas")
        return function(*args, **kwargs)

    return wrapper

@np_exception
def np_float(num):
    try:
        return np.float64(num)
    except:
        return np.nan

@np_exception
def np_date(dt):
    try:
        return np.datetime64(dt)
    except:
        pass

    try:
        dt = datetime.strptime(dt, "%d-%b-%Y").date()
        return np.datetime64(dt)
    except:
        pass

    try:
        dt = datetime.strptime(dt, "%d %b %Y").date()
        return np.datetime64(dt)
    except:
        pass

    return np.datetime64('nat')

@np_exception
def np_int(num):
    try:
        return np.int64(num)
    except:
        return 0

def break_dates(from_date, to_date):
    if from_date.replace(day=1) == to_date.replace(day=1):
        return [(from_date, to_date)]
    date_ranges = []
    month_start = from_date
    month_end = month_start.replace(day=calendar.monthrange(month_start.year, from_date.month)[1])
    while(month_end < to_date):
        date_ranges.append((month_start, month_end))
        month_start = month_end + timedelta(days=1)
        month_end = month_start.replace(day=calendar.monthrange(month_start.year, month_start.month)[1])
        if month_end >= to_date:
            date_ranges.append((month_start, to_date))
    return date_ranges

def kw_to_fname(**kw):
    name = "-".join([str(kw[k]) for k in sorted(kw) if k != "self"])
    return name

def cached(app_name):
    """
        Note to self:
            This is a russian doll
            wrapper - actual caching mechanism
            _cached - actual decorator
            cached - wrapper around decorator to make 'app_name' dynamic
    """
    def _cached(function):
        def wrapper(*args, **kw):
            kw.update(zip(function.__code__.co_varnames, args))
            env_dir = os.environ.get("J_CACHE_DIR")
            if not env_dir:
                cache_dir = user_cache_dir(app_name, app_name)
            else:
                cache_dir = os.path.join(env_dir, app_name)

            file_name = kw_to_fname(**kw)
            path = os.path.join(cache_dir, file_name)
            if not os.path.isfile(path):
                if not os.path.exists(cache_dir):
                    os.makedirs(cache_dir)
                j = function(**kw)
                with open(path, 'wb') as fp:
                    pickle.dump(j, fp)
            else:
                with open(path, 'rb') as fp:
                    j = pickle.load(fp)
            return j
        return wrapper
    return _cached

def pool(function, params, use_threads=True, max_workers=2):
    if use_threads:
        with ThreadPoolExecutor(max_workers=max_workers) as ex:
            dfs = ex.map(function, *zip(*params))
    else:
        dfs = []
        for param in params:
            try:
                r = function(*param)
            except:
                raise
            dfs.append(r)
    return dfs

def live_cache(app_name):
    """Caches the output for time_out specified. This is done in order to
    prevent hitting live quote requests to NSE too frequently. This wrapper
    will fetch the quote/live result first time and return the same result for
    any calls within 'time_out' seconds.

    Logic:
        key = concat of args
        try:
            cached_value = self._cache[key]
            if now - self._cache['tstamp'] < time_out
                return cached_value['value']
        except AttributeError: # _cache attribute has not been created yet
            self._cache = {}
        finally:
            val = fetch-new-value
            new_value = {'tstamp': now, 'value': val}
            self._cache[key] = new_value
            return val

    """
    def wrapper(self, *args, **kwargs):
        """Wrapper function which calls the function only after the timeout,
        otherwise returns value from the cache.

        """
        # Get key by just concating the list of args and kwargs values and hope
        # that it does not break the code :P
        inputs =  [str(a) for a in args] + [str(kwargs[k]) for k in kwargs]
        key = app_name.__name__ + '-'.join(inputs)
        now = datetime.now()
        time_out = self.time_out
        try:
            cache_obj = self._cache[key]
            if now - cache_obj['timestamp'] < timedelta(seconds=time_out):
                return cache_obj['value']
        except:
            self._cache = {}
        value = app_name(self, *args, **kwargs)
        self._cache[key] = {'value': value, 'timestamp': now}
        return value

    return wrapper


from docutils.parsers.rst import Directive, directives
from docutils import nodes
from docutils.core import publish_doctree
from sphinx.errors import SphinxError
import os
import netCDF4
import re

# We store this using a global variable because this much data seems
# to break Sphinx if we store it in an env attribute. (For small
# numbers of parameters config_role an access env via
# inliner.document.settings.env, for larger numbers it cannot.)
pism_data = {}

def make_id(parameter):
    return "config-" + parameter

class config(nodes.literal):
    parameter = None

    def __init__(self, parameter, **kwargs):
        self.parameter = parameter

        nodes.literal.__init__(self, parameter, **kwargs)

# this node makes it possible to add soft hyphens to wrap long parameter names
class softhyphen(nodes.Element):
    pass

def visit_softhyphen_html(self, node):
    self.body.append('')

def depart_softhyphen_html(self, node):
    pass

def visit_softhyphen_latex(self, node):
    self.body.append(r"\-")

def depart_softhyphen_latex(self, node):
    pass

def config_role(typ, rawtext, text, lineno, inliner, options={}, content=[]):
    """Parse :config:`parameter` roles and create `config` nodes resolved
    in resolve_config_links()."""

    # Warn about misspelled parameters names.
    if text not in pism_data:
        msg = inliner.reporter.error('{}: invalid parameter name'.format(rawtext),
                                     line=lineno)
        prb = inliner.problematic(rawtext, rawtext, msg)
        return [prb], [msg]

    name = text

    return [config(name, text=text)], []

class ParameterList(Directive):
    has_content = False
    required_arguments = 0

    option_spec = {"prefix": directives.unchanged,
                   "exclude": directives.unchanged}

    def format_value(self, value, T):
        if T == "string" and len(value) == 0:
            return nodes.emphasis("", "empty")

        if T in ["string", "keyword" ]:
            return nodes.literal(value, value.replace(",", ", "))

        if T in ["number", "integer"]:
            return nodes.Text("{:g}".format(value))

        if T == "flag":
            return nodes.Text(str(value))

    def list_entry(self, name, data):
        "Build an entry for the list of parameters."

        p1 = nodes.paragraph()
        p1 += nodes.target('', '', ids=[make_id(name)])
        p1 += nodes.literal("", name)

        fl = nodes.field_list()

        if True:
            f = nodes.field()
            f += [nodes.field_name("", "Type"),
                  nodes.field_body("", nodes.Text(" " + data["type"]))]
            fl += f

            f = nodes.field()
            value = nodes.paragraph()
            value += self.format_value(data["value"], data["type"])
            if "units" in data:
                value += nodes.emphasis("", " ({})".format(data["units"]))
            f += [nodes.field_name("", "Default value"),
                  nodes.field_body("", value)]
            fl += f

        if "choices" in data:
            choices = self.format_value(data["choices"], "keyword")
            f = nodes.field()
            f += [nodes.field_name("", "Choices"),
                  nodes.field_body("", nodes.paragraph("", "", choices))]
            fl += f

        if "option" in data:
            option = self.format_value("-" + data["option"], "keyword")
            f = nodes.field()
            f += [nodes.field_name("", "Option"),
                  nodes.field_body("", nodes.paragraph("", "", option))]
            fl += f

        p2 = nodes.paragraph()
        doc, _ = self.state.inline_text(data["doc"], self.lineno)
        p2 += doc
        p2 += fl

        return [p1, p2]

    def compact_list_entry(self, name, text, data):
        "Build an entry for the compact list of parameters."

        p1 = nodes.paragraph()
        p1 += config(name, text=text)

        if not (data["type"] == "string" and len(data["value"]) == 0):
            p1 += nodes.Text(" (")
            p1 += self.format_value(data["value"], data["type"])
            if "units" in data:
                if data["units"] not in ["1", "pure number", "count"]:
                    p1 += nodes.emphasis("", " {units}".format(**data))
            p1 += nodes.Text(")")

        doc, _ = self.state.inline_text(data["doc"], self.lineno)

        p1 += nodes.Text(" ")
        p1 += doc

        return p1

    def run(self):
        env = self.state.document.settings.env

        # make sure the current document gets re-built if the config file changes
        env.note_dependency(env.pism_parameters["filename"])

        if "prefix" in self.options:
            prefix = self.options["prefix"]
        else:
            prefix = ""

            # Store the docname corresponding to this parameter list.
            # It is used in resolve_config_links() to build URIs.
            env.pism_parameters["docname"] = env.docname

        if "exclude" in self.options:
            exclude = self.options["exclude"]
        else:
            exclude = None

        full_list = prefix == "" and exclude == None

        parameter_list = nodes.enumerated_list()

        parameters_found = False
        for name in sorted(pism_data.keys()):

            pattern = "^" + prefix
            # skip parameters that don't have the desired prefix
            if not re.match(pattern, name):
                continue

            if exclude and re.match(exclude, name):
                continue

            parameters_found = True

            item = nodes.list_item()

            if full_list:
                # only the full parameter list items become targets
                item += self.list_entry(name, pism_data[name])
            else:
                item += self.compact_list_entry(name,
                                                re.sub(pattern, "", name),
                                                pism_data[name])
            parameter_list += item

        if not parameters_found:
            msg = 'Error in a "{}" directive: no parameters with prefix "{}".'.format(self.name, prefix)
            text_error = nodes.error()
            text_error += nodes.Text(msg)

            reporter = self.state_machine.reporter
            system_error = reporter.error(msg, nodes.literal('', ''), line=self.lineno)
            return [text_error, system_error]

        return [parameter_list]

def resolve_config_links(app, doctree, fromdocname):
    """Replace config nodes with references to items in the list of
    parameters.
    """
    docname = app.builder.env.pism_parameters["docname"]

    for node in doctree.traverse(config):
        reference = nodes.reference('', '')
        reference['internal'] = True
        reference['refdocname'] = docname
        reference['refuri'] = "{}#{}".format(app.builder.get_relative_uri(fromdocname, docname),
                                             make_id(node.parameter))

        # Allow wrapping long parameter names
        words = node.astext().split(".")
        reference += nodes.literal("", words[0])
        for w in words[1:]:
            reference += [softhyphen(), nodes.literal("", "." + w)]

        node.replace_self(reference)

def init_pism_parameters(app):
    """Read and pre-process the list of PISM's configuration parameters."""
    global pism_data
    env = app.builder.env

    filename = os.path.normpath(app.config.pism_config_file)

    if not hasattr(env, "pism_parameters"):
        # the path to the configuration file (used to tell Sphinx to
        # re-build if it changes)
        env.pism_parameters = {"filename" : filename}

    f = netCDF4.Dataset(filename)
    variable = f.variables["pism_config"]
    data = {k : getattr(variable, k) for k in variable.ncattrs()}

    suffixes = ["choices", "doc", "option", "type", "units"]

    def special(name):
        "Return true if 'name' is a 'special' parameter, false otherwise."
        for suffix in suffixes:
            if name.endswith("_" + suffix) or name == "long_name":
                return True
        return False

    for key, value in data.items():
        if special(key):
            continue

        pism_data[key] = {"value" : value}

        for s in suffixes:
            try:
                pism_data[key][s] = data[key + "_" + s]
            except:
                pass

def check_consistency(app, env):
    """Check if we have a full parameter list with link targets"""
    if not "docname" in env.pism_parameters:
        raise SphinxError("make sure that this document contains a pism-parameters directive")

def env_purge_doc(app, env, docname):
    """Update the environment if the file containing the full list changed"""
    if "docname" in env.pism_parameters and docname == env.pism_parameters["docname"]:
        del env.pism_parameters["docname"]

def setup(app):
    app.add_config_value('pism_config_file', "pism_config.json", 'env')

    app.add_node(config)
    app.add_node(softhyphen,
                 html=(visit_softhyphen_html, depart_softhyphen_html),
                 latex=(visit_softhyphen_latex, depart_softhyphen_latex))
    app.add_role('config', config_role)
    app.add_directive('pism-parameters', ParameterList)

    app.connect("builder-inited", init_pism_parameters)
    app.connect("env-purge-doc", env_purge_doc)
    app.connect("env-check-consistency", check_consistency)
    app.connect('doctree-resolved', resolve_config_links)

    return {
        'version': '0.1',
        'parallel_read_safe': False,
        'parallel_write_safe': True,
    }


import json
import os
import os.path
import re
import unittest

import six

from gherkin_parser import ParseError, parse_lines

class ParserTestCaseMixin(object):
    def parse(self, feature_string):
        return parse_lines(feature_string.split('\n'))

    def assertErrorMessage(self, feature_string, message):
        with self.assertRaises(ParseError) as cm:
            self.parse(feature_string)
        self.assertEqual(str(cm.exception), message)

class FeatureTagsTestCase(ParserTestCaseMixin, unittest.TestCase):
    def test_broken_tags(self):
        self.assertErrorMessage(u"""
            @tag broken tags
        """, "Tags must start with a '@' character, and not contain spaces (line 2)")

class FeatureTitleTestCase(ParserTestCaseMixin, unittest.TestCase):
    def test_empty_title(self):
        parsed = self.parse(u"""
            Feature:
        """)
        self.assertEqual(parsed['title']['content'], '')

    def test_missing_feature_title(self):
        self.assertErrorMessage(u"""
            Not a feature
        """, 'Expected feature title (line 2)')

    def test_multiple_feature_titles(self):
        self.assertErrorMessage(u"""
            Feature: Feature 1
                Scenario:
                    Given a given

            Feature: Feature 2
        """, "Unexpected {!r}. File must not contain multiple features (line 6)".format(six.text_type('Feature: Feature 2')))

class FeatureBackgroundTestCase(ParserTestCaseMixin, unittest.TestCase):
    def test_background_after_scenario(self):
        self.assertErrorMessage(u"""
            Feature:
                Scenario:

                Background:
                    Description 1
        """, "Unexpected {!r}. Background must not occur after scenarios (line 5)".format(six.text_type('Background:')))

class ScenarioTestCase(ParserTestCaseMixin, unittest.TestCase):
    def test_with_examples(self):
        self.assertErrorMessage(u"""
            Feature: Title
                Scenario: Title
                    Given I have a given

                    Examples:
        """, "Scenario block has examples section - should be 'scenario outline' (line 3)")

class ScenarioOutlineTestCase(ParserTestCaseMixin, unittest.TestCase):
    def test_without_examples(self):
        self.assertErrorMessage(u"""
            Feature: Title
                Scenario Outline: Title
                    Given I have a given
        """, "Scenario Outline block is missing an examples section (line 3)")


from flask import (
    Blueprint,
    render_template,
    request,
    Response,
)
from .configloader import config
import os
from .utils import *
import random
import traceback

router = Blueprint("router", __name__, url_prefix="/")
root_folder = os.path.dirname(os.path.abspath(__file__))

# 404 Page
@router.app_errorhandler(404)
@router.app_errorhandler(405)
def not_found(e):
    return render_template("error.html", title="404 - Page not found", message="Ala ala, are you lost ?"), 404

@router.app_errorhandler(403)
def unauthorized(e):
    return render_template("error.html", title="403 - Unauthorized", message="Ala ala, are you allowed to see this page ?"), 403

if config["env"] == "production":
    @router.errorhandler(Exception)
    def server_error(e):
        print(traceback.format_exc())
        return render_template("error.html", title="500 - Internal server error", message="Ala ala, an error occured, try in few minutes"), 500

@router.app_errorhandler(500)
def server_error(e):
    return render_template("error.html", title="500 - Internal server error", message="Ala ala, an error occured, try in few minutes"), 500

# Home route
@router.route("/", methods=["GET"])
def home():
    return render_template("home.html", title="File uploader", max_size=config["upload"]["max_size"], git_hash=git_hash())

# FAQ route
@router.route("/faq", methods=["GET"])
def faq():
    return render_template("faq.html", title="FAQ", ext_unauthorized=config["upload"]["unauthorized_ext"], git_hash=git_hash())

# ShareX route
@router.route("/sharex", methods=["GET"])
def sharex():
    return render_template("sharex.html", title="ShareX", git_hash=git_hash())

# Upload files
@router.route("/upload", methods=["POST"])
def upload_files():
    # Get upload domain
    if "UPLOAD_DOMAIN" in request.form:
        if request.form["UPLOAD_DOMAIN"] not in config["upload"]["domain"]:
            return json_with_statuscode({
                "success": False,
                "error": "Invalid upload domain."
            }, 400)
        else:
            upload_domain = request.form["UPLOAD_DOMAIN"]
    else:
        upload_domain = random.choice(config["upload"]["domain"])

    # Get (and create if not exist) upload dir
    upload_dir = os.path.normpath(os.path.join(
        root_folder, config["upload"]["folder"]))
    if not os.path.exists(upload_dir):
        os.mkdir(upload_dir)

    # check if user has uploaded a file
    if len(request.files.getlist("upload[]")) < 1:
        return json_with_statuscode({
            "success": False,
            "error": "You must upload least than one file."
        }, 400)

    files = request.files.getlist("upload[]")
    process_files = []

    # process files
    for file in files:
        # Check file size
        size = uploaded_file_size(file)
        if convert_to_megabites(size) > int(config["upload"]["max_size"]):
            process_files.append({
                "success": False,
                "error": "File too big"
            })
            continue

        # Check file extention
        if "." in file.filename:
            ext = os.path.splitext(file.filename)[1]
        else:
            ext = None

        if ext in config["upload"]["unauthorized_ext"]:
            process_files.append({
                "success": False,
                "error": "Invalid extention"
            })
            continue

        retry_filename = 0
        find_uniq = False

        while retry_filename < int(config["upload"]["retry_filename"]) and not find_uniq:
            upload_filename = generate_filename(ext)

            if not os.path.exists(os.path.join(upload_dir, upload_filename)):
                find_uniq = True

            retry_filename += 1

        if not find_uniq:
            process_files.append({
                "success": False,
                "error": "Unable to find unique name."
            })
            continue

        file.save(os.path.join(upload_dir, upload_filename))

        process_files.append({
            "success": True,
            "original_filename": file.filename,
            "filename": upload_filename,
            "url": "{}/{}".format(upload_domain, upload_filename),
            "size": size
        })

    return {
        "success": True,
        "files": process_files
    }


# https://www.kaggle.com/c/boston-housing
# https://towardsdatascience.com/linear-regression-on-boston-housing-dataset-f409b7e4a155

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns

#l load dataset from scikit-learn
from sklearn.datasets import load_boston
boston_dataset = load_boston()

print(boston_dataset.keys())
# dict_keys(['data', 'target', 'feature_names', 'DESCR', 'filename'])
# 'data' - contains the info for various houses
# 'target' - prices of the house
# 'feature_names' - names of the features
# 'DESCR' - describes the dataset

# print more about the dataset
print(boston_dataset.DESCR)

# load data into pandas dataframe
boston = pd.DataFrame(boston_dataset.data, columns=boston_dataset.feature_names)
print(boston.head())

# Prepare the data for training the mode
boston["PRICE"] = boston_dataset.target
boston.isnull().sum() # count the number of missing values

X = boston.drop('PRICE', axis=1)
Y = boston['PRICE']

# split the data into training and testing sets
from sklearn.model_selection import train_test_split
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.2, random_state=5)
print("X_train: ", X_train.shape, "\n",
      "Y_train: ", Y_train.shape, "\n",
      "X_test: ", X_test.shape, "\n",
      "Y_test: ", Y_test.shape)

# X_train:  (404, 2)
# Y_train:  (404,)
# X_test:  (102, 2)
# Y_test:  (102,)

# plot regression
from sklearn.linear_model import LinearRegression
linearRegression = LinearRegression()
linearRegression.fit(X_train, Y_train)

Y_pred = linearRegression.predict(X_test)

plt.scatter(Y_test, Y_pred)
plt.xlabel("Prices")
plt.ylabel("Predicted prices")
plt.title("Boston houses prices")
plt.savefig('DL_hw1.png')

# compute mean squared error
# https://scikit-learn.org/stable/modules/generated/sklearn.metrics.mean_squared_error.html
from sklearn.metrics import mean_squared_error

# number of samples : [102, 404] - it not works
meanSquaredError = mean_squared_error(Y_test, Y_train)
print("mean squared error: ", meanSquaredError)


from django.urls import path

from users import views

urlpatterns = [
    path('signup/<str:public_id>/', views.SignUpView.as_view(), name='signup'),
    path('profile/edit/', views.EditProfileView.as_view(), name='profile_edit'),
    path('currencies/', views.ListProfileCurrencyView.as_view(), name='currencies_list'),
    path('currencies/new/', views.CreateProfileCurrencyView.as_view(), name='currency_create'),
    path('currencies/<int:pk>/delete/', views.DeleteProfileCurrencyView.as_view(), name='currency_delete'),
    path('invitations/', views.ShowInvitationsView.as_view(), name='invitations'),
    path('submissions/', views.BandSubmissionsView.as_view(), name='submissions'),
    path('submission-details/<int:pk>', views.BandSubmissionDetailView.as_view(), name='submission_details'),
    path('trade-requests/', views.TradeRequestListView.as_view(), name='trade_requests'),
    path('trade-details/<int:pk>/', views.TradeRequestDetailView.as_view(), name='trade_details'),
    path('user-trade-details/<int:pk>/', views.UsersTradeRequestDetailView.as_view(), name='user_trade_details'),
]


import os, time, re, glob
import h5py
import nibabel as nib
import numpy as np
from gpiozero import LED

def usb_inserted():

    devices = os.popen('sudo blkid').readlines()

    usbs = []
    for u in devices:
        loc = [u.split(':')[0]]
        if '/dev/sd' not in loc[0]:
              continue # skip
        loc+=re.findall(r'"[^"]+"',u)
        columns = ['loc']+re.findall(r'\b(\w+)=',u)

        usbs.append(dict(zip(columns,loc)))

        if len(usbs) > 0:
            flag = True
        if len(usbs) == 0:
            flag = False
        return flag

def standardize(image):

  standardized_image=np.zeros(image.shape)
  for c in range(image.shape[0]):
    for z in range(image.shape[3]):
      image_slice = image[c,:,:,z]
      centered = image_slice - np.mean(image_slice)
      if np.std(centered) != 0:
                centered_scaled = centered / np.std(centered)
      else:
        centered_scaled=centered
      standardized_image[c, :, :, z] = centered_scaled
  return standardized_image

def saveh5(filename,cbf_data,cbv_data,mtt_data,tmax_data):
    cbf_data=cbf_data.reshape(1,256,256,8)
    cbv_data=cbv_data.reshape(1,256,256,8)
    mtt_data=mtt_data.reshape(1,256,256,8)
    tmax_data=tmax_data.reshape(1,256,256,8)
    image_data=np.concatenate((cbf_data,cbv_data,mtt_data,tmax_data),axis=0)
    image_data = standardize(image_data)

    with h5py.File('/home/pi/brain_stroke/preprocess/'+str(filename),'w') as f:
        f.create_dataset("X_train",data=image_data)
    f.close()

####MAIN#############
print("Waiting for USB...")
led_r = LED(23)
led_g = LED(24)
while True:
    if usb_inserted():
        break
    else:
        led_r.on()
        time.sleep(0.5)
        led_r.off()
        time.sleep(0.5)

print("USB Inserted")
led_r.off()
time.sleep(2)

usb_path = glob.glob('/media/pi/*/')
print(f"USB Path:{usb_path[0]}")
files_in_usb = glob.glob(str(usb_path[0]) + '/*/')
print(f"Files in usb: {files_in_usb}")
files_in_usb = [i for i in files_in_usb if 'System' not in i]

#print(files_in_usb)

for file in files_in_usb:
    cbf_path = glob.glob(file+'/*CBF.*/*.nii')[0]
    cbv_path = glob.glob(file+'/*CBV.*/*.nii')[0]
    mtt_path = glob.glob(file+'/*MTT.*/*.nii')[0]
    tmax_path= glob.glob(file+'/*Tmax*/*.nii')[0]

    file = file.split('/')[-2]

    cbf_obj=nib.load(cbf_path)
    cbv_obj=nib.load(cbv_path)
    mtt_obj=nib.load(mtt_path)
    tmax_obj=nib.load(tmax_path)

    cbf = cbf_obj.get_fdata()
    cbv = cbv_obj.get_fdata()
    mtt = mtt_obj.get_fdata()
    tmax=tmax_obj.get_fdata()

    if cbf.shape[2] != 8:
        print(cbf.shape)
        if cbf.shape[2] > 8:
            n = cbf.shape[2]
            n = n - (n%8)
            for i in range(0,n):
                name = file + str(i)
                saveh5(name,cbf[:,:,(i*8):(i+1)*8],cbv[:,:,(i*8):(i+1)*8],mtt[:,:,(i*8):(i+1)*8],tmax[:,:,(i*8):(i+1)*8])

        if cbf.shape[2] < 8:
            while cbf.shape[2] <= 8:
                cbf = np.concatenate([cbf,cbf],axis=2)
                cbv = np.concatenate([cbv,cbv],axis=2)
                mtt = np.concatenate([mtt,mtt],axis=2)
                tmax= np.concatenate([tmax,tmax],axis=2)
            print(cbf.shape)
            saveh5(file,cbf[:,:,:8],cbv[:,:,:8],mtt[:,:,:8],tmax[:,:,:8])

    if cbf.shape[2] == 8:
        saveh5(file,cbf,cbv,mtt,tmax)


import logging
import shutil
import tempfile
import zipfile
from pathlib import Path

from ..helpers.system import download_file
from .git_url import GitUrl

LOGGER = logging.getLogger(__name__)

def git_download_zip(repo_url: str, target_folder: Path, branch: str, commit: str = ""):
    """Download Repo Zip from Github.

    Parameters
    ----------
    repo_url : str
        Github Repo Url
    target_folder : Path
        Download Target
    branch : str
        BRanch to download
    commit : str, optional
        Specific Commit to download, by default ""

    Raises
    ------
    FileNotFoundError
        If Download failed
    """
    git_url = GitUrl(repo_url)
    download_url = git_url.get_archive_url(ref=commit or branch)
    with tempfile.TemporaryDirectory() as tdir:
        zip_path = Path(tdir) / f"{git_url.name}.zip"
        LOGGER.info("Downloading GitRepo Zip: '%s'", download_url)
        LOGGER.debug("Target Path: '%s' ", zip_path)
        download_file(download_url, zip_path)
        if not zip_path.exists():
            raise FileNotFoundError(f"Could not download Repo Zip from: {download_url}")
        LOGGER.info("Extracting GitRepo Zip: %s", git_url.name)
        ex_location = Path(tdir) / "extract"
        with zipfile.ZipFile(zip_path, "r") as zip_ref:
            zip_ref.extractall(ex_location)
        for path in ex_location.glob("*"):
            LOGGER.info("Moving %s to %s", path.stem, target_folder)
            shutil.rmtree(target_folder, ignore_errors=True)
            path.rename(target_folder)
            break


# https://www.acmicpc.net/problem/11725

import sys
from collections import deque

input = sys.stdin.readline

N = int(input())

Tree = [[] for _ in range(N+1)]

answer = [-1]*(N+1)
visited = [-1]*(N+1)

for i in range(N-1):
    parent, child = map(int, input().split())
    Tree[parent].append(child)
    Tree[child].append(parent)

queue = deque()

queue.append(1)
visited[1] = 0

while queue:
    new_Node = queue.popleft()
    for i in Tree[new_Node]:
        if(visited[i] == -1):
            answer[i] = new_Node
            visited[i] = 0
            queue.append(i)

for i in answer[2:]:
    print(i)


from flask import Flask, request, jsonify

app = Flask(__name__)

# Sample data (for demonstration purposes)
doctors = [
    {
        "id": 1,
        "name": "Dr. John Smith",
        "specialization": "Internal Medicine",
        "available_days": ["Monday", "Tuesday", "Wednesday"],
        "max_patients": 5,
    },
    {
        "id": 2,
        "name": "Dr. Jane Doe",
        "specialization": "Pediatrics",
        "available_days": ["Tuesday", "Thursday", "Friday"],
        "max_patients": 6,
    },
]

# Defining a route to list all doctors
@app.route("/doctors", methods=["GET"])
def list_doctors():
    return jsonify(doctors)

# Defining a route to retrieve details of a specific doctor
@app.route("/doctors/<int:doctor_id>", methods=["GET"])
def get_doctor(doctor_id):
    doctor = next((doc for doc in doctors if doc["id"] == doctor_id), None)
    if doctor:
        return jsonify(doctor)
    return jsonify({"error": "Doctor not found"}), 404

# Defining a route to book an appointment with a doctor
@app.route("/appointments", methods=["POST"])
def book_appointment():
    data = request.get_json()
    doctor_id = data.get("doctor_id")
    appointment_date = data.get("appointment_date")
    return jsonify({"message": "Appointment booked successfully"})

if __name__ == "__main__":
    app.run(debug=True)


from coordinate_system import CoordinateSystem
from fsm import FSM
from vaccine import Vaccine
from operate import Operate
import constants
from report import Report
from multiprocessing import Process
from random import randint

def plan(fileoutName,virusPlotName,runNumber):

	fileT = fileoutName+str(runNumber)+".dat"
	fileT2 = virusPlotName+str(runNumber)

	l = []
	for i in range(0,constants.N_vacc):
		x = randint(constants.cs_minX,constants.cs_maxX)
		y = randint(constants.cs_minY,constants.cs_maxY)
		l.append((x,y))

	operator = Operate(CoordinateSystem(),[FSM(24,1)],Vaccine(l))
	reporter = Report(operator,fileT,fileT2,[0.33,0.42,0.5,0.58,0.67])

	for i in range(constants.N_turns):
		operator.run()
		reporter.update()
		reporter.reveal_virus(i)

	reporter.output()

def main():
	n_cores = 48
	for i in range(100):
		print(i)
		p = Process(target=plan, args=('Feb9_run_','virus_spatial_',i))
		p.start()
		if(i % n_cores == 0 and i != 0):
			p.join()

if __name__ == "__main__":
	main()


##the import "neuralNet" is referred from
##https://towardsdatascience.com/an-introduction-to-neural-networks-with-implementation-from-scratch-using-python-da4b6a45c05b
import neuralNet

import os
import gym
import numpy as np
import random
import ipdb
import math
import matplotlib.pyplot as plt
env = gym.make('MountainCar-v0')

#increase length of episode. Referred from
#https://www.reddit.com/r/reinforcementlearning/comments/agd6j4/how_to_change_episode_length_and_reward/
env._max_episode_steps = 2000

#entire function referred from
#https://www.youtube.com/watch?v=KzsBaqYzNLc&ab_channel=Jakester897
def get_reward(state):
    if(state>=0.5):
        return 2
    else:
        reward = (state+1.2)/1.8 -1
    return reward

replay_mem = []

## initialise the replay memory with 20,000 random samples
def generate_replay():
    for i in range(100):
        observation = env.reset()
        for t in range(200):
            #ipdb.set_trace()
            state1 = observation[0]
            state2 = observation[1]
            action = env.action_space.sample()
            new_observation, reward, done, info = env.step(action)
            new_state = [new_observation[0],new_observation[1]]
            reward = get_reward(new_observation[0])
            attach = ([state1,state2],reward,new_state,action)
            #print(attach)
            replay_mem.append(attach)
            if(done):
                break
            observation = new_observation
    return replay_mem

##create a directory to save results
##https://www.tutorialspoint.com/How-can-I-create-a-directory-if-it-does-not-exist-using-Python
if not os.path.exists('results'):
    os.makedirs('results')

start_lr=0
start_batch=0
reward_batch = []
total_goal = []
#read_text('./progress.txt')
replay_mem = generate_replay()
reward_tot = []
print(len(replay_mem))

eps = 1.0

#declare neural network architecture
layers = [2,150,100,3]

##initialise 2 neural networks
param = neuralNet.initialize_params(layers)
base_param = neuralNet.initialize_params(layers)

flag=0
goal = 0
flag_goal = 0
goal_list = []

for i in range(len(total_goal)):
    if(total_goal[i]==0):
        goal_list.append(list([]))
    else:
        goal_list.append(total_goal[i])

##iterate over different batch and learning rate values
batch = [30,100,500]
lr = [0.01,0.001,0.0001]

for lr_i in range(len(lr)):
    for batch_i in range(len(batch)):
        reward_mean = []
        reward_batch = []
        loss= 0
        eps = 1.0
        goal_lis = []
        flag=0
        goal = 0
        param = neuralNet.initialize_params(layers)
        base_param = neuralNet.initialize_params(layers)
        for i in range(1000):
            observation = env.reset()
            tot_reward = 0
            for t in range(1000):
                #ipdb.set_trace()
                state1 = observation[0]
                state2 = observation[1]
                curr_state = np.reshape(np.array([state1,state2]),(1,2))

                #get q value for current state
                q_val_curr = neuralNet.forward_propagation(np.transpose(curr_state),param)
                q_val_curr = q_val_curr["A3"]

                ##epsilon greedy strategy
                ##following 4 lines referred from
                ##https://github.com/ikvibhav/reinforcement_learning/blob/master/code/mountain_car/mountain_car_sarsa.py
                if np.random.random() > eps:
                    action = np.argmax(q_val_curr)
                else:
                    action = np.random.randint(0, env.action_space.n)

                new_observation, reward, done, info = env.step(action)

                ##get custom reward
                reward = get_reward(new_observation[0])
                tot_reward = tot_reward + reward

                ##termination condition to consider that MountainCar is solved
                if(new_observation[0]>=0.5):
                    if(eps<0.5):
                        goal = goal+1
                        goal_lis.append(i)
                    print("Goal is:",goal)
                    print("Episode is:",i)
                    done = True
                    #terminate if car reached goal 3 times
                    if(goal >= 3):
                        flag = 1
                    break
                new_state1 = new_observation[0]
                new_state2 = new_observation[1]

                ##append to replay memory and pop the first sample
                if(len(replay_mem)>2000):
                    replay_mem.pop(0)
                replay_mem.append(([state1,state2],reward,[new_state1,new_state2],action))

                if(done):
                    break

                ##randomly sample from replay memory.
                ##following line referred from
                ##https://stackoverflow.com/questions/22842289/generate-n-unique-random-numbers-within-a-range
                sample = random.sample(range(0, len(replay_mem)), batch[batch_i])

                input_curr_state = []
                input_next_state = []
                action = []
                reward = []

                ##processing after getting the samples
                for i_curr in range(len(sample)):
                    input_curr_state.append(replay_mem[sample[i_curr]][0])
                    input_next_state.append(replay_mem[sample[i_curr]][2])
                    action.append(int(replay_mem[sample[i_curr]][3]))
                    reward.append(replay_mem[sample[i_curr]][1])

                #ipdb.set_trace()
                input_curr_state = np.array(input_curr_state)
                input_next_state = np.array(input_next_state)

                #get q value for current state and next state
                y_pred_dict = neuralNet.forward_propagation(np.transpose(input_curr_state),param)
                y_pred = (np.transpose(y_pred_dict["A3"]))
                next_state_y = neuralNet.forward_propagation(np.transpose(input_next_state),base_param)
                next_state_y = (np.transpose(next_state_y["A3"]))
                #get the maximum q value for next state
                max_next_state = np.max(next_state_y,axis=1)

                ##determine if next state is terminal state.
                # This is done by looking at reward function.
                # If it is negative then state was terminal.
                for i_curr in range(len(max_next_state)):
                    ##if next state is terminal state then do not consider it.
                    if(reward[i_curr]==-1):
                        max_next_state[i_curr] = 0

                #calculating the expected q value
                y = reward + 0.99*max_next_state

                ##processing to compute loss. I have used L1 loss in this code
                y_final = y_pred.copy()
                y_pred_final = []
                for i_curr in range(len(y_final)):
                    y_pred_final.append(y_final[i_curr][action[i_curr]])
                    y_final[i_curr][action[i_curr]] = y[i_curr]
                try:
                    loss = np.mean(np.abs(y-y_pred_final))
                except Exception as e:
                    print("Exception ocurred")

                ##perform gradient descent
                ##https://towardsdatascience.com/an-introduction-to-neural-networks-with-implementation-from-scratch-using-python-da4b6a45c05b
                grads = neuralNet.backward_propagation(param,y_pred_dict,np.transpose(input_curr_state),np.transpose(y_final))
                ##update parameters
                param = neuralNet.update_params(param, grads, lr[lr_i])
                observation = new_observation

            ##epsilon decay
            ##https://github.com/philtabor/Youtube-Code-Repository/blob/master/ReinforcementLearning/Fundamentals/mountaincar.py
            eps = eps - 2/100 if eps > 0.01 else 0.01

            if(i%100==0):
                print(i,eps,loss,tot_reward)
            if(math.isnan(loss)):
                print(eps)
                print("Gradient Exploded")
                break
            reward_tot.append(tot_reward)

            ##update target network every 1000 episodes. This is done to keep the DQN stable
            if(i%100==0 and i>100):
                base_param = param

            if(flag==1):
                break

            if(i==0):
                reward_mean.append(np.float(tot_reward))
            else:
                reward_mean.append(np.float(reward_mean[-1]+tot_reward))
        mean_print = []
        for i in range(len(reward_mean)):
            if(math.isnan(reward_mean[i])):
                break
            mean_print.append((reward_mean[i])/(i+1))
        reward_mean_copy = mean_print.copy()
        reward_batch.append(reward_mean_copy)
        if(len(goal_lis)==0):
            total_goal.append([0])
        else:
            total_goal.append(goal_lis)

        goal_list.append(goal_lis)
        ##following 2 lines referred from
        ##https://stackoverflow.com/questions/28439701/how-to-save-and-load-numpy-array-data-properly
        np.save(('./results/result_np'+str(lr_i)),np.array(total_goal))
        np.save(('./results/pregress_np'+str(lr_i)+str(batch_i)),np.array(reward_batch))

        print("Batch finished",batch_i)
    start_batch =0
    print("Alpha finished",lr_i)
print("Finished!!!")
print(len(total_goal))
print(total_goal[0])


from django.contrib.auth import authenticate, logout
from django.shortcuts import render, get_object_or_404
from rest_framework import generics, status
from rest_framework.decorators import api_view
from rest_framework.response import Response
from users.models import User, Friendship
from users.serializers import UserDetailSerializer, UsersListSerializer, \
    UserFriendRequestSerializer
from users.utils import friend_accept_util

@api_view(['GET'])
def user_list(request):
    all_users = User.objects.all()
    serializer = UsersListSerializer(all_users, many=True)

    return Response(serializer.data)

@api_view(['GET'])
def user_me(request):
    user = request.user
    serializer = UserDetailSerializer(user)

    return Response(serializer.data)

@api_view(['POST'])
def friend_request(request, user_id):
    auth_user = request.user
    another_user = User.objects.filter(id=user_id).first()
    auth_user.friends.add(another_user)

    serializer = UserFriendRequestSerializer(another_user)

    return Response(serializer.data)

@api_view(['POST'])
def friend_accept(request, user_id):
    to_user_id = user_id
    from_user = request.user
    status_to_change = request.data['status']

    friend_accept_util(from_user, to_user_id, status_to_change)

    return Response(True)

@api_view(['GET'])
def friends(request):

    user = request.user

    serializer = UsersListSerializer(user.friends, many=True)
    return Response(serializer.data)


# import packages
import glob
from pathlib import Path
import os
import numpy as np
from datetime import datetime
from datetime import timedelta
import pandas as pd
import calendar
import geopandas as gpd
import cartopy
import matplotlib.pyplot as plt
import math
from pathos.threading import ThreadPool as Pool
from scipy.optimize import least_squares
import sklearn
from sklearn.linear_model import LinearRegression
from scipy.stats import gaussian_kde
import statsmodels.api as sm
from statsmodels.regression.linear_model import OLS
from statsmodels.tools import add_constant
import random
from scipy.optimize import minimize

def snow_calculation(catch_id,work_dir):
    # load p-ep-tas timeseries
    file = glob.glob(f'{work_dir}/output/forcing_timeseries/processed/daily/{catch_id}*.csv')[0]
    ts = pd.read_csv(f'{file}',index_col=0)
    # ts = ts.loc[start_date:end_date] #this should be based on sr - or do this for the full timeseries???
    p = ts.p.values
    tas = ts.tas.values

    # load mean elevation
    file = glob.glob(f'{work_dir}/output/elevation/stats_hydrosheds/ele_{catch_id}.csv')[0]
    elm = pd.read_csv(f'{file}',index_col=0)

    # load elevation zones
    file = glob.glob(f'{work_dir}/output/elevation/el_zones/{catch_id}*.csv')[0]
    elz = pd.read_csv(f'{file}',index_col=0)

    # snow parameters
    TT = 0
    MF = 2

    # make empty matrices with timseries rows and elevation zones columns
    Pm_el = np.zeros((len(p),len(elz))) # melt water
    Pl_el = np.zeros((len(p),len(elz))) # liquid precipitation
    Ps_el = np.zeros((len(p),len(elz))) # solid precipitation

    for j in range(len(elz)):
        Ss = np.zeros(len(p)) # snow storage
        Pm = np.zeros(len(p)) # melt water
        Pl = np.zeros(len(p)) # liquid precipitation
        Ps = np.zeros(len(p)) # solid precipitation
        T = np.zeros(len(p)) # temperature

        # temperature difference for elevation zone
        el_dif = elm.mean_ele - elz.loc[j]['mean_el']
        dt = (el_dif/1000.)*6.4

        # loop over timesteps and compute ps, pm and pl
        for i in range(0,len(p)):
            Tmean = tas[i]
            T[i] = Tmean+dt

            if T[i]>TT:
                Pl[i] = p[i]
                Ps[i] = 0

                Pm[i] = min(Ss[i],MF*(T[i]-TT))
                Ss[i] = max(0, Ss[i]-Pm[i])
            else:
                Ps[i] = p[i]
                Pl[i] = 0
                Ss[i] = Ss[i]+Ps[i]
            if i<len(p)-1:
                Ss[i+1] = Ss[i]

        # scale to coverage of elevation zone and add to matrix
        Ps = Ps * elz.loc[j]['frac']
        Pm = Pm * elz.loc[j]['frac']
        Pl = Pl * elz.loc[j]['frac']
        Pm_el[:,j] = Pm
        Pl_el[:,j] = Pl
        Ps_el[:,j] = Ps

    # sum fractions of pm, pl and ps
    pm = Pm_el.sum(axis=1)
    pl = Pl_el.sum(axis=1)
    ps = Ps_el.sum(axis=1)

    # add to forcing dataframe
    ts['pm'] = pm
    ts['pl'] = pl
    ts['ps'] = ps
    ts.to_csv(f'{work_dir}/output/snow/timeseries/{catch_id}.csv')

def run_function_parallel_snow(
    catch_list=list,
    work_dir_list=list,
    # threads=None
    threads=100
    ):
    """
    Runs function snow_calculation  in parallel.

    catch_list:  str, list, list of catchmet ids
    work_dir_list:     str, list, list of work dir
    threads:         int,       number of threads (cores), when set to None use all available threads

    Returns: None
    """
    # Set number of threads (cores) used for parallel run and map threads
    if threads is None:
        pool = Pool()
    else:
        pool = Pool(nodes=threads)
    # Run parallel models
    results = pool.map(
        snow_calculation,
        catch_list,
        work_dir_list,
    )


import os
from collections import defaultdict

class UndeliveredMessages:
    """Class to store undelivered messages. The messages are stored in memory and in a file."""
    def __init__(self, filename: str):
        self.filename = filename

        self.undelivered_msg = defaultdict(list) # Map of recipient username to list of (sender, message) for that recipient
        if os.path.exists(filename):
            # Read the file and store the messages in a dictionary by recipient
            with open(self.filename, 'r') as f:
                lines = f.readlines()
            for line in lines:
                if line.strip():
                    recipient, sender, message = line.strip().split(' ', 2)
                    self.undelivered_msg[recipient].append((sender, message))

    def add_message(self, recipient: str, sender: str, message: str):
        """Add a message to the list of undelivered messages for a recipient."""
        self.undelivered_msg[recipient] += [
            (sender, message)]
        with open(self.filename, 'a') as f:
            f.write(f"{recipient} {sender} {message}\n")
            f.flush()

    def get_messages(self):
        """Return a list of (recipient, [(sender, message)]) for all recipients with undelivered messages."""
        return self.undelivered_msg.items()

    def update_messages(self, recipient, message_infos):
        """Update the messages for a recipient. Replaces the message list for that recipient with the given messages."""
        self.undelivered_msg[recipient] = message_infos
        with open(self.filename, 'r') as f:
            lines = f.readlines()
        with open(self.filename, 'w') as f:
            f.writelines(filter(lambda line: line.strip() and line.strip().split()[0] != recipient, lines))
            f.flush()
            for sender, message in message_infos:
                if (not (sender == "" or message == "")):
                    f.write(f"{recipient} {sender} {message}\n")
                    f.flush()

    def clear(self):
        """
        Clears the undelivered messages for testing purposes
        """
        self.undelivered_msg = defaultdict(list)
        open(self.filename, 'w').close()


import os
import types
import unittest
import warnings

import parasail
import pytest

import medaka.align
from medaka import smolecule

root_dir = os.path.abspath(os.path.dirname(__file__))
test_fasta = os.path.join(root_dir, 'data/smolecule.fasta')
test_mfasta = os.path.join(root_dir, 'data/smolecule_multi.fasta')

class TestRead(unittest.TestCase):

    def test_00_read_single(self):
        read = smolecule.Read.from_fastx(test_fasta)
        self.assertIsInstance(read, smolecule.Read)
        self.assertFalse(read._initialized, 'Read is uninitialized')
        self.assertEqual(read.nseqs, 9, 'Read has correct number of subreads.')

    def test_01_read_multi(self):
        reads = smolecule.Read.multi_from_fastx(test_mfasta)
        self.assertIsInstance(reads, types.GeneratorType, 'multi-read gives generator.')
        reads = [x for x in reads]
        n_subreads = [9, 22]
        self.assertEqual(len(reads), len(n_subreads), 'Retrieved correct number of subreads.')
        for i, (nsr, read) in enumerate(zip(n_subreads, reads)):
            self.assertEqual(read.nseqs, nsr, 'Read {} has correct number of subreads.'.format(i))

    @unittest.skipIf(parasail.dnafull is None, "Using fake parasail.")
    def test_10_initialize(self):
        read = smolecule.Read.from_fastx(test_fasta)
        self.assertIsInstance(read, smolecule.Read)
        read.initialize()
        self.assertTrue(read._initialized, 'Read is initialized after .initialize().')
        self.assertFalse(read._alignments is None, '.alignments is not None after .initialize().')
        self.assertTrue(read._alignments_valid, '.alignments_valid is True after .initialize().')
        self.assertFalse(read._orient is None, '.orients is not None after .initialize().')

    @unittest.skipIf(parasail.dnafull is None, "Using fake parasail.")
    def test_12_interleave(self):
        read = smolecule.Read.from_fastx(test_fasta)
        orient, subreads = read.interleaved_subreads
        exp_orient = (True, False, True, False, True, False, True, False, True)
        self.assertEqual(orient, exp_orient, "Orientations interleaved")
        # note, the sort is stable so this test is only useful
        # because some sorting needs to be done on the test set
        orig_order = [r.name for r in subreads]
        new_order = [r.name for r in read.subreads]
        self.assertNotEqual(orig_order, new_order, "Reads are reordered.")

    @unittest.skipIf(parasail.dnafull is None, "Using fake parasail.")
    @pytest.mark.skipif("CITEST" in os.environ, reason="CI instruction-set issue")
    def test_20_basic_consensus(self):
        read = smolecule.Read.from_fastx(test_fasta)
        cons = read.poa_consensus()
        self.assertTrue(read._initialized, 'Read is initialized after poa.')
        self.assertEqual(cons, read.consensus, 'Returned sequence is self.consensus.')
        self.assertFalse(read._alignments_valid, '.alignments_valid is False after .poa_consensus.()')

    @unittest.skipIf(parasail.dnafull is None, "Using fake parasail.")
    def test_30_parasail_align(self):
        revcom = medaka.common.reverse_complement
        seq_mult = 100
        seq = 'ACGACTACGACTACGACT' * seq_mult
        sub_reads = [
            (smolecule.Subread('read_0', seq), 0),
            (smolecule.Subread('read_1', seq), 0),
            (smolecule.Subread('read_2', revcom(seq)), 16)]
        read = smolecule.Read('test', [s[0] for s in sub_reads])

        expected = [
            smolecule.Alignment(
                'test', sr.name, flag, 0,
                sr.seq if flag == 0 else revcom(sr.seq),
                '{}='.format(len(seq)))
            for i, (sr, flag) in enumerate(sub_reads)]

        for aligner in ('align_to_template', 'mappy_to_template'):
            func = getattr(read, aligner)
            alignments = func(sub_reads[0][0].seq, 'test')
            self.assertEqual(len(alignments), len(expected))
            for aln, exp in zip(alignments, expected):
                for attr, exp_attr in zip(aln, exp):
                    # mappy doesn't report equality, just match
                    if aligner == 'mappy_to_template' and exp_attr == '{}='.format(len(seq)):
                        exp_attr = '{}M'.format(len(seq))
                    self.assertEqual(attr, exp_attr)

        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter("always")
            read.mappy_to_template(sub_reads[0][0].seq, 'test', align=False)
            assert issubclass(w[-1].category, DeprecationWarning)


import os

import ogrtest
import pytest
import test_py_scripts

from osgeo import gdal, ogr

pytestmark = pytest.mark.skipif(
    test_py_scripts.get_py_script("gdal_polygonize") is None,
    reason="gdal_polygonize not available",
)

@pytest.fixture()
def script_path():
    return test_py_scripts.get_py_script("gdal_polygonize")

###############################################################################
# Test a fairly simple case, with nodata masking.

@pytest.mark.require_driver("AAIGRID")
def test_gdal_polygonize_1(script_path):

    outfilename = "tmp/poly.shp"
    # Create a OGR datasource to put results in.
    shp_drv = ogr.GetDriverByName("ESRI Shapefile")
    if os.path.exists(outfilename):
        shp_drv.DeleteDataSource(outfilename)

    shp_ds = shp_drv.CreateDataSource(outfilename)

    shp_layer = shp_ds.CreateLayer("poly", None, ogr.wkbPolygon)

    fd = ogr.FieldDefn("DN", ogr.OFTInteger)
    shp_layer.CreateField(fd)

    shp_ds.Destroy()

    # run the algorithm.
    test_py_scripts.run_py_script(
        script_path,
        "gdal_polygonize",
        test_py_scripts.get_data_path("alg") + "polygonize_in.grd tmp poly DN",
    )

    # Confirm we get the set of expected features in the output layer.

    shp_ds = ogr.Open("tmp")
    shp_lyr = shp_ds.GetLayerByName("poly")

    expected_feature_number = 13
    assert shp_lyr.GetFeatureCount() == expected_feature_number

    expect = [107, 123, 115, 115, 140, 148, 123, 140, 100, 101, 102, 156, 103]

    ogrtest.check_features_against_list(shp_lyr, "DN", expect)

    # check at least one geometry.
    shp_lyr.SetAttributeFilter("dn = 156")
    feat_read = shp_lyr.GetNextFeature()
    ogrtest.check_feature_geometry(
        feat_read,
        "POLYGON ((440720 3751200,440900 3751200,440900 3751020,440720 3751020,440720 3751200),(440780 3751140,440780 3751080,440840 3751080,440840 3751140,440780 3751140))",
    )

    feat_read.Destroy()

    shp_ds.Destroy()
    # Reload drv because of side effects of run_py_script()
    shp_drv = ogr.GetDriverByName("ESRI Shapefile")
    shp_drv.DeleteDataSource(outfilename)

###############################################################################
# Test a simple case without masking.

@pytest.mark.require_driver("AAIGRID")
def test_gdal_polygonize_2(script_path):

    outfilename = "tmp/out.geojson"
    if gdal.VSIStatL(outfilename) is not None:
        gdal.Unlink(outfilename)

    # run the algorithm.
    test_py_scripts.run_py_script(
        script_path,
        "gdal_polygonize",
        "-b 1 -q -nomask "
        + test_py_scripts.get_data_path("alg")
        + "polygonize_in.grd "
        + outfilename,
    )

    # Confirm we get the set of expected features in the output layer.
    ds = gdal.OpenEx(outfilename)
    assert ds.GetDriver().ShortName == "GeoJSON"
    lyr = ds.GetLayerByName("out")

    expected_feature_number = 17
    assert lyr.GetFeatureCount() == expected_feature_number

    expect = [
        107,
        123,
        115,
        132,
        115,
        140,
        132,
        132,
        148,
        123,
        140,
        132,
        100,
        101,
        102,
        156,
        103,
    ]

    ogrtest.check_features_against_list(lyr, "DN", expect)

    ds = None

    gdal.Unlink(outfilename)

@pytest.mark.require_driver("GPKG")
def test_gdal_polygonize_3(script_path):

    drv = ogr.GetDriverByName("GPKG")
    outfilename = "tmp/out.gpkg"
    if os.path.exists(outfilename):
        drv.DeleteDataSource(outfilename)

    # run the algorithm.
    test_py_scripts.run_py_script(
        script_path,
        "gdal_polygonize",
        '-b 1 -f "GPKG" -q -nomask -lco FID=myfid '
        + test_py_scripts.get_data_path("alg")
        + "polygonize_in.grd "
        + outfilename,
    )

    # Confirm we get the set of expected features in the output layer.
    gpkg_ds = ogr.Open(outfilename)
    gpkg_lyr = gpkg_ds.GetLayerByName("out")
    assert gpkg_lyr.GetFIDColumn() == "myfid"
    geom_type = gpkg_lyr.GetGeomType()
    geom_is_polygon = geom_type in (ogr.wkbPolygon, ogr.wkbMultiPolygon)

    gpkg_ds.Destroy()
    # Reload drv because of side effects of run_py_script()
    drv = ogr.GetDriverByName("GPKG")
    drv.DeleteDataSource(outfilename)

    if geom_is_polygon:
        return
    pytest.fail(
        "GetGeomType() returned %d instead of %d or %d (ogr.wkbPolygon or ogr.wkbMultiPolygon)"
        % (geom_type, ogr.wkbPolygon, ogr.wkbMultiPolygon)
    )

###############################################################################
# Test -b mask

@pytest.mark.require_driver("GML")
def test_gdal_polygonize_4(script_path):

    outfilename = "tmp/out.gml"
    # Test mask syntax
    test_py_scripts.run_py_script(
        script_path,
        "gdal_polygonize",
        "-q -f GML -b mask "
        + test_py_scripts.get_data_path("gcore")
        + "byte.tif "
        + outfilename,
    )

    content = open(outfilename, "rt").read()

    os.unlink(outfilename)
    os.unlink(outfilename[0:-3] + "xsd")

    assert (
        '<gml:Polygon srsName="urn:ogc:def:crs:EPSG::26711" gml:id="out.geom.0"><gml:exterior><gml:LinearRing><gml:posList>440720 3751320 440720 3750120 441920 3750120 441920 3751320 440720 3751320</gml:posList></gml:LinearRing></gml:exterior></gml:Polygon>'
        in content
    )

    # Test mask,1 syntax
    test_py_scripts.run_py_script(
        script_path,
        "gdal_polygonize",
        "-q -f GML -b mask,1 "
        + test_py_scripts.get_data_path("gcore")
        + "byte.tif "
        + outfilename,
    )

    content = open(outfilename, "rt").read()

    os.unlink(outfilename)
    os.unlink(outfilename[0:-3] + "xsd")

    assert (
        '<gml:Polygon srsName="urn:ogc:def:crs:EPSG::26711" gml:id="out.geom.0"><gml:exterior><gml:LinearRing><gml:posList>440720 3751320 440720 3750120 441920 3750120 441920 3751320 440720 3751320</gml:posList></gml:LinearRing></gml:exterior></gml:Polygon>'
        in content
    )

###############################################################################
# Test -8

def test_gdal_polygonize_minus_8(script_path):

    outfilename = "tmp/out.geojson"
    test_py_scripts.run_py_script(
        script_path,
        "gdal_polygonize",
        "-q -8 " + test_py_scripts.get_data_path("gcore") + "byte.tif " + outfilename,
    )

    ds = gdal.OpenEx(outfilename)
    lyr = ds.GetLayer(0)
    assert lyr.GetFeatureCount() == 229
    ds = None

    os.unlink(outfilename)

###############################################################################
# Test --overwrite

@pytest.mark.parametrize("format", ["geojson", "gpkg"])
def test_gdal_polygonize_overwrite(script_path, format):

    if gdal.GetDriverByName(format) is None:
        pytest.skip(f"driver {format} not available")

    try:
        outfilename = "tmp/out." + format
        test_py_scripts.run_py_script(
            script_path,
            "gdal_polygonize",
            test_py_scripts.get_data_path("gcore") + "byte.tif " + outfilename,
        )

        ds = gdal.OpenEx(outfilename)
        lyr = ds.GetLayer(0)
        initial_value = lyr.GetFeatureCount()
        ds = None

        # Append behavior by default
        test_py_scripts.run_py_script(
            script_path,
            "gdal_polygonize",
            test_py_scripts.get_data_path("gcore") + "byte.tif " + outfilename,
        )

        ds = gdal.OpenEx(outfilename)
        lyr = ds.GetLayer(0)
        assert lyr.GetFeatureCount() == initial_value * 2
        ds = None

        # Let's overwrite
        test_py_scripts.run_py_script(
            script_path,
            "gdal_polygonize",
            " -overwrite "
            + test_py_scripts.get_data_path("gcore")
            + "byte.tif "
            + outfilename,
        )

        ds = gdal.OpenEx(outfilename)
        lyr = ds.GetLayer(0)
        assert lyr.GetFeatureCount() == initial_value
        ds = None

    finally:
        if os.path.exists(outfilename):
            os.unlink(outfilename)


from app.models import User as model_User
from flask import url_for
from flask_restful import Resource, marshal_with
from . import api, parser, default_per_page
from .fields import user_detail_fields, user_list

@api.route('/users/<int:user_id>/')
class User(Resource):
    @marshal_with(user_detail_fields)
    def get(self, user_id):
        user = model_User.query.get_or_404(user_id)
        user.uri = url_for('api.user', user_id=user_id, _external=True)
        return user

@api.route('/users/')
class UserList(Resource):
    @marshal_with(user_list)
    def get(self):
        args = parser.parse_args()
        page = args['page'] or 1
        per_page = args['per_page'] or default_per_page
        pagination = model_User.query.order_by(model_User.id.desc()).paginate(page=page, per_page=per_page)
        items = pagination.items
        prev = None
        if pagination.has_prev:
            prev = url_for('api.userlist', page=page - 1, per_page=per_page, _external=True)
        next = None
        if pagination.has_next:
            next = url_for('api.userlist', page=page + 1, per_page=per_page, _external=True)
        return {
            'items': items,
            'prev': prev,
            'next': next,
            'total': pagination.total,
            'pages_count': pagination.pages,
            'current_page': pagination.page,
            'per_page': per_page
        }


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @carolio7

# import des python librairies
import sys, re, subprocess
import numpy as np
import pandas as pd
import itertools

# import libraries spark
from pyspark import SparkContext, SparkConf
from pyspark.mllib.regression import LabeledPoint
from pyspark.mllib.classification import SVMWithSGD
from pyspark import StorageLevel

#import des librairies pour S3
import s3fs

# initialization de spark et connection en aws S3
conf = SparkConf().setAppName("Solution 2")
sc = SparkContext.getOrCreate(conf=conf)

fs = s3fs.S3FileSystem(anon=False)

# initialisation des fonctions parametrant notre programe
# Chemins AWS S3 du répertoire contenant les features en json
argument =  sys.argv[1]     # ou égal à 's3://projet2oc-rasambatra/exemple/' pour tester

# variable correspondant aux coefficients de split du dataset
separations = np.linspace(0.0003, 0.8, 10)   #Pour varier le nombre de data dans Training set

BUCKET_DESTINATION = 's3://projet2oc-rasambatra/' # le bucket où le programme va mettre les fichieers output

# Fonctions pour notre programme

def lire_json_features(fichier):
    """
        Fonction lisant un fichier .json donné en paramètre
        et retourne une tuple composé du nom du fichier et liste des features

        Attention: le repertoire doit se terminer par un slash /
    """
    label = re.sub(r'[0-9]','', fichier).split('/')[-1]     # Extraire le nom du pet a part du chemin du fichier
    label = label[:-9].strip('_')      # Enleve les extention .jpg.json au nom du fichier
    for line in fs.open('s3://' + fichier, 'r'):
                values = line.strip('[]').split(',')
                values = [float(x) for x in values]

    return (label,values)

def load_features_to_tuple(directory):
    """
    Fonction pour filtrer les fichiers .json dans un répertoire
        et retourne un rdd contenant des tuples (label du fichier, features)

    Attention: le repertoire doit se terminer par un slash /
    """

    les_features_rdd = sc.parallelize(fs.ls(directory))\
                        .filter(lambda fichier: fichier[-4:] == 'json')\
                        .map(lambda les_json: lire_json_features(les_json))

    return les_features_rdd

def etiqueter_one_vs_all(dataset, un_label):
    """
        Fonction lisant un rdd de plusieurs tuples (label, feature):
            - convertit chaque tuple en LabelPoint
                correspondant à la classification One-versus-All
                   » etiquette 1 si le label est égal à l'argument
                   » sinon etiquette 0

            - retourne une liste de LabelPoint de l'argument (Label 1) versus Rest
    """

    dataset_labeled_rdd = dataset\
        .map(lambda untupl: LabeledPoint(1, untupl[1]) if (untupl[0]== un_label) else LabeledPoint(0, untupl[1]))

    return dataset_labeled_rdd

def etiqueter_one_vs_one(dataset, un_label, autre_label):

    """
        Fonction lisant une liste de plusieurs tuples (label, feature):
            - filte les tuples avec label correspondant aux deux arguments seulements
            - convertit chaque tuple en LabelPoint
                correspondant à la classification One-Versus-One
                   » etiquette 1 si le label est égal au premier argument
                   » sinon etiquette 0 pour le second

            - retourne une liste de LabelPoint du 1er arg (Label 1) versus Znd arg (Label 0)
    """

    dataset_labeled_rdd = dataset\
        .filter(lambda line: (line[0]==un_label) or (line[0]==autre_label))\
        .map(lambda untupl: LabeledPoint(1.0, untupl[1]) if (untupl[0]== un_label)\
             else LabeledPoint(0.0, untupl[1]))

    return dataset_labeled_rdd

def modelWithSVM(dataset_labeled, les_splits):
    """
        Entainer le modele en utilisant Support Vector Machines
            avec different split du dataset.
        Retourne un dictionnaire du le nombre de Training set et la précision obenue
    """

    visualizationData = {}

    for split in les_splits:
        trainingData, validationData = dataset_labeled.randomSplit([split, 1.0-split], seed=2)
        trainingData.persist(StorageLevel.MEMORY_AND_DISK)
        validationData.persist(StorageLevel.MEMORY_AND_DISK)

        if ((trainingData.isEmpty() == False) and (validationData.isEmpty() == False)):
            model = SVMWithSGD.train(trainingData, iterations=100, step=1.0, regParam=0.01)
            predict = validationData.map(lambda ad: (ad.label, model.predict(ad.features)))
            totalValidationData = validationData.count()
            correctlyPredicted = predict.filter(lambda x: x[0] == x[1]).count()
            accuracy = float(correctlyPredicted) / totalValidationData

            visualizationData[trainingData.count()] = accuracy

        trainingData.unpersist()
        validationData.unpersist()

    return visualizationData

def apprendre_one_vs_all(dataset_rdd, label, split_list):

    """
        -Etiquetter le dataset avec le label
        -séparer avec le coefficient du split_list parcouru
        -entrainer
        Retourne un pandas dataframe dela précision obenue en fonction du nombre de Training set
        utilisé pour l'entrainement
    """

    data_labeled_ovr = etiqueter_one_vs_all(dataset_rdd, label)
    visualizationData_ovr = modelWithSVM(data_labeled_ovr, split_list)
    print('Performance : ', visualizationData_ovr)
    return pd.DataFrame(visualizationData_ovr, index=[label])

def apprendre_one_vs_one(dataset_rdd, tuple_label, split_list):

    """
        -Etiquetter le dataset avec le label
        -séparer avec le coefficient du split_list parcouru
        -entrainer
        Retourne un pandas dataframe dela précision obenue en fonction du nombre de Training set
        utilisé pour l'entrainement
    """

    data_labeled_ovo = etiqueter_one_vs_one(dataset_rdd, tuple_label[0], tuple_label[1])
    visualizationData_ovo = modelWithSVM(data_labeled_ovo, split_list)
    print('Performance : ', visualizationData_ovo)
    return pd.DataFrame(visualizationData_ovo, index=[tuple_label[0] + '-vs-' + tuple_label[1]])

def _write_dataframe_to_csv_on_s3(dataframe, filename):
    """ Ecrire un dataframe au format CSV sur S3 """
    print("Writing {} records to {}".format(len(dataframe), filename))
    bytes_to_write = dataframe.to_csv(None).encode()
    with fs.open(BUCKET_DESTINATION + filename, 'wb') as f:
        f.write(bytes_to_write)

print('-----------------------------------------------')
print('>>>>>>>>>>>>>>>> Début du programme')

print('»»»»»»»»»»»»»»»»»»»»»        Partie 1    ««««««««««««««««««««««««««')
print('>>>>>>>>>>>>>>> Lecture des features')

features_list_rdd = load_features_to_tuple(argument).persist(StorageLevel.MEMORY_AND_DISK)
# Extraction du nom des pets dans le dataset features_list
les_labels = features_list_rdd.keys().distinct().collect()

print("voici les labels : ")
print(les_labels)

print('>>>> Fin de lecture des features')
print('»»»»»»»»»»»»»»»»»»»»»    Fin - Partie 1    ««««««««««««««««««««««««««')

print('»»»»»»»»»»»»»»»»»»»»»    Partie 2    ««««««««««««««««««««««««««')
print('>>>> Entrainement du model SVMWithSGD One-versus-All <<<<<<<<<<<<<<<<<<<<')

result_ovr = pd.DataFrame()

for animal in les_labels:
    print('*********************   Debut - Entrainement {}-versus-All   *******************'.format(animal))

    un_dataframe_ova = apprendre_one_vs_all(features_list_rdd, animal, separations)

    result_ovr = result_ovr.append(un_dataframe_ova)

    print('*********************   Fin - Entrainement {}-versus-All   *******************'.format(animal))

print(' VOICI LE TABLEAU DES PRECISIONS DU MODEL :')
print(result_ovr)

print('>>>> Fin entrainement du model SVMWithSGD One-versus-All <<<<<<<<<<<<<<<<<<<<')
print('»»»»»»»»»»»»»»»»»»»»»    Fin - Partie 2    ««««««««««««««««««««««««««')

print('»»»»»»»»»»»»»»»»»»»»»    Partie 3    ««««««««««««««««««««««««««')
print('>>>> Entrainement du model SVMWithSGD One-versus-One <<<<<<<<<<<<<<<<<<<<')

result_ovo = pd.DataFrame()
# Création de couple de label deux par deux
paires = list(itertools.combinations(les_labels, r=2))
print('VOICI LES MODELS A ENTRAINER :')
print(paires)

for unePaire in paires:
    print('**********    Debut - Entrainement {}-versus-{}      ************'.format(unePaire[0],unePaire[1]))

    un_dataframe_ovo = apprendre_one_vs_one(features_list_rdd, unePaire, separations)

    result_ovo = result_ovo.append(un_dataframe_ovo)

    print('*************   Fin - Entrainement {}-versus-{}   *************'.format(unePaire[0],unePaire[1]))

print(' VOICI LE TABLEAU DES PRECISIONS DU MODEL :')
print(result_ovo)

print('>>>> Fin entrainement du model SVMWithSGD One-versus-One <<<<<<<<<<<<<<<<<<<<')
print('»»»»»»»»»»»»»»»»»»»»»    Fin - Partie 3    ««««««««««««««««««««««««««')

print('*********************    Sauvegarde des dataframes dans bucket S3      ******************************')

# Sauver les dataframes obtenus dans le bucket S3
_write_dataframe_to_csv_on_s3(result_ovr, 'output/result_ONE_VS_ALL_10_split.csv')

_write_dataframe_to_csv_on_s3(result_ovo, 'output/result_ONE_VS_ONE_10_split.csv')

print('*********************    Fin des sauvegardes      ******************************')

print('>>>>>>>>>>>>>>>>>>>>> Fin du programme')