"""module labeled_image_dataset. Simple
wrapper to load a labeled image dataset

"""
import random
from pathlib import Path

from PIL.Image import Image as PILImage
from typing import List, Any, Callable, Tuple, Union, Dict
import csv
import os

from navalmartin_mir_vision_utils.image_utils import get_img_files
from navalmartin_mir_vision_utils.image_loaders import load_img
from navalmartin_mir_vision_utils.image_enums import (ImageLoadersEnumType, IMAGE_STR_TYPES)
from navalmartin_mir_vision_utils.mir_vision_config import WITH_TORCH, DUMMY_PATH
from navalmartin_mir_vision_utils.exceptions import InvalidConfiguration
from navalmartin_mir_vision_utils.mir_vision_types import TorchTensor
from navalmartin_mir_vision_utils.image_transformers import pil_to_torch_tensor

if WITH_TORCH:
    import torch

class LabeledImageDataset(object):
    """Simple class to load from a specified
    directory images that are organised into
    subdirectories that represent the labels

    """

    @staticmethod
    def as_pytorch_tensor(dataset: "LabeledImageDataset",
                          transformer: Callable = None) -> Tuple[TorchTensor, List[int]]:
        """Returns a tuple with the images in the dataset as
        PyTorch tensors. If the images were loaded using ImageLoadersEnumType.PIL
        then the image is first converted into a PyTorch tensor.
        Acceptable loaders for the given dataset are ImageLoadersEnumType.PIL or
        ImageLoadersEnumType.PYTORCH_TENSOR
        If WITH_TORCH == False, it raises InvalidConfiguration.
        If the dataset loader is not of the right type it raises ValueError

        Parameters
        ----------
        transformer: A list of operations to apply on the returned tensors
        dataset: The dataset to work on

        Returns
        -------

        A tuple of PyTorch tensor, List[int]
        """
        if not WITH_TORCH:
            raise InvalidConfiguration(message="PyTorch is not installed so cannot use pil_to_torch_tensor")

        labels = []
        data: List[TorchTensor] = []

        if dataset.loader_type == ImageLoadersEnumType.PIL:
            for image in dataset:
                img = image[0]
                label = image[1]

                if isinstance(img, torch.Tensor):
                    raise ValueError(f"Dataset loader type is {ImageLoadersEnumType.PIL.name} "
                                     f"but image point is of type torch.Tensor. "
                                     f"Have you applied any transformation to the images?")

                # first convert to pytorch tensor and
                # then apply the transformations
                img = pil_to_torch_tensor(image=img, unsqueeze_dim=None)
                if transformer is not None:
                    img = transformer(img)
                data.append(img)
                labels.append(label)
            return torch.stack(data), labels
        elif dataset.loader_type == ImageLoadersEnumType.PYTORCH_TENSOR:
            for image in dataset:
                img = image[0]
                label = image[1]

                if transformer is not None:
                    img = transformer(img)

                data.append(img)
                labels.append(label)
            return torch.stack(data), labels
        elif dataset.loader_type == ImageLoadersEnumType.FILEPATH:
            for image in dataset:
                img = image[0]
                label = image[1]

                pytorch_image = load_img(path=img, transformer=transformer,
                                         loader=ImageLoadersEnumType.PYTORCH_TENSOR)

                data.append(pytorch_image)
                labels.append(label)
            return torch.stack(data), labels
        else:
            raise ValueError(f"Invalid loader type {dataset.loader_type} "
                             f"not in [ImageLoadersEnumType.PIL, ImageLoadersEnumType.PYTORCH_TENSOR]")

    @classmethod
    def build_from_list(cls, images: List[Tuple[Union[Path, PILImage, TorchTensor], Union[int, str]]],
                        unique_labels: List[tuple],
                        image_labels: List[int],
                        loader_type: ImageLoadersEnumType,
                        image_formats: List[str] = IMAGE_STR_TYPES,
                        transformer: Callable = None) -> "LabeledImageDataset":

        dataset = LabeledImageDataset(unique_labels=unique_labels,
                                      base_path=DUMMY_PATH,
                                      do_load=False,
                                      transformer=transformer,
                                      loader_type=loader_type,
                                      image_formats=image_formats)

        dataset.images = images
        dataset.image_labels = image_labels
        images_per_label = {}

        for img_label in image_labels:
            for item in unique_labels:
                if img_label == item[1]:
                    if item[0] in images_per_label:
                        images_per_label[item[0]] += 1
                    else:
                        images_per_label[item[0]] = 1

        dataset._images_per_label = images_per_label
        return dataset

    @classmethod
    def load_from_csv(cls, csv_filename: Path, base_path: Path, path_creator: Callable = None) -> "LabeledImageDataset":
        """Load the dataset from the given CSV file. The file should
        have the following format ('image_filename', image_label_index, 'image_label_name')

        Parameters
        ----------
        csv_filename: The CSV file to load the dataset from
        base_path: The base (i.e. root) path that dataset resides
        path_creator: Adapt the path formed from the CSV rows
        Returns
        -------

        """

        images: List[Tuple[Union[Path], Union[int]]] = []
        image_labels: List[int] = []
        unique_labels = []
        image_formats = []
        with open(csv_filename, 'r', newline='\n') as csvfile:
            filereader = csv.reader(csvfile, delimiter=",")

            for row in filereader:

                if len(row) != 3:
                    raise ValueError(f"Invalid format. File should have 3 columns but has {len(row)}")

                if path_creator is not None:
                    img_path = path_creator(row)
                else:
                    img_path = Path(str(base_path) + "/" + row[2] + "/" + row[0])
                images.append((img_path, int(row[1])))
                image_labels.append(int(row[1]))
                format_ = Path(row[0]).suffix

                if format_ not in image_formats:
                    image_formats.append(format_)

                if (row[2], int(row[1])) not in unique_labels:
                    unique_labels.append((row[2], int(row[1])))

        dataset = LabeledImageDataset.build_from_list(images=images,
                                                      image_labels=image_labels,
                                                      unique_labels=unique_labels,
                                                      loader_type=ImageLoadersEnumType.FILEPATH,
                                                      image_formats=image_formats)
        dataset.base_path = base_path
        return dataset

    def __init__(self, unique_labels: List[tuple], base_path: Path,
                 do_load: bool = True, *,
                 image_formats: List[str] = IMAGE_STR_TYPES,
                 loader_type: ImageLoadersEnumType = ImageLoadersEnumType.PIL,
                 transformer: Callable = None):
        """Constructor. Initialize the dataset by providing
        the unique labels for the images in a form of [("label_name", idx)]
        and provide the base path to load the images from.
        The path should arrange the images into directories that each
        directory has the 'label_name' from the unique_labels.
        Depending on the loader_type the class will hold images into one of the
        following three options

        - Path -> loader_type == ImageLoadersEnumType.FILENAME
        - PILImage -> loader_type == ImageLoadersEnumType.PIL
        - TorchTensor -> loader_type == ImageLoadersEnumType.PYTORCH_TENSOR

        Parameters
        ----------
        unique_labels: The unique labels for the images
        base_path: The base path to pull images from
        do_load: Flag indicating if the images should be loaded on construction
        image_formats: The formats of the images to consider
        loader_type: What loader to use
        transformer: Whether any transformation should be applied whilst loading the images
        """

        self.unique_labels: List[Tuple[str, int]] = unique_labels
        self.base_path = base_path
        self.image_formats: List[str] = image_formats
        self.images: List[Tuple[Union[Path, PILImage, TorchTensor], Union[int, str]]] = []
        self.image_labels: List[int] = []
        self.loader_type: ImageLoadersEnumType = loader_type
        self._images_per_label: Dict = {}
        self._current_pos: int = -1

        if do_load:
            self.load(loader_type=loader_type, transformer=transformer)

    def __len__(self) -> int:
        return len(self.images)

    def __iter__(self):
        self._current_pos = 0
        return self

    def __next__(self) -> tuple:
        if len(self.images) == 0:
            raise StopIteration

        if self._current_pos < len(self.images):
            result = self.images[self._current_pos]
            self._current_pos += 1
            return result
        else:
            self._current_pos = -1
            raise StopIteration

    def __getitem__(self, key: int) -> tuple:
        """Returns the image, label pair  that corresponds to the given key
        Parameters
        ----------
        key: The index of the image-label to retrieve
        Returns
        -------
        A tuple of the image-label
        """

        return self.images[key]

    def __del__(self) -> None:
        self.clear()

    @property
    def n_images_per_label(self) -> dict:
        """Get a dictionary with the number of images
        per label

        Returns
        -------

        """
        return self._images_per_label

    @property
    def label_names(self) -> List[str]:
        """Returns the names of the labels

        Returns
        -------

        A python list with the names of the labels
        """
        return [label[0] for label in self.unique_labels]

    def get_label_name(self, index: int) -> str:
        """Returns the label name associated with
        the given label index. If the index is not found
        raise ValueError

        Parameters
        ----------
        index: The label index to look for its label

        Returns
        -------

        The label name corresponding to this index
        """

        for label in self.unique_labels:
            if label[1] == index:
                return label[0]
        raise ValueError(f"Index={index} not found in dataset")

    def get_label_idx(self, label_name: str) -> int:
        """Returns the index that corresponds to the given
        label name. Returns -1 if the label name is not found
        in the list of self.unique_labels

        Parameters
        ----------
        label_name: The name of the label to look for its index

        Returns
        -------

        Integer representing the index of the label
        """
        for label in self.unique_labels:
            if label[0] == label_name:
                return label[1]

        return -1

    def clear(self, full_clear: bool = True) -> None:
        """Invalidate the dataset

        Returns
        -------

        """

        if full_clear:
            self.unique_labels = []
            self.base_path = DUMMY_PATH
            self.image_formats = []
            self._current_pos: int = -1

        self.loader_type = ImageLoadersEnumType.INVALID
        self.images = []
        self.image_labels = []
        self._images_per_label = {}

    def shuffle(self) -> None:
        """Randomly shuffle  the contents of the dataset

        Returns
        -------

        """
        random.shuffle(self.images)

    def get_class_images(self, class_name: str) -> List[Tuple[Union[Path, PILImage, TorchTensor], Union[int, str]]]:
        """Get the images corresponding to the class name.
        It raises ValueError if the class_name is not in the dataset

        Parameters
        ----------
        class_name: The class name to get the images from

        Returns
        -------

        Instance of: List[Tuple[Union[Path, PILImage, TorchTensor], Union[int, str]]]
        """
        if class_name not in self._images_per_label:
            raise ValueError(f"Class name {class_name} not in dataset")

        class_idx = self.get_label_idx(class_name)
        return [item for item in self.images if item[1] == class_idx]

    def random_selection(self, size: int) -> List[int]:
        """Returns a list of indices of the given size
        randomly selected

        Parameters
        ----------
        size: The size of the random sample

        Returns
        -------

        Instance of List[int]
        """

        if size >= len(self.images):
            raise ValueError(f"Invalid size parameter. size should be in [0,{len(self.images)}) but is {size}")

        indices = [i for i in range(len(self.images))]
        return random.sample(indices, size)

    def random_selection_for_class(self, size: int, class_name: str) -> List[Tuple[Union[Path, PILImage, TorchTensor], Union[int, str]]]:
        """Returns a random selection of images corresponding to the given
        class name. Raises ValueError if the class_name is not in the dataset.
        Raises ValueError if the given size is greater than or equal to the number
        of images in the dataset that correspond to the given dataset.

        Parameters
        ----------
        size: The size of the random sample to return
        class_name: The class name to get the images from

        Returns
        -------

        Instance of: List[Tuple[Union[Path, PILImage, TorchTensor], Union[int, str]]]
        """

        if class_name not in self._images_per_label:
            raise ValueError(f"Class name {class_name} not in dataset")

        n_images_for_class = self._images_per_label[class_name]

        if size >= n_images_for_class:
            raise ValueError(f"Invalid size parameter. size should be in [0,{n_images_for_class}) but is {size}")

        # get the images for the class
        class_images = self.get_class_images(class_name)
        return random.sample(class_images, size)

    def remove_images(self, images: List[Tuple[Union[Path, PILImage], Union[int, str]]]) -> None:
        """Remove the images specified in the list

        Parameters
        ----------
        images: The images to remove

        Returns
        -------
        """

        for img in images:

            image = img[0]
            if isinstance(image, Path):
                self.images.remove(img)
                label = img[1]
                label_name = self.get_label_name(label)
                self._images_per_label[label_name] -= 1
            elif isinstance(image, PILImage):
                self.images.remove(img)
                label = img[1]
                label_name = self.get_label_name(label)
                self._images_per_label[label_name] -= 1
            else:
                raise ValueError("Cannot remove image. Image should be either Path or PIL.Image")

    def apply_transform(self, transformer: Callable) -> None:
        """Apply the given transformation on all images
        in the dataset. This eventually will transform all the
        images.

        Parameters
        ----------
        transformer: Callable to apply on the images

        Returns
        -------

        """
        self.images = [(transformer(img[0]), img[1]) for img in self.images]

    def add_to_class(self, images: List[Tuple[Union[Path, PILImage, TorchTensor], Union[int, str]]],
                     class_name: str):

        """Append the given images to the class. Raises ValueError
        if the class_name is not in the unique_labels

        Parameters
        ----------
        images: Images to append
        class_name: The class name to append the images

        Returns
        -------

        """

        if len(images) == 0:
            return

        class_item = self.get_label_idx(label_name=class_name)
        if class_item == -1:
            raise ValueError(f"Label {class_name} not in dataset")

        self.images.extend(images)
        self.image_labels.extend([class_item]*len(images))
        self._images_per_label[class_name] += len(images)

    def load(self, loader_type: ImageLoadersEnumType = ImageLoadersEnumType.PIL,
             transformer: Callable = None, force_load: bool = False) -> None:

        if not self.__can_load() and not force_load:
            raise ValueError("Dataset is not empty. Have you called clear()?")
        elif not self.__can_load() and force_load:
            self.clear(full_clear=False)

        if str(self.base_path) == str(DUMMY_PATH):
            raise ValueError(f"Cannot load dataset from DUMMY_PATH={str(DUMMY_PATH)}. Specify a correct data path.")

        self.loader_type = loader_type
        tmp_img_formats = []

        for label in self.unique_labels:

            label_name = label[0]
            label_idx = label[1]
            base_path = Path(str(self.base_path)) / label_name

            # get all the image files
            img_files: List[Path] = get_img_files(base_path=base_path,
                                                  img_formats=self.image_formats)

            label_images = []
            labels = []
            # load every image in the Path
            for img in img_files:

                suffix = img.suffix

                if suffix.lower() not in tmp_img_formats:
                    tmp_img_formats.append(suffix.lower())

                label_images.append((load_img(path=img,
                                              transformer=transformer,
                                              loader=loader_type), label_idx))
                labels.append(label_idx)

            self._images_per_label[label_name] = len(label_images)
            self.images.extend(label_images)
            self.image_labels.extend(labels)
            self.image_formats = tmp_img_formats

    def save_to_csv(self, filename: Path) -> None:
        """Save the given dataset in a CSV format.
        Currently, only a dataset that has
        loader_type == ImageLoadersEnumType.FILENAME

        Parameters
        ----------
        filename: The file to save the dataset

        Returns
        -------
        """

        if self.loader_type != ImageLoadersEnumType.FILEPATH:
            raise ValueError(f"Cannot save a dataset loaded with {self.loader_type.name}. "
                             f"Load dataset using {ImageLoadersEnumType.FILEPATH.name}. ")

        with open(filename, 'w', newline='\n') as csvfile:
            filewriter = csv.writer(csvfile, delimiter=",")

            for img, label in zip(self.images, self.image_labels):
                label_name = self.get_label_name(label)
                image = str(img[0]).split("/")[-1]
                row = [image, label, label_name]

                filewriter.writerow(row)

    def __can_load(self) -> bool:
        """Checks if the right conditions are met to laod
        a dataset

        Returns
        -------
        A flag indicating if the right conditions are met to load
        """

        if len(self.images) != 0:
            return False

        if len(self.image_labels) != 0:
            return False

        return True


import pandas as pd
import numpy as np
import csv
import modelos

from sklearn.preprocessing import MinMaxScaler

from matplotlib import pyplot as plt

from timeit import default_timer as timer

ID_TO_MODELNAME = {0:'lstm', 1:'randomForest', 2:'adaBoost', 3:'svm', 4:'arima', 5:'lstmNoSW'}

def inverse_transform(data, scaler, n_features):
	"""
		Función que invierte el escalamiento de los datos, es decir pasa de datos escalados a datos originales.

		Parámetros:
		- data -- Arreglo de numpy, arreglo con los valores escalados
		- scaler -- instancia de MinMaxScaler, escalador para invertir el escalamiento
		- n_features -- Entero, número de variables en los datos originales

		Retorna:
		- data -- Arreglo de numpy, arreglo con los valores en escala original

	"""
	data = data.copy()
	assert type(data) == np.ndarray
	if(data.ndim == 1): data = data.reshape(-1, 1)
	assert data.ndim == 2
	for i in range(data.shape[1]):
		tmp = np.zeros((data.shape[0], n_features))
		tmp[:, 0] = data[:, i]
		data[:, i] = scaler.inverse_transform(tmp)[:, 0]
	return data

def normalize_data(values, scale=(0,1)):
	"""
		Función para normalizar los datos, es decir, escalarlos en una escala que por *default* es [-1, 1]

		Parámetros:
		- values -- Arreglo de numpy, los datos
		- scaler -- Tupla de 2 valores, escala a la cual se quiere escalar los datos

		Retorna:
		- scaled -- Arreglo de numpy, los datos escalados
		- scaler -- Instancia de MinMaxScaler, para luego revertir el proceso de escalamiento
	"""
	# Be sure all values are numbers
	values = values.astype('float32')
	# scale the data
	scaler = MinMaxScaler(feature_range=scale)
	scaled = scaler.fit_transform(values)
	return scaled, scaler

def series_to_supervised(data, n_in=1, n_out=1, dropnan=True):
	"""
		Función que convierte la serie de tiempo en datos supervisados para el modelo, es decir cambia el formato de (número de ejemplos, número de *features*)
		por (número de ejemplos, número de *lags*, número de *features*)

		Parámetros:
		- data -- Arreglo de numpy, la serie completa de los datos
		- n_in -- Entero, número de *lags* o resagos de tiempo, *default* es 1
		- n_out -- Entero, número de *time steps* a predecir en el futuro, *default* es 1
		- dropnan -- Booleano, indica si se eliminan los valores de Nan del *dataframe* resultante

		Retorna:
		- agg -- *Dataframe* de pandas, *dataframe* con todas las variables en el nuevo formato, sus nombres de columnas son del tipo: "var3(t+2)"

	"""
	n_vars = 1 if type(data) is list else data.shape[1]
	df = pd.DataFrame(data)
	cols, names = list(), list()
	# input sequence (t-n, ... t-1)
	for i in range(n_in, 0, -1):
		cols.append(df.shift(i))
		names += [('var%d(t-%d)' % (j+1, i)) for j in range(n_vars)]
	# forecast sequence (t, t+1, ... t+n)
	for i in range(0, n_out):
		cols.append(df.shift(-i))
		if i == 0:
			names += [('var%d(t)' % (j+1)) for j in range(n_vars)]
		else:
			names += [('var%d(t+%d)' % (j+1, i)) for j in range(n_vars)]
	# put it all together
	agg = pd.concat(cols, axis=1)
	agg.columns = names
	# drop rows with NaN values
	if dropnan:
		agg.dropna(inplace=True)
	return agg

def transform_values(data, n_lags, n_series, dim):
	"""
		Función para preprocesar la serie de entrada, primero le cambia el formato a (número de ejemplos, número de *lags* X número de *features*), luego divide estos datos en
		entrenamiento, validación y *testing* adicionalmente retorna los últimos valores para la predicción.

		Parámetros:
		- data -- Arreglo de numpy, serie de observaciones ordenada según orden cronológico
		- n_lags -- Entero, el número de *lags* que se usaran para entrenar
		- n_series -- Entero, el número de *time steps* a predecir en el futuro
		- dim -- Booleano, denota si se reformatea los valores de entrenamiento resultantes, es decir si se cambia el formato de (número de ejemplos, número de *lags* X número de *features*) a (número de ejemplos, número de *lags*, número de *features*)

		Retorna:
		- train_X -- Arreglo de numpy, datos de entrenamiento
		- val_X -- Arreglo de numpy, datos de valdiación
		- test_X -- Arreglo de numpy, datos de *testing*
		- train_y -- Arreglo de numpy, observaciones de tiempos futuros de entrenamiento
		- val_y -- Arreglo de numpy, observaciones de tiempos futuros de validación
		- test_y -- Arreglo de numpy, observaciones de tiempos futuros de *testing
		- last_values -- Arreglo de numpy, últimos datos apra hacer la predicción *out of sample*

	"""
	train_size = 0.6
	val_size =0.2
	test_size = 0.2
	n_features = data.shape[1]
	reframed = series_to_supervised(data, n_lags, n_series)

	values = reframed.values # if n_lags = 1 then shape = (349, 100), if n_lags = 2 then shape = (348, 150)
	# n_examples for training set
	n_train = int(values.shape[0] * train_size)
	n_val = int(values.shape[0] * val_size)
	train = values[:n_train, :]
	val = values[n_train:n_train + n_val, :]
	test = values[n_train + n_val:, :]
	# observations for training, that is to say, series in the times (t-n_lags:t-1) taking t-1 because observations in time t is for testing
	n_obs = n_lags * n_features

	# for only testing y, y only contains target variable in different times
	cols = ['var1(t)']
	cols += ['var1(t+%d)' % (i) for i in range(1, n_series)]
	y_o = reframed[cols].values
	train_o = y_o[:n_train]
	val_o = y_o[n_train:n_train + n_val]
	test_o = y_o[n_train + n_val:]

	train_X, train_y = train[:, :n_obs], train_o[:, -n_series:]
	val_X, val_y = val[:, :n_obs], val_o[:, -n_series:]
	test_X, test_y = test[:, :n_obs], test_o[:, -n_series:]

	# reshape train data to be 3D [n_examples, n_lags, features]
	if(dim):
		train_X = train_X.reshape((train_X.shape[0], n_lags, n_features))
		val_X = val_X.reshape((val_X.shape[0], n_lags, n_features))
		test_X = test_X.reshape((test_X.shape[0], n_lags, n_features))
		last_values = reframed.loc[list(reframed.index)[-1], list(reframed.columns)[-n_lags*n_features:]].values.reshape(1, n_lags, n_features)
	else:
		last_values = reframed.loc[list(reframed.index)[-1], list(reframed.columns)[-n_lags*n_features:]].values.reshape(1, -1)

	return train_X, val_X, test_X, train_y, val_y, test_y, last_values

def plot_data(data, labels, title):
	"""
		Función para graficar los datos resultantes, solo sirve para las predicciones a 1 *time step*

		Parámetros:
		- data -- Lista de dos valores, lista con las predicciones y las observaciones
		- labels -- Lista de dos valores, lista con las etiquetas de los datos para mostrar en el gráfico
		- title -- String, título del gráfico

		Retorna:
		NADA

	"""
	plt.figure()
	for i in range(len(data)):
		plt.plot(data[i], label=labels[i])
	plt.suptitle(title, fontsize=16)
	plt.legend()
	plt.show()

def plot_data_lagged_blocks(data, labels, title):
	"""
		Función para graficar los datos resultantes, sirve más que todo para las predicciones a más de un *time step*. Lo que hace es ponerle un *padding* a las predicciones
		para que queden en el tiempo que están prediciendo, es decir, si *time steps* es 10, la priemra predicción se hará para el tiempo 0 y tendrá las predicciones hasta el
		tiempo 9, la segunda predicción se hará en el tiempo 10 y tendra las predicciones del tiempo 10 al 19, etc.

		Parámetros:
		- data -- Lista de dos valores, lista con las predicciones y las observaciones, es necesario que la primera posición sean las observaciones y la segunda las predicciones
		- labels -- Lista de dos valores, lista con las etiquetas de los datos para mostrar en el gráfico
		- title -- String, título del gráfico

		Retorna:
		NADA

	"""
	plt.figure()
	plt.plot(data[0], label=labels[0])
	for i in range(0, len(data[1]), len(data[1][0])):
		padding = [None for _ in range(i)]
		plt.plot(padding + list(data[1][i]), label=labels[1] + str(i+1))
	plt.suptitle(title, fontsize=16)
	plt.legend(loc=9, bbox_to_anchor=(0.5, -0.1), ncol=2)
	plt.show()

def diff(values):
	"""
		Función que sirve para diferenciar una serie

		Parámetros:
		- values -- Arreglo de numpy | lista, serie que va a ser diferenciada

		Retorna:
		- new -- Lista, lista con la serie diferenciada

	"""
	new = np.zeros(len(values)-1)
	for i in range(len(new)):
		new[i] = values[i+1] - values[i]
	return new

def calculate_diff_level_for_stationarity(values, scaler, maxi):
	"""
		Función que sirve para calcular el nivel de diferenciación necesario para que una serie sea estacionaria

		Parámetros:
		- values -- Arreglo de numpy, serie sobre la cual se va a calcular el nivel de diferenciación necesario
		- scaler -- Instancia de la clase MinMaxScaler de sklearn, sirve para revertir el escalamiento de la serie
		- maxi -- Entero, valor maximo de diferenciación permitido, si se llega a este limite y la serie no es estacionaria se devolverá maxi como el nivel de diferenciación

		Retorna:
		- maxi | i -- Entero, mínimo nivel de diferenciación para que la serie sea estacionaria, o máximo número en el que se diferenció en el caso de que la serie no se haya logrado poner estacionaria

	"""
	from statsmodels.tsa.stattools import adfuller

	real_values = scaler.inverse_transform(values)
	serie = real_values[:, 0]
	for i in range(maxi):
		result = adfuller(serie)
		if(result[0] < result[4]['5%']):
			return i
		serie = diff(serie)
	return maxi

def objective(params, id_model, values, scaler, n_features, n_series, verbosity, model_file_name, MAX_EVALS, returns):
	"""
		Función objetivo que sirve para la optimización bayesiana, sirve para ejecutar el modelo con los parámetros recibidos, calcular el error de esta ejecución y así decidir
		cuales parámetros son mejores.

		Parámetros:
		- params -- Diccionario, contien los parametros para la ejecución, estos parámetros son dados por la libreria de optimización bayesiana (bayes_opt) dentro de un espacio previamente definido
		- id_model -- Entero, id del modelo que se va a entrenar
		- values -- Arreglo de numpy, datos con los cuales se va a entrenar el modelo, es decir, la serie previamente preprocesada
		- scaler -- Instancia de la clase MinMaxScaler de sklearn, sirve para el escalamiento de los datos y para revertir este escalamiento
		- n_features -- Entero, número de *features* de la serie
		- n_series -- Entero, número de *time steps*
		- verbosity -- Entero, nivel de verbosidad de la ejecución
		- model_file_name -- String, nombre del archivo donde se guardará y/o se cargará el modelo entrenado
		- MAX_EVALS -- Entero, número máximo  de iteraciones de la optimización bayesiana, en esta función sirve para identificar el archivo de los logs

		Retorna:
		- [valor] -- Flotante, dicconario que retorna la "recompensa" de esta ejecución, la idea es maximizarla, por eso el "error" de la ejecución se invierte

	"""
	# Keep track of evals
	global ITERATION

	ITERATION += 1
	out_file = 'trials/gbm_trials_' + ID_TO_MODELNAME[id_model] + '_' + str(MAX_EVALS) + '.csv'
	# print(ITERATION, params)

	calc_val_error = True
	calc_test_error = False
	if(id_model == 0):
		if(model_file_name == None): model_file_name = 'models/trials-lstm.h5'

		# Make sure parameters that need to be integers are integers
		for parameter_name in ['n_lags', 'n_epochs', 'batch_size', 'n_hidden', 'n_dense', 'n_rnn', 'activation']:
			params[parameter_name] = int(params[parameter_name])

		start = timer()
		train_X, val_X, test_X, train_y, val_y, test_y, last_values = transform_values(values, params['n_lags'], n_series, 1)
		rmse, rmse_val, _, _, val_y, y_hat_val, _, dir_acc, _ = modelos.model_lstm(train_X, val_X, test_X, train_y, val_y, test_y, n_series, params['n_epochs'], params['batch_size'], params['n_hidden'], n_features,
														params['n_lags'], scaler, last_values, calc_val_error, calc_test_error, verbosity, False, model_file_name, params['n_rnn'], params['n_dense'], params['activation'], params['drop_p'], returns)
		if(returns):
			mean = np.mean(np.abs(val_y))
			std = np.std(val_y)
			print('rmse:', rmse_val, 'std: ', np.std(y_hat_val))
			print('mean: ', mean, 'std_y: ', std)
			print('inputs -> error: ', np.abs(rmse_val-mean)/mean, 'deviation: ', (np.abs(std-np.std(y_hat_val)))/std)
			rmse = np.abs(rmse_val-mean)/mean + (np.abs(std-np.std(y_hat_val)))/std
		else:
			mean = np.mean(np.abs(val_y))
			rmse = rmse_val/mean + (1 - dir_acc)
		run_time = timer() - start
		# print('no_score: ', rmse)
		# print('time: ', run_time, end='\n\n')
	elif(id_model == 1):
		if(model_file_name == None): model_file_name = 'models/trials-randomForest.joblib'

		# Make sure parameters that need to be integers are integers
		for parameter_name in ['n_lags', 'n_estimators', 'max_features', 'min_samples']:
			params[parameter_name] = int(params[parameter_name])

		start = timer()
		train_X, val_X, test_X, train_y, val_y, test_y, last_values = transform_values(values, params['n_lags'], n_series, 0)
		rmse, rmse_val, _, _, _, _, _, dir_acc, _ = modelos.model_random_forest(train_X, val_X, test_X, train_y, val_y, test_y, n_series, params['n_estimators'], params['max_features'], params['min_samples'],
																n_features, params['n_lags'], scaler, last_values, calc_val_error, calc_test_error, verbosity, False, model_file_name, returns)
		if(returns):
			rmse = rmse_val
		else:
			mean = np.mean(np.abs(val_y))
			rmse = rmse_val/mean + (1 - dir_acc)
		run_time = timer() - start
		# print('no_score: ', rmse)
		# print('time: ', run_time, end='\n\n')
	elif(id_model == 2):
		if(model_file_name == None): model_file_name = 'models/trials-adaBoost.joblib'
		# Make sure parameters that need to be integers are integers
		for parameter_name in ['n_lags', 'n_estimators', 'max_depth']:
			params[parameter_name] = int(params[parameter_name])

		start = timer()
		train_X, val_X, test_X, train_y, val_y, test_y, last_values = transform_values(values, params['n_lags'], n_series, 0)
		rmse, rmse_val, _, _, _, _, _, dir_acc, _ = modelos.model_ada_boost(train_X, val_X, test_X, train_y, val_y, test_y, n_series, params['n_estimators'], params['lr'], params['max_depth'], n_features,
															params['n_lags'], scaler, last_values, calc_val_error, calc_test_error, verbosity, False, model_file_name, returns)
		if(returns):
			rmse = rmse_val
		else:
			mean = np.mean(np.abs(val_y))
			rmse = rmse_val/mean + (1 - dir_acc)
		rmse = rmse_val/mean + (1 - dir_acc)
		run_time = timer() - start
		# print('no_score: ', rmse)
		# print('time: ', run_time, end='\n\n')
	elif(id_model == 3):
		if(model_file_name == None): model_file_name = 'models/trials-svm.joblib'
		# Make sure parameters that need to be integers are integers
		for parameter_name in ['n_lags']:
			params[parameter_name] = int(params[parameter_name])

		start = timer()
		train_X, val_X, test_X, train_y, val_y, test_y, last_values = transform_values(values, params['n_lags'], n_series, 0)
		rmse, rmse_val, _, _, _, _, _, dir_acc, _ = modelos.model_svm(train_X, val_X, test_X, train_y, val_y, test_y, n_series, n_features, params['n_lags'], scaler, last_values, calc_val_error, calc_test_error,
													verbosity, False, model_file_name, returns)
		if(returns):
			rmse = rmse_val
		else:
			mean = np.mean(np.abs(val_y))
			rmse = rmse_val/mean + (1 - dir_acc)
		rmse = rmse_val/mean + (1 - dir_acc)
		run_time = timer() - start
		# print('no_score: ', rmse)
		# print('time: ', run_time, end='\n\n')
	elif(id_model == 4):
		if(model_file_name == None): model_file_name = 'models/arima.pkl'
		# Make sure parameters that need to be integers are integers
		for parameter_name in ['n_lags', 'd', 'q']:
			params[parameter_name] = int(params[parameter_name])

		start = timer()
		wall = int(len(values)*0.6)
		wall_val= int(len(values)*0.2)
		train_X, val_X, test_X, last_values = values[:wall, :], values[wall:wall+wall_val,:], values[wall+wall_val:-1,:], values[-1,:]
		train_y, val_y, test_y = values[1:wall+1,0], values[wall+1:wall+wall_val+1,0], values[wall+wall_val+1:,0]
		start = timer()
		rmse, rmse_val, y, y_hat, y_valset, y_hat_val, last, dir_acc, model = modelos.model_arima(train_X, val_X, test_X, train_y, val_y, test_y, n_series, params['d'], params['q'], n_features, params['n_lags'], scaler, last_values, calc_val_error,
														calc_test_error, verbosity, False, model_file_name, returns)
		if(returns):
			rmse = rmse_val
		else:
			mean = np.mean(np.abs(val_y))
			rmse = rmse_val/mean + (1 - dir_acc)
		run_time = timer() - start
		# print('no_score: ', rmse)
		# print('time: ', run_time, end='\n\n')

	# Write to the csv file
	of_connection = open(out_file, 'a')
	writer = csv.writer(of_connection)
	writer.writerow([rmse, params, ITERATION, run_time])
	of_connection.close()

	return -1 * rmse

def bayes_optimization(id_model, MAX_EVALS, values, scaler, n_features, n_series, original, verbosity, model_file_name, returns):
	"""
		Función para encontrar los parámetros óptimos para un modelo

		Parámetros:
		- id_model -- Entero, id del modelo que se va a entrenar
		- MAX_EVALS -- Entero, número máximo  de iteraciones de la optimización bayesiana
		- values -- Arreglo de numpy, datos con los cuales se va a entrenar el modelo, es decir, la serie previamente preprocesada
		- scaler -- Instancia de la clase MinMaxScaler de sklearn, sirve para el escalamiento de los datos y para revertir este escalamiento
		- n_features -- Entero, número de *features* de la serie
		- n_series -- Entero, número de *time steps*
		- original -- Booleano, denota si se van a usar los *features* originales de la serie o los *features* seleccionados, en esta función serviría para identificar el archivo de salida
		- verbosity -- Entero, nivel de verbosidad de la ejecución
		- model_file_name -- String, nombre del archivo donde se guardará y/o se cargará el modelo entrenado

		Retorna:
		- best -- Diccionario, diccionario con los mejores parámetros encontrados en la optimización bayesiana
	"""
	from bayes_opt import BayesianOptimization

	global ITERATION
	ITERATION = 0

	if(id_model == 0):
		if(n_series == 1):
			space = {'activation': (0.1, 1.9),
					'batch_size': (20, 100),
					'drop_p': (0, 0.75),
					'n_dense': (0, 3),
					'n_epochs': (10, 200),
					'n_hidden': (5, 300),
					'n_lags': (1, (min(30, int(len(values)/2)))),
					'n_rnn': (0, 3)}
		elif(n_series > 1):
			# space = {'activation': (0.1, 1.9),
			# 		'batch_size': (20, 100),
			# 		'drop_p': (0, 0.3),
			# 		'n_dense': (0, 3),
			# 		'n_epochs': (250, 500),
			# 		'n_hidden': (50, 300),
			# 		'n_lags': (2, (min(30, int(len(values)/2)))),
			# 		'n_rnn': (1, 3)}
			space = {'activation': (0.1, 1.9),
					'batch_size': (20, 100),
					'drop_p': (0, 0.8),
					'n_dense': (0, 3),
					'n_epochs': (50, 500),
					'n_hidden': (25, 300),
					'n_lags': (2, (min(30, int(len(values)/2)))),
					'n_rnn': (0, 3)}
		func = lambda activation, batch_size, drop_p, n_dense, n_epochs, n_hidden, n_lags, n_rnn: objective({'activation':activation , 'batch_size':batch_size, 'drop_p':drop_p, 'n_dense':n_dense, 'n_epochs':n_epochs, 'n_hidden':n_hidden, 'n_lags':n_lags, 'n_rnn':n_rnn}, id_model, values, scaler, n_features, n_series, verbosity, model_file_name, MAX_EVALS, returns)
	elif(id_model == 1):
		space = {'max_features': (1, n_features),
				'min_samples': (1, 20),
				'n_estimators': (10, 1000),
				'n_lags': (1, min(50, int(len(values)/2)))}
		func = lambda max_features, min_samples, n_estimators, n_lags: objective({'max_features': max_features, 'min_samples': min_samples, 'n_estimators': n_estimators, 'n_lags': n_lags}, id_model, values, scaler, n_features, n_series, verbosity, model_file_name, MAX_EVALS, returns)
	elif(id_model == 2):
		space = {'lr': (0.00001, 1.0),
				'max_depth': (2, 10),
				'n_estimators': (10, 1000),
				'n_lags': (1, min(50, int(len(values)/2)))}
		func = lambda lr, max_depth, n_estimators, n_lags: objective({'lr':lr, 'max_depth': max_depth, 'n_estimators': n_estimators, 'n_lags': n_lags}, id_model, values, scaler, n_features, n_series, verbosity, model_file_name, MAX_EVALS, returns)
	elif(id_model == 3):
		space = {'n_lags': (1, min(50, int(len(values)/2)))}
		func = lambda  n_lags: objective({'n_lags': n_lags}, id_model, values, scaler, n_features, n_series, verbosity, model_file_name, MAX_EVALS, returns)
	elif(id_model == 4):
		diff_level = calculate_diff_level_for_stationarity(values, scaler, 5)
		space={'n_lags': (1, 12),
				'q': (1, 12)}
		func = lambda  n_lags, q: objective({'d': diff_level, 'n_lags': n_lags, 'q': q}, id_model, values, scaler, n_features, n_series, verbosity, model_file_name, MAX_EVALS, returns)

	# File to save results
	out_file = 'trials/gbm_trials_' + ID_TO_MODELNAME[id_model] + '_' + str(MAX_EVALS) + '.csv'
	of_connection = open(out_file, 'w')
	writer = csv.writer(of_connection)

	# Write the headers to the file
	writer.writerow(['id_model: ' + str(id_model), 'original: ' + str(original), 'returns: ' + str(returns), 'time_steps: ' + str(n_series)])
	writer.writerow(['rmse', 'params', 'iteration', 'train_time'])
	of_connection.close()

	optimizer = BayesianOptimization(f=func, pbounds=space, verbose=2, random_state=np.random.randint(np.random.randint(100)))
	if(id_model == 0):
		#optimizer.probe(params={'activation':1.0, 'batch_size':10.0, 'drop_p':0.0, 'n_dense':0.0, 'n_epochs':300.0, 'n_hidden':50.0, 'n_lags':10.0, 'n_rnn':0.0})
		#optimizer.probe(params={'activation':1.0, 'batch_size':81.0, 'drop_p':0.0, 'n_dense':0.0, 'n_epochs':200.0, 'n_hidden':250.0, 'n_lags':15.0, 'n_rnn':0.0})
		#optimizer.probe(params={'activation':1.0, 'batch_size':81.0, 'drop_p':0.0, 'n_dense':0.0, 'n_epochs':200.0, 'n_hidden':269.0, 'n_lags':9.0, 'n_rnn':0.0})
		#optimizer.probe(params={'activation':1.0, 'batch_size':81.0, 'drop_p':0.0, 'n_dense':0.0, 'n_epochs':200.0, 'n_hidden':269.0, 'n_lags':25.0, 'n_rnn':0.0})
		optimizer.probe(params={'activation':0.0, 'batch_size':47.0, 'drop_p':0.069, 'n_dense':0.0, 'n_epochs':382.0, 'n_hidden':130.0, 'n_lags':5.0, 'n_rnn':1.0})
		optimizer.probe(params={'activation':0.0, 'batch_size':47.0, 'drop_p':0.3, 'n_dense':0.0, 'n_epochs':382.0, 'n_hidden':130.0, 'n_lags':5.0, 'n_rnn':1.0})
		optimizer.probe(params={'activation':0.0, 'batch_size':47.0, 'drop_p':0.5, 'n_dense':0.0, 'n_epochs':382.0, 'n_hidden':130.0, 'n_lags':5.0, 'n_rnn':1.0})
	optimizer.maximize(init_points=10, n_iter=MAX_EVALS, acq='ucb', kappa=5, alpha=1e-3)

	# store best results
	best = optimizer.max['params']
	of_connection = open('trials/bests.txt', 'a')
	writer = csv.writer(of_connection)
	if(id_model == 0):
		writer.writerow([optimizer.max['target'], best['activation'], best['batch_size'], best['drop_p'], best['n_dense'], best['n_epochs'], best['n_hidden'], best['n_lags'], best['n_rnn'], MAX_EVALS])
	elif(id_model == 1):
		writer.writerow([optimizer.max['target'], best['n_lags'], best['n_estimators'], best['max_features'], best['min_samples'], MAX_EVALS])
	elif(id_model == 2):
		writer.writerow([optimizer.max['target'], best['n_lags'], best['n_estimators'], best['lr'], best['max_depth'], MAX_EVALS])
	elif(id_model == 3):
		writer.writerow([optimizer.max['target'], best['n_lags'], MAX_EVALS])
	elif(id_model == 4):
		best.update({'d': diff_level})
		writer.writerow([optimizer.max['target'], best['d'], best['n_lags'], best['q'], MAX_EVALS])
	of_connection.close()

	return best

def get_direction_accuracy(y, y_hat):
	"""
		Función para calcular el % de aciertos en la dirección, teniendo en cuenta las observaciones y las predicciones.
		Por dirección se entiende que si en la observación el valor sube, en la predicción también igualmente si baja.

		Parámetros:
		- y -- Arreglo de numpy, las observaciones
		- y_hat -- Arreglo de numpy, las predicciones

		Retorna:
		- [valor] -- % de aciertos en la dirección (valor entre 0 y 1)
	"""
	assert len(y) == len(y_hat)
	y_dirs = [1 if y[i] < y[i+1] else 0 for i in range(len(y) - 1)]
	y_hat_dirs = [1 if y_hat[i] < y_hat[i+1] else 0 for i in range(len(y_hat) - 1)]
	return sum(np.array(y_dirs) == np.array(y_hat_dirs))/len(y)

def get_returns_direction_accuracy(y, y_hat):
	"""
		Función para calcular el % de aciertos en la dirección de retornos, teniendo en cuenta las observaciones y las predicciones.
		Por dirección se entiende que si en la observación el valor sube, en la predicción también lo haga. De igual forma en el caso de que baje.

		Parámetros:
		- y -- Arreglo de numpy, las observaciones
		- y_hat -- Arreglo de numpy, las predicciones

		Retorna:
		- [valor] -- % de aciertos en la dirección (valor entre 0 y 1)
	"""
	assert len(y) == len(y_hat)
	y_dirs = [1 if y[i]>0 else 0 for i in range(len(y))]
	y_hat_dirs = [1 if y_hat[i]>0 else 0 for i in range(len(y_hat))]
	return sum(np.array(y_dirs) == np.array(y_hat_dirs))/len(y)

def get_returns_values_direction_accuracy(values, returns):
	"""
		Función para calcular el % de aciertos en la dirección de retornos teniendo una serie en valores y otra en retornos, teniendo en cuenta las observaciones y las predicciones.
		Por dirección se entiende que si en la observación el valor sube, en la predicción también lo haga. De igual forma en el caso de que baje.

		Parámetros:
		- values -- Arreglo de numpy, serie de valores
		- returns -- Arreglo de numpy, serie de retornos
		Retorna:
		- [valor] -- % de aciertos en la dirección (valor entre 0 y 1)
	"""
	assert len(values) == len(returns) + 1
	values_dirs = [1 if values[i] < values[i+1] else 0 for i in range(len(values) - 1)]
	returns_dirs = [1 if returns[i]>0 else 0 for i in range(len(returns))]
	return sum(np.array(values_dirs) == np.array(returns_dirs))/len(returns)

def calculate_rmse(y, y_hat):
	"""
		Función que calcula el rmse (raíz del error medio cuadrático) entre dos arreglos de datos

		Parámetros:
		- y -- Arreglo de numpy | Lista, serie de observaciones o valores reales
		- y_hat -- Arreglo de numpy | Lista, serie de predicciones
	"""
	assert len(y) == len(y_hat)
	from sklearn.metrics import mean_squared_error
	return np.sqrt(mean_squared_error(y, y_hat))


from django.shortcuts import render, redirect
from django.http import JsonResponse
from .models import Message

def message_list(request):
    messages = Message.objects.all()
    return render(request, 'message_list.html', {'messages': messages})

def subit(request):
    if request.method == 'POST':
        author = request.user
        content = request.POST.get('content')
        message = Message(author=author, content=content)
        message.save()
        messages = Message.objects.all()
        return render(request, 'message_list.html',{'messages': messages})

def update(request):
    if request.method == 'POST':
        message_id = request.POST.get('message_id')
        new_content = request.POST.get('new_content')

        message = Message.objects.get(id=message_id)
        message.content = new_content
        message.save()

        messages = Message.objects.all()
        return render(request, 'message_list.html', {'messages': messages})

def delete_message(request):
    if request.method == 'POST':
        message_id = request.POST.get('message_id')

        message = Message.objects.get(id=message_id)
        message.content = "訊息已被 " + message.author + " 已收回"
        message.change = 0
        message.save()

        response = {'author': message.author}  # 將 author 包含在回應中
        return JsonResponse(response)


import webapp2
import jinja2
from google.appengine.api import users
from google.appengine.ext import ndb
import os
import csv

from myuser import MyUser
from mySquad import MySquad
from userModel import UserModel
from userSignUp import UserSignUp
from userProfile import UserProfile
from payments import Payments
from scoring import Scoring
from rules import Rules
from playerStats import PlayerStats
from playerProfile import PlayerProfile
from buildSquad import BuildSquad
from betting import Betting
from betResults import BetResults
from bettingMarkets import BettingMarkets
from myBets import MyBets
from adminPage import AdminPage
from addMarkets import AddMarkets
from updateMarkets import UpdateMarkets
from addPlayersData import AddPlayersData
from blobCollection import BlobCollection
from uploadHandler import UploadHandler

JINJA_ENVIRONMENT = jinja2.Environment(
    loader=jinja2.FileSystemLoader(os.path.dirname(__file__)),
    extensions=['jinja2.ext.autoescape'],
    autoescape=True
)

class MainPage(webapp2.RequestHandler):
    def get(self):
        self.response.headers['Content-Type'] = 'text/html'

        url = ''
        url_string = ''
        welcome = 'Welcome back'

        img = '/static/champions.jpg'

        user = users.get_current_user()
        logout = users.create_logout_url('/')
        admin = False

        if user:
            url = users.create_logout_url(self.request.uri)
            url_string = 'logout'

            myuser_key = ndb.Key('MyUser', user.user_id())
            myuser = myuser_key.get()

            if users.is_current_user_admin():
                welcome = 'Welcome'

                myuser = MyUser(id=user.user_id(), email = user.email(), username = 'admin')
                myuser.put()

                admin_key = ndb.Key('UserModel', myuser.username)
                admin_user = admin_key.get()

                admin_user = UserModel(id = myuser.username,
                                        email = user.email(),
                                        username = myuser.username,
                                        name = 'Administrator')
                admin_user.put()
                admin = True

            if myuser == None:
                welcome = 'Welcome'
                myuser = MyUser(id=user.user_id(), email = user.email())
                myuser.put()

        else:
            url = users.create_login_url(self.request.uri)
            url_string = 'login'

        template_values = {'url' : url,
                            'url_string' : url_string,
                            'user' : user,
                            'admin' : admin,
                            'img' : img,
                            'welcome' : welcome,
                            'logout' : logout}

        template = JINJA_ENVIRONMENT.get_template('main.html')
        self.response.write(template.render(template_values))

# starts the web application we specify the full routing table here as well
app = webapp2.WSGIApplication([('/', MainPage),
                                ('/userSignUp', UserSignUp),
                                ('/userProfile', UserProfile),
                                ('/payments', Payments),
                                ('/scoring', Scoring),
                                ('/rules', Rules),
                                ('/playerStats', PlayerStats),
                                ('/playerProfile', PlayerProfile),
                                ('/adminPage', AdminPage),
                                ('/addMarkets', AddMarkets),
                                ('/updateMarkets', UpdateMarkets),
                                ('/addPlayersData', AddPlayersData),
                                ('/buildSquad', BuildSquad),
                                ('/betting', Betting),
                                ('/betResults', BetResults),
                                ('/upload', UploadHandler)], debug=True)


'''To get started, follow these guidelines:

* Create a script which runs your application, in the
  :ref:`taskqueue tutorial <tutorials-taskqueue>` the script is called
  ``manage.py``.
* Create the modules where :ref:`jobs <app-taskqueue-job>` are implemented. It
  can be a directory containing several submodules as explained in the
  :ref:`task paths parameter <app-tasks_path>`.
* Run your script, sit back and relax.

.. _app-taskqueue-config:

Configuration
~~~~~~~~~~~~~~~~
A :class:`TaskQueue` accepts several configuration parameters on top of the
standard :ref:`application settings <settings>`:

.. _app-tasks_path:

* The :ref:`task_paths <setting-task_paths>` parameter specifies
  a list of python paths where to collect :class:`.Job` classes::

      task_paths = ['myjobs','another.moduledir.*']

  The ``*`` at the end of the second module indicates to collect
  :class:`.Job` from all submodules of ``another.moduledir``.

* The :ref:`schedule_periodic <setting-schedule_periodic>` flag indicates
  if the :class:`TaskQueue` can schedule :class:`.PeriodicJob`. Usually,
  only one running :class:`TaskQueue` application is responsible for
  scheduling tasks.

  It can be specified in the command line via the
  ``--schedule-periodic`` flag.

  Default: ``False``.

* The :ref:`task_backend <setting-task_backend>` parameter is a url
  type string which specifies the :ref:`task backend <apps-taskqueue-backend>`
  to use.

  It can be specified in the command line via the
  ``--task-backend ...`` option.

  Default: ``local://``.

* The :ref:`concurrent_tasks <setting-concurrent_tasks>` parameter controls
  the maximum number of concurrent tasks for a given task worker.
  This parameter is important when tasks are asynchronous, that is when
  they perform some sort of I/O and the :ref:`job callable <job-callable>`
  returns and :ref:`asynchronous component <tutorials-coroutine>`.

  It can be specified in the command line via the
  ``--concurrent-tasks ...`` option.

  Default: ``5``.

.. _celery: http://celeryproject.org/
'''
import time

import pulsar
from pulsar import command
from pulsar.utils.config import section_docs
from pulsar.apps.data import (start_store, create_store,
                              DEFAULT_PULSAR_STORE_ADDRESS)

from .models import *
from .backend import *
from .rpc import *
from .states import *

DEFAULT_TASK_BACKEND = 'pulsar://%s/1' % DEFAULT_PULSAR_STORE_ADDRESS

section_docs['Task Consumer'] = '''
This section covers configuration parameters used by CPU bound type
applications such as the :ref:`distributed task queue <apps-taskqueue>` and
the :ref:`test suite <apps-test>`.'''

class TaskSetting(pulsar.Setting):
    virtual = True
    app = 'tasks'
    section = "Task Consumer"

class ConcurrentTasks(TaskSetting):
    name = "concurrent_tasks"
    flags = ["--concurrent-tasks"]
    validator = pulsar.validate_pos_int
    type = int
    default = 5
    desc = """\
        The maximum number of concurrent tasks for a worker.

        When a task worker reach this number it stops polling for more tasks
        until one or more task finish. It should only affect task queues under
        significant load.
        Must be a positive integer. Generally set in the range of 5-10.
        """

class TaskBackendConnection(TaskSetting):
    name = "task_backend"
    flags = ["--task-backend"]
    default = ""
    meta = 'CONNECTION_STRING'
    desc = '''\
        Connection string for the backend storing :class:`.Task`.

        If the value is not available (default) it uses as fallback the
        :ref:`data_store <setting-data_store>` value. If still not
        set, it uses the ``%s`` value.
        ''' % DEFAULT_TASK_BACKEND

class TaskPaths(TaskSetting):
    name = "task_paths"
    validator = pulsar.validate_list
    default = []
    desc = """\
        List of python dotted paths where tasks are located.

        This parameter can only be specified during initialization or in a
        :ref:`config file <setting-config>`.
        """

class SchedulePeriodic(TaskSetting):
    name = 'schedule_periodic'
    flags = ["--schedule-periodic"]
    validator = pulsar.validate_bool
    action = "store_true"
    default = False
    desc = '''\
        Enable scheduling of periodic tasks.

        If enabled, :class:`.PeriodicJob` will produce
        tasks according to their schedule.
        '''

class TaskQueue(pulsar.Application):
    '''A pulsar :class:`.Application` for consuming :class:`.Task`.

    This application can also schedule periodic tasks when the
    :ref:`schedule_periodic <setting-schedule_periodic>` flag is ``True``.
    '''
    backend = None
    '''The :class:`.TaskBackend` for this task queue.

    Available once the :class:`.TaskQueue` has started.
    '''
    name = 'tasks'
    cfg = pulsar.Config(apps=('tasks',), timeout=600)

    def monitor_start(self, monitor):
        '''Starts running the task queue in ``monitor``.

        It calls the :attr:`.Application.callable` (if available)
        and create the :attr:`~.TaskQueue.backend`.
        '''
        if self.cfg.callable:
            self.cfg.callable()
        connection_string = (self.cfg.task_backend or self.cfg.data_store or
                             DEFAULT_TASK_BACKEND)
        store = yield start_store(connection_string, loop=monitor._loop)
        self.get_backend(store)

    def monitor_task(self, monitor):
        '''Override the :meth:`~.Application.monitor_task` callback.

        Check if the :attr:`~.TaskQueue.backend` needs to schedule new tasks.
        '''
        if self.backend and monitor.is_running():
            if self.backend.next_run <= time.time():
                self.backend.tick()

    def monitor_stopping(self, monitor, exc=None):
        if self.backend:
            self.backend.close()

    def worker_start(self, worker, exc=None):
        if not exc:
            self.get_backend().start(worker)

    def worker_stopping(self, worker, exc=None):
        if self.backend:
            self.backend.close()

    def actorparams(self, monitor, params):
        # makes sure workers are only consuming tasks, not scheduling.
        cfg = params['cfg']
        cfg.set('schedule_periodic', False)

    def worker_info(self, worker, info=None):
        be = self.backend
        if be:
            tasks = {'concurrent': list(be.concurrent_tasks),
                     'processed': be.processed}
            info['tasks'] = tasks

    def get_backend(self, store=None):
        if self.backend is None:
            if store is None:
                store = create_store(self.cfg.task_backend)
            else:
                self.cfg.set('task_backend', store.dns)
            task_backend = task_backends.get(store.name)
            if not task_backend:
                raise pulsar.ImproperlyConfigured(
                    'Task backend for %s not available' % store.name)
            self.backend = task_backend(
                store,
                logger=self.logger,
                name=self.name,
                task_paths=self.cfg.task_paths,
                schedule_periodic=self.cfg.schedule_periodic,
                max_tasks=self.cfg.max_requests,
                backlog=self.cfg.concurrent_tasks)
            self.logger.debug('created %s', self.backend)
        return self.backend

@command()
def next_scheduled(request, jobnames=None):
    actor = request.actor
    return actor.app.backend.next_scheduled(jobnames)


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

import sklearn.datasets
from sklearn import tree
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score
from sklearn.metrics import confusion_matrix

# Load dataset
# (data_full, target_full) = sklearn.datasets.fetch_covtype(return_X_y=True, as_frame=True)
data_full = pd.read_csv('beans.csv')
target_full = data_full['Class']
data_full = data_full.drop('Class', axis = 1)

dt = DecisionTreeClassifier()
dt.fit(data_full, target_full)
importance = dt.feature_importances_
plt.bar([x for x in range(len(importance))], importance)

ind = np.argpartition(importance, -10)[-10:]
i_feat = data_full.columns[ind]
data_full = data_full[i_feat]

# Sample dataset (5% subset)
pct = 1
data = data_full.sample(int(len(data_full)*pct))
target = target_full[data.index]

# Split data into training and testing 70:30
split = train_test_split(data, target, test_size = 0.3)
X_train = split[0]
X_test = split[1]
y_train = split[2]
y_test = split[3]

# Pruning example
# https://scikit-learn.org/stable/auto_examples/tree/plot_cost_complexity_pruning.html#sphx-glr-auto-examples-tree-plot-cost-complexity-pruning-py
path = DecisionTreeClassifier().cost_complexity_pruning_path(X_train, y_train)
alphas, impurities = path.ccp_alphas, path.impurities

# Create trees for different alphas
dts = []
roc_aucs = []
i = 1
for alpha in alphas:
    dt = DecisionTreeClassifier(ccp_alpha=alpha)
    dt.fit(X_train, y_train)
    pred = dt.predict(X_test)
    probs = dt.predict_proba(X_test)
    roc_aucs.append(roc_auc_score(y_test, probs, multi_class='ovo'))
    dts.append(dt)

# Removing the case where tree pruned to only one node
dts = dts[:-1]
alphas = alphas[:-1]
roc_aucs = roc_aucs[:-1]

train_scores = [dt.score(X_train, y_train) for dt in dts]
test_scores = [dt.score(X_test, y_test) for dt in dts]

# Create accuracy vs alpha plots
fig, ax = plt.subplots()
ax.set_xlabel("alpha")
ax.set_ylabel("accuracy")
ax.set_title("Accuracy vs alpha for training and testing sets")
ax.plot(alphas, train_scores, marker="o", label="train", drawstyle="steps-post")
ax.plot(alphas, test_scores, marker="o", label="test", drawstyle="steps-post")
ax.legend()
plt.savefig("beans_ava.png")

plt.figure()
plt.title("ROC AUC vs. Alpha")
plt.ylabel("ROC AUC")
plt.xlabel("Alpha")
plt.plot(alphas, roc_aucs)
plt.savefig("beans_rocaucs.png")

i = np.argmax(roc_aucs)
alpha = alphas[i]
print("Max ROC AUC Alpha: ", alpha)
dtree = DecisionTreeClassifier(ccp_alpha = alpha)

# Split data into training and testing 70:30
split = train_test_split(data_full, target_full, test_size = 0.3)
X_train = split[0]
X_test = split[1]
y_train = split[2]
y_test = split[3]

dtree.fit(X_train, y_train)
score = dtree.score(X_test, y_test)
print("Accuracy: ", score)


from flask import Blueprint, render_template, redirect, url_for, request
from flask_login import current_user, login_required
from app import db
from models import Item

selling = Blueprint('selling', __name__)

@selling.route('/create-listing')
@login_required
def listing():
    return render_template('order/listing.html')

@selling.route('/create-listing', methods=['POST'])
@login_required
def listing_post():
    itemname = request.form.get('itemname')
    price = request.form.get('price')
    imageurl = request.form.get('imageurl')
    description = request.form.get('description')
    qty = request.form.get('qty')

    item = Item(itemname=itemname, merchant_id=current_user.id, price=price, imageurl=imageurl, description=description, qty=qty)
    db.session.add(item)
    db.session.commit()

    return redirect(url_for('user_profile.profile'))

@selling.route('/inventory/item/<item_id>')
@login_required
def view_item(item_id):
    item = Item.query.filter_by(id=item_id).first()

    if (current_user.id == item.merchant_id):
        return render_template('order/inventory.html',
            itemname=item.itemname,
            imageurl=item.imageurl,
            description=item.description,
            price="{:,.2f}".format(item.price),
            qty=item.qty
        )
    else:
        return render_template('permission_error.html')

@selling.route('/inventory/item/<item_id>/edit')
@login_required
def edit_item(item_id):
    item = Item.query.filter_by(id=item_id).first()

    if (current_user.id == item.merchant_id):
        return render_template('order/inventory.html',
            itemname=item.itemname,
            imageurl=item.imageurl,
            description=item.description,
            price="{:,.2f}".format(item.price),
            qty=item.qty
        )
    else:
        return render_template('permission_error.html')


import matplotlib.pyplot as plt
import os
import sys
import csv
import tensorflow as tf
import numpy as np
from util import score_linear_regression, score_logistic_regression, pretty
import pandas as pd
import argparse

pd.set_option('display.width', 1000)
np.set_printoptions(suppress=True)

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

np.random.seed(0)

# ################################################## Args
parser = argparse.ArgumentParser(description="Model Metrixs")
parser.add_argument('--sname', type=str, help="script name", required=True)
parser.add_argument('--model', type=str, help="model", default='logistic')
args, unparsed = parser.parse_known_args()
# ##################################################

csvprefix = "./log/"
tfcsvprefix = "./log/tf-" + args.sname + "/tf"
rttcsvprefix = "./log/rtt-" + args.sname + "/rtt"

y_real_file = tfcsvprefix + "-real-Y.csv"
y_pred_file_tf = tfcsvprefix + "-pred-Y.csv"
y_pred_file_rtt = rttcsvprefix + "-pred-Y.csv"

Y = pd.read_csv(y_real_file, sep=',', header=None, names=['L'])
predYtf = pd.read_csv(y_pred_file_tf, sep=',', header=None, names=['L'])
predYrtt = pd.read_csv(y_pred_file_rtt, sep=',', header=None, names=['L'])

print(args.model)
if args.model == 'logistic':
    emetrixs = score_logistic_regression(
        predYtf.to_numpy(), Y.to_numpy(), tag='tensorflow')
    print(pretty(emetrixs))

    emetrixs = score_logistic_regression(
        predYrtt.to_numpy(), Y.to_numpy(), tag='rosetta')
    print(pretty(emetrixs))

    # hist
    plt.title("tensorflow rosetta")
    plt.xlabel("Probability")
    plt.ylabel("Frequency")
    plt.hist(predYtf.to_numpy(), bins=50, label="TensorFlow")
    plt.hist(predYrtt.to_numpy(), bins=50, label="Rosetta")
    plt.legend()
    plt.savefig(csvprefix+"/" + args.sname + "-hist.png")
    plt.clf()

if args.model == 'linear':
    emetrixs = score_linear_regression(
        predYtf.to_numpy(), Y.to_numpy(), tag='tensorflow')
    print(pretty(emetrixs))

    emetrixs = score_linear_regression(
        predYrtt.to_numpy(), Y.to_numpy(), tag='rosetta')
    print(pretty(emetrixs))


import os
import sys
import glob
import random
import logging
import argparse
from os import path, makedirs, environ
from card_generator import create_card
from spotipy import SpotifyClientCredentials, Spotify, SpotifyException

logging.basicConfig(stream=sys.stderr, level=logging.WARNING)

auth_manager = SpotifyClientCredentials(
    environ["SPOTIFY_CLIENT_ID"], environ["SPOTIFY_CLIENT_SECRET"]
)
spotify = Spotify(auth_manager=auth_manager)
PLAYLIST_URL = (
    "https://open.spotify.com/playlist/5tl7iMYPGspVvMgryoA2HS?si=3812eee2d17a4cc5"
)
# playlist = spotify.playlist(PLAYLIST_URL)

# NUM_SONGS = playlist["tracks"]["total"]

PLAYLISTS = [
    "https://open.spotify.com/playlist/02Ze53FXXBgaSTfB9jpV1N?si=1a29aeb0e6b94621",  # 70s 1
    "https://open.spotify.com/playlist/2ZXxE8nsKuXplbOGVXRmk7?si=3508c19bb14343b0",  # 80s 1
    "https://open.spotify.com/playlist/4akSDZd9ppJx6LEhpWUXnJ?si=dce9d73d30714749",  # 90s 1
    "https://open.spotify.com/playlist/4WSK9EZdrSmVn94oh9fC0F?si=489fe486218b462c",  # 00s 1
    "https://open.spotify.com/playlist/340iasv2MTZsqAJBZHvScl?si=da6837376dd94468",  # 00s 2
    "https://open.spotify.com/playlist/6m9nAnMWZZ37nKqb8VxgT3?si=1ab09f9777c6417d",  # 10s 1
    "https://open.spotify.com/playlist/18i1CLpGUR1F0zl4IEWsOm?si=548426d4703d4b24",  # 10s 2
    "https://open.spotify.com/playlist/312n27j1xsU0Ee1NkyDdFw?si=0cca0803da4f44b2",  # 10s 3
]

def get_song_list(playlist_url: str) -> list:
    """Retrieves a Spotify playlist

    Args:
        playlist_url: the url of the desired playlist

    Returns:
        a list of tuples (title, artist) for the entire playlist
    """
    song_list = []
    results = spotify.playlist_tracks(playlist_url)
    tracks = results["items"]
    while results["next"]:
        results = spotify.next(results)
        tracks.extend(results["items"])

    for song in tracks:
        artists = ""
        for artist in song["track"]["artists"]:
            artists += artist["name"] + ", "
        artists = artists[:-2]

        song_list.append((song["track"]["name"], artists))
    return song_list

def generate_24_numbers(playlist: int = None) -> list:
    """Generates 24 unique numbers within the range of the playlist

    Returns:
        a list of numbers
    """
    url = PLAYLISTS[playlist] if playlist is not None else PLAYLIST_URL
    num_songs = spotify.playlist(url)["tracks"]["total"]

    nums = []
    div, mod = divmod(num_songs, 5)

    for i in range(5):
        nums.extend(
            random.sample(
                range(i * div + min(i, mod), (i + 1) * div + min(i + 1, mod)),
                5 if i != 2 else 4,
            )
        )

    return nums

def split_list(list_: list, n: int) -> list:
    """ "Splits a list into n (roughly) equal lists

    Args:
        list_: desired list to split
        n: number of desired sublists
    Returns:
        list of n sublists
    """
    div, mod = divmod(len(list_), n)

    return [
        list_[i * div + min(i, mod) : (i + 1) * div + min(i + 1, mod)] for i in range(n)
    ]

def add_custom_playlist(url: str) -> bool:
    try:
        spotify.playlist(PLAYLIST_URL)
    except SpotifyException:
        return False

    PLAYLISTS.append(url)
    return True

def generate_card(seed: int = None, playlist: int = None) -> int:
    """Generates a bingo card

    Args:
        seed: seed to use for card generation

    Returns:
        the seed of the generated bingo card
    """
    if seed:
        random.seed(int(seed))
    else:
        seed = random.randint(1000000, 99999999)
        logging.info(f"Seed: {seed}")
        random.seed(seed)

    nums = generate_24_numbers(playlist)
    logging.info(f"{nums}")

    song_list = get_song_list(
        PLAYLISTS[playlist] if playlist is not None else PLAYLIST_URL
    )

    # Shuffle list in a repeatable manner (i.e. shuffle the same way every time)
    random.Random(1).shuffle(song_list)

    card_songs = []
    for num in nums:
        card_songs.append(song_list[num])

    if not path.exists("output/"):
        makedirs("output/")

    # Remove previous generated bingo cards
    files = glob.glob("output/*.jpg")
    for f in files:
        os.remove(f)

    create_card(card_songs).save(f"output/bingo_card-{seed}.jpg")
    return int(seed)

if __name__ == "__main__":
    logging.getLogger().setLevel(logging.INFO)

    parser = argparse.ArgumentParser(description="Generate seed and grid.")
    parser.add_argument("--load", dest="seed", action="store")

    args = parser.parse_args()

    seed = generate_card(args.seed)


from flask import Flask
from flask_sqlalchemy import SQLAlchemy
import json
from configs import app

db = SQLAlchemy(app)

class Recipe(db.Model):
    __tablename__ = 'recipes'
    recipe_id = db.Column(db.String, primary_key=True)
    recipe_name = db.Column(db.String, nullable=False)
    recipe_type = db.Column(db.String, nullable=False)
    ingredients = db.Column(db.String, nullable=False)
    instructions = db.Column(db.String, nullable=False)

    def json(self):
        return{
        'recipe_id': self.recipe_id,
        'recipe_name': self.recipe_name,
        'recipe_type': self.recipe_type,
        'ingredients': self.ingredients,
        'instructions': self.instructions
        }

    def get_all_recipes():
        return [Recipe.json(recipes) for recipes in Recipe.query.all()]

    def delete_recipe(_recipe_id):
        is_successful = Recipe.query.filter_by(recipe_id=_recipe_id).delete()
        db.session.commit()
        return bool(is_successful)

    def update_recipe_instructions(_recipe_id, _instructions):
        recipe_to_update = Recipe.query.filter_by(recipe_id=_recipe_id).first()
        recipe_to_update.instructions = _instructions
        db.session.commit()

    def update_recipe_ingredients(_recipe_id, _ingredients):
        recipe_to_update = Recipe.query.filter_by(recipe_id=_recipe_id).first()
        recipe_to_update.ingredients = _ingredients
        db.session.commit()

    def update_recipe_name(_recipe_id, _recipe_name):
        recipe_to_update = Recipe.query.filter_by(recipe_id=_recipe_id).first()
        recipe_to_update.recipe_name = _recipe_name
        db.session.commit()

    def add_recipe(_recipe_id, _recipe_name, _recipe_type, _ingredients, _instructions):
        new_recipe = Recipe(

            recipe_id=_recipe_id,
            recipe_name=_recipe_name,
            recipe_type=_recipe_type,
            ingredients=_ingredients,
            instructions=_instructions
            )

        db.session.add(new_recipe)
        db.session.commit()

    def __repr__(self):
        recipe_object = {
        'recipe_id': self.recipe_id,
        'recipe_name': self.recipe_name,
        'recipe_type': self.recipe_type,
        'ingredients': self.ingredients,
        'instructions': self.instructions
        }

        return json.dumps(recipe_object)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
@date: 2018-Today
@author: jasper.bathmann@ufz.de
"""
from lxml import etree
import importlib.util

class Plant:

    def __init__(self,
                 x,
                 y,
                 species,
                 plant_id,
                 initial_geometry=False,
                 group_name=""):
        self.plant_id = plant_id
        self.species = species
        self.plants = []
        self.x = x
        self.y = y
        self.survival = 1
        self.group_name = group_name
        ## This initialization is only required if networks (root grafts) are
        # simulated
        self.iniNetwork()
        if species == "Avicennia":
            from PopulationLib.Species import Avicennia
            self.geometry, self.parameter = Avicennia.createPlant()
        elif "/" in species:
            try:
                spec = importlib.util.spec_from_file_location("", species)
                foo = importlib.util.module_from_spec(spec)
                spec.loader.exec_module(foo)
                self.geometry, self.parameter = foo.createPlant()
            except FileNotFoundError:
                raise FileNotFoundError("The file " + species +
                                        " does not exist.")
            except AttributeError:
                raise AttributeError("The file " + species + " is not " +
                                     "correctly defining a plant species. "
                                     "Please review the file.")
        else:
            raise KeyError("Species " + species + " unknown!")
        if initial_geometry:
            self.geometry["r_crown"] = initial_geometry["r_crown"]
            self.geometry["r_root"] = initial_geometry["r_root"]
            self.geometry["r_stem"] = initial_geometry["r_stem"]
            self.geometry["h_stem"] = initial_geometry["h_stem"]
        self.growth_concept_information = {}

    def getPosition(self):
        return self.x, self.y

    def getGeometry(self):
        return self.geometry

    def setGeometry(self, geometry):
        self.geometry = geometry

    def getGrowthConceptInformation(self):
        return self.growth_concept_information

    def setGrowthConceptInformation(self, growth_concept_information):
        self.growth_concept_information = growth_concept_information

    def getParameter(self):
        return self.parameter

    def getSurvival(self):
        return self.survival

    def setSurvival(self, survival):
        self.survival = survival

    def getId(self):
        return self.plant_id

    ## This function initializes a dictionary containing parameters required
    # to build a network of grafted plants
    def iniNetwork(self):
        self.network = {}
        ## Counter to track or define the time required for root graft
        # formation, if -1 no root graft formation takes place at the moment
        self.network['rgf'] = -1
        ## List with the names of plants (plant_name) with which an root graft
        # is currently being formed
        self.network['potential_partner'] = []
        # List with the names of plants (plant_name) with which it is connected
        self.network['partner'] = []
        self.network['groupID'] = []
        self.network['node_degree'] = 0
        self.network['water_absorbed'] = []
        self.network['water_available'] = []
        self.network['water_exchanged'] = []
        ## List with lengths of grafted roots (proportional to r_root of
        # adjacent plants
        self.network['weight_gr'] = 0
        self.network['psi_osmo'] = []
        # List with minimum grafted root radius, only for rgf variant V2
        self.network['r_gr_min'] = []
        self.network['r_gr_rgf'] = []
        self.network['l_gr_rgf'] = []
        self.network['variant'] = None

    def getNetwork(self):
        return self.network

    def setNetwork(self, network):
        self.network = network


from django.shortcuts import render

from .core import Page_video

# Create your views here.
def index(request):
    if request.method == 'POST':
        try:
            search_text = request.POST["Search_box"]
            videos=Page_video.search_videos(search_text)
            if(len(videos) == 1):
                col_size = 6
                offset_size = 3
            else:
                col_size = int(12/len(videos))
                offset_size = 0
        except ZeroDivisionError as e:
            if settings.DEBUG:
                return render_template("index.html",error_msg="Nenhum video encontrado")

        return render(request, "app/index.html", {"videos" : videos, "col_size" : col_size,
                                                         "offset_size" : offset_size})

    return render(request,"app/index.html")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from __future__ import (unicode_literals, division, absolute_import,
                        print_function)

##########################################################
# Author: Yichen Huang (Eugene)
# GitHub: https://github.com/yichen0831/opencc-python
# January, 2016 - Original
# January 2017 - Update to run under Python 2 by Hopkins1
##########################################################

import sys
import os
import io

DICT_DIRECTORY = '.'

MER_INPUTS = [
    'TWPhrasesIT.txt',
    'TWPhrasesName.txt',
    'TWPhrasesOther.txt'
]

MER_OUTPUT = 'TWPhrases.txt'

def merge(mer_inputs=MER_INPUTS, mer_output=MER_OUTPUT):
    """
    merge the phrase files into one file
    :param mer_inputs: the phrase files
    :param mer_output: the output file
    :return: None
    """
    dirname = os.path.dirname(__file__)
    output_file = os.path.join(dirname, DICT_DIRECTORY, mer_output)
    lines = []
    for in_file in MER_INPUTS:
        input_file = os.path.join(dirname, DICT_DIRECTORY, in_file)
        with io.open(input_file, 'r', encoding='utf-8') as f:
            for line in f:
                lines.append(line)

    with io.open(output_file, 'w', encoding='utf-8') as f:
        for line in lines:
            f.write(line)

if __name__ == '__main__':
    merge()


"""Lineages module."""

from typing import Union, TypeVar, Optional, Sequence

import pandas as pd

from cellrank import logging as logg
from cellrank.ul._docs import d
from cellrank.tl._utils import TestMethod
from cellrank.tl.kernels import PrecomputedKernel
from cellrank.tl._constants import AbsProbKey, TermStatesKey, TerminalStatesPlot
from cellrank.tl.estimators import GPCCA
from cellrank.tl.estimators._constants import P
from cellrank.tl.kernels._precomputed_kernel import DummyKernel

AnnData = TypeVar("AnnData")

@d.dedent
def lineages(
    adata: AnnData,
    backward: bool = False,
    copy: bool = False,
    return_estimator: bool = False,
    **kwargs,
) -> Optional[AnnData]:
    """
    Compute probabilistic lineage assignment using RNA velocity.

    For each cell `i` in :math:`{1, ..., N}` and %(initial_or_terminal)s state `j` in :math:`{1, ..., M}`,
    the probability is computed that cell `i` is either going to %(terminal)s state `j` (``backward=False``)
    or is coming from %(initial)s state `j` (``backward=True``).

    This function computes the absorption probabilities of a Markov chain towards the %(initial_or_terminal) states
    uncovered by :func:`cellrank.tl.initial_states` or :func:`cellrank.tl.terminal_states` using a highly efficient
    implementation that scales to large cell numbers.

    It's also possible to calculate mean and variance of the time until absorption for all or just a subset
    of the %(initial_or_terminal)s states. This can be seen as a pseudotemporal measure, either towards any terminal
    population of the state change trajectory, or towards specific ones.

    Parameters
    ----------
    %(adata)s
    %(backward)s
    copy
        Whether to update the existing ``adata`` object or to return a copy.
    return_estimator
        Whether to return the estimator. Only available when ``copy=False``.
    kwargs
        Keyword arguments for :meth:`cellrank.tl.estimators.BaseEstimator.compute_absorption_probabilities`.

    Returns
    -------
    :class:`anndata.AnnData`, :class:`cellrank.tl.estimators.BaseEstimator` or :obj:`None`
        Depending on ``copy`` and ``return_estimator``, either updates the existing ``adata`` object,
        returns its copy or returns the estimator.
    """

    if backward:
        lin_key = AbsProbKey.BACKWARD
        fs_key = TermStatesKey.BACKWARD
        fs_key_pretty = TerminalStatesPlot.BACKWARD
    else:
        lin_key = AbsProbKey.FORWARD
        fs_key = TermStatesKey.FORWARD
        fs_key_pretty = TerminalStatesPlot.FORWARD

    try:
        pk = PrecomputedKernel(adata=adata, backward=backward)
    except KeyError as e:
        raise RuntimeError(
            f"Compute transition matrix first as `cellrank.tl.transition_matrix(..., backward={backward})`."
        ) from e

    start = logg.info(f"Computing lineage probabilities towards {fs_key_pretty.s}")
    mc = GPCCA(
        pk, read_from_adata=True, inplace=not copy
    )  # GPCCA is more general than CFLARE, in terms of what is saves
    if mc._get(P.TERM) is None:
        raise RuntimeError(
            f"Compute the states first as `cellrank.tl.{fs_key.s}(..., backward={backward})`."
        )

    # compute the absorption probabilities
    mc.compute_absorption_probabilities(**kwargs)

    logg.info(f"Adding lineages to `adata.obsm[{lin_key.s!r}]`\n    Finish", time=start)

    return mc.adata if copy else mc if return_estimator else None

@d.dedent
def lineage_drivers(
    adata: AnnData,
    backward: bool = False,
    lineages: Optional[Union[Sequence, str]] = None,
    method: str = TestMethod.FISCHER.s,
    cluster_key: Optional[str] = None,
    clusters: Optional[Union[Sequence, str]] = None,
    layer: str = "X",
    use_raw: bool = False,
    confidence_level: float = 0.95,
    n_perms: int = 1000,
    seed: Optional[int] = None,
    return_drivers: bool = True,
    **kwargs,
) -> Optional[pd.DataFrame]:
    """
    %(lineage_drivers.full_desc)s

    Parameters
    ----------
    %(adata)s
    %(backward)s
    %(lineage_drivers.parameters)s

    Returns
    -------
    %(lineage_drivers.returns)s

    References
    ----------
    %(lineage_drivers.references)s
    """  # noqa: D400

    # create dummy kernel and estimator
    pk = DummyKernel(adata, backward=backward)
    g = GPCCA(pk, read_from_adata=True, write_to_adata=False)
    if g._get(P.ABS_PROBS) is None:
        raise RuntimeError(
            f"Compute absorption probabilities first as `cellrank.tl.lineages(..., backward={backward})`."
        )

    # call the underlying function to compute and store the lineage drivers
    return g.compute_lineage_drivers(
        method=method,
        lineages=lineages,
        cluster_key=cluster_key,
        clusters=clusters,
        layer=layer,
        use_raw=use_raw,
        confidence_level=confidence_level,
        n_perms=n_perms,
        seed=seed,
        return_drivers=return_drivers,
        **kwargs,
    )


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Feb 22 10:03:38 2022

@author: ixl
"""

import numpy as np
import matplotlib.pyplot as plt
import glob

single_list = glob.glob('/Users/ixl/Desktop/Cones/Single/*')
many_list = glob.glob('/Users/ixl/Desktop/Cones/Many/*')

singles = []
labels = []
many = []
for sin in single_list:
    if 'xy' in sin:
        singles.append(np.loadtxt(sin,delimiter=' ',usecols=(0,1)))
        labels.append(int(sin.split('/')[6].strip('mm_xy.txt')))
'''
for man in many_list:
    many.append(np.loadtxt(man,delimiter=' '))
'''
plt.figure()
for i in range(len(singles)):
    plt.hist(singles[i][:,0],label=labels[i],histtype='barstacked',bins=50)
plt.legend()
'''
plt.plot(singles[0][:,0])
plt.plot(many[0][:,1])
'''
plt.show()


"""QQP dataset."""
import torch
from .data_utils import clean_text
from .abstract_dataset import GLUEAbstractDataset

LABELS = [0, 1]

class QQPDataset(GLUEAbstractDataset):

    def __init__(self, name, datapaths, tokenizer, max_seq_length, data_as_tuple=False,
                 test_label=0):
        self.test_label = test_label
        super().__init__('QQP', name, datapaths,
                         tokenizer, max_seq_length, data_as_tuple)

    def process_samples_from_single_path(self, filename):
        """"Implement abstract method."""
        print(' > Processing {} ...'.format(filename))

        samples = []
        total = 0
        first = True
        is_test = False
        with open(filename, 'r') as f:
            for line in f:
                row = line.strip().split('\t')
                if first:
                    first = False
                    if len(row) == 3:
                        is_test = True
                        print('   reading {}, {}, and {} columns and '
                                     'setting labels to {}'.format(
                                         row[0].strip(), row[1].strip(),
                                         row[2].strip(), self.test_label))
                    else:
                        assert len(row) == 6
                        print('    reading {}, {}, {}, and {} columns'
                                     ' ...'.format(
                                         row[0].strip(), row[3].strip(),
                                         row[4].strip(), row[5].strip()))
                    continue

                if is_test:
                    assert len(row) == 3, 'expected length 3: {}'.format(row)
                    uid = int(row[0].strip())
                    text_a = clean_text(row[1].strip())
                    text_b = clean_text(row[2].strip())
                    label = self.test_label
                    assert len(text_a) > 0
                    assert len(text_b) > 0
                else:
                    if len(row) == 6:
                        uid = int(row[0].strip())
                        text_a = clean_text(row[3].strip())
                        text_b = clean_text(row[4].strip())
                        label = int(row[5].strip())
                    else:
                        print('***WARNING*** index error, ' 'skipping: {}'.format(row))
                        continue
                    if len(text_a) == 0:
                        print('***WARNING*** zero length a, ' 'skipping: {}'.format(row))
                        continue
                    if len(text_b) == 0:
                        print('***WARNING*** zero length b, ' 'skipping: {}'.format(row))
                        continue
                assert label in LABELS
                assert uid >= 0
                sample = {'uid': uid,
                          'text_a': text_a,
                          'text_b': text_b,
                          'label': label}
                total += 1
                samples.append(sample)

                if total % 50000 == 0:
                    print('  > processed {} so far ...'.format(total))

        print(' >> processed {} samples.'.format(len(samples)))
        return samples

def get_glue_qqp_train_data_loader(args, tokenizer, num_workers=0):
    train_dataset = QQPDataset('training', args.train_data, tokenizer, args.seq_length)
    train_sampler = torch.utils.data.RandomSampler(train_dataset)
    train_data_loader = torch.utils.data.DataLoader(train_dataset,
                                                    batch_size=args.batch_size,
                                                    sampler=train_sampler,
                                                    shuffle=False,
                                                    num_workers=num_workers,
                                                    drop_last=True,
                                                    pin_memory=True,
                                                    collate_fn=None)
    return train_data_loader

def get_glue_qqp_test_data_loader(args, tokenizer, num_workers=0):
    test_dataset = QQPDataset('testing', args.test_data, tokenizer, args.seq_length)
    test_sampler = torch.utils.data.RandomSampler(test_dataset)
    test_data_loader = torch.utils.data.DataLoader(test_dataset,
                                                    batch_size=args.batch_size,
                                                    sampler=test_sampler,
                                                    shuffle=False,
                                                    num_workers=num_workers,
                                                    drop_last=True,
                                                    pin_memory=True,
                                                    collate_fn=None)
    return test_data_loader


#!/usr/local/bin/python
# -*- coding: utf-8 -*-

import requests
import json
# import base64
import datetime
import numpy as np

from robo.Util import util

URL = 'http://127.0.0.1:8000/api/v1/article/'

def post_news(info):
    date_now = datetime.datetime.now()
    if(info['date'][0] > date_now):
        date = info['date'][0]
    else:
        date = date_now

    str_date = date.strftime("%Y-%m-%d %H:%M:%S")

    news = info['body'][0]
    reduced_news = util.get_reduced_news(news)
    content = reduced_news + '...'

    if(info['thumb'] == '0'):
        img = ''
    else:
        img = info['thumb']

    payload = {
            "title": info['title'][0],
            "slug": info['slug'][0],
            "body": content,
            "date": str_date,
            "thumb": img,
            "link": info['link'][0],
            "author": str(1),
            "categories": info['categories']
        }

    r = requests.request("POST", URL, data=payload)
    print('POST = ' + str(r))
    print(r.text)


import json
from datetime import datetime
from pathlib import Path
from typing import List, Optional, Iterator
from uuid import UUID

from shell_commands.commands import CommandRepository, CommandData
from shell_commands.history import HistoryRepository, HistoryEntry
from shell_commands.packages import PackageRepository
from shell_commands.utils import RunCommandOutput, serialize_datetime, deserialize_datetime

class JSONRepository(CommandRepository, PackageRepository, HistoryRepository):
    SCHEMA_VERSION = 3

    def __init__(self, dbfile: Path):
        super(JSONRepository, self).__init__()
        self.dbfile = dbfile
        if not self.dbfile.exists():
            self._write({
                'schemaVersion': self.SCHEMA_VERSION,
                'commands': {},
                'packages': [],
                'history': [],
            })

    def _load(self) -> dict:
        with self.dbfile.open('r') as fp:
            data = json.load(fp)
        assert 'schemaVersion' in data
        if data['schemaVersion'] != self.SCHEMA_VERSION:
            if data['schemaVersion'] == 1:
                # Upgrade from schema 1 to 2
                data['packages'] = []
                data['schemaVersion'] = 2
            if data['schemaVersion'] == 2:
                data['history'] = []
                data['schemaVersion'] = 3
            assert data['schemaVersion'] == self.SCHEMA_VERSION
            self._write(data)
        return data

    def _write(self, data: dict):
        with self.dbfile.open('w') as fp:
            json.dump(data, fp, indent=2)

    def save_command(self, name: str, user: str, cwd: str, command: List[str]):
        data = self._load()
        assert name not in data['commands'], 'Command with this name already exists'
        data['commands'][name] = {
            'user': user,
            'cwd': cwd,
            'command': command,
        }
        self._write(data)

    def get_command(self, name: str) -> CommandData:
        data = self._load()
        assert name in data['commands'], 'Command with this name does not exist'
        user = data['commands'][name]['user']
        cwd = data['commands'][name]['cwd']
        command = data['commands'][name]['command']
        return CommandData(name=name, user=user, cwd=cwd, command=command)

    def get_all_commands(self) -> List[CommandData]:
        return [self.get_command(name) for name in self._load()['commands'].keys()]

    def delete_command(self, name: str):
        data = self._load()
        assert name in data['commands'], 'Command with this name does not exist'
        del data['commands'][name]
        self._write(data)

    def require_package(self, name: str):
        data = self._load()
        assert name not in data['packages'], 'Package is already required so cannot require again'
        data['packages'].append(name)
        self._write(data)

    def get_required_packages(self) -> List[str]:
        data = self._load()
        return data['packages']

    def unrequire_package(self, name: str):
        data = self._load()
        assert name in data['packages'], 'Package is not required so cannot unrequire'
        data['packages'].remove(name)
        self._write(data)

    def write_to_history(self, entry: HistoryEntry):
        data = self._load()
        data['history'].append({
            'id': entry.id.hex,
            'timestamp': serialize_datetime(entry.timestamp),
            'command_name': entry.command_name,
            'command': entry.command,
            'cwd': entry.cwd,
            'user': entry.user,
            'return_code': entry.return_code,
        })
        self._write(data)

    def iterate_history(self) -> Iterator[HistoryEntry]:
        for entry in self._load()['history']:
            yield HistoryEntry(
                id=UUID(hex=entry['id']),
                timestamp=deserialize_datetime(entry['timestamp']),
                command_name=entry['command_name'],
                command=entry['command'],
                cwd=entry['cwd'],
                user=entry['user'],
                return_code=entry['return_code'],
            )


from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('evaluation', '0033_remove_likert_and_grade_answer'),
    ]

    operations = [
        migrations.AddField(
            model_name='course',
            name='gets_no_grade_documents',
            field=models.BooleanField(default=False, verbose_name='gets no grade documents'),
        ),
    ]


from django.shortcuts import render,redirect
from .form import CategoryCreateForm,DoctorCreateForm
from .models import Category,Doctor

def home(request):
	cats = Category.objects.all()
	dict = {
		"title":"Find A Doctor",
		"cats":cats
	}
	return render(request,"doctor/home.html",dict)

def catcreate(request):
	if request.method == "POST":
		form = CategoryCreateForm(request.POST)
		form.is_valid()
		form.save()
		return redirect('dhome-page')
	else:
		form = CategoryCreateForm

	cats = Category.objects.all()
	dict = {
		"title":"Create Category",
		"form":form,
		"cats":cats
	}
	return render(request,"doctor/catcreate.html",dict)

def createDoctor(request):
	if request.method == "POST":
		form = DoctorCreateForm(request.POST)
		if form.is_valid():
			form.save()
			return redirect('doctor-show-page')
	else:
		form = DoctorCreateForm()

	dict ={
		"title":"Doctor Create Page",
		"form":form
	}

	return render(request,"doctor/doctor_create.html",dict)

def showDoctors(request):
	docts = Doctor.objects.all()
	dict = {
		"title":"Show All Doctors",
		"docts":docts
	}
	return render(request,"doctor/show_all_doctors.html",dict)

def filterDoctors(request,pk):
	docts = Doctor.objects.filter(category=pk)
	print(docts)

	dict = {
		"title":"Find A Doctor",
		"docts":docts
	}

	return render(request,"doctor/show_all_doctors.html",dict)

# def filterDoctors(request):
# 	docts = Doctor.objects.all()
# 	dict = {
# 		"title":"Show All Doctors",
# 		"docts":docts
# 	}
# 	return render(request,"doctor/show_all_doctors.html",dict)

def editDoctor(request,id):
	doctors = Doctor.objects.filter(id=id)
	return render(request,"doctor/show_all_doctors1.html",{"doctors":doctors})

def updateDoctor(request,id):
	doctor1 = Doctor.objects.get(pk=id)
	form = DoctorCreateForm(request.POST,instance=doctor1)

	if form.is_valid():
		form.save()
		return redirect('doctor-show-page')
	else:
		form = DoctorCreateForm(instance=doctor1)

	content = {
		"doctor1":doctor1,
		"form":form
	}
	return render(request,"doctor/doctor_create.html", content)

def deleteDoctor(request,id):
	doctor2 = Doctor.objects.get(pk=id)
	if request.method =="POST":
		doctor2.delete()
		return redirect('doctor-show-page')

	return render(request,"doctor/show_all_doctors.html",{"doctor2":doctor2})

def findDoctor(request):
	dict = {
		"title":"Find A Doctor"
	}

	return render(request,"doctor/findDoctor.html",dict)

# def find1Doctor(request):

# 	names = Doctor.objects.all()
# 	if 'name' in request.GET:
# 		names = Doctor.objects.filter(name__icontains=request.GET['name'])

# 	return render(request,"doctor/show_find_doctors.html",{"names":names})


from __future__ import unicode_literals

from django.utils.encoding import python_2_unicode_compatible
from jsonfield import JSONField

from multigtfs.models.base import models, Base

@python_2_unicode_compatible
class FareRule(Base):
    """Associate a Fare with a Route and/or Zones"""
    fare = models.ForeignKey('Fare', on_delete=models.CASCADE)
    route = models.ForeignKey(
        'Route', null=True, blank=True, on_delete=models.SET_NULL,
        help_text="Fare class is valid for this route.")
    origin = models.ForeignKey(
        'Zone', null=True, blank=True, on_delete=models.SET_NULL,
        related_name='fare_origins',
        help_text="Fare class is valid for travel originating in this zone.")
    destination = models.ForeignKey(
        'Zone', null=True, blank=True, on_delete=models.SET_NULL,
        related_name='fare_destinations',
        help_text="Fare class is valid for travel ending in this zone.")
    contains = models.ForeignKey(
        'Zone', null=True, blank=True, on_delete=models.SET_NULL,
        related_name='fare_contains',
        help_text="Fare class is valid for travel withing this zone.")
    extra_data = JSONField(default={}, blank=True, null=True)

    def __str__(self):
        u = "%d-%s" % (self.fare.feed.id, self.fare.fare_id)
        if self.route:
            u += '-%s' % self.route.route_id
        return u

    class Meta:
        db_table = 'fare_rules'
        app_label = 'multigtfs'

    # For Base import/export
    _column_map = (
        ('fare_id', 'fare__fare_id'),
        ('route_id', 'route__route_id'),
        ('origin_id', 'origin__zone_id'),
        ('destination_id', 'destination__zone_id'),
        ('contains_id', 'contains__zone_id')
    )
    _filename = 'fare_rules.txt'
    _rel_to_feed = 'fare__feed'
    _sort_order = ('route__route_id', 'fare__fare_id')
    _unique_fields = (
        'fare_id', 'route_id', 'origin_id', 'destination_id', 'contains_id')