# -*- coding: utf-8 -*-

"""
Script defining all the functions that are specific to the MNIST dataset
"""

from __future__ import annotations

import os
from time import perf_counter
from typing import Union, Optional

import numpy as np
import matplotlib.pyplot as plt

import utils
from utils import (
    cast,
    check_dtype,
    get_dtype_of_array,
    get_range_of_array,
    vector_to_categorical,
    categorical_to_vector,
    list_to_string,
    display_class_distributions,
    _download_data,
    _validate_selected_classes
)

from core import train_test_split

##############################################################################

def _download_raw_MNIST_dataset() -> str:
    r"""
    Automatically downloads the raw MNIST data (as a single file), and saves
    it to the following location on your disk :
        - on Windows : "C:\Users\YourUsername\.Custom-NN\datasets\MNIST\raw_MNIST_data.npz"
        - on Linux   : "/home/YourUsername/.Custom-NN/datasets/MNIST/raw_MNIST_data.npz"

    This function is basically a wrapper of the `_download_data` function of
    the "utils.py" script

    Naturally, if this is the very first time you call this function, you'll
    need to have an internet connection !

    The downloaded file has a size of about 11 MB, and is retrieved from the
    following URL :
    https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz

    Also, the absolute path of the downloaded data is returned

    Sidenote
    --------
    This function is basically equivalent to the `tensorflow.keras.datasets.mnist.load_data`
    method. The only real difference is that the downloaded data has a different
    location on your disk. The reason why the previous `mnist.load_data`
    method isn't used is simply because we do NOT want to import TensorFlow !
    Indeed, in my opinion, it would be kind of awkward to import TensorFlow in
    a project aiming to implement a Deep Learning model *from scratch* !
    Note that, if you use the `mnist.load_data` method of the TensorFlow module,
    the raw MNIST data will be saved at the following location on your disk :
        - on Windows : "C:\Users\YourUsername\.keras\datasets\mnist.npz"
        - on Linux   : "/home/YourUsername/.keras/datasets/mnist.npz"
    """
    # ---------------------------------------------------------------------- #

    # the raw MNIST data will be downloaded from this URL (by default)

    data_URL = "https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz"

    # ---------------------------------------------------------------------- #

    # creating the default folder that will contain the raw MNIST data (if it
    # doesn't already exist)
    default_data_directory = os.path.join(
        os.path.expanduser("~"),
        ".Custom-NN",
        "datasets",
        "MNIST"
    )
    os.makedirs(default_data_directory, exist_ok=True)

    # here, `default_data_filename` must have a "*.npz" extension
    default_data_filename = "raw_MNIST_data.npz"
    assert os.path.splitext(default_data_filename)[1] == ".npz"

    # defining the absolute path of the downloaded file
    path_of_downloaded_data = os.path.join(
        default_data_directory,
        default_data_filename
    )

    # ---------------------------------------------------------------------- #

    # default SHA-256 hash value of the raw MNIST data

    hash_value = "731c5ac602752760c8e48fbffcf8c3b850d9dc2a2aedcf2cc48468fc17b673d1"

    # ---------------------------------------------------------------------- #

    # actually downloading the raw MNIST data (if it doesn't already exist
    # on your disk)

    _download_data(
        data_URL,
        path_of_downloaded_data,
        hash_value=hash_value
    )

    # ---------------------------------------------------------------------- #

    return path_of_downloaded_data

##############################################################################

def _validate_raw_MNIST_dataset(
        raw_X_train: np.ndarray,
        raw_y_train: np.ndarray,
        raw_X_test:  np.ndarray,
        raw_y_test:  np.ndarray
    ) -> None:
    """
    Checks if the specified raw MNIST data is valid or not. By design, the
    arguments `raw_X_train`, `raw_y_train`, `raw_X_test` and `raw_y_test` are
    meant to be the outputs of the `load_raw_MNIST_dataset_from_disk` function
    (of the "mnist_dataset.py" script)
    """
    assert isinstance(raw_X_train, np.ndarray)
    assert raw_X_train.shape == (60000, 28, 28)
    check_dtype(raw_X_train, np.uint8)

    assert isinstance(raw_y_train, np.ndarray)
    assert raw_y_train.shape == (60000, )
    check_dtype(raw_y_train, np.uint8)

    assert isinstance(raw_X_test, np.ndarray)
    assert raw_X_test.shape == (10000, 28, 28)
    check_dtype(raw_X_test, np.uint8)

    assert isinstance(raw_y_test, np.ndarray)
    assert raw_y_test.shape == (10000, )
    check_dtype(raw_y_test, np.uint8)

    DEFAULT_NB_CLASSES = 10
    expected_classes = np.arange(DEFAULT_NB_CLASSES).astype(np.uint8)
    assert np.allclose(np.unique(raw_y_train), expected_classes)
    assert np.allclose(np.unique(raw_y_test),  expected_classes)

##############################################################################

def load_raw_MNIST_dataset_from_disk(
        *,
        verbose: bool = False
    ) -> tuple[np.ndarray]:
    """
    Loads the raw MNIST data from the disk
    """
    assert isinstance(verbose, bool)

    # ---------------------------------------------------------------------- #

    # downloading the raw MNIST data to the (default) location `path_of_downloaded_data`
    # (cf. the "_download_raw_MNIST_dataset" function of this script), if it
    # hasn't already been done
    path_of_downloaded_data = _download_raw_MNIST_dataset()

    t_beginning_data_loading = perf_counter()

    # actually loading the raw MNIST data from the disk
    with np.load(path_of_downloaded_data, allow_pickle=True) as RAW_MNIST_DATA:
        raw_X_train = RAW_MNIST_DATA["x_train"]
        raw_y_train = RAW_MNIST_DATA["y_train"]
        raw_X_test  = RAW_MNIST_DATA["x_test"]
        raw_y_test  = RAW_MNIST_DATA["y_test"]

    # just to be *100% sure* the loaded data is valid
    _validate_raw_MNIST_dataset(
        raw_X_train,
        raw_y_train,
        raw_X_test,
        raw_y_test
    )

    t_end_data_loading = perf_counter()
    duration_data_loading = t_end_data_loading - t_beginning_data_loading

    # ---------------------------------------------------------------------- #

    if verbose:
        print("\nShapes of the raw MNIST data :")
        print(f"    - X_train : {raw_X_train.shape}")
        print(f"    - y_train : {raw_y_train.shape}")
        print(f"    - X_test  : {raw_X_test.shape}")
        print(f"    - y_test  : {raw_y_test.shape}")

        print("\nTypes of the raw MNIST data :")
        print(f"    - X_train : {get_dtype_of_array(raw_X_train)}")
        print(f"    - y_train : {get_dtype_of_array(raw_y_train)}")
        print(f"    - X_test  : {get_dtype_of_array(raw_X_test)}")
        print(f"    - y_test  : {get_dtype_of_array(raw_y_test)}")

        precision_of_printed_info = 3
        print("\nRanges of the raw MNIST data :")
        print(f"    - X_train : {get_range_of_array(raw_X_train, precision=precision_of_printed_info)}")
        print(f"    - y_train : {get_range_of_array(raw_y_train, precision=precision_of_printed_info)}")
        print(f"    - X_test  : {get_range_of_array(raw_X_test,  precision=precision_of_printed_info)}")
        print(f"    - y_test  : {get_range_of_array(raw_y_test,  precision=precision_of_printed_info)}")

        # displaying the class distribution of the raw MNIST data
        dict_of_label_vectors = {
            "y_train" : raw_y_train,
            "y_test"  : raw_y_test
        }
        display_class_distributions(
            dict_of_label_vectors,
            precision=2
        )

    print(f"\nThe raw MNIST dataset was successfully loaded from the disk. Done in {duration_data_loading:.3f} seconds")

    return raw_X_train, raw_y_train, raw_X_test, raw_y_test

##############################################################################

def plot_random_images_from_raw_MNIST_dataset(
        raw_X_train: np.ndarray,
        raw_y_train: np.ndarray,
        raw_X_test:  np.ndarray,
        raw_y_test:  np.ndarray,
        *,
        seed: Optional[int] = None
    ) -> None:
    """
    Plots a random sample of each of the 10 digits (from the raw MNIST data).
    By design, the arguments `raw_X_train`, `raw_y_train`, `raw_X_test` and
    `raw_y_test` are meant to be the outputs of the `load_raw_MNIST_dataset_from_disk`
    function of this script
    """
    # ---------------------------------------------------------------------- #

    # checking the validity of the specified arguments

    _validate_raw_MNIST_dataset(
        raw_X_train,
        raw_y_train,
        raw_X_test,
        raw_y_test
    )

    assert isinstance(seed, (type(None), int))
    if seed is not None:
        assert seed >= 0

    # ---------------------------------------------------------------------- #

    nb_rows    = 2
    nb_columns = 5

    nb_classes = np.unique(raw_y_train).size # = 10
    assert nb_rows * nb_columns == nb_classes

    data_types = ["train", "test"]

    fig, ax = plt.subplots(nb_rows, nb_columns, figsize=(16, 8))
    plt.suptitle("\nRandom MNIST samples of each digit", fontsize=15)

    np.random.seed(seed)

    for image_index in range(nb_classes):
        random_data_type = data_types[np.random.randint(0, len(data_types))]
        if random_data_type == "train":
            # raw "train" data
            data   = raw_X_train
            labels = raw_y_train
        else:
            # raw "test" data
            data   = raw_X_test
            labels = raw_y_test

        possible_image_indices = np.where(labels == image_index)[0]
        random_image_index = possible_image_indices[np.random.randint(0, possible_image_indices.size)]
        random_image = data[random_image_index]
        assert random_image.ndim == 2

        row_index = image_index // nb_columns
        column_index = image_index % nb_columns
        ax[row_index, column_index].imshow(random_image, cmap="gray")

    # resetting the seed
    np.random.seed(None)

    plt.subplots_adjust(wspace=0.5, top=0.85)

    plt.show()

##############################################################################

def format_raw_MNIST_dataset(
        raw_X_train: np.ndarray,
        raw_y_train: np.ndarray,
        raw_X_test:  np.ndarray,
        raw_y_test:  np.ndarray,
        nb_train_samples: int,
        nb_val_samples:   int,
        nb_test_samples:  int,
        *,
        selected_classes: Union[str, list[int]] = "all",
        dict_of_real_class_names: Optional[dict[int, str]] = None,
        nb_shuffles: int = 20,
        seed: Optional[int] = None,
        verbose: bool = False
    ) -> tuple[Union[np.ndarray, None]]:
    """
    Formats the raw MNIST data so that it can be directly interpreted by a
    regular MLP neural network. By design, the arguments `raw_X_train`,
    `raw_y_train`, `raw_X_test` and `raw_y_test` are meant to be the outputs
    of the `load_raw_MNIST_dataset_from_disk` function of this script

    This function returns `X_train`, `y_train`, `X_val`, `y_val`, `X_test`
    and `y_test`

    If you do not want to get validation data, you can simply set the argument
    `nb_val_samples` to zero. In that case, the outputs `X_val` and `y_val`
    will both be equal to `None`

    The kwarg `selected_classes` can either be :
        - the string "all", if you want to work with all the classes (default)
        - a list/tuple/1D-array containing the specific class indices you want
          to work with (e.g. [2, 4, 7])

    The kwarg `dict_of_real_class_names`, if not set to `None`, is a dictionary
    with :
        - as its keys   : all the selected class indices (as integers)
        - as its values : the REAL names of the associated classes (as strings)
    For instance, if you set `selected_classes` to `[2, 4, 7]`, then you
    could, for instance, set `dict_of_real_class_names` to the following
    dictionary :
    dict_of_real_class_names = {
        2 : "TWO",
        4 : "FOUR",
        7 : "SEVEN"
    }
    By default, if `dict_of_real_class_names` is set to `None`, then the
    class names will simply be the string representations of the (selected)
    class indices
    """
    # ====================================================================== #

    # checking the validity of the specified arguments

    _validate_raw_MNIST_dataset(
        raw_X_train,
        raw_y_train,
        raw_X_test,
        raw_y_test
    )

    nb_classes = np.unique(raw_y_train).size # = 10

    # the 3 arguments `nb_train_samples`, `nb_val_samples` and `nb_test_samples`
    # will be re-checked a bit later in this function, once we check if the
    # `selected_classes` kwarg is equal to "all" or not (so that we can get
    # the actual number of classes in the - selected - raw MNIST data)
    assert isinstance(nb_train_samples, int)
    assert nb_train_samples > 0
    assert isinstance(nb_val_samples, int)
    assert nb_val_samples >= 0
    _return_validation_data = (nb_val_samples > 0)
    assert isinstance(nb_test_samples, int)
    assert nb_test_samples > 0

    selected_classes, _ = _validate_selected_classes(
        selected_classes,
        nb_classes,
        dict_of_real_class_names=dict_of_real_class_names
    )

    assert isinstance(nb_shuffles, int)
    assert nb_shuffles >= 0

    assert isinstance(seed, (type(None), int))
    if seed is not None:
        assert seed >= 0

    assert isinstance(verbose, bool)

    # ====================================================================== #

    t_beginning_data_formatting = perf_counter()

    # ====================================================================== #

    # only keeping the selected classes in the raw MNIST data

    if not(isinstance(selected_classes, str)):
        nb_classes = selected_classes.size

        indices_of_selected_train_classes = []
        indices_of_selected_test_classes  = []

        for selected_class in selected_classes:
            indices_of_selected_train_class = np.where(raw_y_train == selected_class)[0]
            indices_of_selected_train_classes.append(indices_of_selected_train_class)

            indices_of_selected_test_class = np.where(raw_y_test == selected_class)[0]
            indices_of_selected_test_classes.append(indices_of_selected_test_class)

        indices_of_selected_train_classes = np.hstack(tuple(indices_of_selected_train_classes))
        indices_of_selected_test_classes  = np.hstack(tuple(indices_of_selected_test_classes))

        # actually retrieving the selected classes from the raw data
        raw_X_train = raw_X_train[indices_of_selected_train_classes, :]
        raw_y_train = raw_y_train[indices_of_selected_train_classes]
        raw_X_test  = raw_X_test[indices_of_selected_test_classes, :]
        raw_y_test  = raw_y_test[indices_of_selected_test_classes]

        assert np.allclose(np.unique(raw_y_train), selected_classes)
        assert np.allclose(np.unique(raw_y_test),  selected_classes)

        print(f"\nNB : With the selected classes {list_to_string(selected_classes)}, here are the new shapes of the raw \"train\" and \"test\" data (from which the formatted data will be extracted) :")
        print(f"    - X_train : {raw_X_train.shape}")
        print(f"    - y_train : {raw_y_train.shape}")
        print(f"    - X_test  : {raw_X_test.shape}")
        print(f"    - y_test  : {raw_y_test.shape}")

    # ====================================================================== #

    # re-checking the validity of the arguments `nb_train_samples`, `nb_val_samples`
    # and `nb_test_samples` (since the raw data might have been modified if
    # all the classes weren't selected)

    # we want to have each class represented AT LEAST ONCE in the formatted
    # data, therefore the number of samples of the train, val and test datasets
    # CANNOT be strictly less than `nb_classes`
    assert nb_train_samples >= nb_classes
    if _return_validation_data:
        assert nb_val_samples >= nb_classes
    assert nb_test_samples >= nb_classes

    # NB : The validation set will be extracted from the raw training set
    total_nb_of_raw_train_samples = raw_X_train.shape[0] # = 60000 (if ALL the classes are selected)
    assert nb_train_samples + nb_val_samples <= total_nb_of_raw_train_samples

    total_nb_of_raw_test_samples = raw_X_test.shape[0] # = 10000 (if ALL the classes are selected)
    assert nb_test_samples <= total_nb_of_raw_test_samples

    # ====================================================================== #

    # step 1/6 : copying, reshaping, casting and normalizing the data

    # copying the raw data
    X_train = raw_X_train.copy()
    y_train = raw_y_train.copy()
    X_test  = raw_X_test.copy()
    y_test  = raw_y_test.copy()

    # flattening the images/samples
    nb_pixels_per_image = raw_X_train.shape[1] * raw_X_train.shape[2] # = 28 * 28 = 784
    X_train = X_train.reshape((total_nb_of_raw_train_samples, nb_pixels_per_image))
    X_test  = X_test.reshape((total_nb_of_raw_test_samples, nb_pixels_per_image))

    # casting the "uint8" data into `utils.DEFAULT_DATATYPE` data
    X_train = cast(X_train, utils.DEFAULT_DATATYPE)
    X_test  = cast(X_test,  utils.DEFAULT_DATATYPE)

    # casting the "uint8" labels into "int32" labels (optional)
    y_train = np.int_(y_train)
    y_test  = np.int_(y_test)

    # normalizing the data between 0 and 1 (by convention)
    normalizing_factor = cast(1, utils.DEFAULT_DATATYPE) / cast(255, utils.DEFAULT_DATATYPE)
    X_train *= normalizing_factor
    X_test  *= normalizing_factor

    # ====================================================================== #

    # step 2/6 : splitting the data into train/test or train/val/test
    #            subsets, such that the resulting class distributions are
    #            roughly the same as the initial raw data

    if _return_validation_data:
        # getting the formatted "train" and "val" datasets (from the raw "train" data)
        X_train, X_val, y_train, y_val = train_test_split(
            X_train,
            y_train,
            train_size=nb_train_samples,
            test_size=nb_val_samples,
            stratify=True, # keeping the same class distribution as `y_train`
            random_state=seed
        )
    else:
        # getting the formatted "train" dataset (from the raw "train" data)

        if total_nb_of_raw_train_samples - nb_train_samples >= nb_classes:
            X_train, _, y_train, _ = train_test_split(
                X_train,
                y_train,
                train_size=nb_train_samples,
                stratify=True, # keeping the same class distribution as `y_train`
                random_state=seed
            )
        else:
            # in this specific case, the `train_test_split` function breaks
            # (with the arguments used in the previous `if` block)

            np.random.seed(seed)
            train_indices = np.random.choice(
                np.arange(total_nb_of_raw_train_samples),
                size=(nb_train_samples, ),
                replace=False
            )
            np.random.seed(None) # resetting the seed

            X_train = X_train[train_indices, :].copy()
            y_train = y_train[train_indices].copy()

    assert np.unique(y_train).size == nb_classes
    if _return_validation_data:
        assert np.unique(y_val).size == nb_classes

    if seed is not None:
        # updating the seed such that the "randomness" of the test split is
        # different than the one used in the previous train (or train/val )
        # split (just in case)
        seed += 1 # the increment value is arbitrary

    # getting the formatted "test" dataset (from the raw "test" data)
    if total_nb_of_raw_test_samples - nb_test_samples >= nb_classes:
        X_test, _, y_test, _ = train_test_split(
            X_test,
            y_test,
            train_size=nb_test_samples,
            stratify=True, # keeping the same class distribution as `y_test`
            random_state=seed
        )
    else:
        # in this specific case, the `train_test_split` function breaks
        # (with the arguments used in the previous `if` block)

        np.random.seed(seed)
        test_indices = np.random.choice(
            np.arange(total_nb_of_raw_test_samples),
            size=(nb_test_samples, ),
            replace=False
        )
        np.random.seed(None) # resetting the seed

        X_test = X_test[test_indices, :].copy()
        y_test = y_test[test_indices].copy()

    assert np.unique(y_test).size == nb_classes

    if (nb_shuffles > 0) and (seed is not None):
        # updating the seed such that the "randomness" of the shuffle is
        # different than the one used in the previous splits (just in case)
        seed += 1 # the increment value is arbitrary

    # ====================================================================== #

    # step 3/6 : one-hot encoding the resulting label vectors

    y_train = vector_to_categorical(y_train, dtype=utils.DEFAULT_DATATYPE)
    if _return_validation_data:
        y_val = vector_to_categorical(y_val, dtype=utils.DEFAULT_DATATYPE)
    y_test = vector_to_categorical(y_test,  dtype=utils.DEFAULT_DATATYPE)

    # ====================================================================== #

    # step 4/6 : shuffling the data

    if nb_shuffles > 0:
        shuffled_indices_train_data = np.arange(nb_train_samples)
        if _return_validation_data:
            shuffled_indices_validation_data = np.arange(nb_val_samples)
        shuffled_indices_test_data = np.arange(nb_test_samples)

        np.random.seed(seed)
        for shuffle_index in range(nb_shuffles):
            np.random.shuffle(shuffled_indices_train_data)
            if _return_validation_data:
                np.random.shuffle(shuffled_indices_validation_data)
            np.random.shuffle(shuffled_indices_test_data)
        np.random.seed(None) # resetting the seed

        X_train = X_train[shuffled_indices_train_data, :]
        y_train = y_train[shuffled_indices_train_data, :]
        if _return_validation_data:
            X_val = X_val[shuffled_indices_validation_data, :]
            y_val = y_val[shuffled_indices_validation_data, :]
        X_test = X_test[shuffled_indices_test_data, :]
        y_test = y_test[shuffled_indices_test_data, :]

    # ====================================================================== #

    # step 5/6 : checking the shapes of the final formatted data

    assert X_train.shape == (nb_train_samples, nb_pixels_per_image)
    assert y_train.shape == (nb_train_samples, nb_classes)
    if _return_validation_data:
        assert X_val.shape == (nb_val_samples, nb_pixels_per_image)
        assert y_val.shape == (nb_val_samples, nb_classes)
    assert X_test.shape == (nb_test_samples, nb_pixels_per_image)
    assert y_test.shape == (nb_test_samples, nb_classes)

    # ====================================================================== #

    # step 6/6 : checking the datatype of the final formatted data

    check_dtype(X_train, utils.DEFAULT_DATATYPE)
    check_dtype(y_train, utils.DEFAULT_DATATYPE)
    if _return_validation_data:
        check_dtype(X_val, utils.DEFAULT_DATATYPE)
        check_dtype(y_val, utils.DEFAULT_DATATYPE)
    check_dtype(X_test, utils.DEFAULT_DATATYPE)
    check_dtype(y_test, utils.DEFAULT_DATATYPE)

    # ====================================================================== #

    t_end_data_formatting = perf_counter()
    duration_data_formatting = t_end_data_formatting - t_beginning_data_formatting

    # ====================================================================== #

    if verbose:
        print("\nShapes of the formatted MNIST data :")
        print(f"    - X_train : {X_train.shape}")
        print(f"    - y_train : {y_train.shape}")
        if _return_validation_data:
            print(f"    - X_val   : {X_val.shape}")
            print(f"    - y_val   : {y_val.shape}")
        print(f"    - X_test  : {X_test.shape}")
        print(f"    - y_test  : {y_test.shape}")

        print("\nTypes of the formatted MNIST data :")
        print(f"    - X_train : {get_dtype_of_array(X_train)}")
        print(f"    - y_train : {get_dtype_of_array(y_train)}")
        if _return_validation_data:
            print(f"    - X_val   : {get_dtype_of_array(X_val)}")
            print(f"    - y_val   : {get_dtype_of_array(y_val)}")
        print(f"    - X_test  : {get_dtype_of_array(X_test)}")
        print(f"    - y_test  : {get_dtype_of_array(y_test)}")

        precision_of_printed_info = 3
        print("\nRanges of the formatted MNIST data :")
        print(f"    - X_train : {get_range_of_array(X_train, precision=precision_of_printed_info)} (mean={X_train.mean():.{precision_of_printed_info}f}, std={X_train.std():.{precision_of_printed_info}f})")
        print(f"    - y_train : {get_range_of_array(y_train, precision=precision_of_printed_info)} (one-hot encoded)")
        if _return_validation_data:
            print(f"    - X_val   : {get_range_of_array(X_val,   precision=precision_of_printed_info)} (mean={X_val.mean():.{precision_of_printed_info}f}, std={X_val.std():.{precision_of_printed_info}f})")
            print(f"    - y_val   : {get_range_of_array(y_val,   precision=precision_of_printed_info)} (one-hot encoded)")
        print(f"    - X_test  : {get_range_of_array(X_test,  precision=precision_of_printed_info)} (mean={X_test.mean():.{precision_of_printed_info}f}, std={X_test.std():.{precision_of_printed_info}f})")
        print(f"    - y_test  : {get_range_of_array(y_test,  precision=precision_of_printed_info)} (one-hot encoded)")

        # ------------------------------------------------------------------ #

        # displaying the class distributions of the final formatted data

        if _return_validation_data:
            dict_of_label_vectors = {
                "y_train" : categorical_to_vector(y_train, enable_checks=True),
                "y_val"   : categorical_to_vector(y_val,   enable_checks=True),
                "y_test"  : categorical_to_vector(y_test,  enable_checks=True)
            }
        else:
            dict_of_label_vectors = {
                "y_train" : categorical_to_vector(y_train, enable_checks=True),
                "y_test"  : categorical_to_vector(y_test,  enable_checks=True)
            }

        display_class_distributions(
            dict_of_label_vectors,
            selected_classes=selected_classes,
            dict_of_real_class_names=dict_of_real_class_names,
            precision=2
        )

    # ====================================================================== #

    if not(_return_validation_data):
        # checking that `X_val` and `y_val` have not been defined yet (within
        # this scope)
        assert ("X_val" not in locals()) and ("y_val" not in locals())

        # by design
        X_val, y_val = None, None

    print(f"\nThe raw MNIST dataset was successfully formatted. Done in {duration_data_formatting:.3f} seconds")

    return X_train, y_train, X_val, y_val, X_test, y_test


#!/usr/bin/env python

import matplotlib.pyplot as plt
import numpy as np
from lightcurve_fitting.speccal import readspec
from corner import corner
from astropy.table import hstack, Table
from astropy.time import Time
from astropy.constants import c
import argparse
from matplotlib.gridspec import GridSpec
from fitting import MCgauss, gaussian

def closestpt(x, y, refx, refy, ax=None):
    """
    Returns the index of the point in a plotted Line2D that is closest to a mouse click on the plot axes

    Parameters
    ----------
    x : float
        x co-ordinate (in pixels) of the mouse click

    y : float
        y co-ordinate (in pixels) of the mouse click

    refx : np.ndarray
        Numpy array of wavelength values

    refy : np.ndarray
        Numpy array of flux values

    ax : matplotlib.axes.Axes, optional
        Axes of the Line2D plot. Default is current axes.

    Returns
    -------
    i : int
        Index into (refx, refy) that is closest to (x, y) in pixel space
    """
    if ax is None:
        ax = plt.gca()
    pixx, pixy = ax.transData.transform(list(zip(refx, refy))).T
    dist = ((x - pixx) ** 2 + (y - pixy) ** 2) ** 0.5
    i = np.argmin(dist)
    return i

class LinePlot:
    """
    Plot of a spectrum where the fitting region can be chosen interactively, with MCMC trace plots shown below

    Parameters
    ----------
    line : matplotlib.lines.Line2D
        Matplotlib Line2D object containing the spectrum

    refx : np.ndarray
        Numpy array of wavelength values

    refy : np.ndarray
        Numpy array of flux values

    ax2 : matplotlib.axes._subplots.AxesSubplot
        Subplots containing the MCMC traces during burn-in

    ax3 : matplotlib.axes._subplots.AxesSubplot
        Subplots containing the MCMC traces after burn-in

    profile : str
        Type of profile (input by the user)

    linewl : float
        Wavelength of line

    otherwl : float
        Wavelength of second line

    corner : bool, optional
        Makes corner plots if true. Default is False

    Attributes
    ----------
    line : matplotlib.lines.Line2D
        Matplotlib Line2D object containing the spectrum

    refx : np.ndarray
        Numpy array of wavelength values

    refy : np.ndarray
        Numpy array of flux values

    press : bool
        Indicates whether the mouse button is currently pressed

    i0 : int
        Index of (refx, refy) closest to where the mouse button was pressed

    i1 : int
        Index of (refx, refy) closest to where the mouse button was released

    ax2 : matplotlib.axes._subplots.AxesSubplot
        Subplots containing the MCMC traces during burn-in

    ax3 : matplotlib.axes._subplots.AxesSubplot
        Subplots containing the MCMC traces after burn-in

    fits : list
        Line2D objects plotted during the MCMC fitting. These are cleared after each mouse click.

    params : list
        Measurements from the MCMC fit: [mean(velocity), std(velocity), mean(EW), std(EW), mean(flux), std(flux)]

    profile : str
        Type of profile (input by the user)

    linewl : float
        Wavelength of line

    otherwl : float
        Wavelength of second line

    corner : bool
        Makes corner plots if true

    Methods
    -------
    connect()
        connect to all the events needed
    disconnect()
        disconnect all stored connection ids
    """

    def __init__(self, line, refx, refy, ax2, ax3, profile, linewl, otherwl, corner=False):
        self.line = line
        self.refx = refx
        self.refy = refy
        self.press = False
        self.i0 = None
        self.i1 = None
        self.ax2 = ax2
        self.ax3 = ax3
        self.fits = []
        self.params = [np.nan, np.nan, np.nan, np.nan, np.nan, np.nan]
        self.profile = profile
        self.linewl = linewl
        self.otherwl = otherwl
        self.corner = corner

    def connect(self):
        """
        Connect to all the events we need.

        """
        self.cidpress = self.line.figure.canvas.mpl_connect(
            'button_press_event', self._on_press)
        self.cidrelease = self.line.figure.canvas.mpl_connect(
            'button_release_event', self._on_release)
        self.cidmotion = self.line.figure.canvas.mpl_connect(
            'motion_notify_event', self._on_motion)

    def _on_press(self, event):
        """on button press we will see if the mouse is over us and store some data"""
        if event.inaxes != self.line.axes: return
        self.press = True
        self.i0 = closestpt(event.x, event.y, self.refx, self.refy, self.line.axes)
        self.line.set_xdata([self.refx[self.i0]] * 2)
        self.line.set_ydata([self.refy[self.i0]] * 2)
        for ax in self.ax2[:-1] + self.ax3[:-1]:
            ax.cla()
            plt.setp(ax.get_xticklabels(), visible=False)
        self.ax2[-1].cla()
        self.ax3[-1].cla()
        for l in self.fits:
            self.line.axes.lines.remove(l)
        self.fits = []
        self.line.figure.canvas.draw()

    def _on_motion(self, event):
        """on motion we will move the line if the mouse is over us"""
        if not self.press: return
        x0 = self.line.get_xdata()
        y0 = self.line.get_ydata()
        x0[1] = event.xdata
        y0[1] = event.ydata
        self.line.set_xdata(x0)
        self.line.set_ydata(y0)
        self.line.figure.canvas.draw()

    def _on_release(self, event):
        """on release we reset the press data"""
        self.press = False
        self.i1 = closestpt(event.x, event.y, self.refx, self.refy, self.line.axes)
        x0 = self.line.get_xdata()
        y0 = self.line.get_ydata()
        x0[1] = self.refx[self.i1]
        y0[1] = self.refy[self.i1]
        self.line.set_xdata(x0)
        self.line.set_ydata(y0)
        self.line.figure.canvas.draw()

        i0 = min(self.i0, self.i1)
        i1 = max(self.i0, self.i1)
        x = self.refx[i0:i1]
        y = self.refy[i0:i1]
        sampler, pos, model = MCgauss(x, y, profile=self.profile, linewl=self.linewl, otherwl=self.otherwl)
        for i in range(len(self.ax2)):
            self.ax2[i].plot(sampler.chain[:, :, i].T, 'k', alpha=0.2)
        sampler.reset()
        sampler.run_mcmc(pos, 1000)
        for i in range(len(self.ax3)):
            self.ax3[i].plot(sampler.chain[:, :, i].T, 'k', alpha=0.2)

        parameters = {param_name: chain for param_name, chain in zip(model.param_names, sampler.flatchain.T)}
        width = parameters['stddev']
        center = parameters['mean']
        continuum = parameters['intercept']
        slope = parameters.get('slope', 0.)
        amplitude_1 = parameters.get('amplitude_1', 0.)
        center_1 = parameters.get('mean_1', 0.)

        y1 = y[:, np.newaxis]

        # subtract off the second emission line
        if 'two' in self.profile:
            y1 = y1 - gaussian(x[:, np.newaxis], amplitude_1, width, center_1, 0)
            self.fits += self.line.axes.plot(x, y1.mean(axis=-1), color='b')

        # subtract off the flat or linear continuum
        y1 = y1 - continuum - slope * (x[:, np.newaxis] - center)

        if 'pcyg' in self.profile:
            velocity = c * (1. - center_1 / center)  # absorption minimum
        else:
            velocity = 2. * np.sqrt(2. * np.log(2.)) * c * width / center  # FWHM of the emission line
        equivwidth = np.trapz(y1 / continuum, x, axis=0)
        flux = np.trapz(y1, x, axis=0)

        v = np.mean(velocity)
        dv = np.std(velocity)
        ew = np.mean(equivwidth)
        dew = np.std(equivwidth)
        f = np.mean(flux)
        df = np.std(flux)
        self.params = [v, dv, ew, dew, f, df]

        print('FWHM velocity = {:.3f} +/- {:.3f}'.format(v, dv))
        print('equivalent width = {:.3f} +/- {:.3f}'.format(ew, dew))
        print('flux = {:.3e} +/- {:.3e}'.format(f, df))

        ps = [sampler.flatchain[i] for i in np.random.choice(sampler.flatchain.shape[0], 100)]
        for p in ps:
            yfit = model(x, *p)
            l = self.line.axes.plot(x, yfit, color='k', alpha=0.05)
            self.fits.append(l[0])
            if 'two' in self.profile:
                if len(p) == 6:
                    A, s, rA, x0, y0, dx = p
                    m = 0
                elif len(p) == 7:
                    A, s, rA, m, x0, y0, dx = p
                yfitA = gaussian(x, A, s, x0, y0) - m * (x - x0)
                lA = self.line.axes.plot(x, yfitA, color='g', alpha=0.05)
                self.fits.append(lA[0])
                yfitB = gaussian(x, A * rA, s, x0 + dx, y0) - m * (x - x0)
                lB = self.line.axes.plot(x, yfitB, color='r', alpha=0.05)
                self.fits.append(lB[0])

        self.line.figure.canvas.draw()

        if self.corner:
            corner(sampler.flatchain, labels=['A', 's', 'm', 'x0', 'y0', 'dx'])

    def disconnect(self):
        """disconnect all the stored connection ids"""
        self.line.figure.canvas.mpl_disconnect(self.cidpress)
        self.line.figure.canvas.mpl_disconnect(self.cidrelease)
        self.line.figure.canvas.mpl_disconnect(self.cidmotion)

def plot_results(t, ycol='flux'):
    """
    Plot the results of the MCMC fit from the table

    Parameters
    ----------
    t : astropy.table.Table
        The astropy table with the results
    ycol : str, optional
        The column to plot on the vertical axis. Default: 'flux'.
    """

    plt.errorbar(t['phase'], t[ycol], t['d'+ycol], fmt='o')
    plt.xlabel('Phase')
    plt.ylabel(ycol.capitalize())
    plt.show()

def measure_specvels(spectra, profile, linewl, l2=0., viewwidth=20., corner=False):
    """
    Fits a line profile to the spectrum and measures velocity, equivalent width, and flux

    Parameters
    ----------
    spectra : astropy.table.Table
        Astropy table containing the spectra to be fitted

    profile : str
        Type of profile (input by the user)

    linewl : float
        Wavelength of line (units must match the input spectrum)

    l2 : float, optional
        Wavelength of second line, if fitting two lines (units must match the input spectrum).

    viewwidth : float, optional
        Width of the spectrum plot in wavelength (units must match the input spectrum). Default is 20.

    corner : bool, optional
        Makes corner plots if true. Default is False

    Returns
    -------
    tout : astropy.table.Table
        The astropy table with the results
    """

    tparams = Table(names=('v', 'dv', 'EW', 'dEW', 'flux', 'dflux'))
    if profile == 'two':
        nparams = 6
        figsize = (5, 15)
    elif profile == 'pcyg':
        nparams = 6
        figsize = (5, 15)
    elif profile == 'emis':
        nparams = 4
        figsize = (5, 10)
    elif profile == 'bc' or profile == 'rc':
        nparams = 5
        figsize = (5, 12.5)
    elif profile == 'pcygbc':
        nparams = 7
        figsize = (5, 15)
    elif profile == 'twobc':
        nparams = 7
        figsize = (5, 15)
    else:
        raise ValueError('unrecognized profile type')
    hr = [4] + [1] * nparams
    gs = GridSpec(nparams + 1, 2, height_ratios=hr)
    for spec in spectra:
        good = ~np.isnan(spec['flux'])
        wl = spec['wl'][good]
        flux = spec['flux'][good]
        yshown = flux[(wl > linewl - viewwidth) & (wl < linewl + viewwidth)]
        if len(yshown) == 0:
            print('out of range for', spec['filename'], '(phase = {:+.1f})'.format(spec['phase']))
            tparams.add_row([np.nan, np.nan, np.nan, np.nan, np.nan, np.nan])
            continue
        f = plt.figure(figsize=figsize)
        ax1 = plt.subplot(gs[0:2])
        ax1.step(wl, flux)
        ax1.axvline(linewl, color='k', linestyle='-.', zorder=0)
        ax1.axis([linewl - viewwidth, linewl + viewwidth, np.min(yshown), np.max(yshown)])
        ax1.xaxis.tick_top()
        ax1.xaxis.set_ticks_position('both')

        l = plt.Line2D([], [])
        ax1.add_artist(l)

        ax2 = [plt.subplot(gs[2])]
        ax3 = [plt.subplot(gs[3])]
        for row in range(2, nparams + 1):
            ax2.append(plt.subplot(gs[2 * row], sharex=ax2[0]))
            ax3.append(plt.subplot(gs[2 * row + 1], sharex=ax3[0]))
        for ax in ax3:
            ax.yaxis.tick_right()
            ax.yaxis.set_ticks_position('both')
        for ax in ax2[:-1] + ax3[:-1]:
            plt.setp(ax.get_xticklabels(), visible=False)
        plt.subplots_adjust(left=0.11, right=0.89, top=0.97, bottom=0.03, hspace=0, wspace=0)

        dr = LinePlot(l, wl, flux, ax2, ax3, profile, linewl, l2, corner)
        dr.connect()
        plt.show()

        # cont = input('Press enter to accept these values or r to reject them.\n')
        # plt.clf()
        # if dr.corner:
        #     plt.figure(1)
        #     plt.clf()
        # if cont == 'r':
        #     print('fit rejected')
        #     tparams.add_row([np.nan, np.nan, np.nan, np.nan, np.nan, np.nan])
        # else:
        tparams.add_row(dr.params)

    for col in tparams.colnames:
        tparams[col].format = '%.3e'
    tout = hstack([spectra[['filename', 'date', 'telescope', 'instrument', 'phase']], tparams])
    tout[['filename', 'v', 'EW', 'flux']].pprint(max_width=-1, max_lines=-1)
    plot_results(tout)
    return tout

def read_spectra(filenames, redshift, refmjd=0.):
    """
    Read spectra from file and convert to rest wavelength

    Parameters
    ----------
    filenames : list
        List of filenames of the spectra to be read

    redshift : float
        Redshift to correct the spectra

    refmjd : float, optional
        Reference MJD to calculate phase. Default: 0.

    Returns
    -------
    spectra : astropy.table.Table
        Astropy table containing the spectra to be fitted
    """

    wls = []
    fluxes = []
    dates = []
    telescopes = []
    instruments = []
    for fn in filenames:
        wl, flux, date, tel, inst = readspec(fn)
        wl /= (1 + redshift)  # correct to rest wavelength
        if wl[0] > wl[-1]:  # if the spectrum is stored backwards, flip it now
            wl = wl[::-1]
            flux = flux[::-1]
        wls.append(wl)
        fluxes.append(flux)
        dates.append(Time.now() if date is None else date)
        telescopes.append(tel)
        instruments.append(inst)
    if len(wls) and all([len(wl) == len(wls[0]) for wl in wls]):  # stupid astropy workaround
        filenames.append('fake')
        wls.append(np.array([]))
        fluxes.append(np.array([]))
        dates.append(Time.now())
        telescopes.append('fake')
        instruments.append('fake')
        spectra = Table([filenames, dates, telescopes, instruments, wls, fluxes],
                        names=['filename', 'date', 'telescope', 'instrument', 'wl', 'flux'],
                        dtype=[str, object, str, str, np.ndarray, np.ndarray])
        spectra = spectra[:-1]
    else:
        spectra = Table([filenames, dates, telescopes, instruments, wls, fluxes],
                        names=['filename', 'date', 'telescope', 'instrument', 'wl', 'flux'],
                        dtype=[str, object, str, str, np.ndarray, np.ndarray])
    spectra['phase'] = (spectra['date'].mjd - refmjd) / (1. + redshift)
    return spectra

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Measure line velocities and equivalent widths in spectra.')
    parser.add_argument('filenames', nargs='+', help='filenames of the spectra')
    parser.add_argument('-l', '--wavelength', type=float, default=6563., help="wavelength of line")
    parser.add_argument('--l2', type=float, default=0, help="wavelength of second line")
    parser.add_argument('-z', '--redshift', type=float, default=0., help="redshift to correct the spectra")
    parser.add_argument('-t', '--refmjd', type=float, default=0., help="reference MJD to calculate phase")
    parser.add_argument('-w', '--viewwidth', type=float, default=200., help="estimated width of the line")
    parser.add_argument('-p', '--profile', default='emis', choices=['two', 'pcyg', 'emis', 'bc', 'rc', 'pcygbc', 'twobc'],
                        help="emission line (emis) or P Cygni line (pcyg)?")
    parser.add_argument('-c', '--corner', action='store_true', help="show corner plot")
    args = parser.parse_args()

    spectra = read_spectra(args.filenames, args.redshift, refmjd=args.refmjd)
    tout = measure_specvels(spectra, args.profile, args.wavelength, args.l2, args.viewwidth, args.corner)
    savefile = input('Save results as... ')
    if savefile:
        tout.write(savefile, format='ascii.fixed_width_two_line')


# -*- coding: utf-8 -*-
"""
Created on Tue Dec 11 16:42:57 2018

@author: amkle
"""
import cv2
import numpy as np

def dark(img):
    shape_img = np.shape(img)
    for i in range(0, shape_img[0], 1):    # Alle farger under 80,80,80(rgb) blir gjort om til svart
        for j in range(0, shape_img[1], 1):
            a = img[i,j]
            if (80 >= a[0] and  80 >= a[1] and 80 >= a[2]):
                img[i,j]=[0,0,0]
    kernel = np.ones((12,12),np.uint8)
    img = cv2.morphologyEx(img, cv2.MORPH_OPEN, kernel)      #Fjerner alle flekker som er mindre enn 12,12 utenfor objektet.
    kernel = np.ones((7,7),np.uint8)
    img = cv2.morphologyEx(img, cv2.MORPH_CLOSE, kernel)      #Fjerner alle flekker som er mindre enn 7,7 innfor objektet (https://docs.opencv.org/3.0-beta/doc/py_tutorials/py_imgproc/py_morphological_ops/py_morphological_ops.html)

    return(img)


from multiprocessing import Pool, cpu_count
from ReconnectingRTSPGear.ReconnectingRTSPGear import ReconnectingRTSPGear
from vidgear.gears import StreamGear
from vidgear.gears.helper import reducer
import logging
import os

logging.basicConfig(level=logging.INFO)

def start_stream(cameras: dict[int, str]):
    camera_gears: dict[int, ReconnectingRTSPGear] = {}

    # Синхронно захватываем видео в процессе
    for key in cameras.keys():
        print(cameras[key])
        camera_gears[key] = ReconnectingRTSPGear(
            cam_address=cameras[key],
            reset_attempts=20,
            reset_delay=5,
            is_logging=True
        )

    stream_gears: dict[int, StreamGear] = {}

    #  Синхронно стримим на фронт
    for key in cameras.keys():
        stream_params = {"-vf": "scale=1280:-2", "-profile:v": "high444", "-livestream": True, "-streams": [
            {"-resolution": "640x360"}]}
        stream_gears[key] = StreamGear(output=f"streams/hls{key}/cam{key}.m3u8", format="hls",
                                       logging=True, **stream_params)

    while True:
        is_error = False
        for cam_key in cameras.keys():
            frame = camera_gears[cam_key].read()
            if frame is None:
                is_error = True
                break

            reduce_frame = reducer(frame, percentage=50)
            for key in stream_gears.keys():
                stream_gears[key].stream(reduce_frame)

        if is_error:
            for cam_key in cameras.keys():
                camera_gears[cam_key].stop()
                stream_gears[cam_key].terminate()
            break

def process(cameras: dict[int, str]):
    start_stream(cameras)

if __name__ == '__main__':
    num_workers = cpu_count()

    # get this from database
    sources = [({1: "./test_violence_videos/violence2.mp4"},)]

    for t in sources:
        for d in t:
            for cam_id in d.keys():
                if os.path.exists(f'streams/hls{cam_id}'):
                    try:
                        os.rmdir(f'streams/hls{cam_id}')
                        os.mkdir(f'streams/hls{cam_id}')
                        logging.info(f'Made new directory to save stream: streams/hls{cam_id}')
                    except OSError:
                        logging.error('Unable to recreate directory')
                else:
                    try:
                        os.mkdir(f'streams/hls{cam_id}')
                        logging.info(f'Made new directory to save stream: streams/hls{cam_id}')
                    except OSError:
                        logging.error('Unable to create directory')

    with Pool(processes=num_workers) as pool:
        pool.starmap(process,
                     [s for s in sources])


import datasets
from sklearn import linear_model

reg = linear_model.LogisticRegression(C=1e5, max_iter=1000)
reg.fit(datasets.X_TRAIN, datasets.Y_TRAIN)

pred = reg.predict(datasets.X_TEST)

print("Prediction:\n", pred, end="\n\n")
print("Test:\n", datasets.Y_TEST, end="\n\n")


from fastapi import APIRouter, WebSocket
from fastapi.responses import StreamingResponse
from pydantic import BaseModel

from extensions import gpt

class Question(BaseModel):
    question: str

front = APIRouter()

@front.get('/api/conversation/chat', tags=['front'])
async def get_history():
    history = gpt.get_history(username='user')
    if not history:
        return {
            'answer': []
        }
    else:
        return {
            'answer': history
        }

@front.post('/api/conversation/chat', tags=['front'])
async def chat(item: Question):
    answer = await gpt.chat(username='user', version='gpt-3.5-turbo', text=item.question)
    return {'answer': answer}

@front.post('/api/conversation/chat_stream', tags=['front'])
async def chat_stream(item: Question):
    answer = await gpt.chat_stream(username='user', version='gpt-3.5-turbo', text=item.question)
    return StreamingResponse(answer)

@front.websocket('/ws/conversation/chat_stream')
async def chat_stream_ws(websocket: WebSocket):
    await websocket.accept()
    question = await websocket.receive_text()
    answer = await gpt.chat_stream(username='user', version='gpt-3.5-turbo', text=question)
    async for part in answer:
        await websocket.send_text(part)
    await websocket.close()

@front.delete('/api/conversation/chat', tags=['front'])
async def delete_history():
    gpt.delete_history()
    return {'code': 1}


from pi_model import Model
#from speak import speak
from api_handler import AzureHandler, VoiceitHandler
import record
import speech_recognition as sr
from text_to_speech import tts
import os

model = Model("localhost", 'admin', 'silverserver2020?', 'voiceauth')
voiceit = VoiceitHandler("key_d79251d085214874b7479cdf67cd40b8", "tok_3f628df367944320a359510086825836")
r = sr.Recognizer()
m = sr.Microphone()

talkback = {
    'countdown': "Please say the selected phrase, starting recording in 3 ... 2 ... 1. ... Go. ",
    'startedRegistration': "Started registration process",
    'startedVerification': "started verification process"
}

def start_voiceit_registration():
    try:
        voiceit_resp = voiceit.create_user()
        if voiceit_resp['responseCode'] == 'SUCC':
            voiceit_userId = voiceit_resp['userId']

        voiceit_phrases_resp = voiceit.get_supported_phrases()
        if voiceit_phrases_resp['responseCode'] == 'SUCC':
            voiceit_phrases = [x['text'] for x in voiceit_phrases_resp['phrases']]

        resp_dic = {
            'voiceitUserId': voiceit_userId,
            'voiceitPhrases': voiceit_phrases
        }

        return resp_dic
    except Exception as e:
        return "Error: {0}".format(e)

def voiceit_enrollment(uid, phrases):
    iterations = 3
    audio_length = 5
    tts("please say your name", 'name')
    os.system("mpg321 ttsfiles/name.mp3")
    record.record(audio_length, "name.wav")
    name = file_to_text("name.wav")
    print(name)
    model.save_user(name = name, voiceit_id = uid)
    model.add_user_to_group(uid, groupId = 1)
    voiceit.add_user_to_group("grp_bc595294f94341568dc66dd7a09790bc", uid)
    print("user id: ", uid)
    print("supported phrases: \n", pretty_list(phrases))
    count = 1
    while count <= iterations:
        #speak("Please say the selected phrase, starting recording in 3 ... 2 ... 1. ... Go. ")
        os.system("mpg321 ttsfiles/countdown.mp3")
        record.record(audio_length, "test.wav")
        phrase = file_to_text("test.wav")
        res = voiceit.enroll_user(uid, phrase, "test.wav")
        print("response message: ",res['message'])
        print(phrase)
        print("")
        if res['responseCode'] == 'SUCC':
            st = "attempt {0} of enrollment successful".format(count)
            tts(st, 'response')
            count += 1
        else:
            msg = res['message']
            if 'grp_' in msg:
                position = msg.index("grp_")
                gid = msg[position: position+36]
                st = msg.replace(gid, "")
            else:
                st = msg
            tts(st, 'response')
        os.system("mpg321 ttsfiles/response.mp3")

    os.system("mpg321 ttsfiles/EnrollmentSuccess.mp3")

def voiceit_verification(groupId):
    audio_length = 5
    try:
        tts('started verification process', 'startedVerification')
        os.system('mpg321 ttsfiles/startedVerification')
        os.system("mpg321 ttsfiles/countdown.mp3")

        record.record(audio_length, "verify.wav")
        phrase = file_to_text("verify.wav")
        identification_response = voiceit.identify(groupId, phrase, 'verify.wav')
        if identification_response['responseCode'] == 'SUCC':
            print(identification_response)
            uid = identification_response['userId']
            db_response = model.get_user(uid)
            if len(db_response) > 1:
                name = db_response[1]
                s = "First step completed, {0} was successfully identified. Please wait for the complete verification".format(name)
            else:
                s = "First step completed, user was successfully identified. Please wait for the complete verification"

            tts(s, 'response')
            os.system('mpg321 ttsfiles/response.mp3')
            verification_response = voiceit.verify(uid, phrase, 'verify.wav')
            if verification_response['responseCode'] == 'SUCC':
                print(verification_responses)
                confidence = verification_response['confidence']
                st = "successfully verified user with confidence: {0}".format(confidence)
                tts(st, 'response')
            else:
                msg = verification_response['message']
                if 'grp_' in msg:
                    position = msg.index("grp_")
                    gid = msg[position: position+36]
                    st = msg.replace(gid, "")
                else:
                    st = msg

                tts(st, 'response')

            os.system('mpg321 ttsfiles/response.mp3')
        else:
            msg = identification_response['message']
            if 'grp_' in msg:
                position = msg.index("grp_")
                gid = msg[position: position+36]
                st = msg.replace(gid, "")
            else:
                st = msg

            tts(st, 'response')
            os.system('mpg321 ttsfiles/response.mp3')
    except Exception as e:
        return str(e)

def file_to_text(filename):
    with sr.AudioFile(filename) as source:
        audio = r.record(source)

    try:
        s = r.recognize_google(audio)
        return s
    except Exception as e:
        return str(e)

def pretty_list(ls):
    s = ""
    for item in ls:
        s += "\t"+ item +"\n"
    return s


import cv2
from matplotlib import pyplot as plt
from ..utils.media import get_image_path

path = get_image_path('ronaldo.jpg')
img = cv2.imread(path, 0)
edges = cv2.Canny(img, 100, 200)

plt.subplot(121), plt.imshow(img, cmap='gray')
plt.title('Original Image'), plt.xticks([]), plt.yticks([])
plt.subplot(122), plt.imshow(edges, cmap='gray')
plt.title('Edge Image'), plt.xticks([]), plt.yticks([])
plt.show()


import discord
from discord.ext import commands
from discord.ext.commands import cooldown
from discord.ext.commands.cooldowns import BucketType
import time
import asyncio
import asyncpg
from datetime import datetime, timedelta, timezone
from random import randint
from random import choice
import psutil
import os
import sys

colors = [0xE56B6B, 0xDD5151, 0xBA3434, 0xAB1F1F, 0x940808]
online = "<:online_status:506963324391653387>"
offline = "<:offline_status:506963324521414661>"
dnd = "<:dnd_status:506963324634791936>"
idle = "<:idle_status:506963324529803264>"

# git fix again again again

class misc(commands.Cog):
    def __init__(self, bot):
        self.bot = bot

    @commands.command(
        description="Give someone a slap", brief="Slap", usage="slap [user]"
    )
    async def slap(self, ctx, *, user: discord.Member):
        embed = discord.Embed(colour=choice(colors))
        embed.set_author(name=f"{ctx.author.name} slapped {user.name}! Ouchy!")
        embed.set_image(url="https://media.giphy.com/media/RXGNsyRb1hDJm/giphy.gif")
        await ctx.send(embed=embed)

    @commands.command(
        description="Give someone a hug! Awwww", brief="Hug", usage="hug [user]"
    )
    async def hug(self, ctx, *, user: discord.Member):
        embed = discord.Embed(colour=choice(colors))
        embed.set_author(name=f"{ctx.author.name} gave {user.name} hug! How cute!")
        embed.set_image(url="https://media.giphy.com/media/lXiRKBj0SAA0EWvbG/giphy.gif")
        await ctx.send(embed=embed)

    @commands.command(
        aliases=["pong"],
        description="Check the bots latency to Discord Websockets",
        brief="Check the bots ping",
        usage="ping",
    )
    async def ping(self, ctx):
        t_1 = time.perf_counter()
        await ctx.trigger_typing()
        t_2 = time.perf_counter()
        time_delta = round((t_2 - t_1) * 1000)
        await ctx.send(
            f":ping_pong: Websocket: `{round(self.bot.latency*1000)}` ms. Typing: `{time_delta}` ms"
        )

    @commands.command(
        aliases=["avy"],
        description="Get a users or your avatar",
        usage="avatar [user]",
        brief="Get a users or your avatar",
    )
    async def avatar(self, ctx, *, user: discord.Member = None):
        user = user or ctx.author
        embed = discord.Embed(colour=0x7289DA, description=f"[Link]({user.avatar_url})")
        embed.set_author(name=f"{user.name}'s avatar", url=user.avatar_url)
        embed.set_image(url=user.avatar_url_as(static_format="png"))
        await ctx.send(embed=embed)

    @commands.command(
        aliases=["about"],
        description="Get stats about the bot",
        brief="Check bot stats",
        usage="info",
    )
    async def info(self, ctx):
        cmds_used = await self.bot.pool.fetchval("""SELECT * FROM commands_used;""")
        uwulonian = await self.bot.pool.fetchval(
            "SELECT count(user_id) FROM user_settings;"
        )
        delta_uptime = datetime.utcnow() - self.bot.launch_time
        hours, remainder = divmod(int(delta_uptime.total_seconds()), 3600)
        minutes, seconds = divmod(remainder, 60)
        days, hours = divmod(hours, 24)
        cpu_usage = self.bot.process.cpu_percent()
        cpu_count = psutil.cpu_count()
        memory_usage = self.bot.process.memory_full_info().uss / (1024 ** 2)
        version = sys.version_info
        user_count = len(self.bot.users)
        embed = discord.Embed(color=0x7289DA)
        embed.set_author(name="Bot Stats")
        embed.add_field(
            name="Owner", value=f"<@300088143422685185> (mellowmarshe#0001)"
        )
        embed.add_field(
            name="Library",
            value=f"[discord.py\\[rewrite\\]](https://github.com/Rapptz/discord.py/tree/rewrite)",
        )
        embed.add_field(
            name="Language",
            value=f"Python {version.major}.{version.minor}.{version.micro}",
        )
        embed.add_field(name="Uptime", value=f"{days}d {hours}h {minutes}m")
        embed.add_field(name="Servers", value=len(self.bot.guilds))
        embed.add_field(
            name="Process",
            value=f"Memory {round(memory_usage, 2)}MiB\nCPU {cpu_usage/cpu_count}%",
        )
        embed.add_field(
            name="Bot Usage",
            value=f"{self.bot.commands_ran} commands used since boot\n{cmds_used + self.bot.commands_ran} commands used\n{uwulonian} uwulonians",
        )
        embed.add_field(name="Members", value=f"{user_count}")
        embed.add_field(
            name="Links",
            value="[Invite](https://discordapp.com/oauth2/authorize?client_id=508725128427995136&scope=bot&permissions=67501248) | [Support](https://discord.gg/733URZq) | [Donate](https://www.patreon.com/mellOwO) | [Vote](https://discordbots.org/bot/508725128427995136/vote) | [Website](https://uwu-bot.xyz/)",
        )
        await ctx.send(embed=embed)

    @commands.command(
        description="Get the link to the support server",
        brief="Get a link to support server.",
    )
    async def support(self, ctx):
        await ctx.send("The link to the support server is https://discord.gg/733URZq")

    @commands.command(
        description="Check our lovely Patrons!", brief="Check out our lovely patrons."
    )
    async def patrons(self, ctx):
        patrons = await self.bot.pool.fetch("SELECT * FROM p_users")
        e = discord.Embed(
            color=0x7289DA, description="Patreon https://www.patreon.com/mellOwO"
        )
        for i in range(len(patrons)):
            user = self.bot.get_user(patrons[i]["user_id"])
            time_p = datetime.now(timezone.utc) - patrons[i]["patron_time"]
            e.add_field(
                name=user.name,
                value=f"{patrons[i]['tier']} | Has been a patron for `{int(time_p.days)}` days!",
            )
        e.set_author(name="Our awesome Patrons")
        await ctx.send(embed=e)

    @commands.command(description="Send voting link", brief="Send a link to vote")
    async def upvote(self, ctx):
        await ctx.send(
            "You can vote for me here! https://discordbots.org/bot/508725128427995136/vote or <https://botsfordiscord.com/bot/508725128427995136/vote> I greatly appreciate voting as it helps the bot a lot"
        )

    @commands.command(description="Bot rules", aliases=["tos"], brief="Our bot rules")
    async def rules(self, ctx):
        await ctx.send(
            """
uwu rules. You must follow these or we will take action
`-` Don't spam or abuse commands with the intent to harm or slow the bot down. We want the best experience for our users anything to harm this is a problem.
`-` Don't rename or create an uwulonian with a harmful, racist, or any name that can offend someone. We will most likely delete your uwulonian if you do this.
`-` The use of scripts or anything to gain an advantage is forbidden.
`-` DM staff if you are having problems with the bot don't DM regular members unless they say you can.
"""
        )

    @commands.command(description="Get bot invite link", brief="Get invite for bot")
    async def invite(self, ctx):
        permissions = discord.Permissions()

        permissions.update(
            add_reactions=True,
            attach_files=True,
            change_nickname=True,
            embed_links=True,
            external_emojis=True,
            manage_messages=True,
            read_messages=True,
            read_message_history=True,
            send_messages=True,
            send_tts_messages=True,
            view_audit_log=True,
        )

        await ctx.send(
            f"<{discord.utils.oauth_url(self.bot.user.id, permissions=permissions)}>"
        )

    @commands.command(description="Snipe a deleted message", aliases=["snipsnop"])
    async def snipe(self, ctx, channel: discord.TextChannel = None):
        if channel is None:
            channel = ctx.channel
        snop = await self.bot.pool.fetchrow(
            "SELECT * FROM del_snipe WHERE channel_id = $1 AND guild_id = $2",
            channel.id,
            ctx.guild.id,
        )
        if snop is None:
            return await ctx.send("Nothing to snipe", delete_after=30)
        if channel.is_nsfw() is True and ctx.channel.is_nsfw() is False:
            return await ctx.send(
                "No sniping NSFW outside of NSFW channel.", delete_after=30
            )
        user = self.bot.get_user(snop["user_id"])
        guild = self.bot.get_guild(snop["guild_id"])
        chnl = discord.utils.get(guild.text_channels, id=snop["channel_id"])
        del_time = datetime.now(timezone.utc) - snop["msg_time"]
        if snop["msg_type"] == 1:
            e = discord.Embed(
                colour=0x7289DA, description="The image may not be visible"
            )
            e.set_author(name=f"{user.name} sent in {chnl.name}")
            e.set_image(url=snop["message"])
            e.set_footer(text=f"sniped!")
            await ctx.send(embed=e)
        else:
            user = self.bot.get_user(snop["user_id"])
            guild = self.bot.get_guild(snop["guild_id"])
            chnl = discord.utils.get(guild.text_channels, id=snop["channel_id"])
            e = discord.Embed(colour=0x7289DA, description=snop["message"])
            e.set_author(name=f"{user.name} said in {chnl.name}")
            e.set_footer(text=f"sniped!")
            await ctx.send(embed=e)

    @commands.command(description="Report a bug or user")
    async def report(
        self,
        ctx,
        *,
        report: commands.clean_content(
            fix_channel_mentions=False, use_nicknames=False, escape_markdown=True
        ),
    ):
        if len(report) > 512:
            return await ctx.send("Your report was too long")

        supp = self.bot.get_channel(529_722_574_301_560_842)
        e = discord.Embed(
            color=0xFF0000, description=f"```{report}```", timestamp=datetime.utcnow()
        )
        e.set_author(
            name=f"Report by {ctx.author}({ctx.author.id}) in {ctx.guild.name}({ctx.guild.id})"
        )
        e.set_footer(text="Report at")
        await ctx.send(
            "Your report was sent. Join the support server for news on your report!"
        )
        await supp.send(embed=e)

    # Sourced mainly from NekoBot https://github.com/hibikidesu/NekoBot/blob/master/modules/fun.py
    @commands.command()
    async def minesweeper(self, ctx, size: int = 5):
        m_offsets = [
            (-1, -1),
            (0, -1),
            (1, -1),
            (-1, 0),
            (1, 0),
            (-1, 1),
            (0, 1),
            (1, 1),
        ]
        m_numbers = [":one:", ":two:", ":three:", ":four:", ":five:", ":six:"]
        size = max(min(size, 8), 2)
        bombs = [
            [randint(0, size - 1), randint(0, size - 1)] for x in range(int(size - 1))
        ]
        is_on_board = lambda x, y: 0 <= x < size and 0 <= y < size
        has_bomb = lambda x, y: [i for i in bombs if i[0] == x and i[1] == y]
        embed = discord.Embed(color=0x7289DA, description="")
        embed.set_author(name="Press to play")
        for y in range(size):
            for x in range(size):
                tile = f"||{chr(11036)}||"
                if has_bomb(x, y):
                    tile = f"||{chr(128163)}||"
                else:
                    count = 0
                    for xmod, ymod in m_offsets:
                        if is_on_board(x + xmod, y + ymod) and has_bomb(
                            x + xmod, y + ymod
                        ):
                            count += 1
                    if count != 0:
                        tile = f"||{m_numbers[count - 1]}||"
                embed.description += tile
            embed.description += "\n"
        await ctx.send(embed=embed)

    @commands.command(aliases=["8ball"])
    async def eightball(self, ctx, *, question):
        responses = [
            "It is certain.",
            "It is decidedly so.",
            "Without a doubt.",
            "Yes - definitely.",
            "You may rely on it.",
            "As I see it, yes.",
            "Most likely.",
            "Outlook good.",
            "Yes.",
            "Signs point to yes.",
            "Reply hazy, try again.",
            "Ask again later.",
            "Better not tell you now.",
            "Cannot predict now.",
            "Concentrate and ask again.",
            "Don't count on it.",
            "My reply is no.",
            "My sources say no.",
            "Outlook not so good.",
            "Very doubtful.",
        ]
        msg = await ctx.send(f"Q: **{question}...**")
        await asyncio.sleep(3)
        await msg.edit(content=f"A: **{choice(responses)}**")

    @commands.command(aliases=["lovedar"])
    async def love(self, ctx, lover: discord.Member, lovee: discord.Member):
        msg = await ctx.send("**Calculating...**")
        await asyncio.sleep(3)
        seed(int(str(lover.id) + str(lovee.id)))
        if lover == lovee:
            love = 0
        else:
            love = randint(1, 10000) / 100
        await msg.edit(
            content=f"The love calculator says {lover.name} loves {lovee.name} **{love}**%"
        )

def setup(bot):
    bot.add_cog(misc(bot))


from flask_login import current_user, login_required
from flask import render_template, abort, redirect, url_for, request, session
from app.models import Post, Category
from .. import admin
from ..forms import Edit_Post_Form

@admin.route("/posts/")
@admin.route("/posts/<int:page>")
@login_required
def all_posts(page=1):
    posts = current_user.posts
    # return all posts if user is admin.
    if current_user.is_admin():
        posts = Post.get_all_items_pagination_admin(page=page)
    return render_template("all_posts.html", posts=posts)

@admin.route("/posts/new/", methods=["GET", "POST"])
@login_required
def new_post():
    form = Edit_Post_Form()
    post = Post()
    form.category_id.choices = [(category.id, category.name)
                                for category in Category.get_all_items()]
    if form.load_form_to_object(post, query="post"):
        post.add_itself()
        return redirect(post.url())
    return render_template("edit_post.html", form=form)

@admin.route("/posts/<post_id>/delete/", methods=["GET", "POST"])
@login_required
def delete_post(post_id):
    post = Post.get_item_by_id(post_id)
    post.delete_itself()
    return redirect(url_for("admin.all_posts"))

@admin.route("/posts/<post_id>/edit/", methods=["GET", "POST"])
@login_required
def edit_post(post_id):
    if current_user.is_owner_or_admin(post_id):
        form = Edit_Post_Form()
        post = Post.get_item_by_id(post_id)
        form.category_id.choices = [(category.id, category.name)
                                    for category in Category.get_all_items()]
        post.tags_temp = ','.join(t.name for t in post.tags)
        form.load_object_to_form(post)
        if form.load_form_to_object(post, query='post'):
            return redirect(post.url())
        return render_template("edit_post.html", form=form)
    else:
        # raise 403 Forbidden for insuficcient permissions
        abort(403)


import numpy as np
import matplotlib.pyplot as plt
from math import factorial
import os
from pathlib import Path

# Building piecewise constant basis functions
def build_phi_piecewise(param):
    index_list = np.linspace(0, param["nbData"], param["nbFct"]+1)
    phi = np.zeros((param["nbData"],param["nbFct"]))
    for i in range(param["nbFct"]):
        phi[int(index_list[i]):int(index_list[i+1]), i] = 1
    return phi

# Building radial basis functions (RBFs)
def build_phi_rbf(param):
    Ts = np.arange(0,param["nbData"],1)
    bw = param["nbData"] / param["nbFct"]
    avg = bw // 2
    sig = bw
    phi = []
    for i in range(param["nbFct"]):
        phi_k = 1 / (2*np.pi * sig) * np.exp(-1*(Ts-avg)**2 / (2*sig**2))
        phi += [phi_k]
        avg += bw
    return np.asarray(phi).T

# Building Bernstein basis functions
def build_phi_bernstein(param):
    t = np.linspace(0,1,param["nbData"])
    phi = np.zeros((param["nbData"],param["nbFct"]))
    for i in range(param["nbFct"]):
        phi[:,i] = factorial(param["nbFct"]-1) / (factorial(i) * factorial(param["nbFct"]-1-i)) * (1-t)**(param["nbFct"]-1-i) * t**i
    return phi

# Building Fourier basis functions
def build_phi_fourier(param):
    d = np.ceil((param["nbFct"]-1)/2)
    k = np.arange(-d,d+1,1).reshape((-1,1))
    param["nbFct"] = len(k)
    t = np.linspace(0,1,param["nbData"]).reshape((-1,1))
    phi = np.exp( t.T * k * 2 * np.pi * 1j ) / (param["nbData"])
    return phi.T

# General param parameters
# ===============================
param = {
    "nbFct" : 10, # Number of basis functions
    "nbVar" : 2, # Dimension of position data
    "nbData" : 200, # Number of datapoints in a trajectory
    "nbDemos" : 5, # Number of demonstrations
}

# Load handwriting data
FILE_PATH = os.path.dirname(os.path.realpath(__file__))
DATASET_LETTER_FILE = "data/2Dletters/S.npy"
# x = np.load(str(Path(FILE_PATH,DATASET_LETTER_FILE)))[0,:,:2].flatten() #Load single demonstration
X = np.load(str(Path(FILE_PATH,DATASET_LETTER_FILE)))[:param["nbDemos"],:,:2].reshape((param["nbDemos"],param["nbData"]*2)).T #Load multiple demonstrations

t = np.linspace(0,1,param["nbData"]) # Time range

# Fourier basis functions require additional symmetrization (mirroring) if the signal is a discrete motion
# x = np.hstack(( x , x[::-1] ))
# param["nbData"] *= 2

# Generate MP with various basis functions
# ============================================
# phi = build_phi_piecewise(param) #Piecewise constant
# phi = build_phi_rbf(param) #Radial basis functions
phi = build_phi_bernstein(param) #Bernstein polynomials
# phi = build_phi_fourier(param) #Fourier basis functions

psi = np.kron(phi, np.identity(param["nbVar"])) # Transform to multidimensional basis functions

# w = np.linalg.solve(psi.T @ psi + np.identity(param["nbVar"] * param["nbFct"]) * 1e-8, psi.T @ x) # Estimation of superposition weights from data
# w = np.linalg.pinv(psi) @ x # Estimation of superposition weights from data
# w = np.linalg.inv(psi.T @ psi) @ psi.T @ x # Estimation of superposition weights from data
W = np.linalg.pinv(psi) @ X # Estimation of superposition weights from data (for multiple demonstrations)
w = np.mean(W, 1) # Average w

x_hat = psi @ w # Reconstruction data

# Fourier basis functions require de-symmetrization of the signal after processing (for visualization)
# param["nbData"] /= 2
# x = x[:int(param["nbData"]*param["nbVar"])]
# x_hat = x_hat[:int(param["nbData"]*param["nbVar"])]

# Plotting
# =========
fig,axs = plt.subplots(2,1)
# axs[0].plot(x[::2], x[1::2], c='black', label='Demonstration')
axs[0].plot(X[::2], X[1::2], c='black', label='Demonstration')
axs[0].plot(x_hat[::2], x_hat[1::2], c='r', label='Reproduction')
axs[0].axis("off")
axs[0].axis("equal")
axs[0].legend()

axs[1].set_xticks([0,param["nbData"]])
axs[1].set_xticklabels(["0","T"])
for j in range(param["nbFct"]):
    axs[1].plot(phi[:,j])
axs[1].set_ylabel("$\phi_k$")

plt.show()


from enum import Enum

from roid import (
    CommandsBlueprint,
    Response,
    ResponseFlags,
    Interaction,
    ButtonStyle,
    InvokeContext, Option,
)
from roid.objects import Channel, ChannelType, MemberPermissions as MemberPerms
from roid.helpers import check, require_user_permissions
from roid.exceptions import AbortInvoke, Forbidden, NotFound, DiscordServerError

from crunchy.app import CommandHandler
from crunchy.tools import assetloader
from crunchy.config import DISCORD_API, SUPPORT_SERVER_URL

REQUIRED_PERMISSIONS = MemberPerms.MANAGE_GUILD | MemberPerms.MANAGE_WEBHOOKS
NOT_ENOUGH_DATA = AbortInvoke(
    content="<:HimeSad:676087829557936149> Oops! You've not given me enough information to work with here.",
    flags=ResponseFlags.EPHEMERAL,
)

class EventType(Enum):
    """The type of event to register with the API."""

    News = "news"
    Releases = "releases"

events_blueprint = CommandsBlueprint()

async def get_webhook_url(
    app: CommandHandler,
    channel: Channel,
    sub_type: EventType,
) -> str:
    """
    Creates a webhook in the given channel and returns the computed URL
    from the response.

    Any issue with the request is returned with a roid.exception's HTTPException
    subclass.

    Args:
        app:
            The slash commands app which has a http handler.

        channel:
            The channel to target the webhook creation.

        sub_type:
            Which event type the webhook belongs to (news or releases).

    Returns:
        A formatted webhook execution url.
    """

    avatar = assetloader.get_base64_asset("crunchy-128.webp")
    data = await app.http.request(
        "POST",
        f"/channels/{channel.id}/webhooks",
        pass_token=True,
        json={
            "name": f"Crunchy Anime {sub_type.name}",
            "avatar": f"data:image/webp;base64,{avatar}",
        },
    )

    return f"{DISCORD_API}/webhooks/{data['id']}/{data['token']}"

async def submit_webhook(
    app: CommandHandler,
    sub_type: EventType,
    guild_id: int,
    webhook_url: str,
):
    """
    Submits a webhook to the Crunchy API.

    Args:
        app:
            The slash commands app with the client attribute linking to the CrunchyApi
            handler.

        sub_type:
            Which event type the webhook belongs to (news or releases).

        guild_id:
            The id of the guild the webhook belongs to.

        webhook_url:
            The url of the webhook.
    """
    payload = {
        "guild_id": str(guild_id),
        "webhook_url": webhook_url,
    }

    await app.client.request("POST", f"/events/{sub_type.value}/update", json=payload)

@require_user_permissions(REQUIRED_PERMISSIONS)
@events_blueprint.command(
    "add-news-channel",
    description=(
        "Add Crunchy's news webhook to a channel of "
        "your choice to get the latest Anime news."
    ),
    defer_register=False,
)
async def add_news_channel(
    app: CommandHandler,
    interaction: Interaction,
    channel: Channel = Option(channel_types=[ChannelType.GUILD_TEXT]),
) -> Response:
    url = await get_webhook_url(
        app=app,
        channel=channel,
        sub_type=EventType.News,
    )

    await submit_webhook(
        app=app,
        sub_type=EventType.News,
        guild_id=interaction.guild_id,
        webhook_url=url,
    )

    return Response(
        content=(
            f"<:exitment:717784139641651211> All done! I'll send"
            f" to <#{channel.id}> when I get some news."
        ),
        flags=ResponseFlags.EPHEMERAL,
        components=[[test_button]],
        component_context={"channel_id": channel.id, "webhook_url": url},
    )

@require_user_permissions(REQUIRED_PERMISSIONS)
@events_blueprint.command(
    "add-release-channel",
    description=(
        "Add Crunchy's release webhook to a channel of "
        "your choice to get the latest Anime release details."
    ),
    defer_register=False,
)
async def add_release_channel(
    app: CommandHandler,
    interaction: Interaction,
    channel: Channel = Option(channel_types=[ChannelType.GUILD_TEXT]),
) -> Response:
    url = await get_webhook_url(
        app=app,
        channel=channel,
        sub_type=EventType.Releases,
    )

    await submit_webhook(
        app=app,
        sub_type=EventType.Releases,
        guild_id=interaction.guild_id,
        webhook_url=url,
    )

    return Response(
        content=(
            f"<:exitment:717784139641651211> All done! I'll send to "
            f"<#{channel.id}> when a new Anime episode is out!"
        ),
        flags=ResponseFlags.EPHEMERAL,
        components=[[test_button]],
        component_context={"channel_id": channel.id, "webhook_url": url},
    )

@events_blueprint.button("ðŸš€ Test", style=ButtonStyle.PRIMARY, oneshot=True)
async def test_button(app: CommandHandler, ctx: InvokeContext):
    """
    A button that sends a test POST request to the webhook with a test message.

    If The request fails the error handler takes care of the responses otherwise
    it responds as a sucess.
    """
    try:
        channel_id = ctx["channel_id"]
        webhook_url = ctx["webhook_url"]
    except KeyError:
        return Response(
            content=(
                f"Oops! Something has gone awfully wrong in the (events:test_button:KeyError) handler, "
                f"please contact the developer @ {SUPPORT_SERVER_URL}"
            )
        )

    await app.http.request(
        "POST",
        f"{webhook_url}?wait=true",
        pass_token=False,
        json={"content": "ðŸš€ Testing, testing, I think we're ready!"},
    )

    return Response(
        content=f"I've sent a test message! If it appears in <#{channel_id}>",
        flags=ResponseFlags.EPHEMERAL,
    )

@test_button.error
async def on_button_error(_: Interaction, e: Exception):
    if isinstance(e, NotFound):
        return Response(
            content=(
                "<:HimeSad:676087829557936149> Weird!? This webhook seems to have "
                "been delete between me creating and you clicking the test button!"
            ),
            flags=ResponseFlags.EPHEMERAL,
        )

    if isinstance(e, DiscordServerError):
        return Response(
            content=(
                "<:HimeSad:676087829557936149> Oh no! Discord seem to be having some "
                "issues right now, this doesn't mean your event feed hasn't been "
                "added it just means we aren't able to test it right now."
            ),
            flags=ResponseFlags.EPHEMERAL,
        )

@add_news_channel.error
@add_release_channel.error
async def on_command_error(_: Interaction, e: Exception):
    if isinstance(e, Forbidden):
        return Response(
            content=(
                "<:HimeSad:676087829557936149> Oops! Looks like I'm missing permissions "
                "to do this. Make sure I have the permission `MANAGE_WEBHOOKS` and try again."
            ),
            flags=ResponseFlags.EPHEMERAL,
        )

    raise e


from functools import reduce

def run():
    fib_series = lambda n: reduce(lambda x, u: x+[x[-1]+x[-2]],range(n-2), [0, 1])
    # n = 3
    # fib_series = list(reduce(lambda x: x+[x[-1]+x[-2]],range(n-2), [0, 1]))
    print("Fibonacci series upto 2:")
    print(fib_series(3))
    # print(fib_series)

if __name__ == '__main__':
    run()


from __future__ import annotations
import openai
# from asyncstdlib import enumerate, sorted
from discord import Member, User, TextChannel, Thread, Message, Interaction, Embed, Color
from discord.ext.commands import Bot
from naruto.utils.config import Config
from dataclasses import dataclass
from typing import Union, List

@dataclass
class AIThread:
    op: Union[User, Member]
    topic: str
    channel: Thread

    async def close(self, interaction: Interaction) -> AIThread:
        await interaction.response.send_message("Closing...")
        await self.channel.delete()
        ThreadManager.threads.remove(self)
        return self

class ThreadManager:

    threads: List[AIThread] = []

    def __init__(self, bot: Bot) -> None:
        self.client = bot
        openai.api_key = Config.OPENAI_KEY

    async def last_messages(self, limit: int, channel: Thread) -> str:
        messages = channel.history(limit=limit)
        mystr = []
        iterable = [response async for response in messages if response.author.bot]
        for index, message in enumerate(reversed(iterable)):
            if index == 0:
                mystr.append(f'The first message, written by '
                             f'{"a person" if not message.author.bot else "you"}, says: "{message.content}"')
            else:
                print("occurred")
                mystr.append(f'This is the message after that message, written by '
                             f'{message.author.name if not message.author.bot else "you"}: "{message.content}"')
        return '; '.join(mystr[1:])

    async def conversate(self, message: Message, user: Union[User, Member], channel: Thread) -> str:
        thread = await self.client.fetch_channel(channel.id)
        can_respond = await self.can_respond(user, thread)
        text = await self.prompt_gen(can_respond, message, thread)

        return text

    async def prompt_gen(self, can_respond: bool, message: Message, thread: Thread) -> str:
        text_response = ""
        if can_respond:
            limit = 5 if thread.message_count >= 5 else thread.message_count
            memory = await self.last_messages(limit, thread)
            response = openai.Completion.create(
                engine=Config.MODEL,
                prompt=f"These messages are your memory: {memory}."
                       f"Please respond to the prompt: {message.content} as if you were {Config.CHARACTER}. "
                       f"Respond with less than 185 characters. "
                       f"Be friendly but serious in your responses, and act like {Config.CHARACTER} "
                       f"and naturally like them the character, "
                       f"because you ARE {Config.CHARACTER}.",
                max_tokens=2048,
                temperature=0.75
            )
            text_response = response.choices[0].text
        return text_response

    async def create_thread(self, user: Union[User, Member], topic: str, channel: TextChannel) -> AIThread:
        channel = await channel.create_thread(name=topic)
        embed = Embed(title="Naruto Thread", description=f"**Created by**: ``{user.name}``", color=Color.blue())
        await channel.send(embed=embed)
        thread = AIThread(user, topic, channel)
        self.threads.append(thread)
        return thread

    async def get_thread(self, channel: Thread) -> AIThread:
        for thread in self.threads:
            if thread.channel.id == channel.id:
                return thread

    async def can_respond(self, user: Union[User, Member], channel: Thread) -> bool:
        if not user.bot:
            for thread in self.threads:
                if channel.id == thread.channel.id:
                    return True
            return False

class AI:
    pass


import cv2
import argparse
import os

import torch
from torch.utils.data import DataLoader

from apex import amp
from apex.parallel import (
    DistributedDataParallel, convert_syncbn_model)

from ignite.engine import Events
from ignite.handlers import ModelCheckpoint, global_step_from_engine
from ignite.contrib.handlers import (
    create_lr_scheduler_with_warmup, CosineAnnealingScheduler)
from ignite.contrib.handlers import ProgressBar

import albumentations as albu
from albumentations.pytorch import ToTensorV2 as ToTensor

from neural.models.segmentation.contextnet import ContextNet
from neural.engines.segmentation import (
    create_segmentation_trainer, create_segmentation_evaluator)
from neural.data.cityscapes import Cityscapes

from neural.losses import OHEMLoss

from neural.utils.training import (
    setup_distributed, get_datasets_root, create_sampler)

parser = argparse.ArgumentParser()
parser.add_argument('--batch_size', type=int, required=True)
parser.add_argument('--learning_rate', type=float, required=True)
parser.add_argument('--weight_decay', type=float, default=4e-5)
parser.add_argument('--epochs', type=int, required=True)
parser.add_argument('--crop_size', type=int, default=768)
parser.add_argument('--state_dict', type=str, required=False)
parser.add_argument('--width_multiplier', type=int, default=1)

parser.add_argument('--distributed', action='store_true')
parser.add_argument('--local_rank', type=int, default=0)
args = parser.parse_args()

distributed = args.distributed
world_size, world_rank, local_rank = setup_distributed(
    distributed, args.local_rank)

device = torch.device('cuda')

crop_size = args.crop_size

train_tfms = albu.Compose([
    albu.RandomScale([-0.5, 1.0], interpolation=cv2.INTER_CUBIC, always_apply=True),
    albu.RandomCrop(512, 1024),
    albu.HorizontalFlip(),
    albu.HueSaturationValue(),
    albu.Normalize(),
    ToTensor(),
])
val_tfms = albu.Compose([
    albu.Normalize(),
    ToTensor(),
])

dataset_dir = get_datasets_root('cityscapes')
train_dataset = Cityscapes(dataset_dir, split='train', transforms=train_tfms)
val_dataset = Cityscapes(dataset_dir, split='val', transforms=val_tfms)

sampler_args = dict(world_size=world_size,
                    local_rank=local_rank,
                    enable=distributed)

train_loader = DataLoader(
    train_dataset,
    batch_size=args.batch_size,
    drop_last=True,
    num_workers=8,
    sampler=create_sampler(train_dataset, **sampler_args),
    shuffle=not distributed,
)
val_loader = DataLoader(
    val_dataset,
    batch_size=4,
    shuffle=False,
    drop_last=False,
    num_workers=8,
    sampler=create_sampler(val_dataset, training=False, **sampler_args),
)

model = ContextNet(3, 19,
                   scale_factor=4,
                   width_multiplier=args.width_multiplier)

if args.state_dict is not None:
    state_dict = torch.load(args.state_dict, map_location='cpu')
    model.load_state_dict(state_dict, strict=True)

model = model.to(device)

optimizer = torch.optim.AdamW(
    [
        {'params': (p for p in model.parameters() if len(p.shape) != 1),
         'weight_decay': args.weight_decay, },
        {'params': (p for p in model.parameters() if len(p.shape) == 1), },
    ],
    lr=args.learning_rate,
)

loss_fn = OHEMLoss(ignore_index=255)
loss_fn = loss_fn.cuda()

scheduler = CosineAnnealingScheduler(
    optimizer, 'lr',
    args.learning_rate, args.learning_rate / 1000,
    args.epochs * len(train_loader) - 1000,
)
scheduler = create_lr_scheduler_with_warmup(
    scheduler, 0, args.learning_rate, 1000)

model, optimizer = amp.initialize(model, optimizer, opt_level="O2")
if args.distributed:
    model = convert_syncbn_model(model)
    model = DistributedDataParallel(model)

trainer = create_segmentation_trainer(
    model, optimizer, loss_fn,
    device=device,
    use_f16=True,
)
trainer.add_event_handler(Events.ITERATION_COMPLETED, scheduler)

evaluator = create_segmentation_evaluator(
    model,
    device=device,
    num_classes=19,
)

if local_rank == 0:
    ProgressBar(persist=True).attach(trainer, ['loss'])
    ProgressBar(persist=True).attach(evaluator)

@trainer.on(Events.EPOCH_COMPLETED(every=5))
def evaluate(engine):
    evaluator.run(val_loader)

if local_rank == 0:
    checkpointer = ModelCheckpoint(
        dirname=os.path.join('checkpoints', 'contextnet14'),
        filename_prefix='contextnet',
        score_name='miou',
        score_function=lambda engine: engine.state.metrics['miou'],
        n_saved=5,
        global_step_transform=global_step_from_engine(trainer),
    )
    evaluator.add_event_handler(
        Events.COMPLETED, checkpointer,
        to_save={'model': model if not args.distributed else model.module},
    )

trainer.run(train_loader, max_epochs=args.epochs)


# -*- coding: utf-8 -*-
"""
Created on Tue Jun 29 10:11:23 2021

@author: leona
"""
import numpy as np
import matplotlib.pyplot as plt
import cv2

img = cv2.imread("./Fotos/image3d.jpg")
img = cv2.cvtColor(img,cv2.COLOR_BGR2RGB)
imgg = cv2.cvtColor(img,cv2.COLOR_RGB2GRAY)
imgg = np.float32(imgg)

imgs = np.zeros((1280,1280))

row, cols = imgs.shape
imgs[:,:] = 255

for i in np.arange(0,1300,200):
    imgs[:,i] = 0
    imgs[i,:] = 0

plt.figure(2)
plt.imshow(np.uint8(imgs), cmap='gray')

mean = 0.0   # some constant
std = 200    # some constant (standard deviation)
noisy_img = img + np.random.normal(mean, std, img.shape)
# noisy_img_clipped = np.uint8(np.clip(noisy_img, 0, 255) )

# img[dist]=[0,0,255]
plt.figure(1)
plt.imshow(np.uint8(noisy_img))
plt.show()

imgg = imgs
imgg = np.float32(imgg)
Sx = np.array([[-1, 0, 1], [-2, 0, 2],[-1, 0, 1]])
Sy = np.array([[-1, -2, -1], [0, 0, 0],[1, 2, 1]])

Ix = cv2.filter2D(imgg, -1, Sx)
Iy = cv2.filter2D(imgg, -1, Sy)

# Ix = cv2.Sobel(imgg,cv2.CV_64F,1,0,ksize=3)  # x
# Iy = cv2.Sobel(imgg,cv2.CV_64F,0,1,ksize=3)  # y

Ixx = Ix*Ix
Iyy = Iy*Iy
Ixy = Ix*Iy

Sxx = cv2.GaussianBlur(Ixx,(5,5), cv2.BORDER_DEFAULT)
Syy = cv2.GaussianBlur(Iyy,(5,5), cv2.BORDER_DEFAULT)
Sxy = cv2.GaussianBlur(Ixy,(5,5), cv2.BORDER_DEFAULT)

r, c = imgg.shape
k = 0.04

R = (Sxx*Syy - Sxy*Sxy) - k*((Sxx + Syy)**2)

# R = np.zeros((r,c))
# for i in np.arange(0,r):
#     for j in np.arange(0,c):
#         M = [[Sxx[i,j], Sxy[i,j]], [Sxy[i,j],Syy[i,j]]]
#         R[i,j] = np.linalg.det(M) - k*((np.trace(M))**2)

# aa = np.zeros((r,c))
# aa[R>0.04*R.max()]=[255]

# t = 10
# dist = np.full((r, c), True, dtype=bool)

# for i in np.arange(0,r):
#     for j in np.arange(0,c):

#         dist[i:i+t, j:j+t] = R[i:i+t, j:j+t] >= (R[i:i+t, j:j+t].max())

imgt = imgg
imgt[R>0.04*R.max()]=[255]
# img[dist]=[0,0,255]
plt.figure(0)
plt.imshow(imgt, cmap = 'gray')
plt.show()

mean = 0.0   # some constant
std = 200    # some constant (standard deviation)
noisy_img = img + np.random.normal(mean, std, img.shape)
# noisy_img_clipped = np.uint8(np.clip(noisy_img, 0, 255) )

# img[dist]=[0,0,255]
plt.figure(1)
plt.imshow(np.uint8(noisy_img))
plt.show()

# u = [0,0]
# p0 = [0,0]
# p1 = [x,y]

# p1 = u + t*p0
for a in [-2,2]:
    for b in np.arange(0,10,2):

        x = np.linspace(-3, 3, 1000)

        y = a*x + b

        plt.figure(2)

        plt.plot(x, y, color='black');

imgs = np.zeros((1280,1280))

row, cols = imgs.shape
imgs[:,:] = 255

for i in np.arange(0,1300,200):
    imgs[:,i] = 0
    imgs[i,:] = 0

plt.figure(2)
plt.imshow(np.uint8(imgs), cmap='gray')


#!/usr/bin/env python

from runtest import TestBase
import subprocess as sp

TDIR='xxx'

class TestCase(TestBase):
    def __init__(self):
        TestBase.__init__(self, 'sleep', """
{"traceEvents":[
{"ts":56466448731,"ph":"B","pid":32537,"name":"main"},
{"ts":56466448731,"ph":"B","pid":32537,"name":"foo"},
{"ts":56466448742,"ph":"B","pid":32537,"name":"bar"},
{"ts":56466448743,"ph":"B","pid":32537,"name":"usleep"},
{"ts":56466450823,"ph":"E","pid":32537,"name":"usleep"},
{"ts":56466450827,"ph":"E","pid":32537,"name":"bar"},
{"ts":56466450834,"ph":"E","pid":32537,"name":"foo"},
{"ts":56466450835,"ph":"E","pid":32537,"name":"main"}
], "metadata": {
"command_line":"uftrace record -d xxx t-sleep ",
"recorded_time":"Mon Oct  3 22:45:57 2016"
} }
""")

    def pre(self):
        record_cmd = '%s record -d %s %s' % (TestBase.ftrace, TDIR, 't-' + self.name)
        sp.call(record_cmd.split())
        return TestBase.TEST_SUCCESS

    def runcmd(self):
        return '%s dump -d %s -t 1ms --chrome' % (TestBase.ftrace, TDIR)

    def post(self, ret):
        sp.call(['rm', '-rf', TDIR])
        return ret

    def sort(self, output):
        """ This function post-processes output of the test to be compared .
            It ignores blank and comment (#) lines and remaining functions.  """
        import json

        result = []
        o = json.loads(output)
        for ln in o['traceEvents']:
            result.append("%s %s" % (ln['ph'], ln['name']))
        return '\n'.join(result)


from dataclasses import dataclass, field
from pathlib import Path

from tqdm import tqdm

VALID_CHARACTERS = "йцукенгшщзхъфывапролджэячсмитьбю -"

@dataclass(init=False)
class Sentence:
    """
    Represents one sentence of a text
    """

    words: list[str] = field(default_factory=list)

    def __init__(self, sentence: str) -> None:
        sentence = self.__cleanup_sentence(sentence)
        self.words = list(
            filter(
                lambda word: len(word) > 0,
                sentence.split(),
            )
        )

    def __cleanup_sentence(self, sentence: str) -> str:
        """
        Removes all unnecessary characters from a given string

        Parameters
        ----------
        sentence : str

        Returns
        -------
        str
            Cleaned-up sentence
        """
        return "".join(
            filter(
                lambda c: c in VALID_CHARACTERS,
                sentence.lower().replace("ё", "е"),
            )
        ).strip()

    def __len__(self) -> int:
        return len(self.words)

    def __hash__(self) -> int:
        return hash(" ".join(self.words))

@dataclass
class TextFile:
    """
    Opens a file, cleans up the contents and parses the text into sentences.
    """

    input_file: Path
    sentences: list[Sentence] = field(default_factory=list)

    def __post_init__(self):
        text = self.__get_text()
        self.sentences = list(
            map(lambda sentence: Sentence(sentence), self.__split_into_sentences(text))
        )

    def __get_text(self) -> str:
        """
        Reads text from the file and returns it
        """
        text = ""
        with open(self.input_file, "r") as fh:
            text = fh.read()
        return text

    def __split_into_sentences(self, text: str) -> list[str]:
        text = " ".join(text.splitlines())
        return text.split(".")

@dataclass
class DataLoader:
    """
    Loads and parses all data in a given directory.
    """

    data_dir: Path
    parsed_files: list[TextFile] = field(default_factory=list)

    def __post_init__(self):
        print("Parsing files...")

        self.parsed_files = list(
            map(
                lambda path: TextFile(input_file=path),
                tqdm(self.__get_all_file_paths()),
            )
        )

    def __get_files_recursive(self, path: Path) -> list[Path]:
        """
        Recursively traverses a given directory and saves all text files.

        Parameters
        ----------
        path : Path
            Current directory

        Returns
        -------
        list[Path]
            All text files
        """
        result: list[Path] = []
        for child in path.iterdir():
            if child.is_dir():
                result += self.__get_files_recursive(child)
            else:
                if child.suffix == ".txt":
                    result.append(child)
        return result

    def __get_all_file_paths(self) -> list[Path]:
        """
        Returns all text file paths in `data_dir`.

        Returns
        -------
        list[Path]
            Paths to text files with data inside.
        """
        return self.__get_files_recursive(self.data_dir)


import logging
import os
import pathlib
import sys
from datetime import datetime

def create_log_directories(algo_name: str):
    project_abs_path = get_project_path()
    experiment_name = get_algorithm_name_by_time(algo_name)
    create_dir(str(pathlib.Path(project_abs_path, "Runs")))
    create_dir(str(pathlib.Path(project_abs_path, "Runs", experiment_name)))
    create_dir(
        str(pathlib.Path(project_abs_path, "Runs", experiment_name, "checkpoints"))
    )

def create_tensorboard_log_dir(algo_name: str):
    project_abs_path = get_project_path()
    experiment_name = get_algorithm_name_by_time(algo_name)
    create_dir(str(pathlib.Path(project_abs_path, "Runs", experiment_name, "runs")))

def get_project_path():
    return str(pathlib.Path(__file__).parent.resolve().parent.parent)

def get_algorithm_name_by_time(algo_name: str):
    today = datetime.now()
    todays_date_full = (
        str(today.year) + "_" + str(today.month) + "_" + str(today.day) + "_"
    )
    todays_date_full += (
        str(today.hour) + "_" + str(today.minute) + "_" + str(today.second)
    )
    return algo_name + "_" + todays_date_full

def create_dir(path):
    try:
        os.makedirs(path)
    except FileExistsError:
        logging.info("Folder is already exist : ", path)
    except Exception as e:
        logging.error("Could not created the agents folder : ", e)