#!/usr/bin/python3
"""Defines all common attributes/methods for other classes"""
import models
from uuid import uuid4
from datetime import datetime

class BaseModel:
    """Instantiates the basemodel for all other classes.

    Attributes:
        id (string): unique uuid identifier
        created_at (datetime): current datetime when an instance is created
        updated_at (datetime): current datetome when an instance is updated
    """

    def __init__(self, *args, **kwargs):
        """Initializes an instance of BaseModel

        Args:
            args (any): Unused
            kwargs (dict): Key/Value pairs of attributes

        Methods:
            save(self): updates updated_at withe the current datetime
            to_dict(self): returns an updated dict version of the instance
        """
        dateform = "%Y-%m-%dT%H:%M:%S.%f"
        self.id = str(uuid4())
        self.created_at = datetime.today()
        self.updated_at = datetime.today()
        if (len(kwargs) != 0):
            for k, v in kwargs.items():
                if k == 'created_at' or k == 'updated_at':
                    self.__dict__[k] = datetime.strptime(v, dateform)
                else:
                    self.__dict__[k] = v
        else:
            models.storage.new(self)

    def save(self):
        """Updates updated_at with the current datetime"""
        self.updated_at = datetime.today()
        models.storage.save()

    def to_dict(self):
        """Return an updated dict version of the instance"""
        new_dict = self.__dict__.copy()
        new_dict.update({"__class__": self.__class__.__name__})
        new_dict.update({"created_at": self.created_at.isoformat()})
        new_dict.update({"updated_at": self.updated_at.isoformat()})
        return new_dict

    def __str__(self):
        """Returns string representaion of instance"""
        clname = self.__class__.__name__
        return "[{}] ({}) {}".format(clname, self.id, self.__dict__)


from datetime import timedelta
import logging
from .binary_sensors.target_rate import CarbonIntensityTargetRate

from homeassistant.exceptions import ConfigEntryNotReady

from .const import (
  DOMAIN,

  CONFIG_TARGET_NAME,

  DATA_RATES_COORDINATOR
)

_LOGGER = logging.getLogger(__name__)

SCAN_INTERVAL = timedelta(minutes=1)

async def async_setup_entry(hass, entry, async_add_entities):
  """Setup sensors based on our entry"""

  if CONFIG_TARGET_NAME in entry.data:
    if DOMAIN not in hass.data or DATA_RATES_COORDINATOR not in hass.data[DOMAIN]:
      raise ConfigEntryNotReady

    await async_setup_target_sensors(hass, entry, async_add_entities)

  return True

async def async_setup_target_sensors(hass, entry, async_add_entities):
  config = dict(entry.data)

  if entry.options:
    config.update(entry.options)

  coordinator = hass.data[DOMAIN][DATA_RATES_COORDINATOR]

  async_add_entities([CarbonIntensityTargetRate(coordinator, config)], True)


import torch
import torch.nn as nn
import torchvision.transforms.functional as TF
import torch.nn.functional as F

class DoubleConv(nn.Module):
    def __init__(self, in_channels, out_channels):
        super(DoubleConv, self).__init__()
        self.conv = nn.Sequential(
            nn.Conv2d(in_channels, out_channels, 3, 1, 1, bias=False),
            nn.BatchNorm2d(out_channels),
            nn.ReLU(inplace=True),
            nn.Conv2d(out_channels, out_channels, 3, 1, 1, bias=False),
            nn.BatchNorm2d(out_channels),
            nn.ReLU(inplace=True),
        )

    def forward(self, x):
        return self.conv(x)

# Implementation inspired by Aladdin Persson - PyTorch Image Segmentation Tutorial with U-NET: everything from scratch baby
class CNN_with_Unet(nn.Module):
    def __init__(
        self,
        in_channels=5,
        out_channels=1,
        features=[64, 128, 256, 512],
        num_of_class=2,
        dist_from_center=10,
        drop_out=0.3,
        hidden_nodes=512,
    ):
        # UNET Convolution
        super(CNN_with_Unet, self).__init__()
        self.ups = nn.ModuleList()
        self.downs = nn.ModuleList()
        self.pool = nn.MaxPool2d(kernel_size=2, stride=2)

        # Down part of UNET
        for feature in features:
            self.downs.append(DoubleConv(in_channels, feature))
            in_channels = feature

            # Up part of UNET
        for feature in reversed(features):
            self.ups.append(
                nn.ConvTranspose2d(
                    feature * 2,
                    feature,
                    kernel_size=2,
                    stride=2,
                )
            )
            self.ups.append(DoubleConv(feature * 2, feature))

        self.bottleneck = DoubleConv(features[-1], features[-1] * 2)
        self.final_conv = nn.Conv2d(features[0], out_channels, kernel_size=1)

        # Fully Connected Layer
        self.flatten = nn.Flatten()

        self.fc1 = nn.Linear(
            in_features=(2 * dist_from_center) ** 2, out_features=hidden_nodes
        )
        self.drop1 = nn.Dropout(p=drop_out)

        self.fc2 = nn.Linear(in_features=hidden_nodes, out_features=hidden_nodes)
        self.drop2 = nn.Dropout(p=drop_out)

        self.out = nn.Linear(in_features=hidden_nodes, out_features=num_of_class)

    def forward(self, x):
        skip_connections = []

        for down in self.downs:
            x = down(x)
            skip_connections.append(x)
            x = self.pool(x)

        x = self.bottleneck(x)
        skip_connections = skip_connections[::-1]

        for idx in range(0, len(self.ups), 2):
            x = self.ups[idx](x)
            skip_connection = skip_connections[idx // 2]

            if x.shape != skip_connection.shape:
                x = TF.resize(x, size=skip_connection.shape[2:], antialias=True)

            concat_skip = torch.cat((skip_connection, x), dim=1)
            x = self.ups[idx + 1](concat_skip)

        x = self.final_conv(x)

        x = self.flatten(x)

        x = F.relu(self.fc1(x))
        x = self.drop1(x)

        x = F.relu(self.fc2(x))
        x = self.drop2(x)

        x = self.out(x)

        return x


from cffi import FFI
import numpy as np
import time

grid_shape = (512, 512)

ffi = FFI()
ffi.cdef("void evolve(double **in, double **out, double D, double dt);")
# open vscode ex0723_cffi
lib = ffi.dlopen("../ex0721_ffi_ctypes/diffusion.so")

def evolve(grid, out, dt, D=1.0):
    pointer_grid = ffi.cast("double**", grid.ctypes.data)
    pointer_out = ffi.cast("double**", out.ctypes.data)
    lib.evolve(pointer_grid, pointer_out, D, dt)

def run_experiment(num_iterations):
    next_grid = np.zeros(grid_shape)
    grid = np.zeros(grid_shape)

    block_low = int(grid_shape[0] * 0.4)
    block_high = int(grid_shape[0] * 0.5)
    grid[block_low:block_high, block_low:block_high] = 0.005

    start = time.time()
    for i in range(num_iterations):
        evolve(grid, next_grid, 0.1)
        grid, next_grid = next_grid, grid
    return time.time() - start

if __name__ == "__main__":
    print(f"diffusion took {run_experiment(500)} seconds")
    # diffusion took 0.05642199516296387 seconds


import numpy as np
import matplotlib.pyplot as plt
from pyplm.pipelines import model_pipeline, data_pipeline
from pyplm.analyse import tail

plt.style.use("/Users/mk14423/Dropbox/mpl-styles/thesisbody.mplstyle")

file = '/Users/mk14423/Desktop/PaperData/HCP_rsfmri_added_data.hdf5'
group = 'individuals'
# group = 'grouped'
dataset = 'inferredModels'

# pipe = data_pipeline(file, group, dataset)
# print(len(pipe.mods))

# print(pipe.mods['left'].shape)
# hs_indi, Js_indi = pipe.get_parameters('individuals')
# # print(pipe.__dict__.items())
# exit()
fig, ax = plt.subplots()
modpipe = model_pipeline(file, group, dataset)
# print(modpipe.datasets.shape)
# plt.matshow(modpipe.datasets[0])
# plt.show()
modpipe.split_hemispheres(which='left')
# print(modpipe.datasets.shape)
J_meanstds = modpipe.select_parameters(parameter_name='J')
# fig, ax = plt.subplots()
# ax.plot(J_meanstds[0, :], J_meanstds[1, :], ls='none')
# plt.show()
modpipe.transform_to_distributions(
    filter_level=0, bins=100, density=True)

modpipe.tail_distributions(tail='positive')
mcs, xs, yfit = tail.fit_power_laws(modpipe.datasets)
# nMods = len(modpipe.datasets)
nMods = 3
for iMod in range(0, nMods):
    xy = modpipe.datasets[iMod]
    x = xy[0, :]
    y = xy[1, :]
    ax.plot(x, y, label=f'm: {mcs[iMod, 0]:.2f} c {mcs[iMod, 1]:.2f}')
    ax.plot(xs[iMod], yfit[iMod], marker=',', c='k', ls='--')

group = 'grouped'
modpipe = model_pipeline(file, group, dataset)
modpipe.split_hemispheres(which='left')
J_meanstds = modpipe.select_parameters(parameter_name='J')
modpipe.transform_to_distributions(
    filter_level=0, bins=100, density=True)

modpipe.tail_distributions(tail='positive')
mcs, xs, yfit = tail.fit_power_laws(modpipe.datasets)
for iMod in range(0, len(modpipe.datasets)):
    xy = modpipe.datasets[iMod]
    x = xy[0, :]
    y = xy[1, :]
    ax.plot(x, y, label=f'm: {mcs[iMod, 0]:.2f} c {mcs[iMod, 1]:.2f}')
    ax.plot(xs[iMod], yfit[iMod], marker=',', c='k')
plt.xlabel(r'$J_{ij}$')
plt.ylabel(r'$P(J_{ij})$')
plt.legend()
plt.show()
print('-----')
modpipe.tail_distributions(tail='positive')
mcs, xs, yfit = tail.fit_power_laws(modpipe.datasets)
fig, ax = plt.subplots()
for iMod in range(0, len(modpipe.datasets)):
    xy_tail = modpipe.datasets[iMod]
    print(xy_tail.shape)
    x = xy_tail[0, :]
    y = xy_tail[1, :]
    ax.plot(x, y)
plt.show()

print(mcs.shape)
fig, ax = plt.subplots(2, 1)
ax = ax.ravel()
mean = np.mean(mcs[:, 0])
std = np.std(mcs[:, 0])
lbl = f'm: {mean:.2f} pm {std:.2f}'
ax[0].hist(mcs[:, 0], bins=20, label=lbl)
ax[0].legend()

mean = np.mean(mcs[:, 1])
std = np.std(mcs[:, 1])
lbl = f'c: {mean:.2f} pm {std:.2f}'
ax[1].hist(mcs[:, 1], bins=20, label=lbl)
ax[1].legend()
plt.show()
# distros = modpipe.parameter_distributions(Js, 0, bins=101, density=True)
# for distro in distros:
#     plt.plot(distro[0], distro[1])
#     plt.show()
# hs_meanstd = modpipe.parameter_moments(hs)
# Js_meanstd = modpipe.parameter_moments(Js)
# # plt.plot(hs_meanstd[0, :],hs_meanstd[1, :], ls='none')
# plt.plot(Js_meanstd[0, :],Js_meanstd[1, :], ls='none')
# plt.show()
# # print(modpipe.Js.shape, modpipe.hs.shape)
# # print(modpipe.models.shape)


from sqlite3.dbapi2 import Cursor
from sqlite3 import Error
from classes.repository.dbconnection import DbConnection
import sqlite3
import os

class ConnectableMixin():

    def __init__(self):
        self.dbconn = DbConnection()

    def execute(self, SQL, values):
        affectedRows = 0
        try:
            self.dbconn.create_connection()
            cursor = self.dbconn.conn.cursor()
            cursor.execute(SQL, values)
            affectedRows = cursor.rowcount
            self.dbconn.conn.commit()
            cursor.close()
        except (Error) as error:
            print(error)
        finally:
            if self.dbconn.conn:
                self.dbconn.conn.close()

        return affectedRows

    def create_test_database(self) -> None:
        self.dbconn.create_connection()
        fileDir = os.path.dirname(os.path.realpath('__file__'))
        cursor = self.dbconn.conn.cursor()
        file = open(os.path.join(fileDir, 'assets/db/schema.sql'))
        SQL = file.read()
        cursor.executescript(SQL)
        file.close()
        self.dbconn.conn.commit()
        cursor.close()
        self.dbconn.conn.close()

    def fetch_one(self, SQL, values):
        row = None
        try:
            self.dbconn.create_connection()
            cursor = self.dbconn.conn.cursor()
            cursor.execute(SQL, values)
            row = cursor.fetchone()
            cursor.close()
        except (Error) as error:
            print(error)
        finally:
            if self.dbconn.conn:
                self.dbconn.conn.close()

        return row

    def fetch_all(self, SQL, values):
        rows = None
        try:
            self.dbconn.create_connection()
            cursor = self.dbconn.conn.cursor()
            cursor.execute(SQL, values)
            rows = cursor.fetchall()
            cursor.close()
        except (Error) as error:
            print(error)
        finally:
            if self.dbconn.conn:
                self.dbconn.conn.close()

        return rows

    def execute_last_row_id(self, SQL, values):
        last_id = None
        try:
            self.dbconn.create_connection()
            cursor = self.dbconn.conn.cursor()
            cursor.execute(SQL, values)
            self.dbconn.conn.commit()
            last_id = cursor.lastrowid
            cursor.close()
        except (Exception, Error) as error:
            print(error)
        finally:
            if self.dbconn.conn:
                self.dbconn.conn.close()

        return last_id


import os
import asyncio

import discord
from discord.ext import commands

import card_data
import card_art
import card_voice
import sleeves
import deck_code

command_prefix = os.environ["BOT_PREFIX"].split()

intents = discord.Intents.default()
intents.messages = True

bot = commands.Bot(
    command_prefix=command_prefix,
    description=os.environ.get("DESCRIPTION", "Shadowverse info bot"),
    intents=intents,
)

class CardNotFoundError(Exception):
    def __init__(self, query: list):
        self.query = query
        super().__init__(query)

class CardArtError(Exception):
    def __init__(self, card_id: int, card_name: str):
        self.card_id = card_id
        self.card_name = card_name
        super().__init__(card_id, card_name)

class DeckCodeError(Exception):
    def __init__(self, deck_code: str, errors: list):
        self.deck_code = deck_code
        self.errors = errors
        super().__init__(deck_code, errors)

# Hack: avoid raising UnexpectedQuoteError
commands.view._all_quotes = set()

@bot.event
async def on_ready():
    print(f"Logged in as {bot.user}")
    if os.environ.get("DEV") is None:
        status = os.environ.get("STATUS", f"{command_prefix[0]}help")
        activity = discord.Game(name=status)
        await bot.change_presence(activity=activity)

@bot.command()
async def invite(ctx):
    """Generate a link to invite me to your server"""

    app_info = await bot.application_info()
    link = f"https://discord.com/api/oauth2/authorize?client_id={app_info.id}&permissions=18432&scope=bot"
    await ctx.send(f"Invite me to your server: <{link}>")

@bot.command(aliases=["list", "l", "search", "s"])
async def cards(ctx, *query):
    """List all matching cards.

    Keyword search is supported. Examples:
    - search 6pp shadow gold spell
    - search earth sigil
    - search havencraft "repair mode"
    - search legendary "steel rebellion"

    """

    async with ctx.typing():
        cards = await card_data.get()
    results = card_data.find(cards, query)
    if not results:
        raise CardNotFoundError(query)
    else:
        max_results = 20
        if len(results) <= max_results:
            lines = [f"Found {len(results)} cards:"]
            lines += [card_data.effective_card_name(card) for card in results]
            await ctx.send("\n".join(lines))
        else:
            lo = 0
            hi = max_results
            lines = [f"Found {len(results)} cards, displaying {lo+1}-{hi}:"]
            lines += [card_data.effective_card_name(card) for card in results[lo:hi]]
            msg = await ctx.send("\n".join(lines))
            await msg.add_reaction("⬅")
            await msg.add_reaction("➡")

            def check(reaction, user):
                return (
                    user != bot.user
                    and reaction.message.id == msg.id
                    and reaction.emoji in ["➡", "⬅"]
                )

            while True:
                try:
                    reaction, usr = await bot.wait_for(
                        "reaction_add", timeout=30.0, check=check
                    )
                except asyncio.TimeoutError:
                    await msg.remove_reaction("➡", bot.user)
                    await msg.remove_reaction("⬅", bot.user)
                    break
                else:
                    if reaction.emoji == "➡":
                        hi = min(len(results), hi + max_results)
                        lo = hi - max_results
                    elif reaction.emoji == "⬅":
                        lo = max(0, lo - max_results)
                        hi = lo + max_results

                    lines = [f"Found {len(results)} cards, displaying {lo+1}-{hi}:"]
                    lines += [
                        card_data.effective_card_name(card) for card in results[lo:hi]
                    ]
                    await msg.edit(content="\n".join(lines))

@bot.command(aliases=["c", "text", "t"])
async def card(ctx, *query):
    """Display a card's stats and text"""

    async with ctx.typing():
        cards = await card_data.get()
    results = card_data.find(cards, query)
    if not results:
        raise CardNotFoundError(query)
    else:
        result = results[0]
        embed = discord.Embed.from_dict(card_data.info_embed(result))
        await ctx.send(embed=embed)

@bot.command(aliases=["flavourtext", "flavor", "flavour", "flairtext", "flair", "f"])
async def flavortext(ctx, *query):
    """Display a card's flavortext"""

    async with ctx.typing():
        cards = await card_data.get()
    results = card_data.find(cards, query)
    if not results:
        raise CardNotFoundError(query)
    else:
        result = results[0]
        embed = discord.Embed.from_dict(card_data.flavor_embed(result))
        await ctx.send(embed=embed)

async def art_gen(ctx, query: list, which: str):
    """Helper function for art commands"""

    async with ctx.typing():
        cards = await card_data.get()

    results = card_data.find(cards, query)
    if not results:
        raise CardNotFoundError(query)
    result = results[0]
    card_id = result["card_id"]
    card_name = card_data.effective_card_name(result)

    async with ctx.typing():
        image = await card_art.svgdb_get(card_id, which)

    if image is None:
        raise CardArtError(card_id, card_name)
    else:
        await ctx.send(card_name, file=discord.File(image, "0.png"))

@bot.command(aliases=["v"])
async def voice(ctx, *query):
    async with ctx.typing():
        cards = await card_data.get()

    results = card_data.find(cards, query)
    if not results:
        raise CardNotFoundError(query)
    result = results[0]

    async with ctx.typing():
        embed = await card_voice.svgdb_embed(result)

    await ctx.send(embed=discord.Embed.from_dict(embed))

@bot.command(aliases=["img", "a"])
async def art(ctx, *query):
    """Display a card's base art"""

    await art_gen(ctx, query, "0")

@bot.command(aliases=["evoimg", "evo", "e"])
async def evoart(ctx, *query):
    """Display a card's evolved art"""

    await art_gen(ctx, query, "1")

@bot.command()
async def sleeve(ctx, *query):
    """Display sleeve art"""

    # async with ctx.typing():
    #     if not query:
    #         image, sleeve_name = await sleeves.get_random()
    #     else:
    #         sleeve_id, sleeve_name = await sleeves.find_sleeve(query)
    #         if not sleeve_id:
    #             raise CardNotFoundError(query)
    #         image = await sleeves.get_asset(sleeve_id)

    # if image is None:
    #     raise CardArtError(sleeve_id, sleeve_name)
    # else:
    #     await ctx.send(sleeve_name, file=discord.File(image, "0.png"))

    await ctx.send("There is a technical issue with the sleeve command. Sorry!")

@bot.command(aliases=["deck", "code"])
async def deckcode(ctx, code: str):
    """Look up a deck code."""

    async with ctx.typing():
        result = await deck_code.get(code)

    errors = result.get("errors")
    if errors:
        raise DeckCodeError(code, errors)

    embed = discord.Embed.from_dict(deck_code.embed(result))
    await ctx.send(embed=embed)

@bot.command(hidden=True)
async def randomcard(ctx):
    async with ctx.typing():
        cards = await card_data.get()
    card = card_data.random_card(cards)
    embed = discord.Embed.from_dict(card_data.info_embed(card))
    await ctx.send(embed=embed)

@bot.command(hidden=True)
async def update(ctx):
    async with ctx.typing():
        await card_data._update()

@bot.command(hidden=True)
async def eggsplosion(ctx):
    async with ctx.typing():
        cards = await card_data.get()
    card = card_data.random_card(cards, eggsplosion=True)["card_name"]
    await ctx.send(f"{card} dies to Eggsplosion")

@bot.command(aliases=["bugreport", "bug", "report"])
async def feedback(ctx, *, message: str):
    """Report feedback to the bot dev"""

    log_channel = bot.get_channel(int(os.environ["LOG_CHANNEL"]))
    app_info = await bot.application_info()
    await log_channel.send(
        f"<@{app_info.owner.id}> Feedback from {ctx.author} in {ctx.guild} {ctx.channel}: ```{message}```"
    )
    await ctx.send("Feedback sent. Thank you!")

@bot.event
async def on_command_error(ctx, error):
    if isinstance(error, discord.ext.commands.CommandNotFound):
        if ctx.invoked_with in ["art1", "art2", "art3", "art4", "art5", "art6"]:
            await ctx.send(
                f"{error}\n"
                f"Tip: Use `{ctx.prefix}evoart` to look up evolved art. "
                "For alternate card art, you can specify 'alt' or the card set. Examples:\n"
                f"- `{ctx.prefix}art fairy circle alt`\n"
                f"- `{ctx.prefix}art albert leader`\n"
                f"- `{ctx.prefix}art albert prebuilt`\n"
                f"- `{ctx.prefix}art witch snap fate`\n"
            )
        else:
            await ctx.send(
                f"{error}\n"
                f"Tip: Use `{ctx.prefix}card` to look up a card. "
                f"See `{ctx.prefix}help` for other commands."
            )
        return
    elif isinstance(error, discord.ext.commands.CommandInvokeError):
        error = error.original
        if isinstance(error, CardNotFoundError):
            await ctx.send(f"Found no cards matching `{error.query}`")
        elif isinstance(error, CardArtError):
            await ctx.send(f'Failed to get card art for "{error.card_name}"')
        elif isinstance(error, DeckCodeError):
            await ctx.send(f"Failed to get deck code `{error.deck_code}`")
    if "LOG_CHANNEL" in os.environ:
        log_channel = bot.get_channel(int(os.environ["LOG_CHANNEL"]))
        await log_channel.send(
            f"Error from {ctx.author} in {ctx.guild} {ctx.channel}:\n"
            f"{ctx.message.jump_url}\n"
            f"```{repr(error)}```"
        )
    else:
        await commands.Bot.on_command_error(bot, ctx, error)

bot.run(os.environ["DISCORD_TOKEN"])


import os
import serial

serialPort = serial.Serial(port="COM4", baudrate=9600, bytesize=8, timeout=2, stopbits=serial.STOPBITS_ONE)

while True:
    # Wait until there is data waiting in the serial buffer
    if serialPort.in_waiting > 0:
        # Read data out of the buffer until a carriage return / new line is found
        serialString = serialPort.readline()  # STM -> PC
        serialString = serialString.strip()

        # Print the contents of the serial data
        print(serialString)


#!/usr/bin/env python
"""
# Author: Xiong Lei
# Created Time : Sun 17 Nov 2019 03:37:47 PM CST

# File Name: setup.py
# Description:

"""
import pathlib
from setuptools import setup, find_packages

with open('requirements.txt') as f:
    requirements = f.read().splitlines()

HERE = pathlib.Path(__file__).parent
README = (HERE / "README.md").read_text()

import codecs
import os.path

def read(rel_path):
    here = os.path.abspath(os.path.dirname(__file__))
    with codecs.open(os.path.join(here, rel_path), 'r') as fp:
        return fp.read()

def get_version(rel_path):
    for line in read(rel_path).splitlines():
        if line.startswith('__version__'):
            delim = '"' if '"' in line else "'"
            return line.split(delim)[1]
    else:
        raise RuntimeError("Unable to find version string.")

setup(name='scalex',
      version=get_version(HERE / "scalex/__init__.py"),
      packages=find_packages(),
      description='Online single-cell data integration through projecting heterogeneous datasets into a common cell-embedding space',
      long_description=README,

      author='Lei Xiong',
      author_email='jsxlei@gmail.com',
      url='https://github.com/jsxlei/scalex',
      scripts=['SCALEX.py'],
      install_requires=requirements,
      python_requires='>3.6.0',
      license='MIT',

      classifiers=[
          'Development Status :: 4 - Beta',
          'Intended Audience :: Science/Research',
          'License :: OSI Approved :: MIT License',
          'Programming Language :: Python :: 3.7',
          'Operating System :: MacOS :: MacOS X',
          'Operating System :: Microsoft :: Windows',
          'Operating System :: POSIX :: Linux',
          'Topic :: Scientific/Engineering :: Bio-Informatics',
     ],
     )


# -*- coding: utf-8 -*-
"""
Created on Thu Dec 27 10:36:00 2018

@author: ogianan
"""

import os
import pandas as pd
from string import punctuation

from app import app
from flask import render_template, request, redirect, url_for, send_file, send_from_directory
from flask_wtf import FlaskForm
from wtforms import RadioField
from flask_wtf.file import FileField
from werkzeug.utils import secure_filename

class UploadForm(FlaskForm):
    file = FileField()

# Upload folder where xls files will be saved and retrieved by auto-downloader
UPLOAD_FOLDER = '/home/sysadmin/ProServConvert/files/'
ALLOWED_EXTENSIONS = set(['txt', 'csv'])
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER

def load_skus(file, subitems, subcons):
    # Create columns order list
    cols = ['Product ID', 'Quantity', 'Unit Price', 'Unit Cost']
    # Read csv file
    soitems = pd.read_csv(file)
    soitems['Product ID'] = soitems['Product ID'][:].str.strip(punctuation).str.lstrip()
    soitems_sub = soitems[soitems['Product ID'].isin(subitems)]
    soitems_sub = soitems_sub.groupby(['Product ID', 'Unit Price', 'Unit Cost'])['Quantity'].sum().reset_index(name = 'Quantity')
    return soitems, soitems_sub[cols]

def create_proserv(file):
    # Create tables template
    # Create sub-items series
    subitems = pd.Series(['PRO-UC-L1', 'PRO-NET-L1', 'PRO-SVR-L1', 'PRO-PMO-L1', 'Risk', 'PRO-SMARTHANDS-L1','PRO-SMARTHANDS'])
    # Create SUBCON variations series
    subcons = pd.Series(['PRO-SMARTHANDS-L1', 'PRO-SMARTHANDS'])
    # Create output template
    template = { 'Product ID' : ['PKG-PRO-PROFSVCS', 'PKG-PRO-PROFSVCS', 'PKG-PRO-PROFSVCS', 'PKG-PRO-PROFSVCS'] , 'Quantity' : [1, 1, 1, 1] , 'Price' : [0, 0, 0, 0] , 'Cost' : [0, 0, 0, 0] , 'Customer Description' : ['Professional Services: 50%', 'Professional Services: 20%', 'Professional Services: 20%', 'Professional Services: 10%'] }
    out_table = pd.DataFrame(template, columns=['Product ID', 'Quantity', 'Unit Price', 'Unit Cost', 'Customer Description'])
    # Create a Pandas Excel writer using XlsxWriter as the engine.
    #new_file = app.config['UPLOAD_FOLDER'] + 'proserv_' + file

    new_file = app.config['UPLOAD_FOLDER'] + 'proserv_converted.xlsx'

    writer = pd.ExcelWriter(new_file, engine='xlsxwriter')

    # Load soitems file
    df, agg_df = load_skus(file, subitems, subcons)

    # Get total proserv price
    t_proserv_price = df[df['Product Class'].str.contains('Bundle')]['Unit Price'].sum()
    # Get total proserv cost
    t_proserv_cost = df[df['Product Class'].str.contains('Bundle')]['Unit Cost'].sum()

    # Split smarthands from sub_agg
    subcons_df = agg_df.loc[agg_df['Product ID'].isin(subcons)]
    subitems_df = agg_df.loc[~agg_df['Product ID'].isin(subcons)]

    # Get total subcons cost
    t_subcons_cost = (subcons_df['Quantity'] * subcons_df['Unit Cost']).sum()

    # Get adjusted proserv cost
    t_proserv_cost = t_proserv_cost - t_subcons_cost

    # Assign values to out_table
    out_table.at[0, 'Unit Price'] = .5 * t_proserv_price
    out_table.at[1, 'Unit Price'] = .2 * t_proserv_price
    out_table.at[2, 'Unit Price'] = .2 * t_proserv_price
    out_table.at[3, 'Unit Price'] = .1 * t_proserv_price
    out_table.at[0, 'Unit Cost'] =  .5 * t_proserv_cost
    out_table.at[1, 'Unit Cost'] = .2 * t_proserv_cost
    out_table.at[2, 'Unit Cost'] = .2 * t_proserv_cost
    out_table.at[3, 'Unit Cost'] = .1 * t_proserv_cost

    # Zero out subcon pricing
    #subcons_df = subcons_df.replace([subcons_df['Unit Price']], 0)
    # the above code quits when df has only one row
    # Disable warnings
    pd.options.mode.chained_assignment = None
    subcons_df['Unit Price'] = 0
    # the above code outputs a warning
    # rename PRO-SMARTHANDS* to SUBCON
    subcons_df['Product ID'] = "SUBCON"
    # the above code outputs a warning

    # Output final table
    final = pd.concat([out_table.loc[:0], subitems_df, out_table.loc[1:], subcons_df]).reset_index(drop=True)
    final_cols = ['Product ID', 'Quantity', 'Unit Price', 'Unit Cost', 'Customer Description']
    final = final[final_cols]

    # Number of rows of original table
    df_rows = df.shape[0]
    final_start_row = df_rows + 3

    # Writing the dataframes into the worksheet
    df.to_excel(writer, sheet_name = 'Sheet1', index = False)
    final.to_excel(writer, sheet_name = 'Sheet1', startrow=final_start_row, index = False)

    # Conditional formatting
    # Get the xlsxwriter workbook and worksheet objects.
    workbook  = writer.book
    worksheet = writer.sheets['Sheet1']
    # Add a format. Light red fill with dark red text.
    red_format = workbook.add_format({'bg_color': '#FFC7CE', 'font_color': '#9C0006'})

    # Apply format to the cell range.
    marker = df['Product ID'].isin(subitems) | df['Product Class'].str.contains('Bundle')
    # Get index of first TRUE value
    f_in = 1 + marker[marker==True].index.tolist()[0]

    # Get index of last TRUE value
    l_in = 1 +  marker[marker==True].index.tolist()[-1]
    #str('L%d:L%d' % (f_in, l_in))

    for row in range (f_in, l_in+1):
    #    print(row)
        worksheet.set_row(row, 15, red_format)

    #print(marker)
    writer.save()
    return

@app.route('/', methods=['GET', 'POST'])
@app.route('/index')
def upload_file():
    form = UploadForm(csrf_enabled=False)
    if form.validate_on_submit():
	    filename = secure_filename(form.file.data.filename)
	    form.file.data.save(filename)
	    create_proserv(filename)
	    return send_from_directory(app.config['UPLOAD_FOLDER'], 'proserv_converted.xlsx', as_attachment=True)
	    return redirect(url_for('upload_file', filename=filename))
    return render_template('index.html', title='ProServ Conversion Tool', form=form)


"""Catalogs are immutable mappings useful for building indexes.

This module contains definitions of many different catalog types,
all of which implement the interface defined by the Catalog abstract
base class, which is itself implements the mapping protocol.

Rather than constructing Catalog subtypes directly, prefer to use
the CatalogBuilder class which will analyse the contents of the
mapping to find a space and time efficient representation.
"""
from collections import Mapping, Sequence, OrderedDict, Iterable
from fractions import Fraction
from itertools import product

from segpy.sorted_frozen_set import SortedFrozenSet
from segpy.util import (contains_duplicates, measure_stride, make_sorted_distinct_sequence,
                        is_sorted, first)

class CatalogBuilder(object):
    """Use a catalog builder to construct optimised, immutable mappings.

    A CatalogBuilder is useful when, depending on the particular keys and
    values used, a more compact or efficient representation of the mapping
    is possible than, say, a regular dictionary. The CatalogBuilder
    accumulates values and then, once all values have been added, analyzes
    the keys and values to produce a more optimized representation of the
    mapping.
    """

    def __init__(self, mapping=None):
        """Initialize a Catalog Builder.

        Args:
            mapping: An optional mapping (such as a dictionary) of items, or an
                iterable series of 2-tuples containing keys and values.

        Raises:
            TypeError: If mapping is not None and is neither of mapping nor iterable type.
            ValueError: If mapping is an iterable of tuples, and the tuples are not pairs.
        """
        if mapping is None:
            self._catalog = []
        elif isinstance(mapping, Mapping):
            self._catalog = list(mapping.items())
        elif isinstance(mapping, Iterable):
            self._catalog = []
            for pair in mapping:
                if len(pair) != 2:
                    raise ValueError("{!r} is not a pair. Catalogs can only be constructed "
                                     "from iterable series of 2-tuples.")
                self._catalog.append(pair)
        else:
            raise TypeError("Mapping must be either a mapping (e.g. dict), or an iterable "
                            "series of 2-tuples. {!r} does not qualify.")

    def add(self, index, value):
        """Add an item.

        Each index must be unique if create() is to be subsequently
        called successfully, although duplicate index values will be
        accepted by this call without complaint.

        """
        self._catalog.append((index, value))

    def create(self):
        """Create a possibly more optimized representation of the mapping.

        In this worst case, this method returns an object which is
        essentially an immutable dictionary. In the best case, the
        space savings can be vast.

        Returns:
            A mapping, if a unique mapping from indexes to values is
            possible, otherwise None.

        """

        # This method examines the contents of the mapping using
        # various heuristics to come up with a better representation.

        # In-place sort by index
        self._catalog.sort(key=first)

        if contains_duplicates(index for index, value in self._catalog):
            return None

        if all(isinstance(index, Sequence) and (len(index) == 2)
               for index, value in self._catalog):
            return self._create_catalog_2()

        return self._create_catalog_1()

    def _create_catalog_1(self):
        """Create a catalog for one-dimensional integer keys (i.e. scalars)
        """
        if len(self._catalog) <= 1:
            return DictionaryCatalog(self._catalog)

        index_min = self._catalog[0][0]
        index_max = self._catalog[-1][0]
        index_stride = measure_stride(index for index, value in self._catalog)
        assert index_stride != 0

        value_start = self._catalog[0][1]
        value_stop = self._catalog[-1][1]
        value_stride = measure_stride(value for index, value in self._catalog)

        if index_stride is not None and value_stride == 0:
            assert value_start == value_stop
            return RegularConstantCatalog(index_min,
                                          index_max,
                                          index_stride,
                                          value_start)

        if index_stride is None and value_stride == 0:
            assert value_start == value_stop
            return ConstantCatalog(
                    (index for index, value in self._catalog),
                    value_start)

        if index_stride is not None and value_stride is None:
            # Regular index - regular keys and arbitrary values
            return RegularCatalog(index_min,
                                  index_max,
                                  index_stride,
                                  (value for index, value in self._catalog))

        if (index_stride is not None) and (value_stride is not None):
            assert value_stride != 0
            return LinearRegularCatalog(index_min,
                                        index_max,
                                        index_stride,
                                        value_start,
                                        value_stop,
                                        value_stride)

        return DictionaryCatalog(self._catalog)

    def _create_catalog_2(self):
        """Create a catalog for two-dimensional integer keys.

        Each key must be a two-element sequence.
        """
        return (self.make_last_index_varies_quickest_catalog_2d()
             or self.make_first_index_varies_quickest_catalog_2d()
             or self.make_dictionary_catalog_2d())

    def make_last_index_varies_quickest_catalog_2d(self):
        self._catalog.sort(key=_first_then_second_index)
        sorted_sequences = self.make_sorted_ranges()
        if sorted_sequences is None:
            return None
        return LastIndexVariesQuickestCatalog2D(*sorted_sequences)

    def make_first_index_varies_quickest_catalog_2d(self):
        self._catalog.sort(key=_second_then_first_index)
        sorted_sequences = self.make_sorted_ranges()
        if sorted_sequences is None:
            return None
        return FirstIndexVariesQuickestCatalog2D(*sorted_sequences)

    def make_dictionary_catalog_2d(self):
        self._catalog.sort(key=_first_then_second_index)
        i_sorted = make_sorted_distinct_sequence(i for (i, _), _ in self._catalog)
        j_sorted = make_sorted_distinct_sequence(j for (_, j), _ in self._catalog)
        return DictionaryCatalog2D(i_sorted, j_sorted, self._catalog)

    def make_sorted_ranges(self):
        i_sorted = make_sorted_distinct_sequence(i for (i, _), _ in self._catalog)
        j_sorted = make_sorted_distinct_sequence(j for (_, j), _ in self._catalog)
        if len(i_sorted) * len(j_sorted) != len(self._catalog):
            # The do not match so use a dictionary-based mapping
            return None
        vs = [v for (_, _), v in self._catalog]
        # Are the values unique and in ascending or descending order?
        if not (is_sorted(vs, reverse=False, distinct=True) or is_sorted(vs, reverse=True, distinct=True)):
            # The values are not both unique and sorted, so use a dictionary-based mapping
            return None
        v_sorted = make_sorted_distinct_sequence(vs, sense=None)
        i_is_regular = isinstance(i_sorted, range)
        j_is_regular = isinstance(j_sorted, range)
        v_is_regular = isinstance(v_sorted, range)
        if not (i_is_regular and j_is_regular and v_is_regular):
            # The i, j and v values are not regularly spaced, so use a dictionary-based mapping
            return None
        return i_sorted, j_sorted, v_sorted

class Catalog2D(Mapping):
    """An abstract base class for 2D catalogs.
    """

    def __init__(self, i_range, j_range):
        """Initialize a Catalog2D.

        Args:
            i_range: A range which can generate all and only valid i indexes.
            j_range: A range which can generate all and only valid j indexes.

        Raises:
            ValueError: If either i_range or j_range are not totally sorted.
        """
        if not is_sorted(i_range, distinct=True):
            raise ValueError("i indexes must be sorted and unique")
        if not is_sorted(j_range, distinct=True):
            raise ValueError("j indexes must be sorted and unique")
        self._i_range = i_range
        self._j_range = j_range

    @property
    def i_range(self):
        # TODO: This should be renamed, as it's not necessarily a range - just a sorted container (sequence?)
        return self._i_range

    @property
    def j_range(self):
        # TODO: This should be renamed, as it's not necessarily a range - just a sorted container (sequence?)
        return self._j_range

    @property
    def i_min(self):
        """Minimum i value"""
        return self._i_range[0]

    @property
    def i_max(self):
        """Maximum i value"""
        return self._i_range[-1]

    @property
    def j_min(self):
        """Minimum j value"""
        return self._j_range[0]

    @property
    def j_max(self):
        """Maximum j value"""
        return self._j_range[-1]

    def key_min(self):
        """Minimum (i, j) key"""
        return self.i_min, self.j_min

    def key_max(self):
        """Maximum (i, j) key"""
        return self.i_max, self.j_max

    def value_first(self):
        """Minimum value at key_min"""
        return self[self.key_min()]

    def value_last(self):
        """Maximum value at key_max"""
        return self[self.key_max()]

class LastIndexVariesQuickestCatalog2D(Catalog2D):

    def __init__(self, i_range, j_range, v_range):
        """
        Args:
            i_range: An ordered collection of evenly-spaced integers
                which can be used to generate valid i keys.

            j_range: An ordered collection of evenly-spaced integers
                which can be used to generate valid j keys.

            v_range: An ordered collection of evenly-spaced integers
                which can be used to generate valid values.
        """
        num_indices = len(i_range) * len(j_range)
        if num_indices != len(v_range):
            raise ValueError("i_range={} and j_range={} totalling {} indices are incompatible with "
                             "v_range={} with length {}".format(i_range, j_range, num_indices, v_range, len(v_range)))
        super().__init__(i_range, j_range)
        self._v_range = v_range

    @property
    def v_range(self):
        return self._v_range

    def __getitem__(self, key):
        i, j = key
        if not self._contains(i, j):
            raise KeyError("Key {!r} not in {!r}".format(key, self))
        i_index = self.i_range.index(i)
        j_index = self.j_range.index(j)
        v_index = i_index * len(self.j_range) + j_index
        v = self._v_range[v_index]
        return v

    def __contains__(self, key):
        i, j = key
        return self._contains(i, j)

    def _contains(self, i, j):
        return (i in self.i_range) and (j in self.j_range)

    def __len__(self):
        return len(self.i_range) * len(self.j_range)

    def __iter__(self):
        return product(self.i_range, self.j_range)

    def __repr__(self):
        return '{}(i_range={}, j_range={}, v_range={})'.format(
            self.__class__.__name__,
            self.i_range, self.j_range, self._v_range)

    def key(self, value):
        """Given a value, get the corresponding key.

        Args:
            value: The value for which to find the key.

        Returns:
            A 2-tuple containing the (i, j) values corresponding
            to the given value.

        Raises:
            ValueError: If there is no corresponding key.
        """

        try:
            v_index = self._v_range.index(value)
        except ValueError:
            raise ValueError("{!r} is not a value of {!r}".format(value, self))
        i_index, j_index = divmod(v_index, len(self.j_range))
        i = self.i_range[i_index]
        j = self.j_range[j_index]
        return (i, j)

class FirstIndexVariesQuickestCatalog2D(Catalog2D):

    def __init__(self, i_range, j_range, v_range):
        """
        Args:
            i_range: An ordered collection of evenly-spaced integers
                which can be used to generate valid i keys.

            j_range: An ordered collection of evenly-spaced integers
                which can be used to generate valid j keys.

            v_range: An ordered collection of evenly-spaced integers
                which can be used to generate valid values.
        """
        num_indices = len(i_range) * len(j_range)
        if num_indices != len(v_range):
            raise ValueError("i_range={} and j_range={} totalling {} indices are incompatible with "
                             "v_range={} with length {}".format(i_range, j_range, num_indices, v_range, len(v_range)))
        super().__init__(i_range, j_range)
        self._v_range = v_range

    @property
    def v_range(self):
        return self._v_range

    def __getitem__(self, key):
        i, j = key
        if not self._contains(i, j):
            raise KeyError("Key {!r} not in {!r}".format(key, self))
        i_index = self.i_range.index(i)
        j_index = self.j_range.index(j)
        v_index = j_index * len(self.i_range) + i_index
        v = self._v_range[v_index]
        return v

    def __contains__(self, key):
        i, j = key
        return self._contains(i, j)

    def _contains(self, i, j):
        return (i in self.i_range) and (j in self.j_range)

    def __len__(self):
        return len(self.i_range) * len(self.j_range)

    def __iter__(self):
        return ((i, j) for (j, i) in product(self.j_range, self.i_range))

    def __repr__(self):
        return '{}(i_range={}, j_range={}, v_range={})'.format(
            self.__class__.__name__,
            self.i_range, self.j_range, self._v_range)

    def key(self, value):
        """Given a value, get the corresponding key.

        Args:
            value: The value for which to find the key.

        Returns:
            A 2-tuple containing the (i, j) values corresponding
            to the given value.

        Raises:
            ValueError: If there is no corresponding key.
        """

        try:
            v_index = self._v_range.index(value)
        except ValueError:
            raise ValueError("{!r} is not a value of {!r}".format(value, self))
        j_index, i_index = divmod(v_index, len(self.i_range))
        i = self.i_range[i_index]
        j = self.j_range[j_index]
        return (i, j)

class DictionaryCatalog(Mapping):
    """An immutable, ordered, dictionary mapping.
    """

    def __init__(self, items):
        self._items = OrderedDict(items)

    def __getitem__(self, key):
        return self._items[key]

    def __iter__(self):
        return iter(self._items)

    def __len__(self):
        return len(self._items)

    def __contains__(self, item):
        return item in self._items

    def __repr__(self):
        return '{}(items=[<{} items>])'.format(
            self.__class__.__name__, len(self._items))

class DictionaryCatalog2D(Catalog2D):
    """An immutable, ordered, dictionary mapping for 2D keys.
    """

    def __init__(self, i_range, j_range, items):
        """Initialize a DictionaryCatalog2D.

        Args:
            i_range: An ordered collection of evenly-spaced integers
                which can be used to generate valid i values.

            j_range: An ordered collection of evenly-spaced integers
                which can be used to generate valid j values.

            items: Either a) a mapping (dict) where the keys are 2-tuples containing i and j values
                contained within the respective ranges, and the value is an integer, or b) an
                iterable series of tuple structures ((i, j), v).

        Raises:
            TypeError: If items is not a dictionary or iterable of tuples.
            ValueError: If the i and j components of the keys in items are not contained within
                the respective i_range and j_range collections.
        """
        super().__init__(i_range, j_range)
        self._items = OrderedDict()
        if isinstance(items, Mapping):
            mapping = items.items()
        elif isinstance(items, Iterable):
            mapping = items
        else:
            raise TypeError("{} must be constructed from a mapping or an iterable of ((i, j), v) tuples, "
                            "not {}".format(self.__class__.__name__, items))

        for key, value in mapping:
            i, j = key
            if i not in i_range:
                raise ValueError("i={} not in i_range".format(i))
            if j not in j_range:
                raise ValueError("j={} not in j_range".format(j))
            self._items[key] = value

    def __getitem__(self, key):
        return self._items[key]

    def __iter__(self):
        return iter(self._items)

    def __len__(self):
        return len(self._items)

    def __contains__(self, item):
        return item in self._items

    def __repr__(self):
        return '{}(i_range={}, j_range={}, items=[<{} items>])'.format(
            self.__class__.__name__,
            self.i_range, self.j_range,
            len(self._items))

class RegularConstantCatalog(Mapping):
    """Mapping with keys ordered with regular spacing along the number line.

    The values associated with the keys are constant.
    """

    def __init__(self, key_min, key_max, key_stride, value):
        """Initialize a RegularConstantCatalog.

        The catalog is initialized by a description of how the keys
        are distributed along the number line, and a value which
        corresponds with all keys.

        Args:
            key_min: The minimum key.
            key_max: The maximum key.
            key_stride: The difference between successive keys.
            value: A value associated with all keys.
        """
        if key_min > key_max:
            raise ValueError("key_min {} is greater than key_max {}, but key_min must be equal or less-than key_max."
                             .format(key_min, key_max))

        key_range = key_max - key_min
        if key_range % key_stride != 0:
            raise ValueError("RegularIndex key range {!r} is not "
                             "a multiple of stride {!r}".format(
                                 key_stride, key_range))

        self._key_min = key_min
        self._key_max = key_max
        self._key_stride = key_stride
        self._value = value

    def __getitem__(self, key):
        if key not in self:
            raise KeyError("{!r} does not contain key {!r}".format(self, key))
        return self._value

    def __len__(self):
        return 1 + (self._key_max - self._key_min) // self._key_stride

    def __contains__(self, key):
        return (self._key_min <= key <= self._key_max) and \
               ((key - self._key_min) % self._key_stride == 0)

    def __iter__(self):
        return iter(range(self._key_min,
                          self._key_max + 1,
                          self._key_stride))

    def __repr__(self):
        return '{}(key_min={}, key_max={}, key_stride={}, value={})'.format(
            self.__class__.__name__,
            self._key_min,
            self._key_max,
            self._key_stride,
            self._value)

class ConstantCatalog(Mapping):
    """Mapping with arbitrary keys and a single constant value.
    """

    def __init__(self, keys, value):
        """Initialize a ConstantCatalog.

        The catalog is initialized by a description with an iterable series of
        keys and a constant value to be associated with all the keys.

        Args:
            keys: An iterable series of keys.
            value: A value associated with all keys.
        """
        self._keys = SortedFrozenSet(keys)
        self._value = value

    def __getitem__(self, key):
        if key not in self:
            raise KeyError("{!r} does not contain key {!r}".format(self, key))
        return self._value

    def __len__(self):
        return len(self._keys)

    def __contains__(self, key):
        return key in self._keys

    def __iter__(self):
        return iter(self._keys)

    def __repr__(self):
        return '{}(keys=[{} items], value={})'.format(
            self.__class__.__name__,
            len(self._keys),
            self._value)

class RegularCatalog(Mapping):
    """Mapping with keys ordered with regular spacing along the number line.

    The values associated with the keys are arbitrary.
    """

    def __init__(self, key_min, key_max, key_stride, values):
        """Initialize a RegularCatalog.

        The catalog is initialized by a description of how the keys
        are distributed along the number line, and an iterable series of
        corresponding values.

        Args:
            key_min: The minimum key.
            key_max: The maximum key.
            key_stride: The difference between successive keys.
            values: An iterable series of values corresponding to the keys.

        Raises:
            ValueError: There is any inconsistency in the keys, stride,
                and/or values.
        """
        if key_min > key_max:
            raise ValueError("key_min {} is greater than key_max {}, but key_min must be equal or less-than key_max."
                             .format(key_min, key_max))

        key_range = key_max - key_min
        if key_range % key_stride != 0:
            raise ValueError("{} key range {!r} is not "
                             "a multiple of stride {!r}".format(self.__class__.__name__,
                                                                key_stride, key_range))
        self._key_min = key_min
        self._key_max = key_max
        self._key_stride = key_stride
        self._values = list(values)
        num_keys = 1 + key_range // key_stride
        if num_keys != len(self._values):
            raise ValueError("{} key range with length {} and values with length {} are inconsistent"
                             .format(self.__class__.__name__, num_keys, len(self._values)))

    def __getitem__(self, key):
        if not (self._key_min <= key <= self._key_max):
            raise KeyError("{!r} key {!r} out of range".format(self, key))
        offset = key - self._key_min
        if offset % self._key_stride != 0:
            raise KeyError("{!r} does not contain key {!r}".format(self, key))
        index = offset // self._key_stride
        return self._values[index]

    def __len__(self):
        return len(self._values)

    def __contains__(self, key):
        return (self._key_min <= key <= self._key_max) and \
               ((key - self._key_min) % self._key_stride == 0)

    def __iter__(self):
        return iter(range(self._key_min,
                          self._key_max + 1,
                          self._key_stride))

    def __repr__(self):
        return '{}(key_min={}, key_max={}, key_stride={}, values=[{} items])'.format(
            self.__class__.__name__,
            self._key_min,
            self._key_max,
            self._key_stride,
            len(self._values))

class LinearRegularCatalog(Mapping):
    """A mapping which assumes a linear relationship between keys and values.

    A LinearRegularCatalog predicts the value v from the key according to the
    following formula:

        v = (value_max - value_min) / (key_max - key_min) * (key - key_min) + value_min
    """

    def __init__(self,
                 key_min,
                 key_max,
                 key_stride,
                 value_start,
                 value_stop,
                 value_stride):
        """Initialize a LinearRegularCatalog.

        The catalog must contain at least two items.

        Args:
            key_min: The minimum key.
            key_max: The maximum key.
            key_stride: The difference between successive keys.
            value_start: The value corresponding to the minimum key.
            value_stop: The value corresponding to the maximum key.
            value_stride:

        Raises:
            ValueError: There is any inconsistency in the keys, strides,
                and/or values.
        """
        if key_min >= key_max:
            raise ValueError("key_min {} is greater-than or equal-to key_max {}, but key_min must be less-than key_max."
                             .format(key_min, key_max))

        if key_stride <= 0:
            raise ValueError("key_stride {} is not positive".format(key_stride))

        key_range = key_max - key_min
        if key_range % key_stride != 0:
            raise ValueError("{} key range {!r} is not "
                             "a multiple of key stride {!r}".format(
                                 self.__class__.__name__,
                                 key_stride,
                                 key_range))
        self._key_stride = key_stride

        if value_stride == 0:
            raise ValueError("value_stride {} cannot be zero".format(value_stride))

        value_range = value_stop - value_start
        if value_range % value_stride != 0:
            raise ValueError("{} value range {!r} is not "
                             "a multiple of value stride {!r}".format(
                                 self.__class__.__name__,
                                 value_stride,
                                 value_range))
        self._value_stride = value_stride

        self._key_min = key_min
        self._key_max = key_max
        self._value_start = value_start
        self._value_stop = value_stop

        num_keys = 1 + (self._key_max - self._key_min) // self._key_stride
        num_values = 1 + (self._value_stop - self._value_start) // self._value_stride
        if num_keys != num_values:
            raise ValueError("{} inconsistent number of "
                             "keys {} and values {}".format(
                                 self.__class__.__name__,
                                 num_keys,
                                 num_values))

        self._m = Fraction(self._value_stop - self._value_start,
                           self._key_max - self._key_min)

    def __getitem__(self, key):
        if not (self._key_min <= key <= self._key_max):
            raise KeyError("{!r} key {!r} out of range".format(self, key))
        offset = key - self._key_min
        if offset % self._key_stride != 0:
            raise KeyError("{!r} does not contain key {!r}".format(self, key))

        v = self._m * (key - self._key_min) + self._value_start
        assert v.denominator == 1
        return v.numerator

    def __len__(self):
        return 1 + (self._key_max - self._key_min) // self._key_stride

    def __contains__(self, key):
        return (self._key_min <= key <= self._key_max) and \
               ((key - self._key_min) % self._key_stride == 0)

    def __iter__(self):
        return iter(range(self._key_min, self._key_max + 1, self._key_stride))

    def __repr__(self):
        return '{}(key_min={}, key_max={}, key_stride={}, value_first={}, value_last={}, value_stride={})'.format(
            self.__class__.__name__,
            self._key_min,
            self._key_max,
            self._key_stride,
            self._value_start,
            self._value_stop,
            self._value_stride)

def _first_then_second_index(key):
    return (key[0][0], key[0][1])

def _second_then_first_index(key):
    return (key[0][1], key[0][0])


from flasgger import swag_from
from flask import Blueprint
from flask_jwt_extended import jwt_required
from ..database.crud.playlist import (
    create_playlist, create_many, get_playlist, delete_playlist, get_playlists
)
from ..database.schema.playlist import (
    Playlist as PlaylistSchema, Playlists as PlaylistsSchema, GetPlaylist, GetPlaylists
)
from ..database.database import get_db
from http import HTTPStatus
from pydantic import ValidationError
from flask import request

#from ..decorators import admin_token_required

playlists = Blueprint("playlists", __name__)

@swag_from("./docs/add.yml", endpoint="playlists.add", methods=["POST"])
@playlists.route("/playlist", methods=["POST"])
def add():
    try:
        playlist_data = PlaylistSchema(**request.json)
    except ValidationError:
        return {'Error': 'Invalid playlist data!'}, HTTPStatus.BAD_REQUEST
    playlist = get_playlist(session=get_db, playlist_data=playlist_data)
    if playlist:
        return {'Error': f'playlist with id {playlist_data.playlist_id} already exists'}, HTTPStatus.CONFLICT
    playlist = create_playlist(playlist_data=playlist_data, session=get_db)
    resp = PlaylistSchema(
        playlist_description=playlist.playlist_description,
        playlist_id=playlist.playlist_id,
        playlist_title=playlist.playlist_title,
        playlist_thumbnail=playlist.playlist_thumbnail,
        published_at=playlist.published_at,
        privacy_status=playlist.privacy_status,
        videos_count=playlist.videos_count,
        channel_id=playlist.channel_id
    )

    return resp.model_dump_json(indent=4), HTTPStatus.CREATED

@swag_from("./docs/update.yml", endpoint="playlists.update", methods=["PUT"])
@playlists.route("/playlist", methods=["PUT"])
def update():
    return {'success':'playlist'}, HTTPStatus.CREATED

@swag_from("./docs/delete.yml", endpoint="playlists.delete", methods=["DELETE"])
@playlists.route("/playlist", methods=["DELETE"])
def delete():
    try:
        playlist_data = GetPlaylist(playlist_id=request.args.get('playlist_id'))
    except ValidationError:
        return {'error': 'Invalid input: you probably did not include the playlist id.'}, HTTPStatus.BAD_REQUEST
    playlist = get_playlist(session=get_db, playlist_data=playlist_data)
    if not playlist:
        return {'Error': f'playlist with id {playlist_data.playlist_id} does not exists'}, HTTPStatus.NOT_FOUND
    playlist = delete_playlist(get_db, GetPlaylist(playlist_id=request.args.get('playlist_id')))
    resp = PlaylistSchema(
        playlist_description=playlist.playlist_description,
        playlist_id=playlist.playlist_id,
        playlist_title=playlist.playlist_title,
        playlist_thumbnail=playlist.playlist_thumbnail,
        published_at=playlist.published_at,
        privacy_status=playlist.privacy_status,
        videos_count=playlist.videos_count,
        channel_id=playlist.channel_id
    )
    return resp.model_dump_json(indent=4), HTTPStatus.OK

@swag_from("./docs/get.yml", endpoint="playlists.get", methods=["GET"])
@playlists.route("/playlist", methods=["GET"])
def get():
    try:
        playlist_data = GetPlaylist(playlist_id=request.args.get('playlist_id'))
    except ValidationError:
        return {'error': 'Invalid input: you probably did not include the playlist id.'}, HTTPStatus.BAD_REQUEST
    playlist = get_playlist(session=get_db, playlist_data=playlist_data)
    if playlist:
        resp = PlaylistSchema(
            playlist_description=playlist.playlist_description,
            playlist_id=playlist.playlist_id,
            playlist_title=playlist.playlist_title,
            playlist_thumbnail=playlist.playlist_thumbnail,
            published_at=playlist.published_at,
            privacy_status=playlist.privacy_status,
            videos_count=playlist.videos_count,
            channel_id=playlist.channel_id
        )
        return resp.model_dump_json(indent=4), HTTPStatus.OK

    return {'Error':f'No channel with id {playlist_data.playlist_id}'}, HTTPStatus.NOT_FOUND

@playlists.route("/", methods=["POST"])
# @admin_token_required
@swag_from("./docs/add_many.yml", endpoint="playlists.add_many", methods=["POST"])
def add_many():
    try:
        playlists_data = [PlaylistSchema(**data) for data in request.json['playlists']]
        playlists = PlaylistsSchema(playlists=playlists_data)
    except ValidationError:
        return {'Error': 'Invalid playlist data!'}, HTTPStatus.BAD_REQUEST
    count: int = create_many(session=get_db, playlists_data=playlists)
    return {'Success':f'{count} playlist(s) added.'}, HTTPStatus.CREATED

@swag_from("./docs/playlists.yml", endpoint="playlists.list_all_playlists", methods=["GET"])
@playlists.route("/", methods=["GET"])
def list_all_playlists():
    offset: str = request.args.get('offset')
    limit: str = request.args.get('limit')
    playlists = get_playlists(get_db, GetPlaylists(offset=offset, limit=limit))
    playlists = [
        PlaylistSchema(
            playlist_description=playlist.playlist_description,
            playlist_id=playlist.playlist_id,
            playlist_title=playlist.playlist_title,
            playlist_thumbnail=playlist.playlist_thumbnail,
            published_at=playlist.published_at,
            privacy_status=playlist.privacy_status,
            videos_count=playlist.videos_count,
            channel_id=playlist.channel_id
        ).dict()
        for playlist in playlists
    ]
    return playlists, HTTPStatus.OK

@swag_from("./docs/playlist_videos.yml", endpoint="playlists.videos", methods=["GET"])
@playlists.route("/playlist/videos", methods=["GET"])
def videos():
    return {'success':'playlist'}, HTTPStatus.CREATED


# import the necessary packages
import os
import gc
from imutils import paths
import argparse
import cv2
import numpy as np
from osgeo import gdal, ogr, osr
import time
import easygui
import fnmatch
import sys
import re
import subprocess
import mmap
from multiprocessing import Pool
from multiprocessing import cpu_count
from timeit import default_timer as timer
import time
import subprocess
from tqdm import tqdm

def Gdal_Func(work_data):
    epsg = str(work_data[6])
    tif_out = str(work_data[4])
    vrt = str(work_data[5])
    X1=int(work_data[0])
    X2=int(work_data[1])
    X3=int(work_data[2])
    X4=int(work_data[3])
    gsd = str(work_data[7]/100)

    opt= gdal.WarpOptions(options=['TFW=YES','TILED=YES','BLOCKXSIZE=512'], format='GTiff', outputBounds=[X1,X2,X3,X4], xRes=gsd, yRes=gsd, dstSRS=epsg, multithread=True, resampleAlg=gdal.GRIORA_Bilinear)
    gdal.Warp(tif_out,vrt,options=opt)

def pool_handler(work):
    p = Pool(nbr_cpu)
    for _ in tqdm(p.imap_unordered(Gdal_Func, work), total=len(work)):
        pass

def striplist(l):
    return([x.strip() for x in l])

def cls():
    os.system('cls' if os.name=='nt' else 'clear')

def compLat_Long(degs, mins, secs, comp_dir):
    return (degs + (mins / 60) + (secs / 3600)) * comp_dir

def mapcount(filename):
    f = open(filename, "r+")
    buf = mmap.mmap(f.fileno(), 0)
    lines = 0
    readline = buf.readline
    while readline():
        lines += 1
    return lines-1

def getSignOf(chifre):
    if chifre >= 0:
        return 1
    else:
        return -1

def update_progress(progress):
    barLength = 30 # Modify this to change the length of the progress bar

    block = int(round(barLength*progress))
    text = "\rPercent: [{0}] {1}% ".format( "#"*block + "-"*(barLength-block), int(progress*100))
    sys.stdout.write(text)
    sys.stdout.flush()
    #print('\n')

def epsgtoepsg(EPSG_in, lon, lat, EPSG_out):

 InSR = osr.SpatialReference()
 InSR.ImportFromEPSG(EPSG_in)
 OutSR = osr.SpatialReference()
 OutSR.ImportFromEPSG(EPSG_out)

 Point = ogr.Geometry(ogr.wkbPoint)
 Point.AddPoint(lon,lat) # use your coordinates here
 Point.AssignSpatialReference(InSR)    # tell the point what coordinates it's in
 Point.TransformTo(OutSR)              # project it to the out spatial reference

 return (Point.GetX(),Point.GetY())

def time_elapsed(start):
       end = timer()
       hours, rem = divmod(end-start, 3600)
       minutes, seconds = divmod(rem, 60)
       print('Time elapsed (hh:mm:ss.ms) {:0>2}:{:0>2}:{:05.2f}'.format(int(hours),int(minutes),seconds))

if __name__ == '__main__':

 dirphoto = easygui.diropenbox(msg=None, title="Selectionez le dossier des orthos à decouper et translater", default=None )
 list = fnmatch.filter(os.listdir(dirphoto), '*.tif')
 total_con=len(list)
 D1 = str(total_con)
 msg = str(total_con) +" orthos voulez-vous continuer?"
 title = "Merci de confirmer"
 if easygui.ynbox(msg, title, ('Yes', 'No')): # show a Continue/Cancel dialog
     pass # user chose Continue else: # user chose Cancel
 else:
     exit(0)

 x = [os.path.join(r,file) for r,d,f in os.walk(dirphoto) for file in f if file.endswith(".tif")]
 dirmosaic = easygui.diropenbox(msg=None, title="Selectionez le dossier de sortie (sur un autre volume svp)", default=None )

 cl=0
 ci=0
 cls()

 nbr_cpu = cpu_count()

 cpu_in = easygui.integerbox(msg='Vous avez %i cores, combien souhaitez-vous en allouer?' %nbr_cpu, title='Cores CPU', default=2, lowerbound=2, upperbound=nbr_cpu-2, image=None, root=None)

 eps_in = easygui.integerbox(msg='Entrez le code EPSG des orthos originalles', title='Code EPSG', default=3946, lowerbound=0, upperbound=99999, image=None, root=None)
 source = osr.SpatialReference()
 error = source.ImportFromEPSG(eps_in)
 if error==0:
  pass
 else:
   easygui.msgbox("Le code EPSG n'est pas valide, l'application va se fermer!!")
   sys.exit(0)

 eps_out = easygui.integerbox(msg='Entrez le code EPSG des orthos en sorties', title='Code EPSG', default=2154, lowerbound=0, upperbound=99999, image=None, root=None)
 source = osr.SpatialReference()
 error = source.ImportFromEPSG(eps_out)
 if error==0:
  pass
 else:
   easygui.msgbox("Le code EPSG n'est pas valide, l'application va se fermer!!")
   sys.exit(0)

 cut_shp = easygui.fileopenbox(msg='Selectionez le fichier shp (polygone englobant) en code EPSG:%i de l\'emprise des orthos en sorties' %eps_out, title='Shp',default=dirmosaic+'\\*.shp', filetypes=['*.shp'])
 driver = ogr.GetDriverByName('ESRI Shapefile')
 dataSource = driver.Open(cut_shp, 0)
 # Check to see if shapefile is found.
 if dataSource is None:
    easygui.msgbox("Impossible d\'ouvrir %s l'application va se fermer!!!"%cut_shp)
    sys.exit(0)
 else:
    layer = dataSource.GetLayer()
    featureCount = layer.GetFeatureCount()
    spatialRef = layer.GetSpatialRef()
    epsg_shp = (spatialRef.GetAuthorityCode(None))

    if str(epsg_shp) != str(eps_out):
       easygui.msgbox('Attention ce shp n\'est pas de la même projection aue votre ortho en sortie l\'application va se fermer!!!')
       sys.exit(0)

    if featureCount>1:
        easygui.msgbox('Attention ce shp contient plus de 1 polygone l\'application va se fermer!!!')
        sys.exit(0)
    else:
        for feature in layer:
             geom = feature.GetGeometryRef()
             Wkt  = geom.ExportToWkt()
             geo = geom.GetGeometryName()
             if geo == 'POLYGON':
                pass
             else:
                easygui.msgbox('Attention ce shp n\'est pas un polygone, l\'application va se fermer!!!')
                sys.exit(0)

 ortho_res = easygui.integerbox(msg='Entrez le GSD en centimètre ', title='GSD', default=5, lowerbound=1, upperbound=1000, image=None, root=None)

 ortho_size = easygui.integerbox(msg='Entrez la taille de la dalle en mètres', title='Taille de la dalle', default=200, lowerbound=100, upperbound=10000, image=None, root=None)

 epsgin = "EPSG:" + str(eps_in)
 epsgout = "EPSG:" + str(eps_out)

 print('creation du fichier vrt, patientez...')
 vrt_options = gdal.BuildVRTOptions(resampleAlg='cubic', outputSRS=epsgin)
 mon_vrt = dirphoto+'\cutline.vrt'

 gdal.BuildVRT(mon_vrt,x, options=vrt_options)
 #gdal.BuildVRT(mon_vrt,x)
 #rast_obj = gdal.BuildVRT('/vsimem/inmem.vrt',x,options=vrt_options)

 rast_obj = gdal.Open(mon_vrt)

 ulx, xres, xskew, uly, yskew, yres  = rast_obj.GetGeoTransform()

 ncols = rast_obj.RasterXSize
 nrows = rast_obj.RasterYSize
 del rast_obj
 x1 = ulx
 y1 = uly - (xres*nrows)
 x2 = ulx + (xres*ncols)
 y2 = uly

 x1_t0,y1_t0 = epsgtoepsg(eps_in,x1, y1,eps_out)
 x2_t0,y2_t0 = epsgtoepsg(eps_in,x2, y2,eps_out)

 x1_t0 = int(x1_t0 /ortho_size)*ortho_size
 y1_t0 = int(y1_t0 /ortho_size)*ortho_size
 x2_t0 = int(x2_t0 /ortho_size)*ortho_size+ortho_size
 y2_t0 = int(y2_t0 /ortho_size)*ortho_size+ortho_size
 print(x1_t0,y1_t0,x2_t0,y2_t0)
 xdalles = int((x2_t0-x1_t0)/ortho_size)
 ydalles = int((y2_t0-y1_t0)/ortho_size)
 total_dalles = int(xdalles*ydalles)

 cls()
 print('Creation de la liste des dalles pour le traitement sur %i cores'%cpu_in)
 time.sleep(1)
 cls()
 #####################################################
 Photos = np.empty((total_dalles, 8), dtype=object)
 #####################################################

 num = 0

 for i in range(0, xdalles-1):
   x_t = x1_t0+(i*ortho_size)
   X1=str(int(x_t))
   X2=str(int(x_t+ortho_size))
   for ii in range(0, ydalles-1):
      ci  += 1

      y_t = y1_t0+(ii*ortho_size)
      Y1=str(int(y_t))
      Y2=str(int(y_t+ortho_size))

      layer.SetSpatialFilterRect(float(X1),float(Y1),float(X2),float(Y2))
      num = layer.GetFeatureCount()
      if num > 0:
         tif_out = dirmosaic+'\\'+str(X1[:5])+'_'+str(Y2[:6])+'.tif'

         Photos[cl,0] = X1
         Photos[cl,1] = Y1
         Photos[cl,2] = X2
         Photos[cl,3] = Y2
         Photos[cl,4] = tif_out
         Photos[cl,5] = mon_vrt
         Photos[cl,6] = epsgout
         Photos[cl,7] = ortho_res
         cl  += 1

      num = 0
      layer.SetSpatialFilter(None)
      update_progress(ci/total_dalles)

 result = Photos[:cl]

 cls()
 print('Demarage de GdalWarp sur : '+ str(len(result)) +' photos soyez patient...')
 time.sleep(2)
 pool_handler(result)
 exit(0)
 cls()
 print('Fin')


import numpy as np
from sklearn.metrics import pairwise_distances

class t_SNE:
    def __init__(self, n_components=2, perplexity=25, learning_rate=0.01, n_epochs=200):
        self.perplexity = perplexity
        self.n_components = n_components
        self.learning_rate = learning_rate
        self.n_epochs = n_epochs
        self.affinities = None
        self.affinities_Y = None

    def __pairwise_affinities(self, norms):
        affinities = np.zeros_like(norms)
        sigma = 2 * self.perplexity
        for i in range(norms.shape[0]):
            numerator = np.exp(-norms[i] / sigma)
            numerator[i] = 0
            denominator = np.sum(numerator)
            affinities[i] = numerator / denominator

        return affinities

    def fit_transform(self, X):
        n, m = X.shape

        distance_matrix = pairwise_distances(X)
        norms = distance_matrix
        #norms = np.power(distance_matrix, 2)
        self.affinities = self.__pairwise_affinities(norms)

        Y = np.random.randn(n, self.n_components)
        gains = np.ones((n, self.n_components))

        for k in range(self.n_epochs):
            distances_Y = pairwise_distances(Y)
            #norms_Y = np.power(distances_Y, 2)
            norms_Y = distances_Y
            self.affinities_Y = self.__pairwise_affinities(norms_Y)

            grad = np.zeros((n, self.n_components))
            for i in range(n):
                diff = self.affinities_Y[i] - self.affinities[i]
                grad[i] = 4.0 * np.dot(diff, Y[i] - Y)

            gains = (gains + 0.2) * ((grad > 0.0) != (gains > 0.0)) + (gains * 0.8) * (
                        (grad > 0.0) == (gains > 0.0))
            gains[gains < 0.01] = 0.01
            Y = Y - self.learning_rate * (gains * grad)
            if k == 250:
                self.learning_rate /= 2.0

        return self.affinities, self.affinities_Y

    def KL_divergence(self, P, Q):
        return np.sum(P * np.log(P / Q))


'''Object to read user input in order to command mario'''

import pygame
from pygame.locals import *
import sys

class Input:
    def __init__(self, entity):
        self.entity = entity

    def checkForInput(self):
        '''checks if button is pressed for mario controls'''
        pressedKeys = pygame.key.get_pressed()

        if pressedKeys[K_RSHIFT] and not self.entity.pause:
            self.entity.sound.music_channel.pause()
            self.entity.sound.play_sfx("pause")
            self.entity.pause = True
        elif pressedKeys[K_a] and not pressedKeys[K_d] and not self.entity.lookingDown:
            self.entity.traits["goTrait"].direction = -1
        elif pressedKeys[K_d] and not pressedKeys[K_a] and not self.entity.lookingDown:
            self.entity.traits["goTrait"].direction = 1
        else:
            self.entity.traits['goTrait'].direction = 0
        if pressedKeys[K_ESCAPE]:
            self.entity.restart = True
        if pressedKeys[K_COMMA]:
            self.entity.GODMODE(True)
        if pressedKeys[K_PERIOD]:
            self.entity.GODMODE(False)
        isJumping = pressedKeys[K_SPACE]
        self.entity.jump(isJumping)
        self.entity.throw = pressedKeys[K_LSHIFT]
        self.entity.traits['goTrait'].boost = pressedKeys[K_LSHIFT]
        self.entity.traits['jumpTrait'].boost(pressedKeys[K_LSHIFT])
        self.entity.lookingLeft = pressedKeys[K_a]
        self.entity.lookingRight = pressedKeys[K_d]
        self.entity.lookingUp = pressedKeys[K_w]
        self.entity.lookingDown = pressedKeys[K_s]

    def checkForExit(self):
        '''specifically checks if player closes'''
        events = pygame.event.get()
        for event in events:
            if event.type == pygame.QUIT:
                pygame.quit()
                sys.exit()


#%%
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms

#%%
torch.manual_seed(4242)

#%%
train_loader = torch.utils.data.DataLoader(
    datasets.MNIST('./data', train=True, download=True,
                   transform=transforms.Compose([
                       transforms.ToTensor(),
                       transforms.Normalize((0.1307,), (0.3081,))
                   ])),
    batch_size=64, shuffle=True)

#%%
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(1, 10, kernel_size=5)
        self.conv2 = nn.Conv2d(10, 20, kernel_size=5)
        self.conv2_drop = nn.Dropout2d()
        self.fc1 = nn.Linear(320, 50)
        self.fc2 = nn.Linear(50, 10)

    def forward(self, x):
        x = F.relu(F.max_pool2d(self.conv1(x), 2))
        x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))
        x = x.view(-1, 320)
        x = F.relu(self.fc1(x))
        x = F.dropout(x, training=self.training)
        x = self.fc2(x)
        return F.log_softmax(x, dim=1)

#%%
model = Net()

#%%
optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5)

#%%
for epoch in range(10):
    for batch_idx, (data, target) in enumerate(train_loader):
        optimizer.zero_grad()
        output = model(data)
        loss = F.nll_loss(output, target)
        print('Current loss', float(loss))
        loss.backward()
        optimizer.step()

#%%
torch.save(model.state_dict(), 'mnist.pth')

#%%
pretrained_model = Net()
pretrained_model.load_state_dict(torch.load('mnist.pth'))


from github import UnknownObjectException
from sqlalchemy.testing.config import skip_test

from tests.base_test import BaseTest
from unittest.mock import Mock

from crc.services.workflow_processor import WorkflowProcessor
from crc.services.user_file_service import UserFileService

class FakeGithubCreates(Mock):
    def get_user(var):
        class FakeUser(Mock):
            def get_repo(var, name):
                class FakeRepo(Mock):
                    def get_contents(var, filename, ref):
                        raise UnknownObjectException(status='Failure', data='Failed data', headers=[])
                    def update_file(var, path, message, content, sha, branch):
                        pass
                return FakeRepo()
        return FakeUser()

class FakeGithub(Mock):
    def get_user(var):
        class FakeUser(Mock):
            def get_repo(var, name):
                class FakeRepo(Mock):
                    def get_contents(var, filename, ref):
                        fake_file = Mock()
                        fake_file.decoded_content = b'Some bytes'
                        fake_file.path = '/el/path/'
                        fake_file.data = 'Serious data'
                        fake_file.sha = 'Sha'
                        return fake_file
                    def get_branches(var):
                        branch1 = Mock()
                        branch1.name = 'branch1'
                        branch2 = Mock()
                        branch2.name = 'branch2'
                        master = Mock()
                        master.name = 'master'
                        return [branch1, branch2, master]
                    def update_file(var, path, message, content, sha, branch):
                        pass
                return FakeRepo()
        return FakeUser()

class TestFileService(BaseTest):
    """Largely tested via the test_file_api, and time is tight, but adding new tests here."""

    def test_add_file_from_task_never_archives_original_on_subsequent_add(self):
        workflow = self.create_workflow('file_upload_form')
        processor = WorkflowProcessor(workflow)
        task = processor.next_task()
        irb_code = "UVACompl_PRCAppr"  # The first file referenced in pb required docs.
        UserFileService.add_workflow_file(workflow_id=workflow.id,
                                          task_spec_name=task.get_name(),
                                          name="anything.png", content_type="text",
                                          binary_data=b'1234', irb_doc_code=irb_code)
        # Add the file again with different data
        UserFileService.add_workflow_file(workflow_id=workflow.id,
                                          task_spec_name=task.get_name(),
                                          name="anything.png", content_type="text",
                                          binary_data=b'5678', irb_doc_code=irb_code)

        file_models = UserFileService.get_workflow_files(workflow_id=workflow.id)
        # Both models should exist
        self.assertEqual(2, len(file_models))

        file_data = UserFileService.get_workflow_data_files(workflow_id=workflow.id)
        self.assertEqual(2, len(file_data))
        self.assertFalse(file_data[0].archived)
        self.assertFalse(file_data[1].archived)
        self.assertEqual(4, file_data[0].size)  # File data size is included.

    def test_add_file_from_form_allows_multiple_files_with_different_names(self):
        workflow = self.create_workflow('file_upload_form')
        processor = WorkflowProcessor(workflow)
        task = processor.next_task()
        irb_code = "UVACompl_PRCAppr"  # The first file referenced in pb required docs.
        UserFileService.add_workflow_file(workflow_id=workflow.id,
                                      irb_doc_code=irb_code,
                                      task_spec_name=task.get_name(),
                                      name="anything.png", content_type="text",
                                      binary_data=b'1234')
        # Add the file again with different data
        UserFileService.add_workflow_file(workflow_id=workflow.id,
                                      irb_doc_code=irb_code,
                                      task_spec_name=task.get_name(),
                                      name="a_different_thing.png", content_type="text",
                                      binary_data=b'5678')
        file_models = UserFileService.get_workflow_files(workflow_id=workflow.id)
        self.assertEqual(2, len(file_models))


"""
    @name core file
    @info   The core file includes all the implemented code that needs to be accessed across the other files.
    @organization: Laval University
    @author     Parham Nooralishahi
    @email      parham.nooralishahi.1@ulaval.ca
    @professor  Pascal Germain
    @semester   Winter 2021
"""

from deeplib.datasets import load_mnist_v2
from torch.utils.data import Subset, DataLoader
from torchvision.datasets.mnist import MNIST
import torch.nn.functional as F
import torch.nn as nn
import torch.optim as optim
import torch

import imageio
import numpy as np
import math

import matplotlib.pyplot as plt
from matplotlib.backends.backend_agg import FigureCanvasAgg

from poutyne import Callback, Dict

plt.rcParams['figure.dpi'] = 150

def save_model(model, file_path):
    """save_model saves the trained model in the specified path

    Args:
        model (nn.Module): the model to be saved.
        file_path (str): file path.
    """

    torch.save(model, file_path)

def load_model(file_path):
    """load_model loads the saved model from specified path.

    Args:
        file_path (str): file path to the trained model.

    Returns:
        nn.Module: the trained model.
    """
    return torch.load(file_path)

def load_datasets(split_percent=0.8):
    """load_datasets loads the dataset and split it into training, validation, and testing subsets based on the given split percentage.

    Args:
        split_percent (float, optional): The split ratio. Defaults to 0.8.

    Returns:
        train_dataset: the training subset of MNIST dataset,
        valid_dataset: the validation subset of MNIST dataset,
        testing_dataset: the testing subset of MNIST dataset
    """

    # Load MNIST
    print('Loading MNIST Dataset ...')
    training_ds, testing_dataset = load_mnist_v2()
    # Shuffle the data
    num_data = len(training_ds)
    print('Number of Loaded Data: %d' % num_data)
    indices = list(range(num_data))
    np.random.shuffle(indices)
    split_len = math.floor(split_percent * num_data)
    # Training dataset
    train_indices = indices[:split_len]
    train_dataset = Subset(training_ds, train_indices)
    # Validation dataset
    valid_indices = indices[split_len:]
    valid_dataset = Subset(training_ds, valid_indices)

    return train_dataset, valid_dataset, testing_dataset

def load_datasets_q4():
    """load_datasets loads the dataset.

    Returns:
        train_dataset: the training subset of MNIST dataset,
        testing_dataset: the testing subset of MNIST dataset
    """

    # Load MNIST
    print('Loading MNIST Dataset ...')
    training_ds, testing_dataset = load_mnist_v2()
    # Shuffle the data
    num_data = len(training_ds)
    print('Number of Loaded Data: %d' % num_data)

    return training_ds, testing_dataset

class PhmMnist(nn.Module):
    """ PhmMnist is the designed nn model for learning MNIST dataset. """

    mnist_image_size = 28*28

    def __init__(self, num_classes=10):
        """
        Args:
            num_classes (int, optional): the number of classes need to be recognized. Defaults to 10.
        """

        super().__init__()
        self.layer_input = nn.Flatten()
        self.layer_h1 = nn.Linear(self.mnist_image_size, 256)
        self.layer_h2 = nn.Linear(256, 128)
        self.layer_h3 = nn.Linear(128, 64)
        self.layer_h4 = nn.Linear(64, num_classes)

    def forward(self, x):
        x = self.layer_input(x)
        x = F.relu(self.layer_h1(x))
        x = F.relu(self.layer_h2(x))
        x = F.relu(self.layer_h3(x))
        x = F.relu(self.layer_h4(x))
        return x

class PhmPlottingCallback(Callback):
    """ PhmPlottingCallback is the class containing the codes for visualizing the model's results.
        The class also extends Callback class makes it able to attach to Poutyne experiment and register for the raised events.
    """

    def __init__(self):
        super().__init__()
        self.report = list()
        self.fig_list = list()
        self.fig, (self.loss_ax, self.acc_ax) = plt.subplots(2, 1)
        self.fig.tight_layout(pad=1.0)
        self.fig.suptitle('Statistics of training model')
        mng = plt.get_current_fig_manager()
        mng.full_screen_toggle()
        plt.ion()
        plt.show()

    def on_train_begin(self, logs: Dict):
        """ on_train_begin is a method invoked in the begining of each train. """
        pass

    def _plot(self, axis, epoch, train_data, valid_data, label, maxv=1.0, minv=0.0, stepv=0.05):
        axis.clear()
        plt.yticks(fontsize=11)
        axis.plot(epoch, train_data, '.-', label='Train')
        axis.plot(epoch, valid_data, '.-', label='Validation')
        axis.set_xlabel('epoch')
        axis.set_ylabel(label)
        # axis.set_yticks(np.arange(minv, maxv, stepv), minor = True)
        axis.set_yticks(np.arange(min(train_data + valid_data),
                                  max(train_data + valid_data), stepv), minor=True)
        axis.legend()
        axis.grid(True, which='both', axis='both', linestyle='--')

    def on_epoch_end(self, epoch_number: int, logs: Dict):
        """ on_epoch_end is a method invoked in the end of each epoch. """

        self.report.append(logs)

        epoch = [item['epoch'] for item in self.report]
        train_loss = [item['loss'] for item in self.report]
        val_loss = [item['val_loss'] for item in self.report]
        train_acc = [item['acc'] for item in self.report]
        val_acc = [item['val_acc'] for item in self.report]

        canvas = FigureCanvasAgg(self.fig)
        # Loss
        self._plot(self.loss_ax, epoch, train_loss, val_loss, 'Loss')
        # Accuracy
        self._plot(self.acc_ax, epoch, train_acc,
                   val_acc, 'Accuracy', 100, 0, 5)

        plt.show()
        plt.pause(0.05)

        canvas.draw()
        fimg = np.array(canvas.renderer.buffer_rgba())
        self.fig_list.append(fimg)

    def on_train_end(self, logs: Dict):
        """ on_train_end is the method invoked in the end of each training. """

        if len(self.fig_list) > 0:
            imageio.mimsave("result.gif", self.fig_list, 'GIF')
            imageio.imwrite('result.png', self.fig_list[-1])

class DeadNeuronHook:
    """ DeadNeuronHook is a hook class designed to be used to measure the ratio of dead neurons.
        This class contains two hook method: (a) forward pass hook, (b) backward pass hook.
        The hooks are similar to event handlers for network's layers.
        For instance, when a hook is registered for a layer for backward pass,
        this method will be invoked for each input data after the gradient is calculated. The same is correct
        for forward pass, which means the forward pass hook will be invoked for each given input after the output of the neuron
        for all the neurons in the layer is calcualted.
    """

    def __init__(self, name, thresh=0.99):
        self._name = name
        # dead_signals is a matrix with similar dimensions to the layer. The values shows the number of times the specific neuron shows
        # the signs of being dead (zero gradient for backpass, zero output for forward pass).
        self.dead_signals = None
        # number_signals keep the total number of input samples.
        self.number_signals = 0
        # dead_thresh determines the ratio of tolerance to call a model perfect with 0% error rate.
        self.dead_thresh = thresh

    @property
    def name(self):
        """ layer's name """
        return self._name

    def calculate_dead_rate(self):
        """ calculate_dead_rate calculates the ratio of dead neurons for each layer.
        """

        # calculate the ratio of inactiveness for the neurons in the layer.
        # dead_signals is a matrix with similar dimensions to the layer. The values shows the number of times the specific neuron shows
        # the signs of being dead (zero gradient for backpass, zero output for forward pass).
        # number_signals != 0 is checked to make sure if the calculate_dead_rate called by a user before any hook is called, the method
        # doesn't raise any exception.
        active_ratio = self.dead_signals / \
            self.number_signals if self.number_signals != 0 else 0
        dead_neurons = np.zeros(active_ratio.shape)
        # dead_neurons are detected by detecting the neurons that were inactive (or with zero gradient) for all the samples.
        dead_neurons[active_ratio >= self.dead_thresh] = 1
        # layer_ratio calculated by dividing the number of dead neurons to the total number of neurons (np.max(dead_neurons.shape))
        # in the layer.
        layer_ratio = (np.sum(dead_neurons) /
                       np.max(dead_neurons.shape)) * 100.0
        return layer_ratio, active_ratio, dead_neurons

    def hook_backward_dead_neuron(self, module, gradInput, gradOutput):
        """ hook_backward_dead_neuron is a hook method for backward pass call """

        grad = gradOutput[0].cpu().numpy()
        if self.dead_signals is None:
            self.dead_signals = np.zeros(grad.shape)

        tmp = np.zeros(grad.shape)
        tmp[grad == 0] = 1
        self.dead_signals += tmp
        self.number_signals += 1

    def hook_forward_dead_neuron(self, module, inputs, outputs):
        """ hook_backward_dead_neuron is a hook method for forward pass call """
        out = F.relu(outputs)
        out = out.detach().numpy()
        if self.dead_signals is None:
            self.dead_signals = np.zeros(out.shape)

        tmp = np.zeros(out.shape)
        tmp[out == 0] = 1
        self.dead_signals += tmp
        self.number_signals += 1

def calculate_dead_neoron_ratio(model, dataset, forward_check = False, thresh=0.99, batch_size=1):
    """ calculate_dead_neoron_ratio calculate the ratio of dead neurons for all layers.

    Args:
        model (nn.Module): the targeted model.
        dataset (Dataset): the loaded dataset.
        forward_check (bool, optional): determines whether it is a forward pass (True) or backward pass (False). Defaults to False.
        thresh (float, optional): the acceptable tolerance to the error in the model. Defaults to 0.99.
        batch_size (int, optional): the size of the batch. Defaults to 1.

    Returns:
        dict: a dictionary containing the ratio of dead neurons for all layers.
    """

    # Create the data loader
    data_loader = DataLoader(
        dataset, batch_size=batch_size, num_workers=2)

    # Loss Function Initialization
    loss_function = nn.CrossEntropyLoss()
    # Initialize the hooks for calculating dead neuron ratio
    layers = [module for module in model.modules() if type(module) == nn.Linear]
    stats = dict()
    hook_list = []
    # Create a hook (DeadNeuronHook) for tracking the dead neurons.
    for index in range(0, len(layers)):
        l = layers[index]
        name = 'layer %d' % index
        h = DeadNeuronHook(name, thresh=thresh)
        hn = None
        if forward_check:
            # Register forward pass hook.
            hn = l.register_forward_hook(h.hook_forward_dead_neuron)
        else:
            # Register backward pass hook.
            hn = l.register_backward_hook(h.hook_backward_dead_neuron)
        hook_list.append(hn)
        stats[name] = h

    model.train()
    # Feed the model with the given testing samples
    for inputs, targets in data_loader:
        # Reset the gradients
        model.zero_grad()
        # forward pass
        output = model(inputs)
        # Calculate the loss
        loss = loss_function(output, targets)
        # Calculate the gradients
        loss.backward()

    # Calculate the statistics
    res = dict()
    for key, hook in stats.items():
        # Calculate the ratio of dead neurons for the layers.
        layer_ratio, _, _ = hook.calculate_dead_rate()
        res[key] = layer_ratio

    # Remove the hook from the layers (Required for repeated  )
    for hn in hook_list:
        hn.remove()

    return res


# -*- coding: utf-8 -*-
import gzip
import os
import random

import numpy as np
from collections import defaultdict

from tqdm import tqdm

def parse(path):
    g = gzip.open(path, 'r')
    for l in g:
        yield eval(l)

countU = defaultdict(lambda: 0)
countP = defaultdict(lambda: 0)
line = 0

dataset_name = 'Movies_and_TV_5'
if not os.path.exists(dataset_name):
    os.mkdir(dataset_name)

vocab = set()
for l in tqdm(parse('reviews_' + dataset_name + '.json.gz'), total=line):
    line += 1
    asin = l['asin']
    rev = l['reviewerID']
    countU[rev] += 1
    countP[asin] += 1
    vocab.add(asin)
with open(os.path.join(dataset_name, 'vocab.txt'), 'w') as f:
    f.write("[PAD]\n[UNK]\n[CLS]\n[SEP]\n[MASK]\n")
    for asin in vocab:
        f.write(asin + "\n")

print(len(countU), len(countP))
line = 0
User = dict()
for l in tqdm(parse('reviews_' + dataset_name + '.json.gz'), total=line):
    line += 1
    asin = l['asin']
    rev = l['reviewerID']
    time = l['unixReviewTime']
    if countU[rev] < 10 or countP[asin] < 10:
        continue
    if rev not in list(User.keys()):
        User[rev] = []
    User[rev].append([time, asin])

# sort reviews in User according to time
print('start sort')
for userid in list(User.keys()):
    User[userid].sort(key=lambda x: x[0])

with open(os.path.join(dataset_name, 'data_' + dataset_name + '_10_core.txt'), 'w') as f:
    for userid in list(User.keys()):
        f.write(" ".join([str(u[1]) for u in User[userid]]) + "\n")

seq_lens = np.asarray([len(u) for u in User.values()])
print(np.amax(seq_lens), np.amin(seq_lens), np.mean(seq_lens), np.median(seq_lens), np.std(seq_lens))

interval_res = []
for u in User.values():
    time_interval = [i[0] for i in u]
    time_interval = [(time_interval[i] - time_interval[i - 1]) / 3600 for i in range(1, len(time_interval))]
    time_interval.insert(0, 0)
    interval_res.append(
        [np.amax(time_interval), np.amin(time_interval), np.mean(time_interval), np.median(time_interval),
         np.std(time_interval)])
print(np.amax(interval_res, axis=0), np.mean(interval_res, axis=0), np.median(interval_res, axis=0))

actions = [u[0] for s in User.keys() for u in s]

def train_test_split(data_pairs, test_ratio, shuffle=False):
    n_total = len(data_pairs)
    offset = int(n_total * test_ratio)
    if n_total == 0 or offset < 1:
        return [], data_pairs
    if shuffle:
        random.shuffle(data_pairs)
    test = data_pairs[:offset]
    train = data_pairs[offset:]
    return train, test

pretrain_ratio = 0.5
pretrain, test = train_test_split(list(User.values()), pretrain_ratio, True)
print(len(pretrain), len(test))

seqs = []
seq_lens = []
tim_itv = 800 * 3600
max_length = 62
min_length = 10
with open(os.path.join(dataset_name, 'pretrain_' + dataset_name + '.txt'), 'w') as f:
    for user in tqdm(pretrain):
        seq = []
        start = len(user)
        end = len(user)
        action = [u[1] for u in user]
        # time_interval = [u[0] for u in user]
        # for t in range(len(time_interval) - 1, 0, -1):
        #     if time_interval[t] - time_interval[t - 1] > tim_itv:
        #         start = t
        #         if end - start > 10:
        #             seq.append(action[start:end])
        #             end = start
        nbatch = len(action) // max_length
        for i in range(nbatch):
            seq.append(action[len(action) - (i + 1) * max_length:len(action) - i * max_length])
        if len(action) > nbatch * max_length + 10:
            seq.append(action[0:len(action) - nbatch * max_length])
        seq.reverse()
        seqs.append(seq)
        for s in seq:
            seq_lens.append(len(s))
            f.write(' '.join(s) + '\n')
        if len(seq) > 0:
            f.write('\n')

print(np.sum(seq_lens), np.amax(seq_lens), np.amin(seq_lens), np.mean(seq_lens), np.median(seq_lens), np.std(seq_lens))
seq_lens = []
with open(os.path.join(dataset_name, 'test_' + dataset_name + '.txt'), 'w') as f:
    for user in test:
        seq_lens.append(len(user))
        f.write(' '.join([u[1] for u in user]) + '\n')

print(np.sum(seq_lens), np.amax(seq_lens), np.amin(seq_lens), np.mean(seq_lens), np.median(seq_lens), np.std(seq_lens))

# user item
# action

# book
# 603668    367982
# 8898041

# digital music
# 5541  3568
# 64706