import csv
import copy
import json
import datetime
import logging

from RequestForQuotes import *
from ExtMaterials import ExtMaterials

from pymongo import MongoClient

class RFQCreator:

    def __init__(self):
        """clase que crea RFQs en el formator necesario para guardar en la DB"""
        self.rfq = RequestForQuotes()
        self.client = MongoClient('localhost', 27017)
        self.db = self.client.conpancol
        self.collection = self.db.mnmaterials
        self.rfqcollection = self.db.rfquotes
        logging.basicConfig(filename='logs/rfqcreator.log', level=logging.DEBUG)

    def lookForMaterial(self, itemCode):

        if self.collection.find({'itemcode': itemCode}).count() > 0 :
            material = self.collection.find_one({'itemcode': itemCode})
            return material
        else:
            raise AttributeError('Material not found')

    def alreadyExists(self, newID):
        if self.rfqcollection.find({'internalCode': newID}).count() > 0:
            logging.info('RFQ already exists  \t' + str(newID))
            return True
        else:
            logging.info('RFQ does not exists  \t' + str(newID))
            return False

    def setRFQInformation(self, internalCode, externalCode, sender, company, receivedDate):
        self.rfq.setIntenalCode(internalCode)
        self.rfq.setExternalCode(externalCode)
        self.rfq.setSender(sender)
        self.rfq.setCompany(company)
        self.rfq.setReceivedDate(receivedDate)
        dt = datetime.datetime.now()
        self.rfq.setProcessedDate(dt.strftime('%d/%m/%Y'))

    def addRFQNote(self, note):
        self.rfq.setNote(note)

    def createRFQfromCSV(self, csvfile):
        try:
            with open(csvfile, 'r', encoding='utf-8') as f:
                reader = csv.reader(f, dialect="excel-tab")
                extmaterials = []
                for row in reader:
                    try:
                        orderNum = row[0]
                        itemCode = row[1]
                        quantity = float(row[2])
                        unit = row[3]

                        material = self.lookForMaterial(itemCode)
                        extendedMaterial = ExtMaterials(material)
                        extendedMaterial.setOrderNumber(orderNum)
                        extendedMaterial.setUnit(unit)
                        extendedMaterial.setQuantity(quantity)
                        extmaterials.append(extendedMaterial)
                        logging.info('Material found in DB \t' + itemCode + '\t' + "OK")

                    except AttributeError as ex:
                        logging.info('Material code not found in DB \t' + itemCode + '\t' + "NA")
                        continue

                    except ValueError as ex:
                        logging.info('There is a wrong data format entry. Please check')
                        continue

                self.rfq.setMaterialList(extmaterials)

            rfq_json = self.rfq.__dict__
            rfq_json['materialList'] = self.rfq.to_json()
            output = copy.deepcopy(rfq_json)
            print(json.dumps(output))
            totalMaterials = len(rfq_json['materialList'])
            logging.info('End of RFQ creation  \t' + str(totalMaterials))
            self.rfqcollection.insert_one(output)

        except IOError:
            logging.info('Error opening input file')

    def exportRFQtoCSV(self, code, csvfile):

        labels = []
        labels.append('Id')
        labels.append('OrderId')
        labels.append('ItemId')
        labels.append('Description')
        labels.append('Type')
        labels.append('Quantity')
        labels.append('Unit')

        try:
            path = "./data/export/" + csvfile
            with open(path, 'w', encoding='utf-8', newline='') as f:
                writer = csv.writer(f, dialect="excel", delimiter=';')

                if self.alreadyExists(code):

                    cursor = self.rfqcollection.find({'internalCode': code})

                    rfq = cursor.next()

                    header = 'RFQ internal code: ' + str(rfq["internalCode"])
                    writer.writerow([header])
                    info = 'Originator: ' + str(rfq["sender"]) + ' ' + str(rfq["company"])
                    writer.writerow([info])
                    details = 'Received date: ' + str(rfq["receivedDate"])
                    writer.writerow([details])
                    note = 'Note: ' + str(rfq["note"])

                    writer.writerow([note])
                    writer.writerow([''])
                    labels_rwo = '\t'.join(labels)
                    print(labels_rwo)
                    writer.writerow(labels)

                    items = rfq["materialList"]

                    id = 1

                    for item in items:
                        rwo = []
                        rwo.append(str(id))
                        rwo.append(item["orderNumber"])
                        rwo.append(item["itemcode"])
                        description = item["description"]
                        rwo.append(description)
                        rwo.append(item["type"])
                        rwo.append(str(item["quantity"]))
                        rwo.append(str(item["unit"]))

                        if item["category"] == "PLATE":
                            nplates = self.getNumberPlates(item["dimensions"], item["quantity"])
                            rwo.append(str(nplates))
                            rwo.append('PC')

                        writer.writerow(rwo)
                        id += 1

                    for i in range(1,3):
                        writer.writerow([' '])
                    bottom_txt_file = open('./data/Bottom_conditions_EN_FOB.txt', 'r')
                    bottom_txt_rows = bottom_txt_file.readlines()
                    print(bottom_txt_rows)
                    for line in bottom_txt_rows:
                        row = line.replace('\n', ' ')
                        writer.writerow([row])
                    bottom_txt_file.close()

                else:
                    logging.info('RFQ does not exists in DB')

            f.close()

        except IOError:
            logging.info('Problem with file creation')

    def getNumberPlates(self, dimensions, total_area):

        if dimensions.find("MM"):
            dim_values = []
            dims = dimensions.split(',')[1].split('X')
            for dd in dims:

                try:
                    dim_values.append(float( dd.replace(' ','').replace('MM','')))
                except ValueError:
                    dim_values.append(0.0)

            area = dim_values[0] * dim_values[1] * 0.000001
            nplates = format( total_area / (area), '.2f')
            print(dim_values, total_area, nplates)

        return nplates


from ast import alias
from turtle import title
import discord
from discord.ext import commands

from youtube_dl import YoutubeDL

class musicCog(commands.Cog):
    def __init__(self, bot):
        self.bot = bot

        self.isPlaying = False
        self.isPaused = False

        self.musicQueue = []
        self.ydlOptions = {'format': 'bestaudio', 'noplaylist':'True'}
        self.ffmpegOptions = {'before_options': '-reconnect 1 -reconnect_streamed 1 -reconnect_delay_max 5', 'options': '-vn'}

        self.vc = None

    def searchYT(self, item):
        with YoutubeDL(self.ydlOptions) as ydl:
            try:
                info = ydl.extract_info("ytsearch:%s" % item, download=False)['entries'][0]
            except Exception:
                return False

        return {'source': info['formats'][0]['url'], 'title': info['title']}

    def playNext(self):
        if len(self.musicQueue) > 0:
            self.isPlaying = True

            m_url = self.musicQueue[0][0]['source']

            self.musicQueue.pop(0)

            self.vc.play(discord.FFmpegPCMAudio(m_url, **self.ffmpegOptions), after=lambda e: self.playNext())
        else:
            self.isPlaying = False

    async def playSong(self, ctx):
        if len(self.musicQueue) > 0:
            self.isPlaying = True

            m_url = self.musicQueue[0][0]['source']

            if self.vc == None or not self.vc.is_connected():
                self.vc = await self.musicQueue[0][1].connect()

                if self.vc == None:
                    await ctx.send("I CAN'T JOIN, ASSHOLE.")
                    return
            else:
                await self.vc.move_to(self.musicQueue[0][1])

            self.musicQueue.pop(0)

            self.vc.play(discord.FFmpegPCMAudio(m_url, **self.ffmpegOptions), after=lambda e: self.playNext())
        else:
            self.isPlaying = False

    @commands.command(name="play", aliases=["p","playing"])
    async def play(self, ctx, *args):
        query = " ".join(args)

        voice_channel = ctx.author.voice.channel
        if voice_channel is False:
            await ctx.send("COULD YOU USE YOUR THINKING FLESH MOUND AND JOIN A CHANNEL FIRST?")
        elif self.isPaused:
            self.vc.resume()
        else:
            song = self.searchYT(query)
            if type(song) == type(True):
                await ctx.send("COULDN'T GET WHATEVER THE FUCK THAT WAS. REMEMBER: NO PLAYLISTS AND YOUTUBE ONLY, ASSCLOWN.")
            else:
                await ctx.send("ADDED THAT SHIT TO THE QUEUE.")
                self.musicQueue.append([song, voice_channel])

                if self.isPlaying == False:
                    await self.playSong(ctx)

    @commands.command(name="pause")
    async def pause(self, ctx, *args):
        if self.isPlaying:
            self.isPlaying = False
            self.isPaused = True
            await ctx.send("PAUSING, OR WHATEVER THE FUCK.")
            self.vc.pause()
        elif self.isPaused:
            self.isPaused = False
            self.isPlaying = True
            self.vc.resume()

    @commands.command(name = "resume", aliases=["r"])
    async def resume(self, ctx, *args):
        if self.isPaused:
            self.isPaused = False
            self.isPlaying = True
            await ctx.send("RESUMING, OR SOMETHING. I DON'T CARE.")
            self.vc.resume()

    @commands.command(name="skip", aliases=["s"])
    async def skip(self, ctx):
        if self.vc != None and self.vc:
            await ctx.send("DON'T CARE, NEXT.")
            self.vc.stop()

            await self.playSong(ctx)

    @commands.command(name="queue", aliases=["q"])
    async def queue(self, ctx):
        queue = ""
        await ctx.send("UP NEXT ON THE PRIMITIVE, SHAMEFUL RAZZMATAZZ OF GO FUCK YOURSELF IS: \n")
        for i in range(0, len(self.musicQueue)):
            queue += f"{i+1}. {self.musicQueue[i][0]['title']} \n"

        if queue != "":
            await ctx.send(queue)
        else:
            await ctx.send("NOTHING. NO UPCOMING SONGS.")

    @commands.command(name="clear", aliases=["c", "bin"])
    async def clear(self, ctx):
        self.musicQueue = []
        await ctx.send("HOLY SHIT. THANK FUCK.")

    @commands.command(name="leave", aliases=["disconnect", "l", "d"])
    async def dc(self, ctx):
        self.isPlaying = False
        self.isPaused = False
        await ctx.send("WHATEVER.")
        await self.vc.disconnect()
        self.vc = None
        self.musicQueue = []


from django.contrib import admin
from .models import *

# Register your models here.

class ExerciseAdmin(admin.ModelAdmin):
    list_display = ("for_lecture", )
    list_display_links = ("for_lecture", )
    search_fields = ("for_lecture", "task")

class AnswerAdmin(admin.ModelAdmin):
    list_display = ("for_exercise", )
    list_display_links = ("for_exercise", )
    search_fields = ("for_exercise", "option", "correct")

admin.site.register(Exercise, ExerciseAdmin)
admin.site.register(Answer, AnswerAdmin)


from django.urls import path
from . import views

urlpatterns = [
    path('', views.index, name='index'),
    path('index.html', views.contact2, name='contact2'),
    path('seven.html', views.seven, name='seven'),

    path('about.html', views.about, name='about'),
    path('car.html', views.car, name='car'),
    path('contact.html', views.contact, name='contact'),
    path('detail.html', views.detail, name='detail'),
    path('service.html', views.service, name='service'),
    path('message.html', views.contact, name="contact"),

    path('meetings.html', views.meetings, name='meetings'),
    path('meeting-details.html', views.meetingdetails, name='meetingdetails'),
]


from google_images_download import google_images_download   #importing the library
import json
import pandas as pd

response = google_images_download.googleimagesdownload()

if __name__ == "__main__":
    filepath = 'data/raw/100VietnameseCelebName.txt'
    df = pd.DataFrame(columns=['Name', 'Pic'])

    name_list = pd.read_csv(filepath, header=None)
    name_list.columns = ['name']
    # name_list.apply(lambda x: x.strip())

    for _, row in name_list.iterrows():
        name = row.values[0]
        paths, err = response.download({"keywords": name, "limit": 10, 'format': 'png'})
        a = json.dumps(paths)
        d = json.loads(a)
        temp = []
        for i in range(0, len(d[name])):
            temp.append(name)
        data = {'Name': temp, 'Pic': d[name]}
        df = df.append(data, ignore_index=True)

    df.to_csv('celeb.csv')


import json
import pathlib
from datetime import datetime
from decimal import Decimal
from typing import List

import matplotlib.pyplot as plt
import matplotlib.transforms as transforms
import pandas as pd

from src.backtest.backtest_data_type import HedgeTrade, HedgeType, LogState
from src.exchange.ftx.ftx_data_type import FtxHedgePair
from src.indicator.base_indicator import BaseIndicator

def logs_to_summary(logs: List[LogState]) -> dict:
    if len(logs) == 0:
        return {"avg_deposit": 0, "profit": 0, "roi": 0, "apr": 0}
    profit = float(logs[-1].profit)
    index = []
    net_deposit = []
    for log in logs:
        index.append(datetime.fromtimestamp(log.timestamp))
        net_deposit.append(log.net_deposit)
    net_deposit_seires = pd.Series(net_deposit, index)
    avg_deposit = net_deposit_seires.resample("1s").last().fillna(method="ffill").mean()
    if avg_deposit > 0:
        roi = profit / avg_deposit
        days = (logs[-1].timestamp - logs[0].timestamp) / 86400
        apr = roi / days * 365
    else:
        roi = 0
        apr = 0
    return {"avg_deposit": avg_deposit, "profit": profit, "roi": roi, "apr": apr}

class DecimalEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, Decimal):
            return str(obj)
        return json.JSONEncoder.default(self, obj)

def save_to_file(summary: dict, save_path: str) -> dict:
    path = pathlib.Path(save_path)
    if not path.exists():
        path.parent.mkdir(parents=True, exist_ok=True)

    with path.open("w") as fp:
        json.dump(summary, fp, indent=2, cls=DecimalEncoder)
    return summary

def save_summary(summary: dict, save_path: str) -> dict:
    save_to_file(summary, save_path)
    print(f"Save summary to {save_path}")
    return summary

def plot_logs(
    logs: List[LogState],
    hedge_trades: List[HedgeTrade],
    save_path: str,
    to_show: bool = False,
):
    if len(logs) == 0:
        return
    path = pathlib.Path(save_path)
    if not path.exists():
        path.parent.mkdir(parents=True, exist_ok=True)
    index = []
    basis = []
    position = []
    net_deposit = []
    profit = []
    for log in logs:
        index.append(datetime.fromtimestamp(log.timestamp))
        basis.append(log.basis)
        position.append(log.spot_position)
        net_deposit.append(log.net_deposit)
        profit.append(log.profit)

    trade_open_index = []
    trade_close_index = []
    trade_open = []
    trade_close = []
    for trade in hedge_trades:
        if trade.hedge_type == HedgeType.OPEN:
            trade_open_index.append(datetime.fromtimestamp(trade.timestamp))
            trade_open.append(trade.basis)
        else:
            trade_close_index.append(datetime.fromtimestamp(trade.timestamp))
            trade_close.append(trade.basis)

    fig, ax = plt.subplots(nrows=3, ncols=1, sharex=True, figsize=(12, 10))

    basis_ax = ax[0]
    position_ax = ax[1]
    profit_ax = ax[2]

    basis_ax.plot(index, basis, alpha=0.8, lw=1)
    offset = lambda p: transforms.ScaledTranslation(0, p / 72.0, fig.dpi_scale_trans)
    trans = basis_ax.transData
    basis_ax.scatter(
        trade_open_index,
        trade_open,
        c="r",
        marker="v",
        s=9,
        transform=trans + offset(5),
    )
    basis_ax.scatter(
        trade_close_index,
        trade_close,
        c="g",
        marker="^",
        s=9,
        transform=trans + offset(-5),
    )
    basis_ax.set_ylabel("basis")

    position_ax.plot(index, position)
    position_ax.set_ylabel("position")

    profit_ax.plot(index, profit)
    profit_ax.set_ylabel("profit")

    if to_show:
        plt.show()

    fig.savefig(save_path)
    print(f"Save plot to {save_path}")
    plt.close(fig)

# def plot_combined_model_logs(
#     logs: List[LogState],
#     hedge_trades: List[CombinedModelHedgeTrade],
#     save_path: str,
#     to_show: bool = True,
# ):
#     path = pathlib.Path(save_path)
#     if not path.exists():
#         path.parent.mkdir(parents=True, exist_ok=True)
#     index = []
#     basis = []
#     position = []
#     net_deposit = []
#     profit = []
#     for log in logs:
#         index.append(datetime.fromtimestamp(log.timestamp))
#         basis.append(log.basis)
#         position.append(log.spot_position)
#         net_deposit.append(log.net_deposit)
#         profit.append(log.profit)

#     both_open_index = []
#     both_open = []
#     m1_open_index = []
#     m1_open = []
#     m2_open_index = []
#     m2_open = []
#     both_close_index = []
#     both_close = []
#     m1_close_index = []
#     m1_close = []
#     m2_close_index = []
#     m2_close = []
#     one_open_the_other_close_index = []
#     one_open_the_other_close = []

#     for trade in hedge_trades:
#         if trade.hedge_type == CombinedModelHedgeType.BOTH_OPEN:
#             both_open_index.append(datetime.fromtimestamp(trade.timestamp))
#             both_open.append(trade.basis)
#         elif trade.hedge_type == CombinedModelHedgeType.MODEL1_OPEN:
#             m1_open_index.append(datetime.fromtimestamp(trade.timestamp))
#             m1_open.append(trade.basis)
#         elif trade.hedge_type == CombinedModelHedgeType.MODEL2_OPEN:
#             m2_open_index.append(datetime.fromtimestamp(trade.timestamp))
#             m2_open.append(trade.basis)
#         elif trade.hedge_type == CombinedModelHedgeType.BOTH_CLOSE:
#             both_close_index.append(datetime.fromtimestamp(trade.timestamp))
#             both_close.append(trade.basis)
#         elif trade.hedge_type == CombinedModelHedgeType.MODEL1_CLOSE:
#             m1_close_index.append(datetime.fromtimestamp(trade.timestamp))
#             m1_close.append(trade.basis)
#         elif trade.hedge_type == CombinedModelHedgeType.MODEL2_CLOSE:
#             m2_close_index.append(datetime.fromtimestamp(trade.timestamp))
#             m2_close.append(trade.basis)
#         elif trade.hedge_type == CombinedModelHedgeType.ONE_OPEN_THE_OTHER_CLOSE:
#             one_open_the_other_close_index.append(
#                 datetime.fromtimestamp(trade.timestamp)
#             )
#             one_open_the_other_close.append(trade.basis)

#     fig, ax = plt.subplots(nrows=3, ncols=1, sharex=True, figsize=(12, 10))

#     basis_ax: plt.Axes = ax[0]
#     position_ax: plt.Axes = ax[1]
#     profit_ax: plt.Axes = ax[2]

#     basis_ax.plot(index, basis, alpha=0.8, lw=1)
#     offset = lambda p: transforms.ScaledTranslation(0, p / 72.0, fig.dpi_scale_trans)
#     trans = basis_ax.transData
#     basis_ax.scatter(
#         both_open_index,
#         both_open,
#         c="darkgreen",
#         marker="v",
#         s=9,
#         transform=trans + offset(5),
#         label="both open",
#     )
#     basis_ax.scatter(
#         m1_open_index,
#         m1_open,
#         c="cyan",
#         marker="v",
#         s=9,
#         transform=trans + offset(5),
#         label="m1 open",
#     )
#     basis_ax.scatter(
#         m2_open_index,
#         m2_open,
#         c="dodgerblue",
#         marker="v",
#         s=9,
#         transform=trans + offset(5),
#         label="m2 open",
#     )
#     basis_ax.scatter(
#         both_close_index,
#         both_close,
#         c="b",
#         marker="^",
#         s=9,
#         transform=trans + offset(-5),
#         label="both close",
#     )
#     basis_ax.scatter(
#         m1_close_index,
#         m1_close,
#         c="orangered",
#         marker="^",
#         s=9,
#         transform=trans + offset(-5),
#         label="m1 close",
#     )
#     basis_ax.scatter(
#         m2_close_index,
#         m2_close,
#         c="magenta",
#         marker="^",
#         s=9,
#         transform=trans + offset(-5),
#         label="m2 close",
#     )
#     basis_ax.scatter(
#         one_open_the_other_close_index,
#         one_open_the_other_close,
#         c="b",
#         marker="X",
#         s=18,
#         label="bad",
#     )
#     basis_ax.set_ylabel("basis")
#     basis_ax.legend()

#     position_ax.plot(index, position)
#     position_ax.set_ylabel("position")

#     profit_ax.plot(index, profit)
#     profit_ax.set_ylabel("profit")

#     if to_show:
#         plt.show()

#     fig.savefig(save_path)
#     print(f"Save plot to {save_path}")
#     plt.close(fig)

def _check_backtest_attr(indicator_backtest: BaseIndicator):
    if not hasattr(indicator_backtest, "hedge_pair"):
        return False
    if not hasattr(indicator_backtest, "config"):
        return False
    if not hasattr(indicator_backtest, "kline_resolution"):
        return False
    return True

def get_trades_path(indicator_backtest: BaseIndicator):
    if not _check_backtest_attr(indicator_backtest):
        raise ValueError("Not backtest Indicator")
    return (
        indicator_backtest.config.save_dir
        + indicator_backtest.config.exchange
        + "/"
        + "merged_trades/"
        + FtxHedgePair.to_dir_name(indicator_backtest.hedge_pair.future)
        + "/"
        + str(indicator_backtest.config.start_timestamp)
        + "_"
        + str(indicator_backtest.config.end_timestamp)
        + ".parquet"
    )

def get_spot_klines_path(indicator_backtest: BaseIndicator):
    if not _check_backtest_attr(indicator_backtest):
        raise ValueError("Not backtest Indicator")
    return (
        indicator_backtest.config.save_dir
        + indicator_backtest.config.exchange
        + "/"
        + "kline/"
        + FtxHedgePair.to_dir_name(indicator_backtest.hedge_pair.spot)
        + "/"
        + str(indicator_backtest.config.start_timestamp)
        + "_"
        + str(indicator_backtest.config.end_timestamp)
        + "_"
        + str(indicator_backtest.kline_resolution.to_pandas_resample_rule())
        + ".parquet"
    )

def get_future_klines_path(indicator_backtest: BaseIndicator):
    if not _check_backtest_attr(indicator_backtest):
        raise ValueError("Not backtest Indicator")
    return (
        indicator_backtest.config.save_dir
        + indicator_backtest.config.exchange
        + "/"
        + "kline/"
        + FtxHedgePair.to_dir_name(indicator_backtest.hedge_pair.future)
        + "/"
        + str(indicator_backtest.config.start_timestamp)
        + "_"
        + str(indicator_backtest.config.end_timestamp)
        + "_"
        + str(indicator_backtest.kline_resolution.to_pandas_resample_rule())
        + ".parquet"
    )

def get_merged_klines_path(indicator_backtest: BaseIndicator):
    if not _check_backtest_attr(indicator_backtest):
        raise ValueError("Not backtest Indicator")
    return (
        indicator_backtest.config.save_dir
        + indicator_backtest.config.exchange
        + "/"
        + "merged_kline/"
        + FtxHedgePair.to_dir_name(indicator_backtest.hedge_pair.future)
        + "/"
        + str(indicator_backtest.config.start_timestamp)
        + "_"
        + str(indicator_backtest.config.end_timestamp)
        + "_"
        + str(indicator_backtest.kline_resolution.to_pandas_resample_rule())
        + ".parquet"
    )


from input_classes.interface import Interface
import pygame

class Human(Interface):
    def __init__(self):
        self.key_converter = {
            1073741906 : "up",
            1073741905 : "down",
            1073741904 : "left",
            1073741903 : "right"
        }

    def get_action(self, world):
        key_l = []
        for event in pygame.event.get():
            pressed_key = "Not pressed"

            if event.type == pygame.locals.QUIT:
                return "Quit"

            elif event.type == pygame.locals.KEYDOWN:
                pressed_key = event.key

            key_l.append(pressed_key)

        if len(key_l) == 0:
            key_l.append("Not pressed")

        action_list = list(filter(lambda x: x!="Not pressed", key_l))

        if len(action_list) == 0:
            return "Not pressed"
        else:
            return self.key_converter.get(action_list[0], "Not pressed")


# Import the dependencies.
from statistics import mean
import pandas as pd
import numpy as np
import datetime as dt
import sqlalchemy
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import Session
from sqlalchemy import create_engine, func

import flask
from flask import Flask , jsonify

#################################################
# Database Setup
#################################################
engine = create_engine("sqlite:///Resources/hawaii.sqlite")

# reflect an existing database into a new model
Base = automap_base()

# reflect the tables
Base.prepare(engine, reflect = True)

# Save references to each table
Measurement = Base.classes.measurement
Station = Base.classes.station

# Create our session (link) from Python to the DB
session = Session(bind=engine)

#################################################
# Flask Setup
#################################################
app = Flask(__name__)

#################################################
# Flask Routes
#################################################

# welcome page
@app.route("/")
def welcome():
    """List all available routes"""
    return (
        f"Available Routes:
"
        f"/api/v1.0/precipitation
"
        f"/api/v1.0/stations
"
        f"/api/v1.0/tobs
"
        f"/api/v1.0/'start_date'/'end_date'
"
    )

# precipitation
@app.route("/api/v1.0/precipitation")
def precipitation():
    # precipitaion analysis
    prcp_results = session.query(Measurement.prcp, Measurement.date).\
    filter(Measurement.date > '2016-08-23').\
    order_by(Measurement.date).all()
    # close session
    session.close()

    # Create a dictionary for prcp results and make a list of the values
    prcp_list = []
    for prcp, date in prcp_results:
        prcp_dict = {}
        prcp_dict["Precipitation"] = prcp
        prcp_dict["Date"] = date
        prcp_list.append(prcp_dict)

    # return jsonify list
    return jsonify(prcp_list)

# stations
@app.route("/api/v1.0/stations")
def station():
    # get stations
    station_results = session.query(Station.station, Station.name).all()
    # close session
    session.close()

    # list
    station_list = []
    # loop like precipitation
    for station, name in station_results:
        station_dict = {}
        station_dict["Station"] = station
        station_dict["Name"] = name
        station_list.append(station_dict)

    # return
    return jsonify(station_list)

# tobs
@app.route("/api/v1.0/tobs")
def tobs():
    # Query the dates and temperature observations of the most-active station for the previous year of data.
    station_data = session.query(Measurement.date, Measurement.tobs).\
    filter(Measurement.station == 'USC00519281').\
    filter(Measurement.date >= '2016-8-18').all()

    # close session
    session.close()

    # tobs list
    tobs_list = []
    # loop like above
    for date, tobs in station_data:
        station_dict = {}
        station_dict["Date"] = date
        station_dict["tobs"] = tobs
        tobs_list.append(station_dict)

    # return
    return jsonify(tobs_list)

# start and end date
@app.route("/api/v1.0/<start>/<end>")
def temperature(start, end):
    # Return a JSON list of the minimum temperature, the average temperature, \
    # and the maximum temperature for a specified start or start-end range.
    temp_results = session.query(Measurement).\
    filter(Measurement.date >= start).\
    filter(Measurement.date <= end)

    # close session
    session.close()

    temp_list = []
    for temp in temp_results:
        temp_list.append(temp.tobs)

    temp_dict = {
        "Min": min(temp_list),
        "Average": mean(temp_list),
        "Max": max(temp_list)
    }
    return jsonify(temp_dict)

# boilerplate
if __name__ == "__main__":
    app.run(debug=True)


import requests, zipfile, StringIO
import numpy as np
import pandas as pd

# -------------------------------------------------------------------------- ##
# load and clean geographic distance data, scale between 0 and 1, invert

dist = pd.read_csv('http://privatewww.essex.ac.uk/~ksg/data/capdist.csv')
dist = dist[['numa', 'numb', 'kmdist']]
dist['kmdist_norm'] = dist['kmdist'] / dist['kmdist'].max()
dist['weight'] = dist['kmdist_norm'] * -1 + np.max(dist['kmdist_norm'])
dist.drop(['kmdist', 'kmdist_norm'], axis=1, inplace=True)
dist.rename(columns = {'numa' : 'cowcode1', 'numb' : 'cowcode2'}, inplace=True)
dist.to_csv('clean_data/geographic_distance.txt', sep=',', index=False)

# -------------------------------------------------------------------------- ##
# load and clean trade distance data, scale between 0 and 1

trade = pd.read_table('clean_data/dyadic_trade_imputed.txt', sep=',')
trade['trade'] = trade['flow1'] + trade['flow2']
trade['trade'] = np.log(trade['trade'] + 1)
trade['weight'] = trade['trade'] / trade['trade'].max()
trade.rename(columns={'ccode1' : 'cowcode1', 'ccode2' : 'cowcode2'}, inplace=True)
trade.dropna(inplace=True)
trade['cowcode1'] = trade['cowcode1'].astype(int)
trade['cowcode2'] = trade['cowcode2'].astype(int)
trade[['cowcode1','cowcode2','weight','year']].to_csv('clean_data/trade_distance.txt', sep=',', index=False)

# -------------------------------------------------------------------------- ##
# load and clean linguistic distance data, fix weight column name

ling = pd.read_table('clean_data/linguistic_distance.txt', sep=',')
if 'connection' in ling.columns.tolist():
	ling.rename(columns={'connection' : 'weight'}, inplace=True)
ling.to_csv('clean_data/linguistic_distance.txt', sep=',', index=False)

# -------------------------------------------------------------------------- ##
# gwf data

# access the data
gwf_data_url = 'http://sites.psu.edu/dictators/wp-content/uploads/sites/12570/2015/04/GWF-Autocratic-Regimes-1.2.zip'
r = requests.get(gwf_data_url)
z = zipfile.ZipFile(StringIO.StringIO(r.content))
z.extract('GWF Autocratic Regimes 1.2/GWFtscs.txt');
gwf = pd.read_table('GWF Autocratic Regimes 1.2/GWFtscs.txt')

# subset to relevant columns
gwf = gwf[['cowcode', 'year', 'gwf_fail', 'gwf_enddate', 'gwf_fail_subsregime', 'gwf_fail_type',
           'gwf_party', 'gwf_personal', 'gwf_military', 'gwf_monarch', 'gwf_duration']]

# apply specific date to observations
gwf['end_date'] = pd.to_datetime(gwf['gwf_enddate'], format='%d%b%Y')
gwf['end_year'] = pd.DatetimeIndex(gwf['end_date']).year
gwf['date_year'] = gwf['end_year'] != gwf['year']
gwf['date'] = np.where(gwf['date_year'] == True, pd.to_datetime('12/31/' + gwf['year'].astype(str)), gwf['end_date'])
gwf['dateInt'] = gwf['date'].apply(lambda x: x.to_datetime().date().toordinal())

# create coercion, democratic transition, and autocratic transition indicators
gwf['coerce'] = (gwf['gwf_fail_type'].isin([4, 5, 6])).astype(int)
gwf['demtrans'] = (gwf['gwf_fail_subsregime'] == 1).astype(int)
gwf['auttrans'] = (gwf['gwf_fail_subsregime'] == 2).astype(int)

# clean up
gwf.rename(columns = {'gwf_fail' : 'failure',
                      'gwf_party' : 'party',
                      'gwf_personal' : 'personal',
                      'gwf_monarch' : 'monarch',
                      'gwf_military' : 'military',
                      'gwf_duration' : 'duration'}, inplace=True)
gwf = gwf[['cowcode', 'year', 'date', 'dateInt', 'failure', 'coerce', 'demtrans', 'auttrans',
           'party', 'personal', 'military', 'monarch', 'duration']]
gwf.to_csv('clean_data/gwf.txt', sep=',', index=False)

# -------------------------------------------------------------------------- ##
# powell & thyne coup data

def to_date(x):
    try:
        return datetime.strptime(x, '%d%b%Y')
    except:
        return pd.NaT

coups = pd.read_table('http://www.uky.edu/~clthyn2/coup_data/powell_thyne_ccode_year.txt', header=0, sep='\t')
coups.rename(columns={'ccode' : 'cowcode'}, inplace=True)
coups.drop('version', axis=1, inplace=True)

c = coups[['cowcode', 'year'] + [col for col in coups.columns if 'coup' in col]]
c = pd.melt(c, id_vars=['cowcode', 'year']).set_index(['cowcode', 'year'])
c.columns = ['cvar', 'coup']
d = coups[['cowcode', 'year'] + [col for col in coups.columns if 'date' in col]]
d = pd.melt(d, id_vars=['cowcode', 'year']).set_index(['cowcode', 'year'])
d.columns = ['dvar', 'date']

coups = pd.concat([c, d], axis=1)
coups['date_coup'] = coups['date'].apply(lambda x: to_date(x))
coups['success'] = (coups['coup'] > 1).astype(int)
coups['attempt'] = (coups['coup'] > 0).astype(int)
coups.drop(['coup', 'cvar', 'dvar', 'date'], axis=1, inplace=True)
coups.reset_index(inplace=True)

coups = coups.groupby(['cowcode', 'year']).first().reset_index()
coups['date_coup'] = np.where(pd.isnull(coups['date_coup']), pd.to_datetime('12/31/' + coups['year'].astype(str)), coups['date_coup'])
coups['dateInt_coup'] = coups['date_coup'].apply(lambda x: x.to_datetime().date().toordinal())
coups.to_csv('clean_data/coups.txt', sep=',', index=False)


from PIL import Image, ImageDraw
import numpy
import random
import matplotlib.pyplot as plt
import string

import numpy as np

from skimage.transform import resize
from skimage.morphology import label
from sklearn.model_selection import train_test_split

import tensorflow as tf

from keras.models import Model, load_model
from keras.layers import Input, BatchNormalization, Activation, Dense, Dropout
from keras.layers.core import Lambda, RepeatVector, Reshape
from keras.layers.convolutional import Conv2D, Conv2DTranspose
from keras.layers.pooling import MaxPooling2D, GlobalMaxPool2D
from keras.layers.merge import concatenate, add
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
from keras.optimizers import Adam
from keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img

def generate_images(amount_of_pictures = 1000, image_shape = (512, 512), rect_bottom = 450, betwin = 20, x_first = 40):

    number_weigt_dict = {7: 42, 6: 53, 5:70, 4:90, 8:35}

    for i in range(amount_of_pictures):
        img = Image.new("L", image_shape, color="white")
        img_label = Image.new("L", image_shape, color="black")
        columns_amount = random.randint(4, 8)
        rect_up = random.randint(40, 400)

        img1 = ImageDraw.Draw(img)
        img1_label = ImageDraw.Draw(img_label)

        img1.rectangle([(x_first, rect_up), (x_first+number_weigt_dict[columns_amount], rect_bottom)], fill ="gray")
        img1_label.rectangle([(x_first, rect_up), (x_first+number_weigt_dict[columns_amount], rect_bottom)], fill ="white")

        cur_x = x_first+number_weigt_dict[columns_amount]+betwin

        for j in range(1, columns_amount):
            rect_up = random.randint(40, 400)

            shape = [(cur_x, rect_up), (cur_x+number_weigt_dict[columns_amount], rect_bottom)]

            img1.rectangle(shape, fill ="gray")
            img1_label.rectangle(shape, fill ="white")

            cur_x = cur_x+number_weigt_dict[columns_amount]+betwin

	#adding random words from letters and numbers for better diagram imitation. It increase quality

        img1.text((470, 100), ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(5)))
        img1.text((470, 150), ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(5)))
        img1.text((470, 200), ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(5)))
        img1.text((470, 250), ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(5)))
        img1.text((470, 300), ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(5)))
        img1.text((470, 350), ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(5)))

        img1.text((50, 470), ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(3)))
        img1.text((150, 470), ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(3)))
        img1.text((250, 470), ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(3)))
        img1.text((350, 470), ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(3)))
        img1.text((450, 470), ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(3)))

        img.save("images/{}.png".format(i))
        img_label.save("labels/{}.png".format(i))

        img.close()
        img_label.close()

def load_model(model_name='model_exp.h5'):
    def conv2d_block(input_tensor, n_filters, kernel_size = 3, batchnorm = True):
        # first layer
        x = Conv2D(filters = n_filters, kernel_size = (kernel_size, kernel_size),\
              kernel_initializer = 'he_normal', padding = 'same')(input_tensor)
        if batchnorm:
            x = BatchNormalization()(x)
        x = Activation('relu')(x)

        # second layer
        x = Conv2D(filters = n_filters, kernel_size = (kernel_size, kernel_size),\
                  kernel_initializer = 'he_normal', padding = 'same')(input_tensor)
        if batchnorm:
            x = BatchNormalization()(x)
        x = Activation('relu')(x)

        return x

    def get_unet(input_img, n_filters = 16, dropout = 0.1, batchnorm = True):
        # Contracting Path
        c1 = conv2d_block(input_img, n_filters * 1, kernel_size = 3, batchnorm = batchnorm)
        p1 = MaxPooling2D((2, 2))(c1)
        p1 = Dropout(dropout)(p1)

        c2 = conv2d_block(p1, n_filters * 2, kernel_size = 3, batchnorm = batchnorm)
        p2 = MaxPooling2D((2, 2))(c2)
        p2 = Dropout(dropout)(p2)

        c3 = conv2d_block(p2, n_filters * 4, kernel_size = 3, batchnorm = batchnorm)
        p3 = MaxPooling2D((2, 2))(c3)
        p3 = Dropout(dropout)(p3)

        c5 = conv2d_block(p3, n_filters = n_filters * 8, kernel_size = 3, batchnorm = batchnorm)

        # Expansive Path

        u7 = Conv2DTranspose(n_filters * 4, (3, 3), strides = (2, 2), padding = 'same')(c5)
        u7 = concatenate([u7, c3])
        u7 = Dropout(dropout)(u7)
        c7 = conv2d_block(u7, n_filters * 4, kernel_size = 3, batchnorm = batchnorm)

        u8 = Conv2DTranspose(n_filters * 2, (3, 3), strides = (2, 2), padding = 'same')(c7)
        u8 = concatenate([u8, c2])
        u8 = Dropout(dropout)(u8)
        c8 = conv2d_block(u8, n_filters * 2, kernel_size = 3, batchnorm = batchnorm)

        u9 = Conv2DTranspose(n_filters * 1, (3, 3), strides = (2, 2), padding = 'same')(c8)
        u9 = concatenate([u9, c1])
        u9 = Dropout(dropout)(u9)
        c9 = conv2d_block(u9, n_filters * 1, kernel_size = 3, batchnorm = batchnorm)

        outputs = Conv2D(1, (1, 1), activation='sigmoid')(c9)
        model = Model(inputs=[input_img], outputs=[outputs])
        return model

    input_img = Input((512, 512, 1), name='img')
    model = get_unet(input_img, n_filters=16, dropout=0.05, batchnorm=True)
    model.compile(optimizer=Adam(), loss="binary_crossentropy", metrics=["accuracy"])

    model.load_weights(model_name)
    return model

def get_mask(model, image_name, image_size=(512, 512, 1)):
    X_graph = []
    img_graph = img_to_array(load_img(image_name, grayscale=True))
    img_graph = resize(img_graph, image_size, mode = 'constant', preserve_range= True)
    img_graph = img_graph/255.0
    X_graph.append(img_graph)
    X_graph = np.array(X_graph)

    preds_graph = model.predict(X_graph, verbose=1)
    preds_graph_t = (preds_graph > 0.5).astype(np.uint8)

    return preds_graph_t

def plot_result(X, binary_preds, ix=0):
    fig, ax = plt.subplots(1, 2, figsize=(20, 10))
    ax[0].imshow(X[ix, ..., 0], cmap='seismic')
    ax[1].imshow(binary_preds[ix].squeeze(), vmin=0, vmax=1)


"""
This module aims to be the refactored code of old function "reviewapp/signals.py:check_trial_fields"

The method trial_validator.register() was created in replacement to STEP_FORM_MATRIX
"""

import re

from django.template.defaultfilters import slugify
from django.utils import simplejson

from reviewapp.consts import STEP_STATES, REMARK, MISSING, PARTIAL, COMPLETE, TRIAL_FORMS
from polyglot.multilingual_forms import BaseMultilingualWidget

FIELDS = {
    TRIAL_FORMS[0]: {
        'scientific_title': {'required': True, 'type': 'text', 'poly': True},
        'scientific_acronym': {'required': False, 'type': 'text', 'poly': True},
        'scientific_acronym_expansion': {'required': False, 'type': 'text', 'poly': True},
        'public_title': {'required': True, 'type': 'text', 'poly': True},
        'acronym': {'required': False, 'type': 'text', 'poly': True},
        'acronym_expansion': {'required': False, 'type': 'text', 'poly': True},
        'utrn_number': {'required': True, 'type': 'text', 'poly': False},
    },
    TRIAL_FORMS[1]: {
        'primary_sponsor': {'required': True, 'type': 'text', 'poly': False}
    },
    TRIAL_FORMS[2]: {
        'hc_freetext': {'required': True, 'type': 'text', 'poly': True}
    },
    TRIAL_FORMS[3]: {
        'i_freetext': {'required': True, 'type': 'text', 'poly': True},
        'i_code': {'required': True, 'type': 'mult', 'poly': False, 'queryset': None}
    },
    TRIAL_FORMS[4]: {
        'recruitment_status': {'required': True, 'type': 'text', 'poly': False},
        'recruitment_country': {'required': True, 'type': 'mult', 'poly': False, 'queryset': None},
        'enrollment_start_date': {'required': True, 'type': 'datetime', 'poly': False},
        #'enrollment_end_planned': {'required': True, 'type': 'text', 'poly': False},
        'target_sample_size': {'required': True, 'type': 'text', 'poly': False},
        'inclusion_criteria': {'required': True, 'type': 'text', 'poly': True},
        'exclusion_criteria': {'required': True, 'type': 'text', 'poly': True},
        'gender': {'required': True, 'type': 'text', 'poly': False},
        'agemin_value': {'required': True, 'type': 'text', 'poly': False},
        'agemin_unit': {'required': True, 'type': 'text', 'poly': False},
        'agemax_value': {'required': True, 'type': 'text', 'poly': False},
        'agemax_unit': {'required': True, 'type': 'text', 'poly': False},
    },
    TRIAL_FORMS[5]: {
        #'study_type': {'required': True, 'type': 'text', 'poly': False},
        'study_design': {'required': True, 'type': 'text', 'poly': True},
        'phase': {'required': True, 'type': 'text', 'poly': False},
        #'expanded_access_program': {'required': True, 'type': 'text', 'poly': False},
        'intervention_assignment': {'required': True, 'type': 'text', 'poly': False},
        'number_of_arms': {'required': True, 'type': 'text', 'poly': False},
        'masking': {'required': True, 'type': 'text', 'poly': False},
        'allocation': {'required': True, 'type': 'text', 'poly': False},
        'purpose': {'required': True, 'type': 'text', 'poly': False}
    },
    TRIAL_FORMS[6]: {},
    TRIAL_FORMS[7]: {}
}

class TrialValidator(object):
    steps_forms = None # Replaced STEP_FORM_MATRIX
    model = None

    def __init__(self):
        self.steps_forms = {}

    def register(self, step, forms):
        self.steps_forms[step] = forms

    def validate(self, instance):
        """
        FIXME: Some thoughts about this function:

        - Should have comments/docs/tests
        - Should be refactored (in progress)
        - Seems to be too complex for what it does
        """

        if not hasattr(instance, 'submission'):
            return

        fields_status = {}
        mandatory_languages = [lang.lower() for lang in instance.submission.get_mandatory_languages()]

        # Setting instance queryset on validation fields
        FIELDS[TRIAL_FORMS[3]]['i_code']['queryset'] = instance.intervention_code()
        FIELDS[TRIAL_FORMS[4]]['recruitment_country']['queryset'] = instance.recruitment_country.all()

        # attachment
        remarks = instance.submission.remark_set.filter(status='open').filter(context=slugify(TRIAL_FORMS[8])).count()
        count = instance.submission.attachment_set.all().count()
        for lang in mandatory_languages:
            if remarks > 0:
                fields_status.update({lang: {TRIAL_FORMS[8]: REMARK}})
            elif count == 0:
                fields_status.update({lang: {TRIAL_FORMS[8]: PARTIAL}})
            else:
                fields_status.update({lang: {TRIAL_FORMS[8]: COMPLETE}})

        for step, forms in self.steps_forms.items():

            step_status = {}

            remarks = instance.submission.remark_set.filter(status='open').filter(context=slugify(step)).count()
            if remarks > 0:
                for lang in mandatory_languages:
                    step_status.update({lang: REMARK})
            else:
                for form in forms:
                    if hasattr(form.Meta,'queryset'):
                        count = form.Meta.queryset.filter(trial=instance).count()

                        for lang in mandatory_languages:
                            if count < form.Meta.min_required:
                                step_status.update({lang: MISSING})
                            elif count == 0:
                                if step_status.get(lang, '') != MISSING:
                                    step_status.update({lang: PARTIAL})
                            else:
                                if not form.Meta.polyglot:
                                    if step_status.get(lang, '') == '':
                                        step_status.update({lang: COMPLETE})
                                else:
                                    for obj in form.Meta.queryset.filter(trial=instance):
                                        for field in form.Meta.polyglot_fields:
                                            if lang == 'en':
                                                value = [getattr(obj, field)]
                                            else:
                                                list_value = obj.translations.filter(language=lang)
                                                if len(list_value) > 0:
                                                    value = getattr(list_value[0], field)
                                                else:
                                                    value = None

                                            if value is None:
                                                if form.Meta.min_required != 0:
                                                    step_status.update({lang: MISSING})
                                                else:
                                                    if step_status.get(lang, '') != MISSING:
                                                        step_status.update({lang: PARTIAL})
                                            elif type(value) is str or type(value) is unicode:
                                                if re.match('^\s*$', value):
                                                    if step_status.get(lang, '') != MISSING:
                                                        step_status.update({lang: MISSING})
                                                    else:
                                                        if step_status.get(lang, '') != MISSING:
                                                            step_status.update({lang: PARTIAL})
                                                else:
                                                    if step_status.get(lang, '') == '':
                                                        step_status.update({lang: COMPLETE})
                                            else:
                                                if len(str(value)) == 0:
                                                    if form.Meta.min_required != 0:
                                                        step_status.update({lang: MISSING})
                                                    else:
                                                        if step_status.get(lang, '') != MISSING:
                                                            step_status.update({lang: PARTIAL})
                                                else:
                                                    if step_status.get(lang, '') == '':
                                                        step_status.update({lang: COMPLETE})

                    else:
                        if hasattr(form.Meta,'model'):
                            if form.Meta.model == self.model:
                                check_fields = FIELDS[step]
                                for field in form.base_fields.keys():
                                    values = {}
                                    if field in check_fields.keys():
                                        if check_fields[field]['type'] == 'text':
                                            values.update({'en': getattr(instance, field)})

                                            for trans in instance.translations.all():
                                                if trans.language.lower() in mandatory_languages:
                                                    if check_fields[field]['poly']:
                                                        values.update({trans.language.lower(): getattr(trans, field)})
                                                    else:
                                                        values.update({trans.language.lower(): values['en']})

                                            for lang, value in values.items():
                                                if value is None:
                                                    if check_fields[field]['required']:
                                                        step_status.update({lang: MISSING})
                                                    else:
                                                        if step_status.get(lang, '') != MISSING:
                                                            step_status.update({lang: PARTIAL})
                                                elif type(value) is str or type(value) is unicode:
                                                    if re.match('^\s*$', value):
                                                        if check_fields[field]['required']:
                                                            step_status.update({lang: MISSING})
                                                        else:
                                                            if step_status.get(lang, '') != MISSING:
                                                                step_status.update({lang: PARTIAL})
                                                    else:
                                                        if step_status.get(lang, '') == '':
                                                            step_status.update({lang: COMPLETE})
                                                else:
                                                    if len(str(value)) == 0:
                                                        if check_fields[field]['required']:
                                                            step_status.update({lang: MISSING})
                                                        else:
                                                            if step_status.get(lang, '') != MISSING:
                                                                step_status.update({lang: PARTIAL})
                                                    else:
                                                        if step_status.get(lang, '') == '':
                                                            step_status.update({lang: COMPLETE})

                                        else:
                                            if check_fields[field]['type'] == 'mult':
                                                count = check_fields[field]['queryset'].count()

                                                for lang in mandatory_languages:
                                                    if count < 1:
                                                        step_status.update({lang: MISSING})
                                                    else:
                                                        if step_status.get(lang, '') == '':
                                                            step_status.update({lang: COMPLETE})

            for language, status in step_status.items():
                if fields_status.get(language) is None:
                    fields_status[language] = {step: status}
                else:
                    fields_status[language].update({step: status})

        # Stores the serialized dictionary with fields statuses in the submission, to be
        # used in further later, when showing the form (also to block before the trial go
        # to the next state)
        instance.submission.fields_status = simplejson.dumps(fields_status)
        instance.submission.save()

    def field_is_required(self, form, field_name):
        """Returns a boolean value about this is a required field according to validation rules"""

        # Returns this is or is not a required field
        try:
            rules = self.get_rules_by_field(form, field_name)

            if rules:
                return rules['required']
        except KeyError:
            pass

    def get_rules_by_field(self, form, field_name):
        """Returns a dictionary with rules from constant FIELDS for a given field."""

        # Gets step where the given form is
        step_in = None
        for step, forms in self.steps_forms.items():
            if form.__class__ in forms:
                step_in = step
                break

        # Returns the rules from a dictionary
        try:
            return FIELDS[step_in][field_name]
        except KeyError:
            pass

    def get_field_status(self, form, field_name, instance):
        """Returns the status of a given field, for a given instance.submission"""

        rules = self.get_rules_by_field(form, field_name)

        if rules and rules['required']:
            if field_name in form.fields:
                field = form.fields[field_name]
                field_value = form.initial.get(field_name, None)

                # For multilingual fields
                if isinstance(field.widget, BaseMultilingualWidget):
                    # Default language value (english)
                    values = [field_value]

                    # Values for available languages (except to english)
                    for lang in field.available_languages:
                        if lang != 'en':
                            values.append(field.widget.get_value_by_language(field_name, field_value, lang))

                    # Returns "required" if any of translation values is not valid
                    if not all(values):
                        return 'required'

                # For common fields
                elif not field_value:
                    return 'required'

        return ''

trial_validator = TrialValidator()


from ..common.common_imports import *
from ..common.files import *
from dataclasses import dataclass
import os
from pathlib import Path
from evdspy.EVDSlocal.messages.error_classes import BucketFromSeriesFolderCreateError
from evdspy.EVDSlocal.config.config import *
from ..config.apikey_class import ApikeyClass
from ..initial.start_args import Args
from ..components.options_class import Options, load_options, SingletonOptions, read_user_options_on_load
from evdspy.EVDSlocal.initial.start_options import default_data_folder_name, Default_Prefix_
from evdspy.EVDSlocal.components.bucket_from_series import BucketFromSeriesFile, null_BucketFromSeriesFile
from evdspy.EVDSlocal.components.url_class import URLClass
from evdspy.EVDSlocal.components.api_params import Series, DateStart, DateEnd, dataTypeParam, dataTypeEnum
from typing import Tuple, Union, List
from rich import inspect

# ------------------------------------------------------------------------------
# -------------------------------------------------EvdsSeri----------------------
@dataclass
class EvdsSeri:
    """
    EvdsSeri
    """
    ID: str
    bfs: BucketFromSeriesFile
    name: str = ""
    subject: str = ""

    def __repr__(self):
        content = f"""
        <EvdsSeri>
            ID : {self.ID}
            name : {self.name}
            subject : {self.subject}
            bfs : {self.bfs}
        </EvdsSeri>
"""
        # print(content)
        return content

#   ----------------------------------------------------------    / EvdsSeri
liste = ('TP.IHRACATBEC.9999', 'TP.IHRACATBEC.31', 'TP.IHRACATBEC.41')
seri_evds_test_objs = tuple(EvdsSeri(x, bfs=null_BucketFromSeriesFile) for x in liste)

# ------------------------------------------------------------------------------
# /*
#               EvdsSeriesRequest
#
# ------------------------------------------------------------------------------
# */
@dataclass
class EvdsSeriesRequest:
    bfs: BucketFromSeriesFile
    options_: Optional[Options] = load_options()
    # series_list: field(default_factory=list) = field(default_factory=list[seri_evds_test_objs])
    series_list: field(default_factory=list) = tuple(seri_evds_test_objs)
    complete_url_instance: URLClass = None

    def __post_init__(self):
        read_user_options_on_load()
        self.start_date: str = SingletonOptions().get_valid_value("default_start_date")
        self.end_date: str = SingletonOptions().get_valid_value("default_end_date")
        self.update_url_instance()

    def update_url_instance(self):
        """_summary_
        update_url_instance
        """

        date_start: DateStart = DateStart(value=self.start_date)
        date_end: DateEnd = DateEnd(value=self.end_date)
        type_param = dataTypeParam(value=dataTypeEnum.csv)

        slist: Tuple[str] = tuple(x.ID for x in self.series_list)
        series_instance = Series(slist)
        full_list = [series_instance, date_start, date_end, type_param]

        complete_url_instance = URLClass(full_list)

        number_of_series = len(slist)
        # ADD other params from file
        complete_url_instance.add_formulas(self.bfs.formulas, number_of_series)
        complete_url_instance.add_frequency(self.bfs.frequency)
        complete_url_instance.add_aggtype(self.bfs.aggregateType, number_of_series)
        api_key = ApikeyClass().get_valid_api_key()
        complete_url_instance.add_apikey(api_key)
        self.complete_url_instance = URLClass(complete_url_instance.url_items)
        self.complete_url_instance.refresh_url()
        self.complete_url_instance.create_report()

#   ----------------------------------------------------------    / EvdsSeriesRequest
test_series_ = EvdsSeriesRequest(options_=load_options(), series_list=seri_evds_test_objs,
                                 bfs=null_BucketFromSeriesFile)
# ------------------------------------------------------------------------------
# /*
#               EvdsSeriesRequestWrapper(s)
#
# ------------------------------------------------------------------------------
# */
from abc import ABC, abstractmethod

@dataclass
class EvdsSeriesRequestWrapper():
    name: str
    subject: str
    EvdsSeriesRequest_: EvdsSeriesRequest
    bfs: BucketFromSeriesFile = None  # field(default=null_BucketFromSeriesFile)

    # bfs: BucketFromSeriesFile = null_BucketFromSeriesFile

    def __post_init__(self):
        self.EvdsSeriesRequest_.bfs = self.bfs

def EvdsSeriesRequestWrapperBasic(name: str, subject: str, EvdsSeriesRequest_: EvdsSeriesRequest):
    bfs = BucketFromSeriesFile(name, subject, prefix=Default_Prefix_)
    return EvdsSeriesRequestWrapper(name, subject, EvdsSeriesRequest_, bfs)

def EvdsSeriesRequestWrapperFromFile(bfs: BucketFromSeriesFile, EvdsSeriesRequest_: EvdsSeriesRequest):
    return EvdsSeriesRequestWrapper(bfs.subject, bfs.subject, EvdsSeriesRequest_, bfs)

#   ----------------------------------------------------------    / EvdsSeriesRequestWrapper
__all__ = [
    'EvdsSeriesRequest',
    'test_series_',
    'EvdsSeri',
    'EvdsSeriesRequestWrapper',
    'EvdsSeriesRequestWrapperFromFile',
    'EvdsSeriesRequestWrapperBasic',
]


from enum import Enum

class OrderStatus(Enum):
    PENDING = {
        "id": 1,
        "name": "PENDING",
    }
    CONFIRMED = {
        "id": 2,
        "name": "CONFIRMED",
    }
    DELIVERING = {
        "id": 3,
        "name": "DELIVERING",
    }
    DELIVERED = {
        "id": 4,
        "name": "DELIVERED",
    }
    CANCELED = {
        "id": 5,
        "name": "CANCELED",
    }
    COMPLETED = {
        "id": 6,
        "name": "COMPLETED"
    }
    NOTRECEIVED = {
        "id": 7,
        "name": "NOTRECEIVED"
    }

class PaymentMethod(Enum):
    DIRECT = {
        "id": 1,
        "name": "DIRECT"
    }
    PAYPAL = {
        "id": 2,
        "name": "PAYPAL"
    }


import pygame
from modelo.Bala import *

class Nave(pygame.sprite.Sprite):
    def __init__(self):
        pygame.sprite.Sprite.__init__(self)
        self.image = pygame.image.load('img/nave/ovni-min.png')
        self.rect = self.image.get_rect()
        self.rect.centerx = 512
        self.rect.centery = 560
        self.speed = 0.3
        self.listaDisparo = []
        self.vida = True
        self.hearts = 3
        self.soundShot = pygame.mixer.Sound('sounds/disparo2.wav')

    def moverIZQ(self,time):
        if self.vida == True:
            if self.rect.left >= 0:
                self.rect.centerx -= self.speed * time

    def moverDER(self,time):
        if self.vida == True:
            if self.rect.right <= 1024:
                self.rect.centerx += self.speed * time

    def luces(self,timer,time):
        if int(timer/1000) == self.timeChange :
            self.timeChange += 1
            self.posimg += 1

            if self.posimg > len(self.lstimg) -1:
                self.posimg = 0
    def disparar(self):
        bala = Bala(self.rect.centerx,self.rect.top)
        self.listaDisparo.append(bala)
        self.soundShot.play()

    def dibujar(self,superficie):
        superficie.blit(self.image,self.rect)


import torch
import torch.nn as nn

"""
Build loss functions for SegDetector.
Details about the built functions:
Input:
    pred: A dict which contains predictions.
        prob: The text segmentation prediction.
        thresh: The threshold prediction
        binary: Value produced by `step_function(prob - thresh)`.
    target:
        gt: Text regions bitmap gt.
        mask: Ignore mask, pixels where value is 1 indicates no contribution to loss.
        thresh_mask: Mask indicates regions cared by thresh supervision.
        thresh_map: Threshold gt.
Return:
    (loss, metrics).
    loss: A scalar loss value.
    metrics: A dict contraining partial loss values.
"""

class BalanceBCELoss(nn.Module):
    '''
    DiceLoss on prob.
    For SegDetector without adaptive module.
    '''

    def __init__(self, eps=1e-6):
        super(BalanceBCELoss, self).__init__()
        from .balance_cross_entropy_loss import BalanceCrossEntropyLoss
        self.loss = BalanceCrossEntropyLoss(eps=eps)

    def forward(self, pred, target):
        loss = self.loss(pred['prob'], target['gt'], target['mask'])
        return loss, dict(dice_loss=loss)

class AdaptiveDiceLoss(nn.Module):
    '''
    Integration of DiceLoss on both prob prediction and binary prediction.
    '''

    def __init__(self, eps=1e-6):
        super(AdaptiveDiceLoss, self).__init__()
        from .dice_loss import DiceLoss
        self.main_loss = DiceLoss(eps)
        self.binary_loss = DiceLoss(eps)

    def forward(self, pred, target):
        prob = pred['prob']
        binary = pred['binary']
        gt = target['gt']
        mask = target['mask']
        main_loss = self.main_loss(prob, gt, mask)
        binary_loss = self.binary_loss(binary, gt, mask)
        loss = main_loss + binary_loss
        return loss, dict(main_loss=main_loss, binary_loss=binary_loss)

class AdaptiveInstanceDiceLoss(nn.Module):
    '''
    InstanceDiceLoss on both prob and bianry.
    '''

    def __init__(self, iou_thresh=0.2, thresh=0.3):
        super(AdaptiveInstanceDiceLoss, self).__init__()
        from .dice_loss import InstanceDiceLoss, DiceLoss
        self.main_loss = DiceLoss()
        self.main_instance_loss = InstanceDiceLoss(threshold=thresh, iou_thresh=iou_thresh)
        self.binary_loss = DiceLoss()
        self.binary_instance_loss = InstanceDiceLoss(threshold=thresh, iou_thresh=iou_thresh)
        self.weights = nn.ParameterDict(
            dict(main=nn.Parameter(torch.ones(1)),
                 thresh=nn.Parameter(torch.ones(1)),
                 main_instance=nn.Parameter(torch.ones(1)),
                 binary_instance=nn.Parameter(torch.ones(1))
            )
        )

    def partial_loss(self, weight, loss):
        return loss / weight + 0.5 * torch.log(weight)

    def forward(self, pred, target):
        main_loss = self.main_loss(pred['prob'], target['gt'], target['mask'])
        binary_loss = self.binary_loss(pred['binary'], target['gt'], target['mask'])
        main_instance_loss = self.main_instance_loss(pred['prob'], target['gt'], target['mask'])
        binary_instance_loss = self.binary_instance_loss(pred['binary'], target['gt'], target['mask'])
        loss = self.partial_loss(self.weights['main'], main_loss) \
               + self.partial_loss(self.weights['thresh'], binary_loss) \
               + self.partial_loss(self.weights['main_instance'], main_instance_loss) \
               + self.partial_loss(self.weights['binary_instance'], binary_instance_loss)
        metrics = dict(main_loss=main_loss,
                       binary_loss=binary_loss,
                       main_instance_loss=main_instance_loss,
                       binary_instance_loss=binary_instance_loss)

        metrics.update(self.weights)
        return loss, metrics

class L1DiceLoss(nn.Module):
    '''
    L1Loss on thresh, DiceLoss on prob and binary.
    '''

    def __init__(self, eps=1e-6, l1_scale=10):
        super(L1DiceLoss, self).__init__()
        self.dice_loss = AdaptiveDiceLoss(eps=eps)
        from .l1_loss import MaskL1Loss
        self.l1_loss = MaskL1Loss()
        self.l1_scale = l1_scale

    def forward(self, pred, target):
        dice_loss, metrics = self.dice_loss(pred, target)
        l1_loss, l1_metric = self.l1_loss(pred['thresh'], target['thresh_map'], target['thresh_mask'])

        loss = dice_loss + self.l1_scale * l1_loss
        metrics.update(**l1_metric)
        return loss, metrics

class FullL1DiceLoss(L1DiceLoss):
    '''
    L1loss on thresh, pixels with topk losses in non-text regions are also counted.
    DiceLoss on binary and prob.
    '''

    def __init__(self, eps=1e-6, l1_scale=10):
        nn.Module.__init__(self)
        self.dice_loss = AdaptiveDiceLoss(eps=eps)
        from .l1_loss import BalanceL1Loss
        self.l1_loss = BalanceL1Loss()
        self.l1_scale = l1_scale

class L1BalanceCELoss(nn.Module):
    '''
    Balanced CrossEntropy Loss on `prob`,
    MaskL1Loss on `thresh`,
    DiceLoss on `binary`.
    '''
    def __init__(self, eps=1e-6, l1_scale=10, bce_scale=5):
        super(L1BalanceCELoss, self).__init__()
        from .dice_loss import DiceLossV2
        from .l1_loss import MaskL1Loss
        from .balance_cross_entropy_loss import BalanceCrossEntropyLossV2
        self.dice_loss = DiceLossV2(eps=eps)
        self.l1_loss = MaskL1Loss()
        self.bce_loss = BalanceCrossEntropyLossV2()

        self.l1_scale = l1_scale
        self.bce_scale = bce_scale

    def forward(self, pred, target):
        bce_loss = self.bce_loss(pred['prob'], target['gt'], target['mask'])
        metrics = dict(bce_loss=bce_loss)
        if 'thresh' in pred:
            l1_loss, l1_metric = self.l1_loss(pred['thresh'], target['thresh_map'], target['thresh_mask'])
            dice_loss = self.dice_loss(pred['binary'], target['gt'], target['mask'])
            metrics['binary_loss'] = dice_loss
            loss = dice_loss + self.l1_scale * l1_loss + bce_loss * self.bce_scale
            metrics.update(**l1_metric)
        else:
            loss = bce_loss
        return loss, metrics

class L1BCEMiningLoss(nn.Module):
    '''
    Basicly the same with L1BalanceCELoss, where the bce loss map is used as
        attention weights for DiceLoss
    '''

    def __init__(self, eps=1e-6, l1_scale=10, bce_scale=5):
        super(L1BCEMiningLoss, self).__init__()
        from .dice_loss import DiceLoss
        from .l1_loss import MaskL1Loss
        from .balance_cross_entropy_loss import BalanceCrossEntropyLoss
        self.dice_loss = DiceLoss(eps=eps)
        self.l1_loss = MaskL1Loss()
        self.bce_loss = BalanceCrossEntropyLoss()

        self.l1_scale = l1_scale
        self.bce_scale = bce_scale

    def forward(self, pred, target):
        bce_loss, bce_map = self.bce_loss(pred['prob'], target['gt'], target['mask'], return_origin=True)
        l1_loss, l1_metric = self.l1_loss(pred['thresh'], target['thresh_map'], target['thresh_mask'])
        bce_map = (bce_map - bce_map.min()) / (bce_map.max() - bce_map.min())
        dice_loss = self.dice_loss(pred['binary'], target['gt'], target['mask'], weights=bce_map + 1)
        metrics = dict(bce_loss=bce_loss, binary_loss=dice_loss)
        loss = dice_loss + self.l1_scale * l1_loss + bce_loss * self.bce_scale
        metrics.update(**l1_metric)
        return loss, metrics

class L1LeakyDiceLoss(nn.Module):
    '''
    LeakyDiceLoss on binary, MaskL1Loss on thresh, DiceLoss on binary.
    '''

    def __init__(self, eps=1e-6, coverage_scale=5, l1_scale=10):
        super(L1LeakyDiceLoss, self).__init__()
        from .dice_loss import DiceLoss, LeakyDiceLoss
        from .l1_loss import MaskL1Loss
        self.main_loss = LeakyDiceLoss(coverage_scale=coverage_scale)
        self.l1_loss = MaskL1Loss()
        self.binary_loss = DiceLoss(eps=eps)

        self.l1_scale = l1_scale

    def forward(self, pred, target):
        main_loss, metrics = self.main_loss(pred['prob'], target['gt'], target['mask'])
        binary_loss = self.binary_loss(pred['binary'], target['gt'], target['mask'])
        l1_loss, l1_metric = self.l1_loss(pred['thresh'], target['thresh_map'], target['thresh_mask'])
        metrics.update(**l1_metric, binary_loss=binary_loss)
        loss = main_loss + binary_loss + l1_loss * self.l1_scale
        return loss, metrics


# -*- coding: utf-8 -*-
# ---
# jupyter:
#   jupytext:
#     formats: ipynb,py:light
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.4.2
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# **This notebook was written while moving from 04b4716457b134d8d74b4ff5f626435e1b02dc85 to 1ad206d92fc300ea5843438f36fe903b3c0e2477**
#
# Not runnable in current versions

# +
import torch
import torchvision
from PIL import Image
ngpu = 2
from matplotlib import pyplot as plt
import numpy as np
import torchvision.utils as vutils
from tqdm.autonotebook import tqdm
import os
import src.data.make_dataset as md
from src.models.util import GaussianNoise,MemBatch

device = torch.device("cuda:0")

# +
class ConvG(torch.nn.Module):

    def __init__(self):
        super().__init__()
        self.deconv = torch.nn.Sequential(
            torch.nn.ConvTranspose2d(100,256,4,1),
            torch.nn.LeakyReLU(0.2),
            torch.nn.ConvTranspose2d(256,256,3,1,1),
            torch.nn.BatchNorm2d(256),
            torch.nn.LeakyReLU(0.2),
            torch.nn.ConvTranspose2d(256,128,4,1),
            torch.nn.BatchNorm2d(128),
            torch.nn.LeakyReLU(0.2),
            torch.nn.ConvTranspose2d(128,64,4,2),
            torch.nn.BatchNorm2d(64),
            torch.nn.LeakyReLU(0.2),
            torch.nn.ConvTranspose2d(64,64,3,2,1),
            torch.nn.BatchNorm2d(64),
            torch.nn.LeakyReLU(0.2),
            torch.nn.ConvTranspose2d(64,32,3,1,1),
            torch.nn.BatchNorm2d(32),
            torch.nn.LeakyReLU(0.2),
            torch.nn.ConvTranspose2d(32,3,2,1),
            torch.nn.Tanh()
        )
        self.dense = torch.nn.Sequential(
            torch.nn.Linear(100,100),
            torch.nn.LeakyReLU(0.2),
            torch.nn.Linear(100,100),
            torch.nn.BatchNorm1d(100)
        )

    def forward(self,x):
        z = (self.dense(x)+x).unsqueeze(-1).unsqueeze(-1)
        return self.deconv(z)

class ConvD(torch.nn.Module):

    def __init__(self):
        super().__init__()
        self.conv = torch.nn.Sequential(
            torch.nn.Conv2d(3,16,2,2),
            torch.nn.BatchNorm2d(16),
            torch.nn.LeakyReLU(0.2),
            torch.nn.Conv2d(16,64,4,2),
            torch.nn.BatchNorm2d(64),
            torch.nn.LeakyReLU(0.2),
            torch.nn.Conv2d(64,128,4,1),
            torch.nn.BatchNorm2d(128),
            torch.nn.LeakyReLU(0.2),
            torch.nn.Conv2d(128,256,4,1),
            torch.nn.BatchNorm2d(256),
            torch.nn.LeakyReLU(0.2),
        )

        self.dense = torch.nn.Sequential(
            torch.nn.Linear(256,1),
            torch.nn.Sigmoid()
        )

    def forward(self,x):
        z = self.conv(x)
        return self.dense(z.view(-1,256))
# -

gw=torch.load("../models/dcgan32v1/model_weights/checkpointG.2020_04_26")
dw=torch.load("../models/dcgan32v1/model_weights/checkpointD.2020_04_26")

G=ConvG().cuda()
G.load_state_dict(gw["model_state_dict"])

# +
x = torch.zeros(64,100,device=device).normal_(0,.7)
Batch= G(x)

plt.figure(figsize=(10,10))
plt.imshow(np.transpose(vutils.make_grid(Batch, padding=2, normalize=True).detach().cpu(),(1,2,0)))
plt.show()
# -

# # Generating a dataset of z,img

import os

# !rm -rf dcgan32_inversion_dataset

dataset_path = "dcgan32_inversion_dataset"
image_dir = "imgs"
vector_dir = "vecs"
os.makedirs(dataset_path,exist_ok=False)
os.makedirs(os.path.join(dataset_path,image_dir),exist_ok=False)
os.makedirs(os.path.join(dataset_path,vector_dir),exist_ok=False)

nbatch = 32
imageconverter = torchvision.transforms.Compose([
    torchvision.transforms.Normalize((-1, -1, -1), (2, 2, 2)),
    torchvision.transforms.ToPILImage()
])

# +
i0 = 32
torch.manual_seed(42)
x = torch.zeros(nbatch,100).normal_(0,1.).to(device)
batch = G(x).cpu()
x = x.cpu()

for i,b in enumerate(batch):
    imageconverter(b).save(os.path.join(dataset_path,image_dir,"{}.png".format(i+i0)))
    torch.save(x[i],os.path.join(dataset_path,vector_dir,"{}.pkl".format(i+i0)))
# -

imageconverter(batch[2])

b2=batch[2].permute(1,2,0).detach().numpy()
plt.imshow((b2+1)/2)

# !ls dcgan32_inversion_dataset/vecs/

# ## Dataset generation command

# We've put all of the above in a nice command as part of the src library

from src.data.dcgan32 import generate_dcgan32_inversion_dataset
import os

# This is supposed to be ran from the project root (ultimately with a make inversion_data command)
os.chdir('/home/ndeutsch/face_gans/face_gans')
print(os.getcwd())
generate_dcgan32_inversion_dataset(torch_seed=42)

from PIL import Image

# !ls data/processed/dcgan32_inversion/imgs/1.png

Image.open("data/processed/dcgan32_inversion/imgs/12.png")

# ## Testing our dataset importer class

from torchvision.datasets.vision import VisionDataset
from torchvision.datasets.folder import default_loader
from src.data.image_target_folder_dataset import ImageTargetFoldersDataset
from PIL import Image
os.chdir('/home/ndeutsch/face_gans/face_gans')

# +
my_transform=torchvision.transforms.Compose([
                               torchvision.transforms.ToTensor(),
                               torchvision.transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
                           ])

dataset = ImageTargetFoldersDataset("data/processed/dcgan32_inversion","imgs","vecs",transform=my_transform)
# -

dataset.images.index("/home/ndeutsch/face_gans/face_gans/data/processed/dcgan32_inversion/imgs/2.png")

im=dataset[22576][0]
plt.imshow(im.permute(1,2,0)*0.5+0.5)
plt.show()

from src.models.dcgan32 import DCGen32
os.chdir('/home/ndeutsch/face_gans/face_gans')
G=DCGen32(load_weights=True)
G.eval()
v=dataset[22576][1]
plt.imshow(G(v.unsqueeze(0)).detach()[0].permute(1,2,0)*0.5+0.5)

G(v.unsqueeze(0)).shape

pilim=Image.open("/home/ndeutsch/face_gans/face_gans/data/processed/dcgan32_inversion/imgs/2.png")

plt.imshow(pilim)

# # Testing the dataloader

import src.data.datasets as datasets
import os

os.chdir('/home/ndeutsch/face_gans/face_gans')
dataloader = datasets.create_dcgan32_inversion_dataloader()

batch=next(iter(dataloader))

plt.imshow(batch[0][0].permute(1,2,0)*0.5+0.5)

from src.models.dcgan32 import DCGen32
os.chdir('/home/ndeutsch/face_gans/face_gans')
G=DCGen32(load_weights=True)
G.eval()
plt.imshow(G(batch[1][0].unsqueeze(0)).detach()[0].permute(1,2,0)*0.5+0.5)

# # Prototyping an inverser network

os.chdir('/home/ndeutsch/face_gans/face_gans')
dataloader = datasets.create_dcgan32_inversion_dataloader()
batch=next(iter(dataloader))

batch[0][0].shape

x = batch[0][:1]
print(x.shape)
x=torch.nn.Conv2d(3,32,3,1,1)(x)
print(x.shape)
x=torch.nn.Conv2d(32,128,3,2,1)(x)
print(x.shape)
x=torch.nn.Conv2d(128,256,2,2,1)(x)
print(x.shape)
x=torch.nn.Conv2d(256,256,2,2)(x)
print(x.shape)
x=torch.nn.Conv2d(256,512,2,4)(x)
print(x.shape)
x=x.squeeze(-1).squeeze(-1)
print(x.shape)
x

class DCGAN32Inverter(torch.nn.Module):
    def __init__(self,channels=32):
        super(DCGAN32Inverter,self).__init__()

        self.conv = torch.nn.Sequential(
            torch.nn.BatchNorm2d(3),
            torch.nn.Conv2d(3,channels,3,1,1),
            torch.nn.BatchNorm2d(channels),
            torch.nn.ReLU(),
            torch.nn.Conv2d(channels,channels*2,3,2,1),
            torch.nn.BatchNorm2d(channels*2),
            torch.nn.ReLU(),
            torch.nn.Conv2d(channels*2,channels*4,2,2,1),
            torch.nn.BatchNorm2d(channels*4),
            torch.nn.ReLU(),
            torch.nn.Conv2d(channels*4,channels*8,2,2),
            torch.nn.BatchNorm2d(channels*8),
            torch.nn.ReLU(),
            torch.nn.Conv2d(channels*8,channels*16,2,4),
            torch.nn.BatchNorm2d(channels*16),
            torch.nn.ReLU()
        )

        self.dense = torch.nn.Sequential(
            torch.nn.Linear(16*channels,16*channels),
            torch.nn.BatchNorm1d(channels*16),
            torch.nn.ReLU(),
            torch.nn.Linear(16*channels,100)
        )

    def forward(self,x):
        return self.dense( self.conv(x).squeeze(-1).squeeze(-1))

invgan=DCGAN32Inverter(32)
invgan(batch[0]).shape

batch[1].shape

# ### Setting up training

# +
import torch
import torchvision
from PIL import Image
ngpu = 2
from matplotlib import pyplot as plt
import numpy as np
import torchvision.utils as vutils
from tqdm.autonotebook import tqdm
import os
import src.data.make_dataset as md
from src.models.util import GaussianNoise,MemBatch

device = torch.device("cuda:0")

device = torch.device("cuda")
os.chdir('/home/ndeutsch/face_gans/face_gans')
dataloader = datasets.create_dcgan32_inversion_dataloader(batch_size=512,num_workers=6)

invgan=DCGAN32Inverter(32).to(device)
invgan.to(device)
loss = torch.nn.MSELoss()
optim = torch.optim.Adam(invgan.parameters())

n_epochs = 1

Losses = []

for epoch in range(n_epochs):
    print("Epoch ",epoch)
    for i,data in enumerate(dataloader):
        imgs = data[0].to(device)
        vecs_tgt = data[1].to(device)

        optim.zero_grad()
        vecs = invgan(imgs)
        L = loss(vecs, vecs_tgt)
        L.backward()
        optim.step()
        Loss = L.detach().cpu().item()
        Losses.append(Loss)
        print("Epoch: ",epoch,"Step: ",i,"Loss: ",Loss)

# +
invgan=DCGAN32Inverter(32).to(device)
invgan.to(device)
loss = torch.nn.MSELoss()
optim = torch.optim.Adam(invgan.parameters())

n_epochs = 1

Losses = []

for epoch in range(n_epochs):
    print("Epoch ",epoch)
    for i,data in enumerate(dataloader):
        imgs = data[0].to(device)
        vecs_tgt = data[1].to(device)

        optim.zero_grad()
        vecs = invgan(imgs)
        L = loss(vecs, vecs_tgt)
        L.backward()
        optim.step()
        Loss = L.detach().cpu().item()
        Losses.append(Loss)
        print("Epoch: ",epoch,"Step: ",i,"Loss: ",Loss)

# -
# This is much too slow


import logging
import sys
import time
from logging import Logger

import matplotlib.pyplot as plt
from adjustText import adjust_text

from cobras_ts.cobras import COBRAS
from cobras_ts.cobras_logger import ClusteringLogger
from datasets import Dataset
from evaluate_clusterings.evaluation.ari import get_ARI
from generate_clusterings.algorithms.my_mpckmeans import MyMPCKMeans
from generate_clusterings.algorithms.my_npu import NPU
from generate_clusterings.algorithms.noise_robust_npu import NoiseRobustNPU
from querier.labelquerier import LabelQuerier
from querier.noisy_labelquerier import ProbabilisticNoisyQuerier

counter = 0

def plotting(logger, ground_truth_querier):
    # use latex for font rendering
    # mpl.rcParams['text.usetex'] = True
    global counter
    # plt.ion()
    fig, phase_plot = plt.subplots(1, 1, sharex='all', figsize=(18, 8))
    # ari_plot.plot(ari)

    # all the user constraints that have been received
    all_user_constraint = logger.all_user_constraints
    all_noisy_user_constraints = [(idx, con) for idx, con in enumerate(all_user_constraint) if
                                  ground_truth_querier._query_points(*con.get_instance_tuple()) != con.is_ML()]

    # the constraints that are detected as noisy
    detected_noisy_data = logger.detected_noisy_constraint_data

    phase_data = logger.algorithm_phases

    phase_plot.plot(phase_data)

    texts = []
    # plot constraint reusage
    # for x, (correctly_reused, wrongly_reused, _) in enumerate(logger.constraint_reusing_data):
    #     if correctly_reused > 0:
    #         phase_plot.scatter([x], [phase_data[x]], marker='o', c='tab:blue', s=40)
    #     if wrongly_reused > 0:
    #         phase_plot.scatter([x], [phase_data[x]], marker='o', c='tab:red', s=30)

    for (x, detected_noisy_constraint) in detected_noisy_data:

        if x < len(phase_data):

            # print(detected_noise, constraint_type, i1, i2)

            real_constraint_type = ground_truth_querier._query_points(detected_noisy_constraint.i1,
                                                                      detected_noisy_constraint.i2)
            phase_plot.scatter([x], [phase_data[x]], marker='D', c='g', s=90)
            if real_constraint_type != detected_noisy_constraint.is_ML():  # the detected noise was indeed noisy
                text = phase_plot.text(x, phase_data[x], str(detected_noisy_constraint), color="green", ha="center",
                                       va="center")
            else:
                text = phase_plot.text(x, phase_data[x], str(detected_noisy_constraint), color="tab:orange",
                                       ha="center",
                                       va="center")
            texts.append(text)
    for idx, noisy_constraint in all_noisy_user_constraints:
        constraint_str = str(noisy_constraint)
        phase_plot.scatter([idx], [phase_data[idx]], marker='x', c='r', s=50)
        text = phase_plot.text(idx, phase_data[idx], constraint_str, color='red', ha='center', va='center')
        texts.append(text)
    adjust_text(texts, arrowprops=dict(arrowstyle='->', color='black'), only_move={'points': 'y', 'text': 'y'})
    phase_plot.set_xticks(range(0, 61, 1), [str(n) if n % 5 == 0 else "" for n in range(0, 61)])
    plt.ylabel("Algorithm phase")
    plt.xlabel("#queries")
    plt.tight_layout()
    plt.show()
    counter += 1

    # plt.waitforbuttonpress()

    plt.close(fig)

def simpleRun(dataset_name):
    start = time.time()
    dataset = Dataset(dataset_name)

    basic_logger = Logger("cobras", logging.DEBUG)
    handler = logging.StreamHandler(sys.stderr)
    handler.setFormatter(logging.Formatter('[%(levelname)s] %(message)s'))
    basic_logger.addHandler(handler)

    cobras_logger = ClusteringLogger()
    ALGO_TO_USE = "cobras"
    if ALGO_TO_USE == "cobras":
        clusterer = COBRAS(minimum_approximation_order=3, maximum_approximation_order=5, noise_probability=0.05,
                           certainty_threshold=0.95, correct_noise=True,
                           logger=basic_logger, cobras_logger=cobras_logger)

        # logger.vis = ConstraintGraphVisualisation(clusterer)
        clusterings, _, _, _ = clusterer.fit(dataset.data, -1, None,
                                             ProbabilisticNoisyQuerier(cobras_logger, dataset.target, 0.05, 100))

        logger = clusterer._cobras_log

    elif ALGO_TO_USE == 'nnpu-mpck':
        clusterer = NoiseRobustNPU(MyMPCKMeans(), 3, 8, 0.05, 0.95, cobras_logger)
        clusterings, _, _, _ = clusterer.fit(dataset.data, dataset.number_of_classes(), None,
                                             ProbabilisticNoisyQuerier(cobras_logger, dataset.target, 0.05, 50))
        logger = clusterer.logger

    elif ALGO_TO_USE == 'npu-mpck':
        clusterer = NPU(MyMPCKMeans())
        clusterings, _,_,_ = clusterer.fit(dataset.data, dataset.number_of_classes(), None, ProbabilisticNoisyQuerier(None, dataset.target, 0.05,50))
        logger = None
    else:
        raise Exception(f"unknown algorithm to use {ALGO_TO_USE}")
    # print(clustering, dataset.target)
    end = time.time()
    ground_truth_querier =LabelQuerier(None, dataset.target, None)
    print("took: ", end - start)
    # print("correctly reused", sum([x for x, _, _ in logger.constraint_reusing_data]))
    # print("wrongly reused", sum([x for _, x, _ in logger.constraint_reusing_data]))
    print("produced ARI:", get_ARI(clusterings[-1], dataset.target))
    aris = [get_ARI(cluster, dataset.target) for cluster in clusterings]
    fig, ax = plt.subplots(1,1)
    ax.plot(aris)
    fig.show()
    if logger is not None:
        print("corrected_constraint_set + mistakes:")
        # logger.add_mistake_information(ground_truth_querier)
        # for constraint_number, _, mistakes in logger.corrected_constraint_sets:
        #     print("\t", constraint_number, ")", len(mistakes), "mistakes ", mistakes)
        plotting(logger, ground_truth_querier)

if __name__ == '__main__':
    # while True:
    simpleRun("iris")


from __future__ import annotations

from pathlib import Path
from typing import Any

import albumentations as A
import cv2
import numpy as np
import torch
from omegaconf import DictConfig
from torch import Tensor, nn

from anomalib_extend.data import TaskType
from anomalib_extend.data.utils.boxes import masks_to_boxes

from anomalib_extend.deploy.inferencers.base_inferencer import Inferencer

class TorchInferencer(Inferencer):
    """PyTorch implementation for the inference.

    Args:
        path (str | Path): Path to Torch model weights.
        device (str): Device to use for inference. Options are auto, cpu, cuda. Defaults to "auto".
    """

    def __init__(
        self,
        path: str | Path,
        device: str = "auto",
    ) -> None:
        self.device = self._get_device(device)

        # Load the model weights.
        self.model = self.load_model(path)
        self.metadata = self._load_metadata(path)
        self.transform = A.from_dict(self.metadata["transform"])

    @staticmethod
    def _get_device(device: str) -> torch.device:
        """Get the device to use for inference.

        Args:
            device (str): Device to use for inference. Options are auto, cpu, cuda.

        Returns:
            torch.device: Device to use for inference.
        """
        if device not in ("auto", "cpu", "cuda", "gpu"):
            raise ValueError(f"Unknown device {device}")

        if device == "auto":
            device = "cuda" if torch.cuda.is_available() else "cpu"
        elif device == "gpu":
            device = "cuda"
        return torch.device(device)

    def _load_metadata(self, path: str | Path | None = None) -> dict | DictConfig:
        """Load metadata from file.

        Args:
            path (str | Path): Path to the model pt file.

        Returns:
            dict: Dictionary containing the metadata.
        """
        metadata = torch.load(path, map_location=self.device)["metadata"] if path else {}
        return metadata

    def load_model(self, path: str | Path) -> nn.Module:
        """Load the PyTorch model.

        Args:
            path (str | Path): Path to model ckpt file.

        Returns:
            (AnomalyModule): PyTorch Lightning model.
        """

        model = torch.load(path, map_location=self.device)["model"]
        model.eval()
        return model.to(self.device)

    def pre_process(self, image: np.ndarray) -> Tensor:
        """Pre process the input image by applying transformations.

        Args:
            image (np.ndarray): Input image

        Returns:
            Tensor: pre-processed image.
        """
        processed_image = self.transform(image=image)["image"]

        if len(processed_image) == 3:
            processed_image = processed_image.unsqueeze(0)

        return processed_image.to(self.device)

    def forward(self, image: Tensor) -> Tensor:
        """Forward-Pass input tensor to the model.

        Args:
            image (Tensor): Input tensor.

        Returns:
            Tensor: Output predictions.
        """
        return self.model(image)

    def post_process(self, predictions: Tensor, metadata: dict | DictConfig | None = None) -> dict[str, Any]:
        """Post process the output predictions.

        Args:
            predictions (Tensor): Raw output predicted by the model.
            metadata (dict, optional): Meta data. Post-processing step sometimes requires
                additional meta data such as image shape. This variable comprises such info.
                Defaults to None.

        Returns:
            dict[str, str | float | np.ndarray]: Post processed prediction results.
        """
        if metadata is None:
            metadata = self.metadata

        if isinstance(predictions, Tensor):
            anomaly_map = predictions.detach().cpu().numpy()
            pred_score = anomaly_map.reshape(-1).max()
        else:
            # NOTE: Patchcore `forward`` returns heatmap and score.
            #   We need to add the following check to ensure the variables
            #   are properly assigned. Without this check, the code
            #   throws an error regarding type mismatch torch vs np.
            if isinstance(predictions[1], (Tensor)):
                anomaly_map, pred_score = predictions
                anomaly_map = anomaly_map.detach().cpu().numpy()
                pred_score = pred_score.detach().cpu().numpy()
            else:
                anomaly_map, pred_score = predictions
                pred_score = pred_score.detach()

        # Common practice in anomaly detection is to assign anomalous
        # label to the prediction if the prediction score is greater
        # than the image threshold.
        pred_label: str | None = None
        if "image_threshold" in metadata:
            pred_idx = pred_score >= metadata["image_threshold"]
            pred_label = "Anomalous" if pred_idx else "Normal"

        pred_mask: np.ndarray | None = None
        if "pixel_threshold" in metadata:
            pred_mask = (anomaly_map >= metadata["pixel_threshold"]).squeeze().astype(np.uint8)

        anomaly_map = anomaly_map.squeeze()
        anomaly_map, pred_score = self._normalize(anomaly_maps=anomaly_map, pred_scores=pred_score, metadata=metadata)

        if isinstance(anomaly_map, Tensor):
            anomaly_map = anomaly_map.detach().cpu().numpy()

        if "image_shape" in metadata and anomaly_map.shape != metadata["image_shape"]:
            image_height = metadata["image_shape"][0]
            image_width = metadata["image_shape"][1]
            anomaly_map = cv2.resize(anomaly_map, (image_width, image_height))

            if pred_mask is not None:
                pred_mask = cv2.resize(pred_mask, (image_width, image_height))

        if self.metadata["task"] == TaskType.DETECTION:
            pred_boxes = masks_to_boxes(torch.from_numpy(pred_mask))[0][0].numpy()
            box_labels = np.ones(pred_boxes.shape[0])
        else:
            pred_boxes = None
            box_labels = None

        return {
            "anomaly_map": anomaly_map,
            "pred_label": pred_label,
            "pred_score": pred_score,
            "pred_mask": pred_mask,
            "pred_boxes": pred_boxes,
            "box_labels": box_labels,
        }