import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import argparse

'''
Get a list of all useful row subsets of a given dataframe
Parameters:
	df - The dataframe
Returns:
	dct - The dictionary of row subsets
		All - df
		Arc - df['Plan type'] == 'arc'
		IMRT - df['Plan type'] == 'imrt'
		Gated - df['Gated'] == 'y'
		Non-Gated - df['Gated'] == 'n'
		813 - df['RTOG'] == '813'
		915 - df['RTOG'] == '915'
		DT < 200 - df['Total Time'] < 200
		DT 200-300 - df['Total Time'] in [200, 300)
		DT > 300 - df['Total Time'] >= 300
		TV < 20 - df['Tumor Volume (cc)'] < 20
		TV 20-40 - df['Tumor Volume (cc)'] in [20, 40)
		TV 40-60 - df['Tumor Volume (cc)'] in [40, 60)
		TV > 60 - df['Tumor Volume (cc)'] >= 60
'''
def get_all_subsets(df):
	dct = {'All': df}
	dct['Arc'] = df.loc[df['Plan type'] == 'arc']
	dct['IMRT'] = df.loc[df['Plan type'] == 'imrt']
	dct['Gated'] = df.loc[df['Gated'] == 'y']
	dct['Non-Gated'] = df.loc[df['Gated'] == 'n']
	dct['813'] = df.loc[df['RTOG'] == 813]
	dct['915'] = df.loc[df['RTOG'] == 915]
	dct['DT < 200'] = df.loc[df['Total Time'] < 200]
	dct['DT 200-300'] = df.loc[(df['Total Time'] >=  200) & (df['Total Time'] < 300)]
	dct['DT > 300'] = df.loc[df['Total Time'] >= 300]
	dct['TV < 20'] = df.loc[df['Tumor Volume (cc)'] < 20]
	dct['TV 20-40'] = df.loc[(df['Tumor Volume (cc)'] >= 20) & (df['Tumor Volume (cc)'] < 40)]
	dct['TV 40-60'] = df.loc[(df['Tumor Volume (cc)'] >= 40) & (df['Tumor Volume (cc)'] < 60)]
	dct['TV > 60'] = df.loc[df['Tumor Volume (cc)'] >= 60]
	return dct

'''
Print the average blood fraction receiving each dose
Parameters:
	df - The dataframe
'''
def get_average_fractions(df):
	frac_cols = [col for col in df.columns if 'Gy' in col]
	means = df[frac_cols].mean()

	outstr1 = ''
	outstr2 = ''
	for col in frac_cols:
		outstr1 += '%10s ' % col
		outstr2 += '%10g ' % means[col]
	print(outstr1)
	print(outstr2)

'''
Print accuracy metrics (absolute LYA difference, percentage difference, min diff, max diff, num < 0.1, 0.3, 0.5) for the dataframe
Parameters:
	df - The dataframe
'''
def get_accuracy_info(df):
	post_tx = {}
	post_tx['flq'] = df['Pre-Tx LYA'] * (1 - df['Predicted KP (FLQ)'])
	post_tx['lq'] = df['Pre-Tx LYA'] * (1 - df['Predicted KP (LQ)'])
	post_tx['lin'] = df['Pre-Tx LYA'] * (1 - df['Predicted KP (Lin)'])
	post_tx['nak'] = df['Pre-Tx LYA'] * (1 - df['Predicted KP (Nak)'])

	print('%8s  %8s  %8s  %8s  %8s  %8s  %8s  %8s' % ('Func', 'Abs Diff', '% Diff', 'Min', 'Max', 'D<0.1', 'D<0.3', 'D<0.5'))
	for key in post_tx:
		diff = np.abs(post_tx[key] - df['Post-Tx LYA'])
		abs_diff = np.mean(diff)
		perc_diff = np.mean(diff / df['Pre-Tx LYA']) * 100
		min_diff = np.min(diff)
		max_diff = np.max(diff)
		nl01 = np.sum(diff <= 0.1)
		nl03 = np.sum(diff <= 0.3)
		nl05 = np.sum(diff <= 0.5)
		print('%8s  %8g  %8g  %8g  %8g  %8g  %8g  %8g' % (key, abs_diff, perc_diff, min_diff, max_diff, nl01, nl03, nl05))

'''
Generate a plot of LYA difference (sim vs. measured) vs Pre-Tx LYA
Parameters:
	dct - The dictionary of subsets from get_all_subsets
	subsets - The subset keys to plot
'''
def get_accuracy_plot(dct, subsets):
	fig = plt.figure()
	max_x = 0
	for subset in subsets:
		df = dct[subset]
		post_tx = df['Pre-Tx LYA'] * (1 - df['Predicted KP (FLQ)'])
		diff = post_tx - df['Post-Tx LYA']
		max_x = np.max(df['Pre-Tx LYA'])
		error = df['Pre-Tx LYA'] * df['Error (FLQ)']
		plt.errorbar(df['Pre-Tx LYA'], diff, yerr=error, fmt='o', label=subset)
	plt.plot([0, max_x*1.1], [0, 0], color='black')
	plt.xlim([0, max_x*1.1])
	plt.xlabel(r'Pre-Treatment LYA (cells/L x $10^9$)')
	plt.ylabel(r'LYA difference (Simulated vs. Measured) (cells/L x $10^9$')
	if len(subsets) > 1:
		plt.legend()
	plt.show()

'''
The spreadsheet has the following columns:
	Patient - The patient initials
	Plan type - arc or imrt
	Beams - Number of beams
	Total Time - Total beam on time (not including gating)
	Gated - y/n
	RTOG - 813 (central) or 915 (peripheral)
	Tumor Volume (cc)
	Measurement Day
	Patient Age
	Pre-Tx LYA
	Post-Tx LYA
	Predicted KP (FLQ) - Predicted kill percentage (Fractionated LQ)
	Error (FLQ)
	Predicted KP (LQ) - Predicted kill percentage (LQ)
	Error (LQ)
	Predicted KP (Lin) - Predicted kill percentage (Linear Spline)
	Error (Lin)
	Predicted KP (Nak) - Predicted kill percentage (Nakamura best fit)
	< 0.5 Gy
	0.5-0.6 Gy
	0.6-0.7 Gy
	0.7-0.8 Gy
	0.8-0.9 Gy
	0.9-1.0 Gy
	1.0-2.0 Gy
	2.0-3.0 Gy
	> 3.0 Gy
'''
if __name__=='__main__':
	parser = argparse.ArgumentParser()
	parser.add_argument('spreadsheet', type=str, help='Path to spreadsheet')
	args = parser.parse_args()

	df = pd.read_csv(args.spreadsheet)
	df['RTOG'] = pd.to_numeric(df['RTOG'], errors='coerce')
	df['Tumor Volume (cc)'] = pd.to_numeric(df['Tumor Volume (cc)'], errors='coerce')

	subsets = get_all_subsets(df)
	for subset in subsets:
		print('%s\tN=%d' % (subset, len(subsets[subset])))
		get_average_fractions(subsets[subset])
		get_accuracy_info(subsets[subset])
	get_accuracy_plot(subsets, ['All'])
	get_accuracy_plot(subsets, [s for s in subsets if 'Gated' in s])
	get_accuracy_plot(subsets, [s for s in subsets if 'TV' in s])
	get_accuracy_plot(subsets, [s for s in subsets if 'DT' in s])


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Jul 12 17:09:06 2023

@author: tsengallen
"""
import pandas as pd
import numpy as np
import os
import matplotlib.pyplot as plt
import seaborn as sns
import matplotlib as mpl
from matplotlib.colors import ListedColormap, LinearSegmentedColormap
plt.rcParams['font.sans-serif'] = ['Arial Unicode MS']
top = mpl.colormaps['Greens']
bottom = mpl.colormaps['Reds']
newcolors = np.vstack((top(np.linspace(1, 0, 128)),bottom(np.linspace(0, 1, 128))))
newcmp = ListedColormap(newcolors, name='OrangeBlue')

def stripX(x):
    x = str(x)
    if x[-1] == '*':
        return x[:-1]
    return x

def selectCol(arr,s):
    arr = [i.split('_') for i in arr]
    result = [arr[0][0],arr[1][0]]
    for a in arr[2:]:
        judge = []
        for i in range(len(s)):
            if s[i] == 'all':
                judge += [True]
            else:
                judge += [a[i]==s[i]]
        if all(judge):
            result+=['_'.join(a)]
    return result

def plotHeat_window(aos,df,ind,st,im,fil,rd):
    newdf = df.loc[df['績效指標']==ind,selectCol(df.columns.to_list(),
                    [st,im,fil,rd])]
    dd = newdf.sort_index(ascending=False).set_index(['窗格比例']).iloc[:,1:]
    if aos == 'Rgv' or aos == 'Rga':
        cname = '標竿PR'
        c = ['100','99','95','90']
    else:
        cname = '延伸圈數'
        c = ['0','1','2','3']
    dd.index = ['51','41','31','21','11']
    if im != 'all' and fil != 'all':
        dd.columns = c
        dd.columns.name = cname
        t = '策略：{}，商品：{}，以{}篩選'.format(st,im,fil)
    elif fil == 'all':
        dd.columns = ['_'.join(i.split('_')[2:]) for i in dd.columns.to_list()]
        dd.columns.name = '篩選指標_'+cname
        t = '策略：{}，商品：{}'.format(st,im)
    elif im == 'all':
        dd.columns = [i.split('_')[1]+'_'+i.split('_')[3] for i in dd.columns.to_list()]
        dd.columns.name = '商品_'+cname
        t = '策略：{}，以{}篩選'.format(st,fil)
    if im != 'all' and fil != 'all':
        ro = 0
        ansize = 15
        figx = 14;figy = 6
        xlabel = dd.columns.name;ylabel = '窗格比例'
    else:
        ro = 0
        ansize = 12
        figx = 8;figy = 10
        ylabel = dd.columns.name;xlabel = '窗格比例'
        dd = dd.T.loc[:,['11','21','31','41','51']]
    fig, ax = plt.subplots(figsize=(figx, figy))
    sns.heatmap(dd,cmap=newcmp, center=0,annot=True, linewidth=.5, annot_kws={'rotation': ro ,'size':ansize})
    ax.set_xlabel(xlabel,fontsize=18,weight='bold',color= '#717e94')
    ax.set_ylabel(ylabel,fontsize=18,weight='bold',color= '#717e94')
    ax.tick_params(axis='both', which='major', labelsize=12,colors= '#717e94')
#     plt.title(t, fontsize=20,x=0.55,y=1.07,weight='bold')
    print(t)
    plt.tight_layout()
    name = './Result2/{}/{}/{}/{}.png'.format(st,st+'_'+im,aos,fil)
#     name = '../'+fil+'.png'
    plt.savefig(name,dpi=400)

def plotHeat_rd(aos,df,ind,st,im,fil,rd):
    newdf = df.loc[df['績效指標']==ind,selectCol(df.columns.to_list(),
                    [st,im,fil,rd])]
    if aos == 'Rgv' or aos == 'Rga':
        dd = newdf.sort_index(ascending=False).set_index(['標竿PR']).iloc[:,1:]
        dd.index = ['90','95','99','100']
        label = '標竿PR'
        clist = ['100','99','95','90']
    else:
        dd = newdf.sort_index(ascending=False).set_index(['延伸圈數']).iloc[:,1:]
        dd.index = ['3','2','1','0']
        label = '延伸圈數'
        clist = ['0','1','2','3']
    if im != 'all' and fil != 'all':
        dd.columns = [i.split('_')[-1] for i in dd.columns.to_list()]
        dd.columns.name = '窗格比例'
        t = '策略：{}，商品：{}，以{}篩選'.format(st,im,fil)
    elif fil == 'all':
        dd.columns = ['_'.join(i.split('_')[2:]) for i in dd.columns.to_list()]
        dd.columns.name = '篩選指標_窗格比例'
        t = '策略：{}，商品：{}'.format(st,im)
    elif im == 'all':
        dd.columns = [i.split('_')[1]+'_'+i.split('_')[3] for i in dd.columns.to_list()]
        dd.columns.name = '商品_窗格比例'
        t = '策略：{}，以{}篩選'.format(st,fil)
    if im != 'all' and fil != 'all':
        ro = 0
        ansize = 15
        figx = 14;figy = 6
        xlabel = dd.columns.name;ylabel = label
    else:
        ro = 0
        ansize = 12
        figx = 8;figy = 10
        ylabel = dd.columns.name;xlabel = label
        dd = dd.T.loc[:,clist]
    fig, ax = plt.subplots(figsize=(figx, figy))
    sns.heatmap(dd,cmap=newcmp, center=0, annot=True, linewidth=.5, annot_kws={'rotation': ro ,'size':ansize})
    ax.set_xlabel(xlabel,fontsize=18,weight='bold',color= '#717e94')
    ax.set_ylabel(ylabel,fontsize=18,weight='bold',color= '#717e94')
    ax.tick_params(axis='both', which='major', labelsize=12,colors= '#717e94')
#     plt.title(t, fontsize=20,x=0.55,y=1.07,weight='bold')
    print(t)
    plt.tight_layout()
    name = './Result2/{}/{}/{}/{}.png'.format(st,st+'_'+im,aos,'rd')
#     name = '../temp.png'
    plt.savefig(name,dpi=400)


#!/usr/bin/python
## CPCantalapiedra 2019

import os, sys
import json

from . import gmgc_mongodb
from django.conf import settings

#
def init(config_fn):
        # DB connection

        MONGO_HOST, MONGO_PORT = load_mongo_config(config_fn)
        sys.stderr.write("gmgc_queries: connecting to db...\n")

        gmgc_mongodb.connectdb(MONGO_HOST, MONGO_PORT)

        sys.stderr.write("gmgc_queries: connected\n")

        return

#
def load_mongo_config(config_fn):
        host = None
        port = None
        with open(config_fn, 'r') as mongo_config:
                config_data = json.load(mongo_config)
                if "MONGO_HOST" in config_data:
                        host = config_data["MONGO_HOST"]
                if "MONGO_PORT" in config_data:
                        port = config_data["MONGO_PORT"]

        sys.stderr.write("gmgc_mongodb "+str(host)+":"+str(port)+"\n")

        return host, port

#
def get_cluster_data(cluster_id):

        query = {"cl":cluster_id}

        results = gmgc_mongodb.gmgcdb_clusters_find_one(query)
        #results = [result for result in results if result is not None]

        for ret in results:
                if ret is not None and "_id" in ret:
                        del ret["_id"]

        return results
#
def get_unigene_data(unigene_id):

        ret = None

        query = {"u":unigene_id}

        results = gmgc_mongodb.gmgcdb_unigenes_find_one(query)
        #results = [result for result in results if result is not None]
        for ret in results:
                if ret is not None and "_id" in ret:
                        del ret["_id"]

        return results

def get_mgs_gene_data(mgs_id):
        ret = None

        query = {"mgs": mgs_id}

        result = gmgc_mongodb.gmgcdb_unigenes_mgs_find_one(query)

        if result is not None and "_id" in result:
                del result["_id"]

        return result

## END


import regex as re

class TextClean():
    def __init__(self, stop_words, verbose, phrase_lens, **kwargs):
        super().__init__(**kwargs)
        self.stop_words = stop_words
        self.verbose = verbose
        self.phrase_lens = phrase_lens

    def formatting_cleaner(self, text_list):

        # removes formatting tokens from text #
        # returns the cleaned list of texts #

        cleaned_text_list = []
        for text in text_list:
            text = re.sub(r'\[[0-9]"\]', ' ', text)
            text = re.sub(r'<br>', ' ', text)
            text = re.sub(r'\t', ' ', text)
            text = re.sub(r'\d', ' ', text)
            text = re.sub(r'\s+', ' ', text)
            cleaned_text_list.append(text)

        return cleaned_text_list

    def bracket_removal(self, text_list):

        # removes brackets #
        # returns list of texts cleaned of bracketed text #

        cleaned_text_list = []
        for text in text_list:
            cleaned_text_list.append(re.sub(r'\([^()]*\)', '', text))

        return cleaned_text_list

    def punctuation_removal(self, text_list):

        # removes punctuation from text #
        # returns new list of texts #

        cleaned_text_list = []
        for text in text_list:
            new_text = ''
            for i in range(len(text)):
                if text[i] not in ['"', ',', '.', '?', '!', '(', ')', '*',
                                    "'", '-', ':', '/', '£']:
                    new_text += text[i]
            cleaned_text_list.append(new_text)

        return cleaned_text_list

    def decapitalise(self, text_list):

        # removes capital letters from texts #
        # returns clean list of texts #

        decap_dict = {'A':'a', 'B':'b', 'C':'c', 'D':'d', 'E':'e', 'F':'f',
                        'G':'g', 'H':'h', 'I':'i', 'J':'j',
                        'K':'k', 'L':'l', 'M':'m', 'N':'n', 'O':'o', 'P':'p',
                        'Q':'q', 'R':'r', 'S':'s', 'T':'t',
                        'U':'u', 'V':'v', 'W':'w', 'X':'x', 'Y':'y', 'Z':'z'}
        cleaned_text_list = []
        for text in text_list:
            new_text = ''
            for i in range(len(text)):
                if re.match(r'[A-Z]', text[i]):
                    new_text += decap_dict[text[i]]
                else:
                    new_text += text[i]
            cleaned_text_list.append(new_text)

        return cleaned_text_list

    def phrase_splitter(self, words, phrase_len):

        # takes words and turns into phrases #
        # returns a list of phrases #

        phrases = []
        for i in range(len(words) - phrase_len):
            if i >= 0:
                phrase = ''
                for j in range(phrase_len):
                    #if words[i + j] not in self.stop_words:
                    if j == (phrase_len - 1):
                        phrase += words[i + j]
                    else:
                        phrase += f'{words[i + j]} '
                if len(phrase) > 0 and phrase not in self.stop_words:
                    if phrase[-1] == ' ':
                        phrase = phrase[0:-1]
                    elif phrase[0] == ' ':
                        phrase = phrase[1:]
                    phrases.append(phrase)

        return phrases

    def stop_word_removal(self, text_list):

        # removes stop words from text #
        # returns text in list format without stop words present #

        split_text_list = []
        word_count = 0
        stop_word_count = 0
        for text in text_list:
            if len(text) > 0:
                ngrams = text.split(' ')
                ngrams_no_stop = []
                for ngram in ngrams:
                    word_count += 1
                    if ngram not in self.stop_words:
                        ngrams_no_stop.append(ngram)
                    else:
                        stop_word_count += 1
                all_phrases = []
                for phrase_len in self.phrase_lens:
                    all_phrases += self.phrase_splitter(ngrams_no_stop, phrase_len)
                new_split_text = []
                for ngram in all_phrases:
                    if ' ' in ngram:
                        word_count += 1
                    if ngram not in self.stop_words and len(ngram) > 0:
                        new_split_text.append(ngram)
                    else:
                        stop_word_count += 1
                split_text_list.append(new_split_text)

        return split_text_list, word_count, stop_word_count

    def sentence_splitter(self, text_list):

        # splits texts into sentences for use in word2vec #
        # returns list of sentences #

        sentence_list = []
        for text in text_list:
            sentence = ''
            for i in range(len(text)):
                if text[i] not in ['!', '.', '?', '...', ':']:
                    sentence += text[i]
                else:
                    sentence_list.append(sentence)
                    sentence = ''

        return sentence_list

    def clean_up(self, text_list, classifier, verbose):

        # runs groups of functions for cleaning up text #
        # returns the cleaned up list of actor speech #

        print('')
        print('... cleaning ...')
        print('')

        text_list = self.formatting_cleaner(text_list)
        text_list = self.bracket_removal(text_list)
        text_list = self.punctuation_removal(text_list)
        text_list = self.decapitalise(text_list)
        text_list, word_count, stop_word_count = self.stop_word_removal(text_list)

        if verbose == True:
            print(f'Number of reviews left after cleaning: {len(text_list)}')
            print(f'Percentage of words removed using stop word list: {round(100 * (stop_word_count / word_count), 1)}%')
            print(f'Average num of ngrams in text after cleaning: {round(word_count / len(text_list), 1)}')

        return text_list, classifier


import logging
import os

from pyramid.i18n import get_localizer, TranslationStringFactory
from pyramid.events import subscriber, BeforeRender, NewRequest
from pyramid_formalchemy.events import subscriber as fa_subscriber, \
        IAfterSyncEvent, IBeforeDeleteEvent

from pygall.models import Photo
from pygall.lib.imageprocessing import ip, get_info, ORIG

log = logging.getLogger(__name__)

@subscriber(BeforeRender)
def add_renderer_globals(event):
    request = event.get('request')
    if request:
        event['_'] = request.translate
        event['localizer'] = request.localizer

tsf = TranslationStringFactory('PyGall')

@subscriber(NewRequest)
def save_locale(event):
    locale = event.request.params.get('_LOCALE_', None)
    if locale:
        event.request.response.set_cookie('_LOCALE_', locale)

@subscriber(NewRequest)
def add_localizer(event):
    request = event.request
    localizer = get_localizer(request)
    def auto_translate(string):
        return localizer.translate(tsf(string))
    request.localizer = localizer
    request.translate = auto_translate

@fa_subscriber([Photo, IAfterSyncEvent])
def after_photo_sync(context, event):
    if context.time and context.md5sum:
        return
    uri = context.uri
    info = get_info(os.path.join(
        event.request.registry.settings['photos_dir'], ORIG, uri))
    if not context.time:
        context.time = info['date']
        log.debug("context.time = %s" % info['date'])
    if not context.md5sum:
        context.md5sum = info['md5sum']
        log.debug("context.md5sum = %s" % info['md5sum'])

@fa_subscriber([Photo, IBeforeDeleteEvent])
def before_photo_delete(context, event):
    ip.remove_image(context.uri)
    log.debug('ip.remove_image(%s)' % context.uri)


from typing import List, Optional, Union

from cognite.client import ClientConfig
from cognite.client._api_client import APIClient
from cognite.client.utils._auxiliary import assert_type
from cognite.client.utils._identifier import IdentifierSequence

from cognite.experimental.data_classes.hosted_extractors import (
    HostedExtractorsDestination,
    HostedExtractorsDestinationList,
    HostedExtractorsJob,
    HostedExtractorsJobList,
    HostedExtractorsSource,
    HostedExtractorsSourceList,
)

class HostedExtractorsAPI(APIClient):
    _RESOURCE_PATH = "/hostedextractors"

    def __init__(self, config: ClientConfig, api_version: Optional[str], cognite_client: "CogniteClient") -> None:
        super().__init__(config, api_version, cognite_client)
        self.sources = HostedExtractorsSourcesAPI(config, api_version, cognite_client)
        self.jobs = HostedExtractorsJobsAPI(config, api_version, cognite_client)
        self.destinations = HostedExtractorsDestinationsAPI(config, api_version, cognite_client)

class HostedExtractorsJobsAPI(APIClient):
    _RESOURCE_PATH = "/hostedextractors/jobs"

    def __init__(self, config: ClientConfig, api_version: Optional[str], cognite_client: "CogniteClient") -> None:
        super().__init__(config, api_version, cognite_client)

    def list(
        self, source_external_id: Optional[str] = None, destination_external_id: Optional[str] = None, limit: int = 10
    ) -> HostedExtractorsJobList:
        filter = {}
        if source_external_id:
            filter["source_external_id"] = source_external_id
        if destination_external_id:
            filter["destination_external_id"] = destination_external_id
        return self._list(
            list_cls=HostedExtractorsJobList,
            resource_cls=HostedExtractorsJob,
            method="GET",
            limit=limit,
            filter=filter,
            headers={"cdf-version": "beta"},
        )

    def create(
        self, jobs: Union[HostedExtractorsJob, List[HostedExtractorsJob]]
    ) -> Union[HostedExtractorsJob, List[HostedExtractorsJob]]:
        assert_type(jobs, "jobs", [HostedExtractorsJob, list])
        return self._create_multiple(
            items=jobs,
            list_cls=HostedExtractorsJobList,
            resource_cls=HostedExtractorsJob,
            headers={"cdf-version": "beta"},
        )

    def delete(self, external_id: Union[str, List[str]]) -> None:
        self._delete_multiple(
            resource_path="/hostedextractors/jobs",
            identifiers=IdentifierSequence.load(external_ids=external_id),
            wrap_ids=True,
            headers={"cdf-version": "beta"},
        )

class HostedExtractorsSourcesAPI(APIClient):
    _RESOURCE_PATH = "/hostedextractors/sources"

    def __init__(self, config: ClientConfig, api_version: Optional[str], cognite_client: "CogniteClient") -> None:
        super().__init__(config, api_version, cognite_client)

    def list(
        self, source_external_id: Optional[str] = None, destination_external_id: Optional[str] = None, limit: int = 10
    ) -> HostedExtractorsSourceList:
        filter = {}
        if source_external_id:
            filter["source_external_id"] = source_external_id
        if destination_external_id:
            filter["destination_external_id"] = destination_external_id
        return self._list(
            list_cls=HostedExtractorsSourceList,
            resource_cls=HostedExtractorsSource,
            method="GET",
            limit=limit,
            filter=filter,
            headers={"cdf-version": "beta"},
        )

    def create(
        self, sources: Union[HostedExtractorsSource, List[HostedExtractorsSource]]
    ) -> Union[HostedExtractorsSource, List[HostedExtractorsSource]]:
        assert_type(sources, "sources", [HostedExtractorsSource, list])
        return self._create_multiple(
            items=sources,
            list_cls=HostedExtractorsSourceList,
            resource_cls=HostedExtractorsSource,
            headers={"cdf-version": "beta"},
        )

    def delete(self, external_id: Union[str, List[str]], force: bool = None, ignore_unknown_ids: bool = None) -> None:
        extras = {}
        if force is not None:
            extras["force"] = force
        if ignore_unknown_ids is not None:
            extras["ignoreUnknownIds"] = ignore_unknown_ids

        self._delete_multiple(
            identifiers=IdentifierSequence.load(external_ids=external_id),
            wrap_ids=True,
            extra_body_fields=extras if extras else None,
            headers={"cdf-version": "beta"},
        )

class HostedExtractorsDestinationsAPI(APIClient):
    _RESOURCE_PATH = "/hostedextractors/destinations"

    def __init__(self, config: ClientConfig, api_version: Optional[str], cognite_client: "CogniteClient") -> None:
        super().__init__(config, api_version, cognite_client)

    def list(
        self, source_external_id: Optional[str] = None, destination_external_id: Optional[str] = None, limit: int = 10
    ) -> HostedExtractorsDestinationList:
        filter = {}
        if source_external_id:
            filter["source_external_id"] = source_external_id
        if destination_external_id:
            filter["destination_external_id"] = destination_external_id
        return self._list(
            list_cls=HostedExtractorsDestinationList,
            resource_cls=HostedExtractorsDestination,
            method="GET",
            limit=limit,
            filter=filter,
            headers={"cdf-version": "beta"},
        )

    def create(
        self, destinations: Union[HostedExtractorsDestination, List[HostedExtractorsDestination]]
    ) -> Union[HostedExtractorsDestination, List[HostedExtractorsDestination]]:
        assert_type(destinations, "destinations", [HostedExtractorsDestination, list])
        return self._create_multiple(
            items=destinations,
            list_cls=HostedExtractorsDestinationList,
            resource_cls=HostedExtractorsDestination,
            headers={"cdf-version": "beta"},
        )

    def delete(self, external_id: Union[str, List[str]], force: bool = None, ignore_unknown_ids: bool = None) -> None:
        extras = {}
        if force is not None:
            extras["force"] = force
        if ignore_unknown_ids is not None:
            extras["ignoreUnknownIds"] = ignore_unknown_ids

        self._delete_multiple(
            identifiers=IdentifierSequence.load(external_ids=external_id),
            extra_body_fields=extras if extras else None,
            wrap_ids=True,
            headers={"cdf-version": "beta"},
        )


import math
import copy
import numpy as np
import pygame

class Vector:
    components = []
    def __init__(self, dimensions, element_list = None):
        if(element_list is tuple):
            element_list = list(element_list)
        try:
            if (len(element_list) == dimensions):
                valid_element_list = True
                for e in element_list:
                    if (type(e) != int and type(e) != float):
                        valid_element_list = False
                if (valid_element_list):
                    self.components = element_list
                else:
                    raise TypeError
            else:
                raise IndexError
        except:
            if (element_list == None):
                element_list = []
                for i in range(dimensions):
                    element_list.append(0)
                valid_element_list = True
                for e in element_list:
                    if (type(e) != int and type(e) != float):
                        valid_element_list = False
                if (valid_element_list):
                    self.components = element_list
                else:
                    raise TypeError
            elif(type(element_list) == list and dimensions > len(element_list)):
                for i in range(dimensions - len(element_list)):
                    element_list.append(0)
                valid_element_list = True
                for e in element_list:
                    if (type(e) != int and type(e) != float):
                        valid_element_list = False
                if (valid_element_list):
                    self.components = element_list
                else:
                    raise TypeError
            else:
                print("Error: More vector components than dimensions")
                raise TypeError

    def __getitem__(self, dimension):
        try:
            if (type(dimension) == int):
                if(dimension-1 < 0):
                    raise ValueError
                return self.components[dimension - 1]
            elif (dimension == "x"):
                return self.components[0]
            elif (dimension == "y"):
                return self.components[1]
            elif (dimension == "z"):
                return self.components[2]
            elif (dimension == "w"):
                return self.components[3]
            elif (dimension == "t"):
                return self.components[3]
            else:
                raise ValueError
        except:
            print("Error: Invalid Index")
            raise ValueError

    def __setitem__(self, dimension, value):
        try:
            if (type(dimension) == int):
                if (dimension - 1 < 0):
                    raise ValueError
                self.components[dimension-1] = value
            elif (dimension == "x"):
                self.components[0] = value
            elif (dimension == "y"):
                self.components[1] = value
            elif (dimension == "z"):
                self.components[2] = value
            elif (dimension == "w"):
                self.components[3] = value
            elif (dimension == "t"):
                self.components[3] = value
            else:
                raise ValueError
        except:
            print("Error: Invalid Index")
            raise ValueError

    def __add__(self, other):
        try:
            self_copy = copy.copy(self)
            if (isinstance(other,Vector) and isinstance(self,Vector)):
                for i in range(1,len(self.components)+1):
                    self_copy[i] += other[i]
                return self_copy
            else:
                raise TypeError
        except:
            print("Error: Invalid Type")

    def __radd__(self, other):
        try:
            self_copy = copy.copy(self)
            if (isinstance(other,Vector) and isinstance(self,Vector)):
                for i in range(1,len(self.components)+1):
                    self_copy[i] += other[i]
                return self_copy
            else:
                raise TypeError
        except:
            print("Error: Invalid Type")

    def __sub__(self, other):
        try:
            self_copy = copy.copy(self)
            if (isinstance(other, Vector) and isinstance(self, Vector)):
                for i in range(1,len(self.components)+1):
                    self_copy[i] -= other[i]
                return self_copy
            else:
                raise TypeError
        except TypeError:
            print("Error: Invalid Type")

    def __mul__(self, other):
        try:
            self_copy = copy.copy(self)
            if (type(other) is int or type(other) is float):
                for i in range(1,len(self.components)+1):
                    self_copy[i] *= other
                return self_copy
            else:
                raise TypeError
        except TypeError:
            print("Error: Invalid Type")

    def __truediv__(self, other):
        self_copy = copy.copy(self)
        self_copy * (1/other)
        return self_copy

    def __rmul__(self, other):
        try:
            self_copy = copy.copy(self)
            if (type(other) is int or type(other) is float or type(other)):
                for i in range(1,len(self.components)+1):
                    self_copy[i] *= other
                return self_copy
            else:
                raise TypeError
        except TypeError:
            print("INVALID?" + str(other))
            print("Error: Invalid Type")

    def dot(self, other):
        sum = 0
        if (type(other) is type(self)):
            if (len(other) == len(self)):
                for i in range(1,len(self)+1):
                    sum += self[i] * other[i]
                return sum
            else:
                raise IndexError
        else:
            raise TypeError

    def cross(self, other):
        if (type(other) is type(self)):
            if (len(self) == 3 and len(other) == 3):
                first_array = np.array(self.components)
                second_array = np.array(other.components)
                result = np.cross(first_array,second_array)
                new_results = []
                for i in range(len(result)):
                    r = str(result[i])
                    new_results.append(float(r))
                result_vector = Vector(len(self),new_results)
                return result_vector
            else:
                print("Error: Invalid Vector Size")
                raise IndexError
        else:
            raise TypeError

    def __len__(self):
        return len(self.components)

    def mag(self):
        radicand = 0
        for i in range(1, len(self) + 1):
            radicand += math.pow(self[i],2)
        magnitude = math.sqrt(radicand)
        return magnitude

    def magnitude(self):
        radicand = 0
        for i in range(1, len(self) + 1):
            radicand += math.pow(self[i],2)
        magnitude = math.sqrt(radicand)
        return magnitude

    def __copy__(self):
        return Vector(len(self.components), list(self.components).copy())

    def unit(self):
        if(self.mag() != 0):
            self_copy = copy.copy(self)
            return self_copy * (1 / self.mag())
        else:
            print("Cannot normalize, as magnitude is 0.")

    def normalize(self):
        if (self.mag() != 0):
            self_copy = copy.copy(self)
            return self_copy * (1 / self.mag())
        else:
            print("Cannot normalize, as magnitude is 0.")

    def rotateDegrees(self, angle_degrees):
        self_copy = copy.copy(self)
        if(len(self.components) == 2):
            angle_radians = angle_degrees * (math.pi / 180)
            new_components = [self.components[0] * math.cos(angle_radians) - self.components[1] * math.sin(angle_radians), self.components[0] * math.sin(angle_radians) +  self.components[1] * math.cos(angle_radians)]
            self_copy.components = new_components
            return self_copy
        else:
            raise ValueError

    def rotateRadians(self, angle_radians):
        self_copy = copy.copy(self)
        if(len(self.components) == 2):
            new_components = [self.components[0] * math.cos(angle_radians) - self.components[1] * math.sin(angle_radians), self.components[0] * math.sin(angle_radians) +  self.components[1] * math.cos(angle_radians)]
            self_copy.components = new_components
            return self_copy
        else:
            raise ValueError

    def reflect(self,dimension):
        self_copy = copy.copy(self)
        new_components = []
        count = 1
        for component in self.components:
            if(count == dimension):
                new_components.append(component)
            else:
                new_components.append(-1 * component)
            count += 1
        self_copy.components = new_components
        return self_copy

    def __abs__(self):
        self_copy = copy.copy(self)
        new_components = []
        for component in self.components:
            new_components.append(abs(component))
        self_copy.components = new_components
        return self_copy

    def __str__(self):
        return str(self.components)

    def __round__(self, n=None):
        for i in range(len(self.components)):
            self.components[i] = round(self.components[i], n)
        return self

    def print(self):
        print(self)

    def draw(self,coordinateSystem, initialPositionVector,color = pygame.Color(255,0,0)):
        pygame.draw.line(pygame.display.get_surface(),color,coordinateSystem.convertToPixelPositionVector(initialPositionVector).components, coordinateSystem.convertToPixelPositionVector((self + initialPositionVector)).components)


import torch
import torch.nn as nn
import torchvision.datasets as dsets
import torchvision.transforms as transforms
import random

learning_rate = 0.001
training_epochs = 15
batch_size = 100
drop_prob = 0.3

# MNIST dataset
mnist_train = dsets.MNIST(root='MNIST_data/',
                          train=True,
                          transform=transforms.ToTensor(),
                          download=True)

#test 데이터셋.
mnist_test = dsets.MNIST(root='MNIST_data/',
                         train=False,
                         transform=transforms.ToTensor(),
                         download=True)

data_loader = torch.utils.data.DataLoader(dataset=mnist_train,
                                          batch_size=batch_size,
                                          shuffle=True,
                                          drop_last=True)

linear1 = nn.Linear(784, 512, bias=True)
linear2 = nn.Linear(512, 512, bias=True)
linear3 = nn.Linear(512, 512, bias=True)
linear4 = nn.Linear(512, 512, bias=True)
linear5 = nn.Linear(512, 10, bias=True)
relu = nn.ReLU()
dropout = nn.Dropout(p=drop_prob)

nn.init.xavier_uniform_(linear1.weight)
nn.init.xavier_uniform_(linear2.weight)
nn.init.xavier_uniform_(linear3.weight)
nn.init.xavier_uniform_(linear4.weight)
nn.init.xavier_uniform_(linear5.weight)

model = nn.Sequential(linear1, relu, dropout,
                            linear2, relu, dropout,
                            linear3, relu, dropout,
                            linear4, relu, dropout,
                            linear5)

criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)

total_batch = len(data_loader)
model.train()
for epoch in range(training_epochs):
    avg_cost = 0

    for X, Y in data_loader:
        X = X.view(100, 28 * 28)
        Y = Y

        optimizer.zero_grad()
        hypothesis = model(X)
        cost = criterion(hypothesis, Y)
        cost.backward()
        optimizer.step()

        avg_cost += cost / total_batch

    print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.9f}'.format(avg_cost))

print('Learning finished')

# Test model and check accuracy
with torch.no_grad():
    model.eval()    # set the model to evaluation mode (dropout=False)

    # Test the model using test sets
    X_test = mnist_test.test_data.view(-1, 28 * 28).float()
    Y_test = mnist_test.test_labels

    prediction = model(X_test)
    correct_prediction = torch.argmax(prediction, 1) == Y_test
    accuracy = correct_prediction.float().mean()
    print('Accuracy:', accuracy.item())

    # Get one and predict
    r = random.randint(0, len(mnist_test) - 1)
    X_single_data = mnist_test.test_data[r:r + 1].view(-1, 28 * 28).float()
    Y_single_data = mnist_test.test_labels[r:r + 1]

    print('Label: ', Y_single_data.item())
    single_prediction = model(X_single_data)
    print('Prediction: ', torch.argmax(single_prediction, 1).item())


import requests

def check_hash(url, hash_):
    """
    Checks that the given double 256-hash exists at
    twitter url.
    """
    try:
        resp = requests.get(url)
        content = resp.text
        present = hash_ in content
        return present, ""
    except Exception:
        return None

if __name__ == '__main__':
    URL = "https://twitter.com/HashesSparkl/status/1008182155057360898"
    HASH = "a8e26fa85a95a531ebd98b4454d17790430cc4a11ce157e2ad051f7c3c128ce8"
    print(check_hash(URL, HASH))


'''
Data Scientist.: Dr.Eddy Giusepe Chirinos Isidro
'''

'''
ChatGPT é uma variante do modelo de linguagem GPT-3, projetado especificamente
para geração de linguagem de conversação. Para usar o ChatGPT em Python, você precisará
instalar o cliente OpenAI API e obter uma chave de API.
'''
import os
import openai
import gradio as gr
from dotenv import load_dotenv

print('Minha chave Key: ', load_dotenv())
Eddy_API_KEY = os.environ['API_KEY']

# Configurar o cliente OpenAI API
openai.api_key = Eddy_API_KEY

# Configure o modelo e o prompt
model_engine = "text-davinci-003"
prompt = "Olá, como vai você, hoje?"

# Gerar uma resposta
completion = openai.Completion.create(
    engine=model_engine,
    prompt=prompt,
    max_tokens=1024,
    n=1,
    stop=None, # Especifica string ou sequência de strings que, se encontradas no texto gerado, farão com que o modelo pare de gerar mais texto.
    # Isso pode ser útil para controlar o comprimento do texto gerado ou para garantir que o modelo não gere conteúdo inapropriado.
    temperature=0.5, # Mais alta resultará em respostas mais variadas e potencialmente menos coerentes, mais baixa resultará em respostas mais
    # previsíveis e potencialmente mais coerentes.
)

response = completion.choices[0].text # A resposta retorna como uma string nesta variável response.
print(response)


from contextlib import closing
import pickle
import unittest
import json
import mock
import __builtin__

from docido_sdk.toolbox.file_ext import (
    FileProperties,
    iterator_to_file,
)
from docido_sdk.toolbox.http_ext import delayed_request
from docido_sdk.index.test import CustomJSONEncoder

class TestStreamFromRequest(unittest.TestCase):
    def test_pickle(self):
        s = delayed_request('http://google.com')
        json.dumps(repr(s))
        pickle.dumps(s)

    def test_json(self):
        s = delayed_request('http://google.com', param=dict(test='test'))
        self.assertEqual(
            '"{\\"param\\": {\\"test\\": \\"test\\"}}"',
            json.dumps(s, cls=CustomJSONEncoder)
        )
        with mock.patch.object(__builtin__, 'repr',
                               return_value=None) as test:
            CustomJSONEncoder().default('Test')
        test.assert_called_once_with('Test')

    def test_fetch_google(self):
        with delayed_request('http://google.com').open() as istr:
            istr = iterator_to_file(iter(istr))
            all_content = content = istr.read(50)
            while len(content) == 50:
                content = istr.read(50)
                all_content += content
        self.assertTrue(all_content.endswith(''))

        with delayed_request('http://google.com').open() as istr:
            istr = iterator_to_file(iter(istr))
            all_content_at_once = istr.read()
        self.assertTrue(all_content_at_once.endswith(''))

    def test_iterate(self):
        content = ['foo', 'bar']
        result = []
        with closing(iterator_to_file(iter(content))) as istr:
            for word in istr:
                result.append(word)
        self.assertEqual(content, result)

class TestFileProperties(unittest.TestCase):
    def test_file_type(self):
        self.assertEquals(FileProperties.file_type('foo.png'), 'image')
        self.assertEquals(FileProperties.file_type(None), 'other')
        self.assertEquals(FileProperties.file_type('foo.unknown'), 'other')
        self.assertEquals(
            FileProperties.file_type('foo.unknown', 'image/jpeg'),
            'image'
        )

    def test_mime_type(self):
        self.assertEquals(FileProperties.mime_type('foo.png'), 'image/png')
        self.assertEquals(FileProperties.mime_type('foo.unknown'), None)

if __name__ == '__main__':
    unittest.main()


#Used to make UI for the game
import pygame
#Randomly chooses which way the ball will start moving
import random

#frame rate per second
FPS = 60

#size of our window
WINDOW_WIDTH = 800
WINDOW_HEIGHT = 600

#size of our paddle
PADDLE_WIDTH = 10
PADDLE_HEIGHT = 60
#distance from the edge of the window
PADDLE_BUFFER = 10

#size of our ball
BALL_WIDTH = 10
BALL_HEIGHT = 10

#speeds of paddle and ball
PADDLE_SPEED = 4
BALL_X_SPEED = 5
BALL_Y_SPEED = 4

#RGB for paddle and ball
WHITE = (255, 255, 255)
BLACK = (0, 0, 0)

screen = pygame.display.set_mode((WINDOW_WIDTH, WINDOW_HEIGHT))
pygame.display.set_caption("PingAndPong")

def drawBall(ballXPos, ballYPos):
    ball = pygame.Rect(ballXPos, ballYPos, BALL_WIDTH, BALL_HEIGHT)
    pygame.draw.rect(screen, WHITE, ball)

def drawPaddle1(paddle1YPos):
    paddle1 = pygame.Rect(PADDLE_BUFFER, paddle1YPos, PADDLE_WIDTH, PADDLE_HEIGHT)
    pygame.draw.rect(screen, WHITE, paddle1)

def drawPaddle2(paddle2YPos):
    paddle2 = pygame.Rect(WINDOW_WIDTH - PADDLE_BUFFER - PADDLE_WIDTH, paddle2YPos, PADDLE_WIDTH, PADDLE_HEIGHT)
    pygame.draw.rect(screen, WHITE, paddle2)

def updateBall(paddle1YPos, paddle2YPos, ballXPos, ballYPos, ballXDirection, ballYDirection):

    ballXPos = ballXPos + ballXDirection * BALL_X_SPEED
    ballYPos = ballYPos + ballYDirection * BALL_Y_SPEED
    score = 0

    #checks for a collision, if the ball hits the left side
    if (
                        ballXPos <= PADDLE_BUFFER + PADDLE_WIDTH and ballYPos + BALL_HEIGHT >= paddle1YPos and ballYPos - BALL_HEIGHT <= paddle1YPos + PADDLE_HEIGHT):
        ballXDirection = 1
    elif (ballXPos <= 0):
        ballXDirection = 1
        score = -1
        return [score, paddle1YPos, paddle2YPos, ballXPos, ballYPos, ballXDirection, ballYDirection]

    if (
                        ballXPos >= WINDOW_WIDTH - PADDLE_WIDTH - PADDLE_BUFFER and ballYPos + BALL_HEIGHT >= paddle2YPos and ballYPos - BALL_HEIGHT <= paddle2YPos + PADDLE_HEIGHT):

        ballXDirection = -1

    elif (ballXPos >= WINDOW_WIDTH - BALL_WIDTH):

        ballXDirection = -1
        score = 1
        return [score, paddle1YPos, paddle2YPos, ballXPos, ballYPos, ballXDirection, ballYDirection]

    if (ballYPos <= 0):
        ballYPos = 0;
        ballYDirection = 1;

    elif (ballYPos >= WINDOW_HEIGHT - BALL_HEIGHT):
        ballYPos = WINDOW_HEIGHT - BALL_HEIGHT
        ballYDirection = -1
    return [score, paddle1YPos, paddle2YPos, ballXPos, ballYPos, ballXDirection, ballYDirection]

def updatePaddle1(action, paddle1YPos):

    if (action[1] == 1):
        paddle1YPos = paddle1YPos - PADDLE_SPEED

    if (action[2] == 1):
        paddle1YPos = paddle1YPos + PADDLE_SPEED

    if (paddle1YPos < 0):
        paddle1YPos = 0
    if (paddle1YPos > WINDOW_HEIGHT - PADDLE_HEIGHT):
        paddle1YPos = WINDOW_HEIGHT - PADDLE_HEIGHT
    return paddle1YPos

def updatePaddle2(paddle2YPos, ballYPos):
    if (paddle2YPos + PADDLE_HEIGHT/2 < ballYPos + BALL_HEIGHT/2):
        paddle2YPos = paddle2YPos + PADDLE_SPEED
    if (paddle2YPos + PADDLE_HEIGHT/2 > ballYPos + BALL_HEIGHT/2):
        paddle2YPos = paddle2YPos - PADDLE_SPEED
    if (paddle2YPos < 0):
        paddle2YPos = 0
    if (paddle2YPos > WINDOW_HEIGHT - PADDLE_HEIGHT):
        paddle2YPos = WINDOW_HEIGHT - PADDLE_HEIGHT
    return paddle2YPos

#game class
class PongGame:
    def __init__(self):
        num = random.randint(0,9)
        self.grzesiu = 0
        self.paddle1YPos = WINDOW_HEIGHT / 2 - PADDLE_HEIGHT / 2
        self.paddle2YPos = WINDOW_HEIGHT / 2 - PADDLE_HEIGHT / 2

        self.ballXDirection = 1
        self.ballYDirection = 1

        self.ballXPos = WINDOW_WIDTH/2 - BALL_WIDTH/2

        if(0 < num < 3):
            self.ballXDirection = 1
            self.ballYDirection = 1
        if (3 <= num < 5):
            self.ballXDirection = -1
            self.ballYDirection = 1
        if (5 <= num < 8):
            self.ballXDirection = 1
            self.ballYDirection = -1
        if (8 <= num < 10):
            self.ballXDirection = -1
            self.ballYDirection = -1

        num = random.randint(0,9)

        self.ballYPos = num*(WINDOW_HEIGHT - BALL_HEIGHT)/9

    def getPresentFrame(self):
        pygame.event.pump()
        screen.fill(BLACK)
        drawPaddle1(self.paddle1YPos)
        drawPaddle2(self.paddle2YPos)
        drawBall(self.ballXPos, self.ballYPos)
        image_data = pygame.surfarray.array3d(pygame.display.get_surface())
        pygame.display.flip()
        return image_data

    def getNextFrame(self, action):
        pygame.event.pump()
        score = 0
        screen.fill(BLACK)
        self.paddle1YPos = updatePaddle1(action, self.paddle1YPos)
        drawPaddle1(self.paddle1YPos)
        self.paddle2YPos = updatePaddle2(self.paddle2YPos, self.ballYPos)
        drawPaddle2(self.paddle2YPos)
        [score, self.paddle1YPos, self.paddle2YPos, self.ballXPos, self.ballYPos, self.ballXDirection, self.ballYDirection] = updateBall(self.paddle1YPos, self.paddle2YPos, self.ballXPos, self.ballYPos, self.ballXDirection, self.ballYDirection)
        drawBall(self.ballXPos, self.ballYPos)
        image_data = pygame.surfarray.array3d(pygame.display.get_surface())
        pygame.display.flip()
        self.grzesiu = self.grzesiu + score
        print ("Grzesiu is " + str(self.grzesiu))
        return [score, image_data]


import taichi as ti
import numpy as np
import os
import utils
from engine.mpm_solver import MPMSolver

write_to_disk = False

if write_to_disk:
    os.makedirs('outputs', exist_ok=True)

# Try to run on GPU
ti.init(arch=ti.cuda, device_memory_GB=4.0)

gui = ti.GUI("Taichi Elements", res=512, background_color=0x112F41)

mpm = MPMSolver(res=(64, 64, 64), size=1)

mpm.set_gravity((0, -20, 0))

mpm.add_sphere_collider(center=(0.25, 0.5, 0.5),
                        radius=0.1,
                        surface=mpm.surface_slip)
mpm.add_sphere_collider(center=(0.5, 0.5, 0.5),
                        radius=0.1,
                        surface=mpm.surface_sticky)
mpm.add_sphere_collider(center=(0.75, 0.5, 0.5),
                        radius=0.1,
                        surface=mpm.surface_separate)

for frame in range(1500):
    mpm.add_cube((0.2, 0.8, 0.45), (0.1, 0.03, 0.1),
                 mpm.material_water,
                 color=0x8888FF)
    mpm.add_cube((0.45, 0.8, 0.45), (0.1, 0.03, 0.1),
                 mpm.material_water,
                 color=0xFF8888)
    mpm.add_cube((0.7, 0.8, 0.45), (0.1, 0.03, 0.1),
                 mpm.material_water,
                 color=0xFFFFFF)
    mpm.step(4e-3)
    particles = mpm.particle_info()
    np_x = particles['position'] / 1.0

    # simple camera transform
    screen_x = ((np_x[:, 0] + np_x[:, 2]) / 2**0.5) - 0.2
    screen_y = (np_x[:, 1])

    screen_pos = np.stack([screen_x, screen_y], axis=-1)

    gui.circles(screen_pos, radius=1.1, color=particles['color'])
    gui.show(f'outputs/{frame:06d}.png' if write_to_disk else None)


import torch
import torch.nn as nn
import torch.nn.functional as F

from tqdm import tqdm

import numpy as np

from dataset import Dataset, EOS, SOS
from model import Seq2Seq

import random
import math
import argparse

def evaluate(model, eval_instances, num_eval_instances=None):
    criterion = nn.CrossEntropyLoss(size_average=False)
    instances = 0
    total_loss = 0
    for i, (word, target) in tqdm(enumerate(eval_instances), total=num_eval_instances):
        loss = model.train_step(word, target, criterion)

        num_preds, loss = rescale_loss(loss, target)
        instances += num_preds
        total_loss += (loss.item() * num_preds)

    avg_loss = total_loss / instances
    print("Eval PPL: {:4.4f}".format(math.exp(avg_loss)))
    return math.exp(avg_loss)

def free_run(model, instances):
    outputs = []
    for word in instances:
        outputs.append(model.free_run(word, return_attentions=True))
    return outputs

def rescale_loss(loss, target):
    #TODO does the loss scaling make sense
    num_preds = target.shape[0] * (target.shape[1] - 1)
    loss = loss / num_preds
    return num_preds, loss

def train(model, dataset, epochs=10, log_every=500, my_dev=[], save_name=None):
    criterion = nn.CrossEntropyLoss(size_average=False)
    params = model.parameters()
    optim = torch.optim.Adam(params, lr=5e-3)
    old_best_dev_ppl = np.inf
    for epoch in range(epochs):
        epoch_loss = 0
        instances = 0
        for i, (word, target) in tqdm(enumerate(dataset.train_epoch()), total=len(dataset.batches)):
            loss = model.train_step(word, target, criterion)

            num_preds, loss = rescale_loss(loss, target)
            instances += num_preds
            epoch_loss += (loss.item() * num_preds) # unscale loss

            loss.backward()
            optim.step()
            optim.zero_grad()

        epoch_loss = epoch_loss / instances
        print("Epoch {} train PPL: {:4.4f}".format(epoch, math.exp(epoch_loss)))
        with torch.no_grad():
            dev_ppl = evaluate(model, dataset.dev_set(), len(dataset.dev))
            if save_name != None and dev_ppl < old_best_dev_ppl:
                print("New Best")
                torch.save(model.state_dict(), save_name)
            old_best_dev_ppl = dev_ppl

def run(model):
    while True:
        word = input('> ')
        output, attentions = free_run(model, [model.dataset.wrap_word(word)])[0]
        arpa_string = ' '.join([model.dataset.ix_to_out[o] for o in output])
        print(arpa_string[:-5])
        print(dataset.translate_arpabet(arpa_string[:-5]))
        attentions = torch.cat(attentions, dim=0)
        print(attentions[:-1, 1:-1])

if __name__ == '__main__':
    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    parser.add_argument("--character-size", dest="character_size", default=64, type=int, help="size of the character embedding")
    parser.add_argument("--encoder", dest="encoder_size", default=64, type=int, help="size of the encoder hidden state")
    parser.add_argument("--epochs", dest="epochs", default=10, type=int, help="number of epochs to train for")
    parser.add_argument("--batch-size", dest="batch_size", default=50, type=int, help="max size of a full batch (batches can be smaller)")
    parser.add_argument("--debug", dest="debug", action="store_true", help="truncate the dataset for faster training")
    parser.add_argument("--save", dest="save", help="path to save best models by dev ppl")
    parser.add_argument("--run", dest="run", action="store_true", help="echos word back in IPA")
    parser.add_argument("--type", dest="type", default="base", help="Decoder type, one of: base, attn")

    args = parser.parse_args()

    random.seed(1)
    torch.manual_seed(1)

    device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
    dataset = Dataset(device, batch_size = args.batch_size, debug=args.debug)

    model = Seq2Seq(dataset, device, args.encoder_size, args.character_size, args.type)

    if args.run:
        #TODO unhardcode and implement a joint encoder-decoder model class
        model.load_state_dict(torch.load('attn_01.model'))
        run(model)
    else:
        try:
            train(model, dataset, args.epochs, log_every=len(dataset.batches) // 10, save_name=args.save)
        except KeyboardInterrupt:
            print("Interruped")


import chainer
import chainer.links as L
import chainer.functions as F
from chainer import cuda,Chain,optimizers,serializers
import numpy as np
import os
import argparse
import pylab
from model import Generator, Discriminator, VGG
from prepare import prepare_dataset

xp=cuda.cupy
cuda.get_device(0).use()

def set_optimizer(model,alpha=0.0002,beta=0.5):
    optimizer = optimizers.Adam(alpha=alpha,beta1=beta)
    optimizer.setup(model)

    return optimizer

def calc_vgg_loss(feat1, feat2):
    _,_,h,w=feat1.shape

    return F.mean_squared_error(feat1,feat2)/(h*w)

parser=argparse.ArgumentParser(description="ESRGAN")
parser.add_argument("--epoch",default=1000,type=int,help="the number of epochs")
parser.add_argument("--batchsize",default=2,type=int,help="batchsize")
parser.add_argument("--testsize",default=2,type=int,help="testsize")
parser.add_argument("--aw",default=0.005,type=float,help="weight of adversarial loss")
parser.add_argument("--l1",default=0.01,type=float,help="weight of l1 loss")
parser.add_argument("--interval",default=1,type=int,help="interval of snapshot")
parser.add_argument("--Ntrain",default=24000,type=int,help="the number of training images")
parser.add_argument("--iterations",default=2000,type=int,help="the numbef of iterations")

args = parser.parse_args()
epochs=args.epoch
batchsize=args.batchsize
testsize=args.testsize
adver_weight=args.aw
l1_weight=args.l1
interval=args.interval
Ntrain=args.Ntrain
iterations=args.iterations

image_path="/coco/"
image_list=os.listdir(image_path)

outdir="./output_train"
if not os.path.exists(outdir):
    os.mkdir(outdir)

test_box=[]
for i in range(testsize):
    rnd = np.random.randint(Ntrain + 1, Ntrain + 100)
    image_name = image_path + image_list[rnd]
    _, sr = prepare_dataset(image_name)
    test_box.append(sr)

x_test=chainer.as_variable(xp.array(test_box).astype(xp.float32))

generator=Generator()
generator.to_gpu()
gen_opt=set_optimizer(generator)
serializers.load_npz("./output_pretrain/generator_pretrain.model",generator)

discriminator=Discriminator()
discriminator.to_gpu()
dis_opt=set_optimizer(discriminator)

vgg=VGG()
vgg.to_gpu()
vgg_opt=set_optimizer(vgg)
vgg.base.disable_update()

for epoch in range(epochs):
    sum_gen_loss=0
    sum_dis_loss=0
    for batch in range(0,iterations,batchsize):
        hr_box=[]
        sr_box=[]
        for index in range(batchsize):
            rnd = np.random.randint(Ntrain)
            image_name = image_path + image_list[rnd]
            hr, sr = prepare_dataset(image_name)
            hr_box.append(hr)
            sr_box.append(sr)

        x=chainer.as_variable(xp.array(sr_box).astype(xp.float32))
        t=chainer.as_variable(xp.array(hr_box).astype(xp.float32))

        y=generator(x)
        y_dis=discriminator(y)
        t_dis=discriminator(t)
        y.unchain_backward()

        # Relativistic loss
        fake_mean=F.broadcast_to(F.mean(y_dis),(batchsize,1))
        real_mean=F.broadcast_to(F.mean(t_dis),(batchsize,1))
        dis_loss=F.mean(F.softplus(-(t_dis-fake_mean)))
        dis_loss+=F.mean(F.softplus(y_dis-real_mean))
        dis_loss /= 2

        # Adversarial loss
        #dis_loss = F.mean(F.softplus(y_dis)) + F.mean(F.softplus(-t_dis))

        discriminator.cleargrads()
        dis_loss.backward()
        dis_opt.update()
        dis_loss.unchain_backward()

        y=generator(x)
        y_dis=discriminator(y)

        # Relativistic loss
        fake_mean=F.broadcast_to(F.mean(y_dis),(batchsize,1))
        real_mean=F.broadcast_to(F.mean(t_dis),(batchsize,1))
        adver_loss=F.mean(F.softplus(-(y_dis-real_mean)))
        adver_loss+=F.mean(F.softplus(t_dis-fake_mean))
        adver_loss /= 2

        # Adversarial loss
        #adver_loss = F.mean(F.softplus(-y_dis))

        fake_feat1=vgg(y)
        real_feat1=vgg(t)
        real_feat1.unchain_backward()
        vgg_loss=calc_vgg_loss(fake_feat1,real_feat1)

        l1_loss=F.mean_absolute_error(y,t)

        gen_loss=vgg_loss+adver_weight*adver_loss+l1_weight*l1_loss

        generator.cleargrads()
        vgg.cleargrads()
        gen_loss.backward()
        gen_opt.update()
        vgg_opt.update()
        gen_loss.unchain_backward()

        sum_dis_loss+=dis_loss.data.get()
        sum_gen_loss+=gen_loss.data.get()

        if epoch%interval==0 and batch==0:
            serializers.save_npz("%s/generator_%d.model"%(outdir,epoch),generator)
            with chainer.using_config("train", False):
                y = generator(x_test)
            y = y.data.get()
            sr = x_test.data.get()
            for i_ in range(testsize):
                tmp = (np.clip((sr[i_,:,:,:])*127.5 + 127.5, 0, 255)).transpose(1,2,0).astype(np.uint8)
                pylab.subplot(testsize,2,2*i_+1)
                pylab.imshow(tmp)
                pylab.axis('off')
                pylab.savefig('%s/visualize_%d.png'%(outdir, epoch))
                tmp = (np.clip((y[i_,:,:,:])*127.5 + 127.5, 0, 255)).transpose(1,2,0).astype(np.uint8)
                pylab.subplot(testsize,2,2*i_+2)
                pylab.imshow(tmp)
                pylab.axis('off')
                pylab.savefig('%s/visualize_%d.png'%(outdir, epoch))

    print("epoch:{}".format(epoch))
    print("Discrimintor loss:{}".format(sum_dis_loss/Ntrain))
    print("Generator loss:{}".format(sum_gen_loss/Ntrain))


import sqlite3
import time
import uuid
import datetime
from typing import Union, List, Tuple

class User(object):
    def __init__(self, chat_id: int, name: str):
        self.id = chat_id
        self.name = name

class Irrigation(object):
    def __init__(self, id, field_id, date, amount):
        self.id = id
        self.field_id = field_id
        self.date = date
        self.amount = amount

class NPK(object):
    def __init__(self, id, field_id, date, npk):
        self.id = id
        self.field_id = field_id
        self.date = date
        self.npk = npk

class Field(object):
    def __init__(
        self,
        id: int,
        name: str,
        creator_id: int,
        latitude: float,
        longitude: float,
        crop_start: datetime.date,
        crop_end: datetime.date,
        crop_name: str,
    ):
        self.id = id
        self.name = name
        self.creator_id = creator_id
        self.latitude = latitude
        self.longitude = longitude
        self.crop_start = crop_start
        self.crop_end = crop_end
        self.crop_name = crop_name

# All async in case we switch to Postgres
class Database(object):
    def __init__(self):
        self._db = sqlite3.connect("/home/db/agrobot.db")
        self._db.row_factory = sqlite3.Row

    async def get_all_users(self) -> List[User]:
        query = """
        SELECT id,
               name
        FROM users
        """
        cursor = self._db.cursor()
        cursor.execute(query)
        rows = cursor.fetchall()
        cursor.close()

        return [User(row["id"], row["name"]) for row in rows]

    async def get_user(self, chat_id) -> Union[User, None]:
        query = """
        SELECT id,
               name
        FROM users
        WHERE id = ?
        """
        args = (chat_id,)
        cursor = self._db.cursor()
        cursor.execute(query, args)
        row = cursor.fetchone()
        cursor.close()

        if row:
            return User(row["id"], row["name"])
        else:
            return None

    async def new_user(self, chat_id: int, name: str) -> User:
        query = """
        INSERT or REPLACE INTO users (id, name)
        VALUES (?, ?)
        """
        args = (chat_id, name)

        cursor = self._db.cursor()
        cursor.execute(query, args)
        self._db.commit()
        cursor.close()

        return User(chat_id, name)

    async def delete_user(self, id):
        query = """
        DELETE
        FROM users
        WHERE id = ?
        """
        args = (id,)
        cursor = self._db.cursor()
        cursor.execute("PRAGMA foreign_keys = ON")
        cursor.execute(query, args)
        self._db.commit()
        cursor.close()

    async def get_user_fields(self, id) -> List[Field]:
        query = """
        SELECT *
        FROM fields
        WHERE creator_id = ?
        """
        args = (id,)
        cursor = self._db.cursor()
        cursor.execute(query, args)
        rows = cursor.fetchall()
        cursor.close()
        return [
            Field(
                row["id"],
                row["name"],
                row["creator_id"],
                row["latitude"],
                row["longitude"],
                row["crop_start"],
                row["crop_end"],
                row["crop_name"],
            )
            for row in rows
        ]

    async def get_user_fields_irrs_npks(self, id):
        all_fields = []
        for field in await self.get_user_fields(id):
            field_irrigations = await self.get_irrigations(field.id)
            field_npks = await self.get_npks(field.id)
            all_fields.append([field, field_irrigations, field_npks])
        return all_fields

    async def get_user_field_dicts(self, id):
        field_dict = {}
        triplets = await self.get_user_fields_irrs_npks(id)
        for field, field_irrigations, field_npks in triplets:
            irrigation_events = []
            irrigation_ammounts = []
            for field_irrigation in field_irrigations:
                irrigation_events.append(field_irrigation.date)
                irrigation_ammounts.append(field_irrigation.amount)

            npk_events = []
            npk = []
            for field_npk in field_npks:
                npk_events.append(field_npk.date)
                npk.append(field_npk.npk)

            field_dict[field.id] = {
                "name": field.name,
                "latitude": field.latitude,
                "longitude": field.longitude,
                "crop_start": field.crop_start,
                "crop_end": field.crop_end,
                "crop_name": field.crop_name,
                "irrigation_events": irrigation_events,
                "irrigation_ammounts": irrigation_ammounts,
                "npk_events": npk_events,
                "npk": npk,
            }
        return field_dict

    async def add_field(
        self,
        id: int,
        name: str,
        creator_id: int,
        latitude: float,
        longitude: float,
        crop_start: datetime.date,
        crop_end: datetime.date,
        crop_name: str,
    ):
        query = """
        INSERT or REPLACE INTO fields(id, name, creator_id, latitude, longitude, crop_start, crop_end, crop_name)
               VALUES (?, ?, ?, ?, ?, ?, ?, ?)
        """
        args = (
            id,
            name,
            creator_id,
            latitude,
            longitude,
            crop_start.strftime("%Y-%m-%d"),
            crop_end.strftime("%Y-%m-%d"),
            crop_name,
        )
        cursor = self._db.cursor()
        cursor.execute(query, args)
        self._db.commit()
        cursor.close()

    async def delete_field(self, id):
        query = """
        DELETE
        FROM fields
        WHERE id = ?
        """
        args = (id,)
        cursor = self._db.cursor()
        cursor.execute("PRAGMA foreign_keys = ON")
        cursor.execute(query, args)
        self._db.commit()
        cursor.close()

    async def get_field(self, id):
        query = """
        SELECT *
        FROM fields
        WHERE id = ?
        """
        args = (id,)
        cursor = self._db.cursor()
        cursor.execute(query, args)
        row = cursor.fetchone()
        cursor.close()
        if row:
            return row
        else:
            return None

    async def get_irrigations(self, field_id) -> List[Field]:
        query = """
        SELECT *
        FROM irrigations
        WHERE field_id = ?
        """
        args = (field_id,)
        cursor = self._db.cursor()
        cursor.execute(query, args)
        rows = cursor.fetchall()
        cursor.close()
        return [
            Irrigation(row["id"], row["field_id"], row["date"], row["amount"])
            for row in rows
        ]

    async def add_irrigation(self, id, field_id, date, amount):
        query = """
        INSERT or REPLACE INTO irrigations(id, field_id, date, amount)
               VALUES (?, ?, ?, ?)
        """
        args = (id, field_id, date.strftime("%Y-%m-%d"), amount)
        cursor = self._db.cursor()
        cursor.execute(query, args)
        self._db.commit()
        cursor.close()

    async def delete_irrigation(self, id: int):
        query = """
        DELETE
        FROM irrigations
        WHERE id = ?
        """
        args = (id,)
        cursor = self._db.cursor()
        cursor.execute(query, args)
        self._db.commit()
        cursor.close()

    async def get_npks(self, id) -> List[NPK]:
        query = """
        SELECT *
        FROM npks
        WHERE field_id = ?
        """
        args = (id,)
        cursor = self._db.cursor()
        cursor.execute(query, args)
        rows = cursor.fetchall()
        cursor.close()
        return [
            NPK(
                row["id"],
                row["field_id"],
                row["date"],
                [float(x) for x in row["npk"].split(" ")],
            )
            for row in rows
        ]

    async def add_npk(self, id, field_id, date, npk):
        query = """
        INSERT or REPLACE INTO npks(id, field_id, date, npk)
               VALUES (?, ?, ?, ?)
        """
        npk_str = " ".join(list(map(str, npk)))  # save as: [50,30,20] -> 50 30 20
        args = (id, field_id, date.strftime("%Y-%m-%d"), npk_str)
        cursor = self._db.cursor()
        cursor.execute(query, args)
        self._db.commit()
        cursor.close()

    async def delete_npk(self, id):
        query = """
        DELETE
        FROM npks
        WHERE id = ?
        """
        args = (id,)
        cursor = self._db.cursor()
        cursor.execute(query, args)
        self._db.commit()
        cursor.close()

    async def add_income(self, id, field_id, income_per_ga):
        query = """
        INSERT or REPLACE INTO income(id, field_id, income_per_ga)
               VALUES (?, ?, ?)
        """
        args = (id, field_id, income_per_ga)
        cursor = self._db.cursor()
        cursor.execute(query, args)
        self._db.commit()
        cursor.close()

    async def get_income(self, id):
        query = """
        SELECT *
        FROM income
        WHERE id = ?
        """
        args = (id,)
        cursor = self._db.cursor()
        cursor.execute(query, args)
        row = cursor.fetchone()
        cursor.close()

        if row:
            return row
        else:
            return None


import keras
from keras import backend as K
from keras.models import Model, load_model, Sequential
from keras.layers import Input, Conv2D, Conv2DTranspose, MaxPooling2D, \
    Lambda, BatchNormalization, Dropout, Activation, \
    UpSampling2D, GlobalMaxPooling1D, TimeDistributed, Dense, \
    Permute, RepeatVector, Dot, Multiply, Maximum, \
    CuDNNGRU, CuDNNLSTM, Bidirectional
from keras.layers.merge import Concatenate, Maximum
from keras.layers.advanced_activations import ELU
from keras.optimizers import RMSprop, Nadam
from keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, \
    EarlyStopping, LearningRateScheduler, History
from keras.regularizers import l2
from keras.objectives import binary_crossentropy, categorical_crossentropy, \
    mean_squared_error

from keras.utils.vis_utils import model_to_dot

smooth = 1e-12

def dice_coef(y_true, y_pred, channels=None):
    intersection = K.sum(y_true * y_pred, axis=[0, 1, 2])
    union = K.sum(y_true + y_pred, axis=[0, 1, 2])
    dice = (intersection + smooth) / (union - intersection + smooth)
    if not channels is None:
        return K.mean(K.gather(dice, channels))
    else:
        return K.mean(dice)

def dice_coef_0(y_true, y_pred):
    return dice_coef(y_true, y_pred, channels=[0])

def dice_coef_1(y_true, y_pred):
    return dice_coef(y_true, y_pred, channels=[1])

def dice_coef_01(y_true, y_pred):
    return dice_coef(y_true, y_pred, channels=[0, 1])

def dice_coef_loss(y_true, y_pred, channels=None):
    return -dice_coef(y_true, y_pred, channels=channels)

def dice_ce_loss(y_true, y_pred):
    return -K.log(dice_coef(y_true, y_pred)) + categorical_crossentropy(y_true, y_pred)

def dice_ce_01_loss(y_true, y_pred):
    return categorical_crossentropy(y_true, y_pred) - 0.5 * K.log(dice_coef_01(y_true, y_pred))

def dice_01_loss(y_true, y_pred):
    return -dice_coef_01(y_true, y_pred)

def fake_loss(y_true, y_pred):
    return K.mean(y_true) - K.mean(y_pred)

def simple_attention(seq_size, input_size, name, features=1):
    model = Sequential()
    model.add(TimeDistributed(Dense(features, activation='sigmoid', name=name+'_dense'),
                              input_shape=(seq_size, input_size)))
    model.add(GlobalMaxPooling1D(name=name+'_pool'))
    return model

def colwise_rowwise_attention(src_shape, depth, in_tensor, name):
    rows_attention = TimeDistributed(simple_attention(src_shape[0],
                                                      depth,
                                                      name+'_rows_att'),
                                     name=name+'_rows_att_td_in')(in_tensor)
    rows_attention_matrix = TimeDistributed(RepeatVector(src_shape[1]),
                                            name=name+'_rows_att_td_out')(rows_attention)
    out_with_row_att = Multiply(name=name+'_rows_att_result')([in_tensor,
                                                               rows_attention_matrix])

    transpose = Permute((2, 1, 3),
                        name=name+'_cols_att_transp_in')(in_tensor)
    cols_attention = TimeDistributed(simple_attention(src_shape[1],
                                                      depth,
                                                      name+'_cols_att'),
                                     name=name+'_cols_att_td_in')(transpose)
    cols_attention_matrix_t = TimeDistributed(RepeatVector(src_shape[0]),
                                              name=name+'_cols_att_td_out')(cols_attention)
    cols_attention_matrix = Permute((2, 1, 3),
                                    name=name+'_cols_att_transp_out')(cols_attention_matrix_t)
    out_with_col_att = Multiply(name=name+'_cols_att_result')([in_tensor, cols_attention_matrix])

    out_with_full_att = Multiply(name=name+'_full_att')([out_with_col_att, out_with_row_att])
    return out_with_full_att

def conv_block(inp, kernels_n, name, dilations=[1], add_pool=True, upconv=False, add_upconv_input=None, out_att=False):
    if upconv or not add_upconv_input is None:
        up = UpSampling2D(name=name+'_up')(inp)
        if not add_upconv_input is None:
            inp = Concatenate(name=name+'_up_concat')([up, add_upconv_input])
        else:
            inp = up

    conv_outs = []
    for direction, k_shape in (('h', (3, 1)), ('v', (1, 3))):
        for dilation in dilations:
            conv_outs.append(Conv2D(kernels_n,
                                    k_shape,
                                    dilation_rate=dilation,
                                    padding='same',
                                    name=name+'_conv_{}_conv_dil{}'.format(direction, dilation))(inp))
    if len(conv_outs) > 1:
        conv = Concatenate(name=name+'_conv1_conv')(conv_outs)
    else:
        conv = conv_outs[0]
    conv = BatchNormalization(name=name+'_conv1_bn')(conv)
    conv = ELU(name=name+'_conv1_act')(conv)
#     conv = Activation('relu')(conv)

#    conv_outs = []
#    for dilation in dilations:
#        conv_outs.append(Conv2D(kernels_n,
#                                (1, 3),
#                                dilation_rate=dilation,
#                                padding='same',
#                                name=name+'_conv2_conv_dil{}'.format(dilation))(conv))
#    if len(conv_outs) > 1:
#        conv = Concatenate(name=name+'_conv2_conv')(conv_outs)
#    else:
#        conv = conv_outs[0]
#    conv = BatchNormalization(name=name+'_conv2_bn')(conv)
#    conv = ELU(name=name+'_conv2_act')(conv)
#     conv = Activation('relu')(conv)

    if out_att:
        conv = colwise_rowwise_attention(K.int_shape(conv)[1:-1], kernels_n, conv, name)

    if add_pool:
        pool = MaxPooling2D(pool_size=(2, 2), name=name+'_pool')(conv)
    else:
        pool = None
    return conv, pool

def get_unet(in_shape, out_channels):
    first_conv_n = 16
    inputs = Input(in_shape)

    conv1, pool1 = conv_block(inputs, first_conv_n, 'enc1', dilations=[1])
    conv2, pool2 = conv_block(pool1, first_conv_n * 2, 'enc2', dilations=[2])
#     conv3, pool3 = conv_block(pool2, first_conv_n * 4, 'enc1', dilations=[4])
#     conv4, pool4 = conv_block(pool3, first_conv_n * 8, 'enc1', dilations=[8])

    conv5, _ = conv_block(pool2, first_conv_n * 8, 'bottleneck', add_pool=False)

#     conv6, _ = conv_block(conv5, first_conv_n * 8, 'dec4', add_pool=False, add_upconv_input=conv4)
#    conv7, _ = conv_block(conv5, first_conv_n * 4, 'dec3', dilations=[1], add_pool=False, add_upconv_input=conv3)
    conv8, _ = conv_block(conv5, first_conv_n * 2, 'dec2', dilations=[2], add_pool=False, add_upconv_input=conv2)
    conv9, _ = conv_block(conv8, first_conv_n, 'dec1', dilations=[4], add_pool=False, add_upconv_input=conv1)

    conv10 = Conv2D(out_channels, (1, 1), padding='same', name='out_conv')(conv9)
    out = Activation('softmax', name='out_act')(conv10)

    model = Model(inputs=[inputs], outputs=[out])
    model.compile(optimizer=Nadam(), #RMSprop(1e-2),
                  loss=dice_ce_01_loss,
#                   loss=dice_coef_loss,
#                   loss=categorical_crossentropy,
                  metrics=[dice_coef_0, dice_coef_1, dice_coef_01, categorical_crossentropy])

    return model

def get_unet_row_col_info(in_shape, out_channels):
    first_conv_n = 12
    inputs = Input(in_shape)

    conv1, pool1 = conv_block(inputs, first_conv_n, 'enc1', dilations=[1, 2, 4])
    conv2, pool2 = conv_block(pool1, first_conv_n * 2, 'enc2', dilations=[1, 2, 4])
    conv3, pool3 = conv_block(pool2, first_conv_n * 4, 'enc3', dilations=[1, 2, 4])
    #conv4, pool4 = conv_block(pool3, first_conv_n * 8, 'enc4', dilations=[1, 2, 4])

    conv5, _ = conv_block(pool3, first_conv_n * 8, 'bottleneck', add_pool=False)

    #conv6, _ = conv_block(conv5, first_conv_n * 8, 'dec4', add_pool=False, add_upconv_input=conv4)
    conv7, _ = conv_block(conv5, first_conv_n * 4, 'dec3', dilations=[1, 2, 4], add_pool=False, add_upconv_input=conv3)
    conv8, _ = conv_block(conv7, first_conv_n * 2, 'dec2', dilations=[1, 2, 4], add_pool=False, add_upconv_input=conv2)
    conv9, _ = conv_block(conv8, first_conv_n, 'dec1', dilations=[1, 2, 4], add_pool=False, add_upconv_input=conv1)

    out_with_full_att = colwise_rowwise_attention(in_shape, K.int_shape(conv9)[-1], conv9, 'out_att')

    conv10 = Conv2D(out_channels, (1, 1), padding='same', name='out_conv')(out_with_full_att)
    out = Activation('softmax', name='out_act')(conv10)

    model = Model(inputs=[inputs], outputs=[out])
    model.compile(optimizer=Nadam(), #RMSprop(1e-2),
                  loss=dice_ce_01_loss,
#                   loss=dice_coef_loss,
#                   loss=categorical_crossentropy,
                  metrics=[dice_coef_0, dice_coef_1, dice_coef_01, categorical_crossentropy])

    return model

def get_unet_row_col_info_autoenc(in_shape, out_channels):
    first_conv_n = 12
    inputs = Input(in_shape)

    conv1, pool1 = conv_block(inputs, first_conv_n, 'enc1', dilations=[1, 2])
    conv2, pool2 = conv_block(pool1, first_conv_n * 2, 'enc2', dilations=[1, 2])
    conv3, pool3 = conv_block(pool2, first_conv_n * 4, 'enc3', dilations=[1, 2])
    #conv4, pool4 = conv_block(pool3, first_conv_n * 8, 'enc4', dilations=[1, 2, 4])

    conv5, _ = conv_block(pool3, first_conv_n * 8, 'bottleneck', add_pool=False)

    #conv6, _ = conv_block(conv5, first_conv_n * 8, 'dec4', add_pool=False, add_upconv_input=conv4)
    conv7, _ = conv_block(conv5, first_conv_n * 4, 'dec3', dilations=[1, 2],
                          add_pool=False, add_upconv_input=conv3)
    conv8, _ = conv_block(conv7, first_conv_n * 2, 'dec2', dilations=[1, 2],
                          add_pool=False, add_upconv_input=conv2)
    conv9, _ = conv_block(conv8, first_conv_n, 'dec1', dilations=[1, 2],
                          add_pool=False, add_upconv_input=conv1)

    aux_conv7, _ = conv_block(conv5, first_conv_n * 4, 'aux_dec3', dilations=[1, 2],
                              add_pool=False, upconv=True)
    #aux_loss1 = Lambda(lambda t: mean_squared_error(conv3, aux_conv7)
    aux_conv8, _ = conv_block(aux_conv7, first_conv_n * 2, 'aux_dec2', dilations=[1, 2],
                              add_pool=False, upconv=True)
    #aux_loss2 = mean_squared_error(conv2, aux_conv8)
    aux_conv9, _ = conv_block(aux_conv8, first_conv_n, 'aux_dec1', dilations=[1, 2],
                              add_pool=False, upconv=True)
    #aux_loss3 = mean_squared_error(conv1, aux_conv9)
    aux_conv10 = Conv2D(K.int_shape(inputs)[-1], (1, 1), padding='same', name='aux_out_conv')(aux_conv9)
    #aux_loss4 = mean_squared_error(inputs, aux_conv10)

    out_with_full_att = colwise_rowwise_attention(in_shape, K.int_shape(conv9)[-1], conv9, 'out_att')

    conv10 = Conv2D(out_channels, (1, 1), padding='same', name='out_conv')(out_with_full_att)
    out = Activation('softmax', name='out_act')(conv10)

    #aux_out = 0.1 * K.mean(aux_loss1) + 0.1 * K.mean(aux_loss2) + 0.1 * K.mean(aux_loss3) + 0.5 * K.mean(aux_loss4)

    model = Model(inputs=[inputs], outputs=[out, aux_conv10])
    model.compile(optimizer=Nadam(), #RMSprop(1e-2),
                  loss=[dice_ce_01_loss, mean_squared_error],
                  metrics=[dice_coef_0, dice_coef_1, dice_coef_01, categorical_crossentropy])

    return model

def get_unet_each_row_col_info(in_shape, out_channels):
    first_conv_n = 12
    inputs = Input(in_shape)

    conv1, pool1 = conv_block(inputs, first_conv_n, 'enc1', dilation=1, out_att=True)
    conv2, pool2 = conv_block(pool1, first_conv_n * 2, 'enc2', dilation=2, out_att=True)
    conv3, pool3 = conv_block(pool2, first_conv_n * 4, 'enc3', dilation=4, out_att=True)
    conv4, pool4 = conv_block(pool3, first_conv_n * 8, 'enc4', dilation=8, out_att=True)

    conv5, _ = conv_block(pool4, first_conv_n * 16, 'bottleneck', add_pool=False)

    conv6, _ = conv_block(conv5, first_conv_n * 8, 'dec4', add_pool=False,
                          add_upconv_input=conv4, out_att=True)
    conv7, _ = conv_block(conv6, first_conv_n * 4, 'dec3', dilation=1, add_pool=False,
                          add_upconv_input=conv3, out_att=True)
    conv8, _ = conv_block(conv7, first_conv_n * 2, 'dec2', dilation=2, add_pool=False,
                          add_upconv_input=conv2, out_att=True)
    conv9, _ = conv_block(conv8, first_conv_n, 'dec1', dilation=4, add_pool=False,
                          add_upconv_input=conv1, out_att=True)

    conv10 = Conv2D(out_channels, (1, 1), padding='same', name='out_conv')(conv9)
    out = Activation('softmax', name='out_act')(conv10)

    model = Model(inputs=[inputs], outputs=[out])
    model.compile(optimizer=Nadam(), #RMSprop(1e-2),
                  loss=dice_ce_01_loss,
#                   loss=dice_coef_loss,
#                   loss=categorical_crossentropy,
                  metrics=[dice_coef_0, dice_coef_1, dice_coef_01, categorical_crossentropy])

    return model

def get_unet_lstm(in_shape, out_channels):
    first_conv_n = 8
    inputs = Input(in_shape)

    conv1, pool1 = conv_block(inputs, first_conv_n, 'enc1', dilation=1)
    conv2, pool2 = conv_block(pool1, first_conv_n * 2, 'enc2', dilation=2)
    conv3, pool3 = conv_block(pool2, first_conv_n * 4, 'enc3', dilation=4)
    conv4, pool4 = conv_block(pool3, first_conv_n * 8, 'enc4', dilation=8)

    conv5, _ = conv_block(pool4, first_conv_n * 16, 'bottleneck', add_pool=False)

    conv6, _ = conv_block(conv5, first_conv_n * 8, 'dec4', dilation=1, add_pool=False, add_upconv_input=conv4)
    conv7, _ = conv_block(conv6, first_conv_n * 4, 'dec3', dilation=2, add_pool=False, add_upconv_input=conv3)
    conv8, _ = conv_block(conv7, first_conv_n * 2, 'dec2', dilation=3, add_pool=False, add_upconv_input=conv2)
    conv9, _ = conv_block(conv8, first_conv_n, 'dec1', dilation=4, add_pool=False, add_upconv_input=conv1)

    rows_repr_forward = TimeDistributed(CuDNNLSTM(64, return_sequences=True, name='rows_rnn_forward'),
                                        name='rows_td_forward')(conv9)
    rows_backward = Lambda(lambda t: K.reverse(t, 1), name='rows_reverse_in')(conv9)
    rows_repr_backward_rev = TimeDistributed(CuDNNLSTM(64, return_sequences=True, name='rows_rnn_backward'),
                                             name='rows_td_backward')(rows_backward)
    rows_repr_backward = Lambda(lambda t: K.reverse(t, 1), name='rows_reverse_out')(rows_repr_backward_rev)

    transpose = Permute((2, 1, 3), name='cols_transp_in')(conv9)
    cols_repr_forward_t = TimeDistributed(CuDNNLSTM(64, return_sequences=True, name='cols_rnn_forward'),
                                          name='cols_td_forward')(transpose)
    cols_repr_forward = Permute((2, 1, 3), name='cols_transp_out')(cols_repr_forward_t)

    cols_backward = Lambda(lambda t: K.reverse(t, 1), name='cols_reverse_in')(transpose)
    cols_repr_backward_rev_t = TimeDistributed(CuDNNLSTM(64, return_sequences=True, name='cols_rnn_backward'),
                                               name='cols_td_backward')(cols_backward)
    cols_repr_backward_t = Lambda(lambda t: K.reverse(t, 1), name='cols_reverse_out')(cols_repr_backward_rev_t)
    cols_repr_backward = Permute((2, 1, 3), name='cols_backward_transp_out')(cols_repr_backward_t)

    full_features = Concatenate(-1)([conv9,
                                     cols_repr_forward, cols_repr_backward,
                                     rows_repr_forward, rows_repr_backward])

    conv10 = Conv2D(out_channels, (1, 1), padding='same')(full_features)
    out = Activation('softmax')(conv10)

    model = Model(inputs=[inputs], outputs=[out])
    model.compile(optimizer=Nadam(), #RMSprop(1e-2),
                  loss=dice_ce_01_loss,
#                   loss=dice_coef_loss,
#                   loss=categorical_crossentropy,
                  metrics=[dice_coef_0, dice_coef_1, dice_coef_01, categorical_crossentropy])

    return model


from django.urls import path, include
from .views import ImageList, ImageListAPI, ImageDetailAPI, RandomImageAPI

urlpatterns = [
    path('images', ImageListAPI.as_view()),
    path('images/random/', RandomImageAPI.as_view()),
    path('images/<int:pk>/', ImageDetailAPI.as_view()),
    # path('', ImageList.as_view()),
]


import curses

# Initialize the curses screen
screen = curses.initscr()

# Set some basic screen options
curses.noecho()
curses.cbreak()
screen.keypad(True)

# Add some basic status information to the screen
screen.addstr(0, 0, "Fuzzing Status:")
screen.addstr(1, 0, "================")

# Define a function to update the status screen
def update_status(packet_count, crashes_found, unique_crashes_found):
    screen.addstr(3, 0, f"Packets sent: {packet_count}")
    screen.addstr(4, 0, f"Crashes found: {crashes_found}")
    screen.addstr(5, 0, f"Unique crashes found: {unique_crashes_found}")
    screen.refresh()

# Call the update_status function periodically to update the screen
update_status(100, 2, 1)

# Wait for user input before exiting
screen.getch()

# Restore the terminal settings
curses.nocbreak()
screen.keypad(False)
curses.echo()
curses.endwin()


from collections import Counter
import sys
sys.path.append(".")

import vgmusic
from music21 import note, interval, stream
from typing import Dict, Generator, List
import json

class SongDataTracker:
    def __init__(self):
        self.highest_note = 0
        self.lowest_note = 5000
        self.total_note_length = 0.0
        self.note_count = 0
        self.total_rest_length = 0.0
        self.rest_count = 0
        self.range = 0

    def get_notes_to_rests(self) -> float:
        if self.total_rest_length > 0.0:
            return self.total_note_length / self.total_rest_length
        return 0.0

    def get_avg_note(self) -> float:
        if self.note_count > 0:
            return self.total_note_length / self.note_count
        return 0.0

    def get_avg_rest(self) -> float:
        if self.rest_count > 0:
            return self.total_rest_length / self.rest_count
        return 0.0

    def to_dict(self) -> Dict:
        tmp = {
            "highest_note": self.highest_note,
            "lowest_note": self.lowest_note,
            "total_note_length": self.total_note_length,
            "note_count": self.note_count,
            "total_rest_length": self.total_rest_length,
            "rest_count": self.rest_count,
            "range": self.range
        }

        if self.total_rest_length > 0.0:
            tmp["notes_to_rests"] = self.get_notes_to_rests()

        if self.note_count > 0:
            tmp["avg_note_length"] = self.get_avg_note()

        if self.rest_count > 0:
            tmp["avg_rest_length"] = self.get_avg_rest()

        return tmp

    def from_dict(self, inst: Dict):
        self.highest_note = inst.get("highest_note", 0)
        self.lowest_note = inst.get("lowest_note", 0)
        self.total_note_length = inst.get("total_note_length", 0.0)
        self.note_count = inst.get("note_count", 0)
        self.total_rest_length = inst.get("total_rest_length", 0)
        self.rest_count = inst.get("rest_count", 0)
        self.range = inst.get("range", 0)

    def process_stream(self, song_str: stream.Stream):
        new_range = song_str.analyze("ambitus")
        if new_range.semitones > self.range:
            self.range = new_range.semitones

        for gennote in song_str.flat.getElementsByClass(note.GeneralNote):
            if isinstance(gennote, note.Note):
                new_ps = gennote.pitch.ps
                if self.highest_note < new_ps:
                    self.highest_note = new_ps
                if self.lowest_note > new_ps:
                    self.lowest_note = new_ps

                self.total_note_length += gennote.duration.quarterLength
                self.note_count += 1
            elif isinstance(gennote, note.Rest):
                self.total_rest_length += gennote.duration.quarterLength
                self.rest_count += 1

    def compare_stream(self, song_str: stream.Stream, sdt) -> float:
        #notes_in_range = 0
        #note_count = 0
        #for note_obj in song_str.flat.getElementsByClass(note.Note):
        #    if self.lowest_note <= note_obj.pitch.ps <= self.highest_note:
        #        notes_in_range += 1
        #    note_count += 1
        #perc_in_range = 0.0

        range_diff = abs(sdt.range - self.range)
        range_diff_score = 1.0 - range_diff / 100.0

        ntr_diff_score = abs(sdt.get_notes_to_rests() - self.get_notes_to_rests())

        avg_note_diff = abs(sdt.get_avg_note() - self.get_avg_note())
        avg_rest_diff = abs(sdt.get_avg_rest() - self.get_avg_rest())

        scores = [range_diff_score, ntr_diff_score, avg_note_diff, avg_rest_diff]
        weights = [0.2, 0.2, 0.2, 0.2]

        tmp_sum = 0
        for i in range(len(scores)):
            tmp_sum += scores[i] * weights[i]
        return tmp_sum

class GenreDataTracker:
    def __init__(self):
        self.song_trackers: List[SongDataTracker] = []

    def append(self, tracker: SongDataTracker):
        self.song_trackers.append(tracker)

    def get_median_range(self) -> float:
        ranges = [tracker.range for tracker in self.song_trackers]
        size = len(ranges)
        if size == 1:
            return ranges[0]
        elif size == 0:
            return 0
        elif size % 2 == 0:
            return ranges[size // 2]
        else:
            return (ranges[size // 2] + ranges[size // 2 + 1]) / 2.0

    def compare(self, song: stream.Stream, song_dt: SongDataTracker) -> float:
        # given a stream, test the probability the stream belonds to this genre.
        probs = []
        for sdt in self.song_trackers:
            probs.append(sdt.compare_stream(song, song_dt))
        return sum(probs) / len(probs)

    def to_dict(self):
        tmp = {}

        tmp["tracker_data"] = []
        for tracker in self.song_trackers:
            tmp["tracker_data"].append(tracker.to_dict())

        tmp["median_range"] = self.get_median_range()

        return tmp

    def from_dict(self, inst: Dict):
        tracker_data = inst.get("tracker_data", [])
        for tracker_inst in tracker_data:
            sdt = SongDataTracker()
            sdt.from_dict(tracker_inst)
            self.song_trackers.append(sdt)

def calculate_stats_for_genres():
    genres = {}

    for game in vgmusic.get_games_by_console("nes"):

        if len(game.genres) > 0:
            print("Processing {}...".format(game.name))
            for song in game.fetch_all_songs():
                main_tracker = SongDataTracker()
                main_tracker.process_stream(song)

                for genre in game.genres:
                    if genre not in genres:
                        genres[genre] = GenreDataTracker()
                    genres[genre].append(main_tracker)

    final_dict = {}
    for genre in genres:
        final_dict[genre] = genres[genre].to_dict()
    with open("test_output/nes.json", "w") as f:
        f.write(json.dumps(final_dict, indent=4))

def calculate_for_unknown_games():
    with open("test_output/nes.json", "r") as f:
        raw_data = json.loads(f.read())
    genres: Dict[str, GenreDataTracker] = {}
    for genre in raw_data:
        gdt = GenreDataTracker()
        gdt.from_dict(raw_data[genre])
        genres[genre] = gdt

    def sort_genre_data(genre_data):
        return genre_data[1]

    final_dict = {}
    for game in vgmusic.get_games_by_console("nes"):
        if game.id > 0:
            continue
        print(game.name)
        genres_sum = {}
        genres_total = {}
        for song in game.fetch_all_songs():
            sdt = SongDataTracker()
            sdt.process_stream(song)
            for genre in genres:
                if genre not in genres_sum:
                    genres_sum[genre] = 0.0
                    genres_total[genre] = 0
                genres_sum[genre] += genres[genre].compare(song, sdt)
                genres_total[genre] += 1
        genres_avg = []
        for genre in genres_sum:
            genre_data = (
                genre,
                genres_sum[genre] / genres_total[genre]
            )
            genres_avg.append(genre_data)

        genres_avg.sort(key=sort_genre_data, reverse=True)

        top_3 = genres_avg[0:3]
        final_dict[game.name] = top_3

    with open("test_output/unknown_genres.json", "w") as f:
        f.write(json.dumps(final_dict, indent=4))

if __name__ == "__main__":
    #calculate_stats_for_genres()
    calculate_for_unknown_games()