from dataclasses import dataclass
from typing import Union

#jeder spieler sammelt chips bekommt punkte
#pro zug bekommt spieler zufälligen chips
#punktewert des neuen chips wird zu Punktestand addiert
#mehr als 7 punkte nach weißem chip verloren
#bei orange passiert nichts
#bei rot und einem oder zwei orangenem extra punkte
#bei 3 oder mehr 2 extra
@dataclass
class White:
    points : int

@dataclass
class Orange:
    points : int

@dataclass
class Red:
    points : int

@dataclass
class GameState:
    stack : [White,Red,Orange]
    total : int

def add_chip(state : GameState, chips:Union[White,Orange,Red]):
    if isinstance(chips,Red):
        for chip in state.stack:
            if chip == Orange:
                chip = Orange
                if chip.points > 2:
                    stack.total += 2
                else:
                    stack.total += 1
    if isinstance(chips,White):
        old = state.total
        state.total +=  chips.points
        state.stack.append(chips)
        return state if old <= 7 and state.total < 8 else None

    if isinstance(chips,Orange):
        state.total += chips.points
        state.stack.append(chips)
        return state
    if isinstance(chips,Red):
        state.total += chips.points
        state.stack.append(chips)
        return state
state = GameState([], 0)
state = add_chip(state, White(3))
state = add_chip(state, Orange(3))
state = add_chip(state, Red(1))
stack = add_chip(state, White(1))
print(state.total)
print(state.stack)


import glob
import logging
import os
from pathlib import Path
from tempfile import NamedTemporaryFile

import pandas as pd
import pendulum
from airflow.hooks.base import BaseHook
from minio import Minio

DATA_DIR = "/data"
USE_COLS = ["city_ascii", "lat", "lng", "country", "iso2", "iso3"]

BATCH_SIZE = 5000
BATCH_SIZE = 5

def map_response(json_response: dict, return_data: dict):
    return {
        **return_data,
        "weather_descr": json_response["weather"][0]["description"],
        "weather": json_response["weather"][0]["main"],
        "temp": json_response["main"]["temp"],
        "feels_like": json_response["main"]["feels_like"],
        "wind_speed": json_response["wind"]["speed"],
        "lat": json_response["coord"]["lat"],
        "lng": json_response["coord"]["lon"],
        "timestamp": pendulum.now().to_iso8601_string(),
    }

def read_cities():
    filename = f"{DATA_DIR}/worldcities.csv"

    # keep_default_na because of iso2 of Namibia = NA
    world_cities = pd.read_csv(filename, usecols=USE_COLS, keep_default_na=False)
    # keep the number of requests small
    world_cities = world_cities.sample(25)

    data = world_cities.to_dict(orient="records")

    result = [
        {"batch": data[start : start + BATCH_SIZE]}
        for start in range(0, len(data), BATCH_SIZE)
    ]

    return result

def store_to_temp_file(cities_data, tmp_storage_location, **context):
    task_instance = context["ti"]
    map_index = task_instance.map_index
    logical_date = context["logical_date"]

    Path.mkdir(
        Path(f"{tmp_storage_location}/{logical_date}/"), parents=True, exist_ok=True
    )

    df = pd.DataFrame.from_records(cities_data)
    file_path = f"{tmp_storage_location}/{logical_date}/{map_index}-tmp.json"
    df.to_json(file_path)

def collect_and_upload_s3(s3_conn_id, tmp_storage_location, **context):
    logical_date = context["logical_date"]
    dag = context["dag"]

    all_files = glob.glob(f"{tmp_storage_location}/{logical_date}/*-tmp.json")

    file_dfs = []

    for file in all_files:
        file_df = pd.read_json(file)
        file_dfs.append(file_df)

    df = pd.concat(file_dfs, ignore_index=True)

    conn = BaseHook.get_connection(conn_id=s3_conn_id)

    minio_client = Minio(
        conn.extra_dejson["host"].split("://")[1],
        access_key=conn.login,
        secret_key=conn.password,
        secure=False,
    )

    with NamedTemporaryFile("w+b") as file:
        df.to_csv(file.name, index=False)

        bucket = "weatherdata"
        key = f"{dag.dag_id}/{logical_date}.csv"
        logging.info("Storing object: %s/%s.", bucket, key)

        minio_client.remove_object(bucket_name=bucket, object_name=key)

        result = minio_client.fput_object(
            bucket_name=bucket, object_name=key, file_path=file.name
        )

        logging.info("Result of uploading file:")
        logging.info(result)

def cleanup_files(tmp_storage_location, **context):
    logical_date = context["logical_date"]
    tmp_storage = Path(f"{tmp_storage_location}/{logical_date}/")

    all_files = glob.glob(f"{tmp_storage_location}/{logical_date}/*-tmp.json")

    for file in all_files:
        os.remove(file)

    tmp_storage.rmdir()


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import matplotlib.pyplot as plt
import Advanced_1.confusionMatrix as cm
import pickle

from tensorflow.examples.tutorials.mnist import input_data
from Advanced_1.dataBatcher import DataBatcher
from Advanced_1.learningRateScheduler import LearningRateScheduler
from Advanced_1.network import Network
from Advanced_1.layer import LinearLayer
from Advanced_1.layer import ReLULayer
from Advanced_1.layer import ConvLayer
from Advanced_1.layer import ConvLayerColumns
from Advanced_1.layer import MaxPoolLayer
from Advanced_1.layer import FlattenLayer
from Advanced_1.layer import CrossEntropyLogits

from Advanced_1.convergenceTester import ConvergenceTester

from scipy.misc import toimage
from sklearn.metrics import confusion_matrix

root_dir = 'C:/Users/Dave/Documents/GI13-Advanced/Assignment1';
summaries_dir = root_dir + '/Summaries';
save_dir = root_dir + '/SavedVariables';

def print_confusion_matrix(network, X, y, model_type):
    y_pred = network.get_predictions(X)
    y_true = y.argmax(axis=1)

    cnf_matrix = confusion_matrix(y_true, y_pred)
    cm_filename = model_type.replace(' ','_') + '.p'
    pickle.dump( cnf_matrix, open( cm_filename, "wb" ) )

    class_names = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
    np.set_printoptions(precision=2)

    # Plot non-normalized confusion matrix
    plt.figure()
    cm.plot_confusion_matrix(cnf_matrix, classes=class_names,
                          title=('Confusion matrix - ' + model_type))
    plt.show()

def part_a(network):
    network.addLayer( LinearLayer(10,(10,784),10) )
    return 'P2_a'

def part_b(network):
    network.addLayer( ReLULayer(128,(128,784),128) )
    network.addLayer( LinearLayer(10,(10,128),10) )
    return 'P2_b'

def part_c(network):
    network.addLayer( ReLULayer(256,(256,784),256) )
    network.addLayer( ReLULayer(256,(256,256),256) )
    network.addLayer( LinearLayer(10,(10,256),10) )
    return 'P2_c'

def part_d(network, X_train, X_test, use_saved=True):

    cl1 = ConvLayerColumns(depth=16, filter_size=3, width=28, height=28)

    if use_saved:
        X_train_cols = np.load('X_train_cols.npy')
        X_test_cols = np.load('X_test_cols.npy')
    else:
        X_train_pad = cl1.pre_pad_all_images(X_train)
        X_train_cols = cl1.im2col(X_train_pad)
        np.save('X_train_cols', X_train_cols)

        X_test_pad = cl1.pre_pad_all_images(X_test)
        X_test_cols = cl1.im2col(X_test_pad)
        np.save('X_test_cols', X_test_cols)

    network.addLayer( cl1 )
    network.addLayer( MaxPoolLayer(2, 14, 14, (4,14,14,16), 16 ))
    network.addLayer( ConvLayer(depth=16, filter_size=3, width=14, height=14) )
    network.addLayer( MaxPoolLayer(2, 7, 7, (4,7,7,16), 16) )
    network.addLayer( FlattenLayer((16,7,7)) )
    network.addLayer( ReLULayer(256, (256,784), 256) )
    network.addLayer( LinearLayer(10,(10,256),10) )

    return 'P2_d', X_train_cols, X_test_cols

def run_part2_models(FLAGS):

    # Import data
    mnist = input_data.read_data_sets(FLAGS.data_dir, one_hot=True)

    X_train = mnist.train.images
    y_train = mnist.train.labels
    X_test = mnist.test.images
    y_test = mnist.test.labels

    network = Network()
    network.addLossFunction( CrossEntropyLogits() )

    if FLAGS.model=='P2_a':
        model_type = part_a(network)
        data_batcher = mnist.train
    elif FLAGS.model=='P2_b':
        model_type = part_b(network)
        data_batcher = mnist.train
    elif FLAGS.model=='P2_c':
        model_type = part_c(network)
        data_batcher = mnist.train
    elif FLAGS.model=='P2_d':
        print('P2:d not implemented')
        exit
        model_type, X_train_cols, X_test_cols = part_d(network, X_train, X_test, use_saved=True)
        data_batcher = DataBatcher(X_train_cols, y_train)

#     rs = np.reshape(mnist.test.images[0], (28,28))
#     toimage(rs).show()

    train_accuracy_hist = []
    test_accuracy_hist = []
    conv_tester = ConvergenceTester(0.0005, lookback_window=25) #stop if converged to within 0.01%
    learning_rate = 0.005
    decay = learning_rate / 50000
#     learning_rate = 0.1
#     decay = learning_rate / 2e6

    model_filename = FLAGS.saved_model_dir + '/' + "trained_model_" + model_type + ".p"

    if FLAGS.use_saved:
        network = pickle.load( open( model_filename, "rb" ) )
        train_accuracy = network.get_accuracy(X_train, y_train)
        test_accuracy = network.get_accuracy(X_test, y_test)
    else:
        lrs = LearningRateScheduler(decay)
        for i in range(50000):
            learning_rate = lrs.get_learning_rate(i, learning_rate)
            batch_xs, batch_ys = data_batcher.next_batch(50)
            train_error = network.run_one_train_epoch(batch_xs, batch_ys, learning_rate)

            if i % 100 == 0:
                train_accuracy = network.get_accuracy(X_train, y_train)
                test_accuracy = network.append([i,train_accuracy])
                test_accuracy_hist.append([i,test_accuracy])

                print('{0:d} accuracy train test: {1:0.5f} : {2:0.5f} learning rate {3:0.8f}'.
                      format( i, train_accuracy, test_accuracy, learning_rate))

                if conv_tester.has_converged(test_accuracy):
                    print('converged after ', i, ' epochs')
                    break

        np.savetxt(model_type + "_train_accuracy_hist.csv", train_accuracy_hist, delimiter=",", fmt='%f')
        np.savetxt(model_type + "_test_accuracy_hist.csv", test_accuracy_hist, delimiter=",", fmt='%f')

        pickle.dump( network, open( model_filename, "wb" ) )

    print('Final accuracy train, test: {0:0.5f}, {1:0.5f}'.format( train_accuracy, test_accuracy))
    print_confusion_matrix(network, X_train, y_train, model_type + ' train data')
    print_confusion_matrix(network, X_test, y_test, model_type + ' test data')


from django.db import models
import pprint
from django.conf import settings

# Create your models here.
class Game(models.Model):

    CATEGORY_PC = 1
    CATEGORY_MOBILE = 2
    CATEGORY_CONSOLE = 3
    CATEGORY_OTHER = 4

    GAME_CATEGORY = (
        (CATEGORY_PC, 'PC'),
        (CATEGORY_MOBILE, 'Mobile'),
        (CATEGORY_CONSOLE, 'Console'),
        (CATEGORY_OTHER, 'Other'),
    )

    game_name = models.CharField(max_length=100)
    game_name_jp = models.CharField(max_length=100, null=True)
    game_name_sys = models.CharField(max_length=100, null=True, db_index=True)
    game_category = models.SmallIntegerField(
        choices=GAME_CATEGORY,
        default=CATEGORY_OTHER,
    )
    game_image = models.ImageField(upload_to='streamer', null=True, blank=True)
    game_icon = models.ImageField(upload_to='streamer', null=True, blank=True)
    active = models.BooleanField(default=True)
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

    def game_name_global(self):

        if settings.SERVER_SERVICE == 'jp':
            return self.game_name_jp
        else:
            return self.game_name

    def __str__(self):
        return self.game_name


from loguru import logger
import colorama

class Episode:
    e_id = None
    num = None
    air_date = None
    link = None

    def __init__(self, e_id, num, air_date, link):
        logger.debug(f"Creating episode obj (id: {e_id}, num: {num})")
        self.e_id = e_id
        self.num = num
        self.air_date = air_date
        self.link = link

    def __repr__(self):
        return f"{self.e_id}-{self.num}"

    def __str__(self):
        return self.link

class Related:
    a_id = None
    type = None
    title = None
    slug = None

    def __init__(self, a_id, type, title, slug):
        self.a_id = a_id
        self.type = type
        self.title = title
        self.slug = slug

    def __repr__(self):
        return f"{self.a_id}-{self.title}"

    def get_anime_url(self):
        return f"https://animeunity.it/anime/{self.a_id}-{self.slug}"

class Anime:
    a_id = None
    title = None
    title_eng = None
    thumbnail = None
    cover_image = None
    status = None
    type = None
    slug = None
    year = None
    episodes = None
    episodes_length = None
    related = None

    def __init__(self, a_id, title, type, episodes_length):
        logger.debug(f"Creating Anime obj (id: {a_id}, num: {title})")
        self.a_id = a_id
        self.title = title
        self.episodes_length = episodes_length
        # Se è OVA o ONA e dura più di 50 min lo converto in film
        if int(self.episodes_length) > 50 and (type == 'OVA' or type == 'ONA'):
            self.type = 'Movie'
        else:
            self.type = type

    def __repr__(self):
        return f"{self.a_id}-{self.title}"

    def __str__(self):
        title = f"{colorama.Fore.BLUE} {str(self.a_id)}"
        if self.type != 'TV':
            title = f"{title} {colorama.Fore.CYAN} {self.type}"
        return f"{title} {colorama.Fore.GREEN} {self.title}{('{} ({})'.format(colorama.Fore.CYAN, self.title_eng), '')[self.title_eng is None]} {colorama.Style.RESET_ALL}"

    def get_anime_url(self):
        return f"https://animeunity.it/anime/{self.a_id}-{self.slug}"


from connexion.resolver import Resolver
from connexion.operations import AbstractOperation
import os, traceback
from flask import Flask, Response
import connexion, json, logging
from stratus.handlers.manager import Handlers
from functools import partial
from stratus_endpoint.util.config import Config, StratusLogger
from flask_sqlalchemy import SQLAlchemy
from celery import Celery

class StratusResolver(Resolver):

    def __init__(self, api: str  ):
        Resolver.__init__( self, self.function_resolver )
        self.api = api

    def resolve_operation_id(self, operation: AbstractOperation) -> str:
        return '{}:{}'.format( self.api, operation.operation_id )

    def function_resolver( self, operation_id: str ) :
        return partial( Handlers.processRequest, operation_id )

class StratusApp:
    HERE = os.path.dirname(__file__)
    SETTINGS = os.path.join( HERE, 'settings.ini')

    def __init__(self):
        self.logger = StratusLogger.getLogger()
        self.app = connexion.FlaskApp("stratus", specification_dir='api/', debug=True )
        self.app.add_error_handler( 500, self.render_server_error )
        self.app.app.register_error_handler( TypeError, self.render_server_error )
        settings = os.environ.get( 'STRATUS_SETTINGS', self.SETTINGS )
        config_file = Config(settings)
        flask_parms = config_file.get_map('flask')
        flask_parms[ 'SQLALCHEMY_DATABASE_URI' ] = flask_parms['DATABASE_URI']
        self.app.app.config.update( flask_parms )
        self.parms = config_file.get_map('stratus')
        api = self.getParameter( 'API' )
        handler = self.getParameter( 'HANDLER' )
        celery_parms = config_file.get_map('celery')
        self.app.app.config.update( celery_parms )
        self.celery = self.make_celery( self.app.app )
        self.db = SQLAlchemy( self.app.app )
        self.app.add_api( api + ".yaml", resolver=StratusResolver( api ) )

    def run(self):
        port = self.getParameter( 'PORT', 5000 )
        self.db.create_all( )
        return self.app.run( int( port ) )

    def getParameter(self, name: str, default = None ) -> str:
        parm = self.parms.get( name, default )
        if parm is None: raise Exception( "Missing required stratus parameter in settings.ini: " + name )
        return parm

    @staticmethod
    def render_server_error( ex: Exception ):
        print( str( ex ) )
        traceback.print_exc()
        return Response(response=json.dumps({ 'message': getattr(ex, 'message', repr(ex)), "code": 500, "rid": "", "status": "error" } ), status=500, mimetype="application/json")

    def make_celery( self, app: Flask ):
        celery = Celery( app.import_name, backend=app.config['DATABASE_URI'], broker=app.config['CELERY_BROKER_URL'] )
        celery.conf.update(app.config)

        class ContextTask(celery.TaskHandle):
            def __call__(self, *args, **kwargs):
                with app.app_context():
                    return self.run(*args, **kwargs)

        celery.TaskHandle = ContextTask
        return celery

app = StratusApp()

if __name__ == "__main__":
    app.run()


"""Script to migrate a Maestrowf YAML file to a CoRR YAML file

To run use:

    $ python corr-sync.py --corr-output=corr.yaml --output-path=demo/sample_output

To run the tests:

    $ py.test --doctest-modules migrate_yaml.py

Test reading a sample Maestorwf file

>>> data_path = os.path.join(get_path(), 'lulesh_sample1.yaml')
>>> test_path = os.path.join(get_path(), 'corr-test.yaml')
>>> assert mapping(read_yaml(data_path)) == read_yaml(test_path)

"""

import os

# pylint: disable=redefined-builtin
from toolz.curried import pipe, get, curry, map, filter, tail
import yaml
import glob
import mimetypes
import click

def file_metadata(path):
    """Generate metadata for one file

    >>> from click.testing import CliRunner
    >>> with CliRunner().isolated_filesystem() as dir_:
    ...     test_file = os.path.join(dir_, 'hello.json')
    ...     with open(test_file, 'w') as f:
    ...         _ = f.write('{"hello": "hello"}')
    ...     metadata = file_metadata(test_file)
    >>> assert('hello.json' in metadata['path'])
    >>> assert metadata['metadata'] == {'encoding': 'utf-8',
    ...                                 'mimetype': 'application/json',
    ...                                 'size': 18}

    Args:
      path: the path to the file

    Returns:
      dictionary of metadata
    """
    return {
        'path': os.path.relpath(path),
        'metadata': {
            'encoding': 'utf-8',
            'mimetype': mimetypes.guess_type(path)[0],
            'size': os.path.getsize(path)
        }
    }

def glob_all_files_27(path):
    """Find all files in Py 2.7
    """
    import fnmatch
    import os

    matches = []
    for root, dirnames, filenames in os.walk(path):
        for filename in fnmatch.filter(filenames, '*'):
            matches.append(os.path.join(root, filename))

    return matches

def glob_all_files(path):
    """Find all files

    Args:
      path: the path to search for files

    Returns:
      a list of files
    """
    return pipe(
        os.path.join(path, '**'),
        lambda x: glob.glob(x, recursive=True),
        filter(os.path.isfile),
    )

def outputs(path):
    """Generate outputs metadata based on path

    Args:
      path: the path to the output data

    Returns:
      a list of file metadata

    """

    return pipe(
        path,
        glob_all_files_27,
        map(file_metadata),
        list
    )

@curry
def mapping(output_path, data):
    """Map from Maestrowf data to CoRR data

    Args:
      data: the Maestrowf data
      output_path: the path to the output data

    Returns
      the CoRR data
    """
    return {
        'execution': {
            'parameters': data['global.parameters'],
            'cmd_line': data['study'][-1]['run']['cmd']},
        'system': {'env': data['env']},
        'outputs': outputs(output_path),
        'inputs': [],
        'dependencies': []
    }

def get_path():
    """Return the local file path for this file.

    Returns:
      the filepath
    """
    return pipe(
        __file__,
        os.path.realpath,
        os.path.split,
        get(0)
    )

def read_yaml(filepath):
    """Read a YAML file

    Args:
      filepath: the path to the YAML file

    Returns:
      returns a dictionary
    """
    with open(filepath) as stream:
        data = yaml.safe_load(stream)
    return data

@curry
def write_yaml_data(filepath, data):
    """Write data to YAML file

    Args:
      filepath: the path to the YAML file
      data: a dictionary to write to the YAML file

    """
    with open(filepath, 'w') as stream:
        yaml.safe_dump(data, stream, default_flow_style=False, indent=2)
    return (filepath, data)

def find_yaml_file(path):
    """Find a YAML file in the path

    Args:
      path: find a YAML file on the path

    Returns:
      the path to the YAML file
    """
    return pipe(
        path,
        glob_all_files_27,
        filter(lambda x: tail(5, x) == '.yaml'),
        list,
        get(0)
    )
"""
@click.command()
@click.option('--corr-output',
              'corr_output',
              required=True,
              default='corr.yaml',
              help="Set the CoRR YAML file")
@click.option('--output-path',
              'output_path',
              required=True,
              help="Set the outputs capture path")"""
def main(corr_output, output_path, maestro_spec_path):
    """Read in a Maestrowf YAML, translate and write the CoRR YAML
    """
    return pipe(
        maestro_spec_path,
        read_yaml,
        mapping(output_path),
        write_yaml_data(corr_output)
    )

if __name__ == '__main__':
    # data = main()
    main()


import pandas as pd
from sklearn.linear_model import LinearRegression
data = pd.read_csv("papsep2022.csv")

features = data[["place","area"]]
target = data["price"]

print(features)
print(target)

model = LinearRegression()
model.fit(features , target)


import torch

from survboard.python.utils.utils import neg_par_log_likelihood

class intermediate_fusion_mean_criterion(torch.nn.Module):
    def forward(
        self,
        predictions,
        target,
        alpha=0,
    ):
        joint_log_hazard, unimodal_log_hazards = predictions
        time, event = target[:, 0], target[:, 1]
        unicox = 0
        for i in range(len(unimodal_log_hazards)):
            unicox += neg_par_log_likelihood(
                unimodal_log_hazards[i],
                torch.unsqueeze(time, 1).float(),
                torch.unsqueeze(event, 1).float(),
            )
        joint_cox = neg_par_log_likelihood(
            joint_log_hazard,
            torch.unsqueeze(time, 1).float(),
            torch.unsqueeze(event, 1).float(),
        )
        return joint_cox + alpha * unicox

class naive_neural_criterion(torch.nn.Module):
    def forward(self, predicted, target):
        time, event = target[:, 0], target[:, 1]
        cox_loss = neg_par_log_likelihood(
            predicted,
            torch.unsqueeze(time, 1).float(),
            torch.unsqueeze(event, 1).float(),
        )
        return cox_loss


from flask import Flask,render_template,request
import pickle

from sympy import re
model=pickle.load(open('stroke.pkl','rb'))
app=Flask(__name__)
@app.route('/',methods=['GET'])
def home():
    return render_template('stroke.html')
@app.route('/predict',methods=['POST','GET'])
def predict():
    if request.method=='POST':
        gender=request.form['gender']
        if gender=='Male':
            gender=1
        else:
            gender=0
        Age=int(request.form['Age'])
        bp=request.form['bp']
        if bp=='Yes':
            bp=1
        else:
            bp=0
        heart=request.form['heart']
        if heart=='Yes':
            heart=0
        else:
            heart=1
        married=request.form['married']
        if married=='Yes':
            married=1
        else:
            married=0
        work=request.form['work']
        if work=='never wroked':
            work=1
        elif work=='private':
            work=2
        elif work=='self employed':
            work=3
        elif work=='childern':
            work=4
        else:
            work=0
        house=request.form['house']
        if house=='Urban Housing':
            house=1
        else:
            house=0
        glucose=int(request.form['glucose'])
        bmi=int(request.form['bmi'])
        smoking=request.form['smoking']
        if smoking=='Used to Smoke':
            smoking=1
        elif smoking=='Never Smoked':
            smoking=2
        elif smoking=='Smokes':
            smoking=3
        else:
            smoking=0
        pred=model.predict([[gender,Age,bp,heart,married,work,house,glucose,bmi,smoking]])
        if pred==0:
            return render_template('pred.html',text='You will never have a stroke in the near future,maintain good health and diet.')
        else:
            return render_template('pred.html',text='You may have stroke in the near future Please take precautions and lead a stress free life as possible.')
    else:
        return render_template('stroke.html')
if __name__=='__main__':
    app.run(debug=True)


import pandas as pd
from datetime import datetime
from sklearn.neighbors import KNeighborsRegressor

def transform_and_normalize_data(df, stores_df):
    df = pd.merge(df, stores_df, on='Store')
    X = df[['Store', 'Type', 'Size', 'Dept', 'Date', 'IsHoliday']].as_matrix()

    max_size = 0
    max_week = 0
    for i in range(int(X.size/6)):
        X[i][1] = ord(X[i][1]) - ord('A')
        if max_size < X[i][2]: max_size = X[i][2]
        X[i][4] = datetime.strptime(X[i][4], '%Y-%m-%d').isocalendar()[1]
        if max_week < X[i][4]: max_week = X[i][4]
    for i in range(int(X.size/6)):
        X[i][2] /= max_size
        X[i][4] /= max_week

    return X

# Retrieve stores' information.
stores_df = pd.read_csv('stores.csv')

# Retrieve and pre-process the training set.
train_df = pd.read_csv('train.csv')
X = transform_and_normalize_data(train_df, stores_df)
y = train_df[['Weekly_Sales']].as_matrix()

# Train the kNN model.
knn_regressor = KNeighborsRegressor(n_neighbors=50, weights='distance', p=1)
knn_regressor.fit(X, y)

# Retrieve and pre-process the testing set.
test_df = pd.read_csv('test.csv')
T = transform_and_normalize_data(test_df, stores_df)
ids = test_df[['Store', 'Dept', 'Date']].as_matrix()

# Write the prediction results to submission.csv file.
with open('submission.csv', 'w') as file:
    file.write('Id,Weekly_Sales\n')
    for i in range(int(T.size/6)):
        (store, store_type, size, dept, date, is_holiday) = (T[i][0], T[i][1], T[i][2], T[i][3], T[i][4], T[i][5])
        file.write("%s_%s_%s," % (ids[i][0], ids[i][1], ids[i][2]))
        file.write(str(knn_regressor.predict([[store, store_type, size, dept, date, is_holiday]])[0][0]))
        file.write('\n')


import matplotlib
from matplotlib import pyplot as plt
from tqdm import tqdm
import network
import utils
from torchvision import transforms as T
import torch.nn as nn

from PIL import Image
from glob import glob

import colorsys
import os
import time

import numpy as np
import torch
from PIL import ImageDraw, ImageFont

from network.models.detect.centernet import CenterNet as CenterNet_Resnet50
from utils.utils import (cvtColor, get_classes, preprocess_input, resize_image,
                         show_config)
from utils.utils_bbox import decode_bbox, postprocess

# ---------------------------------------------------#
#   检测图片
# ---------------------------------------------------#
def detect_image(image, img_data, model, opts, crop=False, count=False):
    # ---------------------------------------------------#
    #   计算输入图片的高和宽
    # ---------------------------------------------------#
    image_shape = np.array(np.shape(image)[0:2])

    # 图片预处理，归一化。获得的photo的shape为[1, 512, 512, 3]
    # image_data = np.expand_dims(np.transpose(preprocess_input(np.array(image_data, dtype='float32')), (2, 0, 1)), 0)

    with torch.no_grad():
        # 将图像输入网络当中进行预测！
        outputs = model(img_data)

        # 利用预测结果进行解码
        outputs = decode_bbox(outputs[0], outputs[1], outputs[2], opts.confidence, opts.device)

        # -------------------------------------------------------#
        #   对于centernet网络来讲，确立中心非常重要。
        #   对于大目标而言，会存在许多的局部信息。
        #   此时对于同一个大目标，中心点比较难以确定。
        #   使用最大池化的非极大抑制方法无法去除局部框
        #   所以我还是写了另外一段对框进行非极大抑制的代码
        #   实际测试中，hourglass为主干网络时有无额外的nms相差不大，resnet相差较大。
        # -------------------------------------------------------#
        results = postprocess(outputs, opts.nms, image_shape, opts.crop_size, opts.letterbox_image, opts.nms_iou)

        # --------------------------------------#
        #   如果没有检测到物体，则返回原图
        # --------------------------------------#
        if results[0] is None:
            return image

        top_label = np.array(results[0][:, 5], dtype='int32')
        top_conf = results[0][:, 4]
        top_boxes = results[0][:, :4]

    # ---------------------------------------------------------#
    #   设置字体与边框厚度
    # ---------------------------------------------------------#
    font = ImageFont.truetype(font='model_data/simhei.ttf',
                              size=np.floor(3e-2 * np.shape(image)[1] + 0.5 - 10).astype('int32'))
    thickness = max((np.shape(image)[0] + np.shape(image)[1]) // opts.crop_size, 1)
    # ---------------------------------------------------------#
    #   计数
    # ---------------------------------------------------------#
    if count:
        print("top_label:", top_label)
        classes_nums = np.zeros([opts.num_classes])
        for i in range(opts.num_classes):
            num = np.sum(top_label == i)
            if num > 0:
                print(opts.class_names[i], " : ", num)
            classes_nums[i] = num
        print("classes_nums:", classes_nums)
    # ---------------------------------------------------------#
    #   是否进行目标的裁剪
    # ---------------------------------------------------------#
    if crop:
        for i, c in list(enumerate(top_label)):
            top, left, bottom, right = top_boxes[i]
            top = max(0, np.floor(top).astype('int32'))
            left = max(0, np.floor(left).astype('int32'))
            bottom = min(image.size[1], np.floor(bottom).astype('int32'))
            right = min(image.size[0], np.floor(right).astype('int32'))

            dir_save_path = "img_crop"
            if not os.path.exists(dir_save_path):
                os.makedirs(dir_save_path)
            crop_image = image.crop([left, top, right, bottom])
            crop_image.save(os.path.join(dir_save_path, "crop_" + str(i) + ".png"), quality=95, subsampling=0)
            print("save crop_" + str(i) + ".png to " + dir_save_path)
    # ---------------------------------------------------------#
    #   图像绘制
    # ---------------------------------------------------------#
    for i, c in list(enumerate(top_label)):
        predicted_class = opts.class_names[int(c)]
        box = top_boxes[i]
        score = top_conf[i]

        top, left, bottom, right = box

        top = max(0, np.floor(top).astype('int32'))
        left = max(0, np.floor(left).astype('int32'))
        bottom = min(image.size[1], np.floor(bottom).astype('int32'))
        right = min(image.size[0], np.floor(right).astype('int32'))

        label = '{} {:.2f}'.format(predicted_class, score)
        draw = ImageDraw.Draw(image)
        label_size = draw.textsize(label, font)
        label = label.encode('utf-8')
        print(label, top, left, bottom, right)

        if top - label_size[1] >= 0:
            text_origin = np.array([left, top - label_size[1]])
        else:
            text_origin = np.array([left, top + 1])

        for i in range(thickness):
            draw.rectangle([left + i, top + i, right - i, bottom - i], outline=opts.colors[c])
        # draw.rectangle([tuple(text_origin), tuple(text_origin + label_size)], fill=opts.colors[c])
        draw.text(text_origin, str(label, 'UTF-8'), fill=(0, 0, 0), font=font)
        del draw

    return image

def main(opts):
    # Setup dataloader
    image_files = []
    if os.path.isdir(opts.input):
        for ext in ['png', 'jpeg', 'jpg', 'JPEG']:
            files = glob(os.path.join(opts.input, '**/*.%s' % (ext)), recursive=True)
            if len(files) > 0:
                image_files.extend(files)
    elif os.path.isfile(opts.input):
        image_files.append(opts.input)

    # Set up model (all models are 'constructed at network.modeling)

    # det_model
    det_model = network.modeling.__dict__[opts.det_model](num_classes=opts.num_classes, output_stride=opts.output_stride)
    utils.set_bn_momentum(det_model.backbone, momentum=0.01)

    if opts.det_ckpt is not None and os.path.isfile(opts.det_ckpt):
        # https://github.com/VainF/DeepLabV3Plus-Pytorch/issues/8#issuecomment-605601402, @PytaichukBohdan
        checkpoint = torch.load(opts.det_ckpt, map_location=torch.device('cpu'))
        det_model.load_state_dict(checkpoint["model_state"])
        det_model = nn.DataParallel(det_model)
        det_model.to(opts.device)
        print("Resume model from %s" % opts.ckpt)
        del checkpoint
    else:
        print("No ck for predict")

        det_model = nn.DataParallel(det_model)
        det_model.to(opts.device)

    # seg_model
    seg_model = network.modeling.__dict__[opts.seg_model](num_classes=opts.num_classes + 2, output_stride=opts.output_stride)
    if opts.separable_conv and 'plus' in opts.seg_model:
        network.convert_to_separable_conv(seg_model.classifier)
    utils.set_bn_momentum(seg_model.backbone, momentum=0.01)

    if opts.seg_ckpt is not None and os.path.isfile(opts.seg_ckpt):
        # https://github.com/VainF/DeepLabV3Plus-Pytorch/issues/8#issuecomment-605601402, @PytaichukBohdan
        checkpoint = torch.load(opts.seg_ckpt, map_location=torch.device('cpu'))
        seg_model.load_state_dict(checkpoint["model_state"])
        seg_model = nn.DataParallel(seg_model)
        seg_model.to(opts.device)
        print("Resume model from %s" % opts.ckpt)
        del checkpoint
    else:
        print("[!] Retrain")
        seg_model = nn.DataParallel(seg_model)
        seg_model.to(opts.device)

    # 计算总的类的数量  classes_path = 'model_data/voc_house6.txt'
    opts.class_names, opts.num_classes = get_classes(opts.classes_path)

    # 画框设置不同的颜色
    hsv_tuples = [(x / opts.num_classes, 1., 1.) for x in range(opts.num_classes)]
    opts.colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
    opts.colors = list(map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)), opts.colors))

    # trans
    transform = T.Compose([
        T.ToTensor(),
        T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
    ])

    if opts.save_val_results_to is not None:
        os.makedirs(opts.save_val_results_to, exist_ok=True)

    # pre
    with torch.no_grad():
        seg_model = seg_model.eval()
        det_model = det_model.eval()
        for img_path in tqdm(image_files):
            ext = os.path.basename(img_path).split('.')[-1]
            img_name = os.path.basename(img_path)[:-len(ext) - 1]
            img = Image.open(img_path).convert('RGB')  # cvtColor
            re_image = img.resize((opts.crop_size, opts.crop_size), Image.BICUBIC)  # resize
            img_data = transform(re_image).unsqueeze(0)  # To tensor of NCHW
            img_data = img_data.to(opts.device)
            # seg
            pred = seg_model(img_data).max(1)[1].cpu().numpy()[0]  # HW
            colorized_preds = opts.decode_fn(pred).astype('uint8')
            colorized_preds = Image.fromarray(colorized_preds)  # seg_pred_img = Image.fromarray(colorized_preds)

            plt.imshow(re_image)
            plt.axis('off')
            plt.imshow(colorized_preds, alpha=0.7)
            ax = plt.gca()
            ax.xaxis.set_major_locator(matplotlib.ticker.NullLocator())
            ax.yaxis.set_major_locator(matplotlib.ticker.NullLocator())
            plt.savefig(os.path.join(opts.save_val_results_to, '%s_overlay.png' % img_name), bbox_inches='tight', pad_inches=0)
            plt.close()

            mask_img = Image.open(os.path.join(opts.save_val_results_to, '%s_overlay.png' % img_name)).convert('RGB')  # cvtColor
            mask_img = mask_img.resize((opts.crop_size, opts.crop_size), Image.BICUBIC)  # resize

            # det
            mask_img_data = transform(mask_img).unsqueeze(0)
            mask_img_data = mask_img_data.to(opts.device)
            if opts.crop_img:
                pred_img = detect_image(re_image, mask_img_data, det_model, opts=opts, count=True)  # resize img
            else:
                # pred_img = detect_image(img, mask_img_data, det_model, opts=opts, count=True)  # origin img
                pred_img = detect_image(mask_img, mask_img_data, det_model, opts=opts, count=True)
            # save
            if opts.save_val_results_to:
                pred_img.save(os.path.join(opts.save_val_results_to, img_name + '.jpg'))

            print('done')


import contextlib
import functools
import logging
import os
import shutil
import signal
import socket
import sys
import tempfile
import time
from subprocess import CalledProcessError
from subprocess import PIPE
from subprocess import Popen
from unittest import mock

from testifycompat import assert_not_equal
from testifycompat import setup
from testifycompat import teardown
from testifycompat import TestCase
from tron.commands import client
from tron.config import manager
from tron.config import schema

# Used for getting the locations of the executable
test_dir, _ = os.path.split(__file__)
repo_root, _ = os.path.split(test_dir)

log = logging.getLogger(__name__)

def wait_on_sandbox(func, delay=0.1, max_wait=5.0):
    """Poll for func() to return True. Sleeps `delay` seconds between polls
    up to a max of `max_wait` seconds.
    """
    start_time = time.time()
    while time.time() - start_time < max_wait:
        time.sleep(delay)
        if func():
            return
    raise TronSandboxException("Failed %s" % func.__name__)

def wait_on_state(client_func, url, state, field="state"):
    """Use client_func(url) to wait until the resource changes to state."""

    def wait_func():
        return client_func(url)[field] == state

    wait_func.__name__ = f"{url} wait on {state}"
    wait_on_sandbox(wait_func)

def wait_on_proc_terminate(pid):
    def wait_on_terminate():
        try:
            os.kill(pid, 0)
        except Exception:
            return True

    wait_on_terminate.__name__ = "Wait on %s to terminate" % pid
    wait_on_sandbox(wait_on_terminate)

def build_waiter_func(client_func, url):
    return functools.partial(wait_on_state, client_func, url)

def handle_output(cmd, out_err, returncode):
    """Log process output before it is parsed. Raise exception if exit code
    is nonzero.
    """
    stdout, stderr = out_err
    cmd = " ".join(cmd)
    if stdout:
        log.warn("%s STDOUT: %s", cmd, stdout)
    if stderr:
        log.warn("%s STDERR: %s", cmd, stderr)
    if returncode:
        raise CalledProcessError(returncode, cmd)

def find_unused_port():
    """Return a port number that is not in use."""
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    with contextlib.closing(sock) as sock:
        sock.bind(("localhost", 0))
        _, port = sock.getsockname()
    return port

class TronSandboxException(Exception):
    pass

class SandboxTestCase(TestCase):

    _suites = ["sandbox"]

    sandbox = None

    @setup
    def make_sandbox(self):
        verify_environment()
        self.sandbox = TronSandbox()
        self.client = self.sandbox.client

    @teardown
    def delete_sandbox(self):
        if self.sandbox:
            self.sandbox.delete()
            self.sandbox = None

    def start_with_config(self, config):
        self.sandbox.save_config(config)
        self.sandbox.trond()

    def restart_trond(self):
        old_pid = self.sandbox.get_trond_pid()
        self.sandbox.shutdown_trond()
        wait_on_proc_terminate(self.sandbox.get_trond_pid())

        self.sandbox.trond()
        assert_not_equal(old_pid, self.sandbox.get_trond_pid())

class ClientProxy:
    """Wrap calls to client and raise a TronSandboxException on connection
    failures.
    """

    def __init__(self, client, log_filename):
        self.client = client
        self.log_filename = log_filename

    def log_contents(self):
        """Return the contents of the log file."""
        with open(self.log_filename) as f:
            return f.read()

    def wrap(self, func, *args, **kwargs):
        with mock.patch("tron.commands.client.log", autospec=True):
            try:
                return func(*args, **kwargs)
            except (client.RequestError, ValueError) as e:
                # ValueError for JSONDecode errors
                log_contents = self.log_contents()
                if log_contents:
                    log.warn(f"{e!r}, Log:\n{log_contents}")
                return False

    def __getattr__(self, name):
        attr = getattr(self.client, name)
        if not callable(attr):
            return attr

        return functools.partial(self.wrap, attr)

def verify_environment():
    for env_var in ["SSH_AUTH_SOCK", "PYTHONPATH"]:
        if not os.environ.get(env_var):
            raise TronSandboxException("Missing $%s in test environment." % env_var,)

class TronSandbox:
    """A sandbox for running trond and tron commands in subprocesses."""

    def __init__(self):
        """Set up a temp directory and store paths to relevant binaries"""
        self.tmp_dir = tempfile.mkdtemp(prefix="tron-")
        cmd_path_func = functools.partial(os.path.join, repo_root, "bin")
        cmds = "tronctl", "trond", "tronfig", "tronview"
        self.commands = {cmd: cmd_path_func(cmd) for cmd in cmds}
        self.log_file = self.abs_path("tron.log")
        self.log_conf = self.abs_path("logging.conf")
        self.pid_file = self.abs_path("tron.pid")
        self.config_path = self.abs_path("configs/")
        self.port = find_unused_port()
        self.host = "localhost"
        self.api_uri = f"http://{self.host}:{self.port}"
        cclient = client.Client(self.api_uri)
        self.client = ClientProxy(cclient, self.log_file)
        self.setup_logging_conf()

    def abs_path(self, filename):
        """Return the absolute path for a file in the sandbox."""
        return os.path.join(self.tmp_dir, filename)

    def setup_logging_conf(self):
        config_template = os.path.join(repo_root, "tests/data/logging.conf")
        with open(config_template) as fh:
            config = fh.read()

        with open(self.log_conf, "w") as fh:
            fh.write(config.format(self.log_file))

    def delete(self):
        """Delete the temp directory and shutdown trond."""
        self.shutdown_trond(sig_num=signal.SIGKILL)
        shutil.rmtree(self.tmp_dir)

    def save_config(self, config_text):
        """Save the initial tron configuration."""
        manager.create_new_config(self.config_path, config_text)

    def run_command(self, command_name, args=None, stdin_lines=None):
        """Run the command by name and return (stdout, stderr)."""
        args = args or []
        command = [sys.executable, self.commands[command_name]] + args
        stdin = PIPE if stdin_lines else None
        proc = Popen(command, stdout=PIPE, stderr=PIPE, stdin=stdin)
        streams = proc.communicate(stdin_lines)
        try:
            handle_output(command, streams, proc.returncode)
        except CalledProcessError:
            log.warn(self.client.log_contents())
            raise
        return streams

    def tronctl(self, *args):
        args = list(args) if args else []
        return self.run_command("tronctl", args + ["--server", self.api_uri])

    def tronview(self, *args):
        args = list(args) if args else []
        args += ["--nocolor", "--server", self.api_uri]
        return self.run_command("tronview", args)

    def trond(self, *args):
        args = list(args) if args else []
        args += [
            "--working-dir=%s" % self.tmp_dir,
            "--pid-file=%s" % self.pid_file,
            "--port=%d" % self.port,
            "--host=%s" % self.host,
            "--config-path=%s" % self.config_path,
            "--log-conf=%s" % self.log_conf,
        ]

        self.run_command("trond", args)
        wait_on_sandbox(lambda: bool(self.client.home()))

    def tronfig(
        self, config_content=None, name=schema.MASTER_NAMESPACE,
    ):
        args = ["--server", self.api_uri, name]
        args += ["-"] if config_content else ["-p"]
        return self.run_command("tronfig", args, stdin_lines=config_content)

    def get_trond_pid(self):
        if not os.path.exists(self.pid_file):
            return None
        with open(self.pid_file) as f:
            return int(f.read())

    def shutdown_trond(self, sig_num=signal.SIGTERM):
        trond_pid = self.get_trond_pid()
        if trond_pid:
            os.kill(trond_pid, sig_num)


from __future__ import print_function

from pymongo import MongoClient

db = MongoClient('localhost').condor

reserved_ips = {
    '18.12': 'MIT',
    '23.22': 'AWS',
    '35.9': 'AGLT2',
    '40.78': 'Azure',
    '40.112': 'Azure',
    '50.16': 'AWS',
    '50.17': 'AWS',
    '54.144': 'AWS',
    '54.145': 'AWS',
    '54.157': 'AWS',
    '54.158': 'AWS',
    '54.159': 'AWS',
    '54.161': 'AWS',
    '54.163': 'AWS',
    '54.166': 'AWS',
    '54.167': 'AWS',
    '54.197': 'AWS',
    '54.204': 'AWS',
    '54.205': 'AWS',
    '54.211': 'AWS',
    '54.227': 'AWS',
    '54.243': 'AWS',
    '72.36': 'Illinois',
    '128.9': 'osgconnect',
    '128.55': 'Berkeley',
    '128.84': 'NYSGRID_CORNELL_NYS1',
    '128.104': 'CHTC',
    '128.105': 'CHTC',
    '128.118': 'Bridges',
    '128.120': 'UCD',
    '128.205': 'osgconnect',
    '128.211': 'Purdue-Hadoop',
    '128.227': 'FLTech',
    '128.230': 'Syracuse',
    '129.74': 'NWICG_NDCMS',
    '129.93': 'Nebraska',
    '129.105': 'NUMEP-OSG',
    '129.107': 'UTA_SWT2',
    '129.119': 'SU-OG',
    '129.130': 'Kansas',
    '129.217': 'LIDO_Dortmund',
    '130.74': 'Miss',
    '130.127': 'Clemson-Palmetto',
    '130.199': 'BNL-ATLAS',
    '131.94': 'FLTECH',
    '131.215': 'CIT_CMS_T2',
    '131.225': 'USCMS-FNAL-WC1',
    '132.206': 'CA-MCGILL-CLUMEQ-T2',
    '133.82': 'Japan',
    '134.93': 'mainz',
    '136.145': 'osgconnect',
    '137.99': 'UConn-OSG',
    '137.135': 'Azure',
    '138.23': 'UCRiverside',
    '138.91': 'Azure',
    '141.34': 'DESY-HH',
    '142.150': 'CA-SCINET-T2',
    '142.244': 'Alberta',
    '144.92': 'HEP_WISC',
    '149.165': 'Indiana',
    '155.101': 'Utah',
    '163.118': 'FLTECH',
    '169.228': 'UCSDT2',
    '171.67': 'HOSTED_STANFORD',
    '174.129': 'AWS',
    '184.73': 'AWS',
    '192.5': 'Boston',
    '192.12': 'Colorado',
    '192.41': 'AGLT2',
    '192.84': 'Ultralight',
    '192.168': None,
    '192.170': 'MWT2',
    '193.58': 'T2B_BE_IIHE',
    '193.190': 'T2B_BE_IIHE',
    '198.32': 'osgconnect',
    '198.48': 'Hyak',
    '200.136': 'SPRACE',
    '200.145': 'SPRACE',
    '206.12': 'CA-MCGILL-CLUMEQ-T2',
    '216.47': 'MWT2',
}
reserved_ips.update({'10.%d'%i:None for i in range(256)})
reserved_ips.update({'172.%d'%i:None for i in range(16,32)})

reserved_domains = {
    'aglt2.org': 'AGLT2',
    'bridges.psc.edu': 'Bridges',
    'campuscluster.illinois.edu': 'Illinois',
    'cl.iit.edu': 'MWT2',
    'cm.cluster': 'LIDO_Dortmund',
    'cmsaf.mit.edu': 'MIT',
    'colorado.edu': 'Colorado',
    'cpp.ualberta.ca': 'Alberta',
    'crc.nd.edu': 'NWICG_NDCMS',
    'cci.wisc.edu': 'CHTC',
    'chtc.wisc.edu': 'CHTC',
    'cs.wisc.edu': 'CS_WISC',
    'cse.buffalo.edu': 'osgconnect',
    'discovery.wisc.edu': 'CHTC',
    'ec2.internal': 'AWS',
    'ember.arches': 'Utah',
    'fnal.gov': 'USCMS-FNAL-WC1',
    'grid.tu-dortmund.de': 'LIDO_Dortmund',
    'guillimin.clumeq.ca': 'Guillimin',
    'hcc.unl.edu': 'Crane',
    'hep.caltech.edu': 'CIT_CMS_T2',
    'hep.int': 'osgconnect',
    'hep.olemiss.edu': 'Miss',
    'hep.wisc.edu': 'HEP_WISC',
    'icecube.wisc.edu': 'NPX',
    'ics.psu.edu': 'Bridges',
    'iihe.ac.be': 'T2B_BE_IIHE',
    'internal.cloudapp.net': 'osgconnect',
    'isi.edu': 'osgconnect',
    'iu.edu': 'Indiana',
    'lidocluster.hp': 'LIDO_Dortmund',
    'math.wisc.edu': 'MATH_WISC',
    'mwt2.org': 'MWT2',
    'nut.bu.edu': 'Boston',
    'palmetto.clemson.edu': 'Clemson-Palmetto',
    'panther.net': 'FLTECH',
    'phys.uconn.edu': 'UConn-OSG',
    'rcac.purdue.edu': 'Purdue-Hadoop',
    'research.northwestern.edu': 'NUMEP-OSG',
    'stat.wisc.edu': 'CHTC',
    't2.ucsd.edu': 'UCSDT2',
    'tier3.ucdavis.edu':'UCD',
    'unl.edu': 'Nebraska',
    'uppmax.uu.se': 'Uppsala',
    'usatlas.bnl.gov': 'BNL-ATLAS',
    'wisc.cloudlab.us': 'CLOUD_WISC',
    'wisc.edu': 'WISC',
    'zeuthen.desy.de': 'DESY-ZN',
}

def get_domain(hostname):
    parts = hostname.lower().split('.')
    if '.'.join(parts[-3:]) in reserved_domains:
        return '.'.join(parts[-3:])
    elif '.'.join(parts[-2:]) in reserved_domains:
        return '.'.join(parts[-2:])
    if len(parts) > 3:
        return '.'.join(parts[-3:])
    elif len(parts) > 2:
        return '.'.join(parts[-2:])

def get_ip_range(ip, reserved=False):
    parts = ip.split('.')
    if len(parts) == 4 and all(p.isdigit() for p in parts):
        if '.'.join(parts[:2]) in reserved_ips:
            ret = reserved_ips['.'.join(parts[:2])]
            if ret and reserved:
                return ret
            else:
                return None
        return '.'.join(parts[:3])

# do initial lookup
domain_lookup = {}
ip_lookup = {}
for row in db.condor_history.find(filter={'MATCH_EXP_JOBGLIDEIN_ResourceName':{'$exists': True}},
                                  projection=['LastRemoteHost','StartdPrincipal','MATCH_EXP_JOBGLIDEIN_ResourceName']):
    site = row['MATCH_EXP_JOBGLIDEIN_ResourceName']
    if '.' in site or site.startswith('gzk'):
        continue
    if site == 'Local Job' and 'LastRemoteHost' in row:
        domain = get_domain(row['LastRemoteHost'].split('@')[-1])
        if not domain:
            continue
        if domain.endswith('chtc.wisc.edu'):
            site = 'CHTC'
        elif domain.endswith('hep.wisc.edu'):
            site = 'HEP_WISC'
        elif domain.endswith('cs.wisc.edu'):
            site = 'CS_WISC'
        elif domain.endswith('math.wisc.edu'):
            site = 'MATH_WISC'
        elif domain.endswith('icecube.wisc.edu'):
            site = 'NPX'
        elif domain.endswith('wisc.edu'):
            site = 'WISC'
        elif domain.endswith('wisc.cloudlab.us'):
            site = 'CLOUD_WISC'
        else:
            print('Local Job with strange domain:',domain)
            continue
    if 'LastRemoteHost' in row:
        domain = get_domain(row['LastRemoteHost'].split('@')[-1])
        if domain:
            if domain in reserved_domains:
                continue
            if domain in domain_lookup and domain_lookup[domain] != site:
                print('duplicate site name for domain',domain)
                print(domain_lookup[domain])
                print(site)
            else:
                domain_lookup[domain] = site
    if 'StartdPrincipal' in row:
        ip_range = get_ip_range(row['StartdPrincipal'].split('/')[-1])
        if ip_range:
            if ip_range in ip_lookup and ip_lookup[ip_range] != site:
                print('duplicate site name for ip_range',ip_range)
                print(ip_lookup[ip_range])
                print(site)
            else:
                ip_lookup[ip_range] = site

def fix_site(row):
    site = row['MATCH_EXP_JOBGLIDEIN_ResourceName']
    new_site = None
    if site.startswith('gzk'):
        new_site = 'CHTC'
    if site.endswith('chtc.wisc.edu'):
        new_site = 'CHTC'
    elif site.endswith('hep.wisc.edu'):
        new_site = 'HEP_WISC'
    elif site.endswith('cs.wisc.edu'):
        new_site = 'CS_WISC'
    elif site.endswith('math.wisc.edu'):
        new_site = 'MATH_WISC'
    elif site.endswith('icecube.wisc.edu'):
        new_site = 'NPX'
    elif site.endswith('wisc.cloudlab.us'):
        new_site = 'CLOUD_WISC'
    elif site.endswith('wisc.edu'):
        new_site = 'WISC'
    elif '.' not in site and site != 'Local Job':
        return # fine the way it is
    if new_site:
        filter = {'GlobalJobId':row['GlobalJobId']}
        update = {'$set':{'MATCH_EXP_JOBGLIDEIN_ResourceName':new_site}}
        db.condor_history.update_one(filter, update)
        return
    raise Exception('cannot fix')

def fix_host(row):
    host = row['LastRemoteHost'].split('@')[-1]
    domain = get_domain(host)
    if domain and domain in reserved_domains:
        filter = {'GlobalJobId':row['GlobalJobId']}
        update = {'$set':{'MATCH_EXP_JOBGLIDEIN_ResourceName':reserved_domains[domain]}}
        db.condor_history.update_one(filter, update)
        return
    if domain and domain in domain_lookup:
        filter = {'GlobalJobId':row['GlobalJobId']}
        update = {'$set':{'MATCH_EXP_JOBGLIDEIN_ResourceName':domain_lookup[domain]}}
        db.condor_history.update_one(filter, update)
        return
    print('unknown domain',domain)
    raise Exception('cannot fix')

def fix_ip(row):
    ip = row['StartdPrincipal'].split('/')[-1]
    ip_range = get_ip_range(ip, reserved=True)
    if not ip_range:
        raise Exception('reserved ip')
    if '.' not in ip_range: # reserved range
        filter = {'GlobalJobId':row['GlobalJobId']}
        update = {'$set':{'MATCH_EXP_JOBGLIDEIN_ResourceName':ip_range}}
        db.condor_history.update_one(filter, update)
        return
    if ip_range and ip_range in ip_lookup:
        filter = {'GlobalJobId':row['GlobalJobId']}
        update = {'$set':{'MATCH_EXP_JOBGLIDEIN_ResourceName':ip_lookup[ip_range]}}
        db.condor_history.update_one(filter, update)
        return
    print('unknown ip',ip_range)
    raise Exception('cannot fix')

# remap bad sites
for row in db.condor_history.find(projection=['GlobalJobId','LastRemoteHost','StartdPrincipal','MATCH_EXP_JOBGLIDEIN_ResourceName']):
    if 'MATCH_EXP_JOBGLIDEIN_ResourceName' in row:
        try:
            fix_site(row)
        except:
            pass
        else:
            continue
    if 'LastRemoteHost' in row:
        try:
            fix_host(row)
        except:
            pass
        else:
            continue
    if 'StartdPrincipal' in row:
        try:
            fix_ip(row)
        except:
            pass
        else:
            continue
    print('cannot fix')


# coding: utf-8

# ## For Colab

# In[ ]:

from IPython.display import HTML, display

def set_css():
  display(HTML('''
  <style>
    pre {
        white-space: pre-wrap;
    }
  </style>
  '''))
get_ipython().events.register('pre_run_cell', set_css)

# In[ ]:

from google.colab import drive
drive.mount('/content/drive')

# ## Module Imports

# In[ ]:

import torch
import torchvision
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.optim.lr_scheduler import ReduceLROnPlateau
import numpy as np
import random
from tqdm import tqdm
import os
import nltk
nltk.download('punkt')
nltk.download('stopwords')
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from matplotlib import pyplot as plt
from torch.nn.utils.rnn import pad_sequence
import glob
import itertools
import pickle
import re
import pandas as pd
from tqdm import tqdm
from sklearn.model_selection import train_test_split
get_ipython().system('pip install rouge-score')

# ## Constants
# - remember to point to the right dataset

# In[ ]:

drive_prefix = '/content/drive/MyDrive/ese546/'
PATH_TRAIN_X = drive_prefix + 'cnndm/train.txt.src'
PATH_TRAIN_Y = drive_prefix + 'cnndm/train.txt.tgt.tagged'

# PATH_TRAIN_X = drive_prefix + 'cnndm/small.src'
# PATH_TRAIN_Y = drive_prefix + 'cnndm/small.tagged'

# PATH_VAL_X = drive_prefix + 'cnndm/val.txt.src'
# PATH_VAL_Y = drive_prefix + 'cnndm/val.txt.tgt.tagged'

# CHANGE THIS AFTER
# PATH_TRAIN_X = drive_prefix + 'cnndm/test.txt.src'
# PATH_TRAIN_Y = drive_prefix + 'cnndm/test.txt.tgt.tagged'

STOP_WORDS = set()
EMB_SIZE = 300
GLOVE_EMB = drive_prefix + f'glove.6B.{EMB_SIZE}d.txt'
BATCH_SIZE = 32
NUM_EVAL_BATCHES = 4
NUM_HEADS = 10
DROPOUT_RATE = 0.2

START_CHAR = 'starttoken'
END_CHAR = 'endtoken'
PAD_CHAR = 'padtoken'

MAX_ARTICLE_LEN = 300 # the article can have at most 300 tokens
MAX_LABEL_LEN = 40 # labels can have at most 100 tokens

# NN Hyper-parameters
E_HIDDEN_DIM = 512
D_HIDDEN_DIM = 512
KEY_DIM = 64
VALUE_DIM = 64
LR = 1e-3

EP = 100000
PRINT_EVERY_EP = 200
SAVE_MODEL_EVERY_EP = 10000
FORCE_CREATE_DICT = True # force to recreate the word features from scratch, but doesn't recreate the embedding dict, since it doesn't change

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(device)
torch.cuda.empty_cache()

# ## Helper Functions

# In[ ]:

def create_word_vec(input_text, prefix='train'):
    word_idx_dict = {} # { word : the index of that word in the dictionary}
    idx_word_dict = {} # { index of word : word }

    # 1. Create mapping between words and the corresponding embedding values
    embed_file_path = drive_prefix + f'{EMB_SIZE}d_embed_dict'
    if os.path.exists(embed_file_path):
        print('Embedding dictionary exists, loading from file...')
        embedding_dict = pickle.load(open(embed_file_path, 'rb'))
    else:
        embedding_dict = {}

        for line in glove:
            tokens = line.split()
            embedding_dict[tokens[0]] = np.array(tokens[1:], dtype='float32')
        pickle.dump(embedding_dict, open(embed_file_path, 'wb'))
        print('Saved embedding dictionary')

    # 2. Tokenize the input_text and labels
    if os.path.exists(drive_prefix + f'{prefix}_word_idx_dict') and not FORCE_CREATE_DICT:
        print('Word-to-index dictionary exists, loading from file...')
        word_idx_dict = pickle.load(open(drive_prefix + f'{prefix}_word_idx_dict', 'rb'))
    if os.path.exists(drive_prefix + f'{prefix}_idx_word_dict') and not FORCE_CREATE_DICT:
        print('Index-to-word dictionary exists, loading from file...')
        idx_word_dict = pickle.load(open(drive_prefix + f'{prefix}_idx_word_dict', 'rb'))
    else:
        unique_tokens = set([])
        for text, label in input_text:
            unique_tokens = unique_tokens.union(word_tokenize(text))
            unique_tokens = unique_tokens.union(word_tokenize(label))

        for token in unique_tokens:
            word_idx_dict[token] = len(word_idx_dict)

        # 2.1 Add in the special tokens to the dictionary, note that the START_CHAR and END_CHAR have been added
        # during the preprocessing stage
        word_idx_dict[PAD_CHAR] = len(word_idx_dict)

        idx_word_dict = dict(zip(word_idx_dict.values(), word_idx_dict.keys()))

    # 3. Build the word vector for all the words in our dictionary
    if os.path.exists(drive_prefix + f'{prefix}_word_vector') and not FORCE_CREATE_DICT:
        print('Word Vector exists, loading from file...')
        word_vector = pickle.load(open(drive_prefix + f'{prefix}_word_vector', 'rb'))
    else:
        word_vector = []
        for idx, token in idx_word_dict.items():
            if token in embedding_dict:
                word_vector.append(embedding_dict[token])
            # Append the special tokens to the word vector and assign random values
            elif token in [START_CHAR, END_CHAR, PAD_CHAR]:
                word_vector.append(np.random.normal(0, 1, EMB_SIZE))
            # if the token doesn't have an embedding, we set to 0
            else:
                word_vector.append(np.zeros([EMB_SIZE]))

    ## Save the dictionaries
    pickle.dump(word_idx_dict, open(drive_prefix + f'{prefix}_word_idx_dict', 'wb'))
    pickle.dump(idx_word_dict, open(drive_prefix + f'{prefix}_idx_word_dict', 'wb'))
    pickle.dump(word_vector, open(drive_prefix + f'{prefix}_word_vector', 'wb'))

    # The index in worvalue_dimec corresponds to the article index in the original X_Train array
    return np.array(word_vector), word_idx_dict, idx_word_dict

def sentence_to_idx(sentence, word_to_idx):
    text, label = sentence
    text_tokens = [word_to_idx[token] for token in word_tokenize(text)]
    label_tokens = [word_to_idx[token] for token in word_tokenize(label)]
    return np.array(text_tokens).astype(int), np.array(label_tokens).astype(int)

def decontracted(text):
    '''Clean text by removing unnecessary characters and altering the format of words.'''
    text = text.lower()
    text = re.sub(r"i'm", "i am", text)
    text = re.sub(r"he's", "he is", text)
    text = re.sub(r"she's", "she is", text)
    text = re.sub(r"it's", "it is", text)
    text = re.sub(r"that's", "that is", text)
    text = re.sub(r"what's", "that is", text)
    text = re.sub(r"where's", "where is", text)
    text = re.sub(r"how's", "how is", text)
    text = re.sub(r"\'ll", " will", text)
    text = re.sub(r"\'ve", " have", text)
    text = re.sub(r"\'re", " are", text)
    text = re.sub(r"\'d", " would", text)
    text = re.sub(r"\'re", " are", text)
    text = re.sub(r"won't", "will not", text)
    text = re.sub(r"can't", "cannot", text)
    text = re.sub(r"n't", " not", text)
    text = re.sub(r"n'", "ng", text)
    text = re.sub(r"'bout", "about", text)
    text = re.sub(r"'til", "until", text)
    text = re.sub(r"[-()\"#/@;:<>{}`+=~|.!?,]", "", text)
    return text

def _process(text):
    text = decontracted(text)
    text = text.replace('\\r', ' ')
    text = text.replace('\\"', ' ')
    text = text.replace('\\n', ' ')
    text = re.sub(r"\b[a-zA-Z]\b", " ", text)
    text = re.sub("^\d+\s|\s\d+\s|\s\d+$", " ", text)
    text = re.sub('[^A-Za-z0-9]+', ' ', text)
    text = re.sub('<[^>]*>', '', text)
    return text

def pre_process(articles, labels):
    preprocessed_text = []
    # tqdm is for printing the status bar
    for i in tqdm(range(len(articles))):
        sent = articles[i]
        label = labels[i]

        sent = _process(sent)
        label = _process(label)

        sent = sent.lower().strip().split()
        label = label.lower().strip().split()

        # trim longer items and ignore shorter ones
        if len(sent) < MAX_ARTICLE_LEN or len(label) < MAX_LABEL_LEN:
            continue

        sent = ' '.join(sent[:MAX_ARTICLE_LEN])
        label = ' '.join(label[:MAX_LABEL_LEN])
        label = f"{START_CHAR} {label} {END_CHAR}"
        preprocessed_text.append([sent, label])
    return preprocessed_text

def get_batch(data):
    while True:
        for i in range(int((data.shape[0] - BATCH_SIZE) / BATCH_SIZE)):
            yield data[i * BATCH_SIZE: i * BATCH_SIZE + BATCH_SIZE]

def patch_trg(trg):
    # Need to understand
    trg, gold = trg[:, :-1], trg[:, 1:].contiguous().view(-1)
    return trg, gold

def plot(train_loss, val_loss):
    plt.plot(train_loss, label='Train')
    plt.plot(val_loss, label='Val')
    plt.title('Training vs Validation Loss')
    plt.xlabel('Episodes')
    plt.ylabel('Loss')
    plt.legend()
    plt.show()

# ## Read Data

# In[ ]:

with open(PATH_TRAIN_X, 'r', encoding='utf-8') as f:
    X = f.readlines()[:100000]

with open(PATH_TRAIN_Y, 'r', encoding='utf-8') as f:
    Y = f.readlines()[:100000]

with open(GLOVE_EMB, 'r', encoding='utf-8') as f:
    glove = f.readlines()

X_Train, X_Val, Y_Train, Y_Val = train_test_split(X, Y, test_size=0.3, shuffle=True)

# ## Features

# In[ ]:

def get_features(processed_data, prefix):
    # note that the prefix variable is only used for naming dictionaries that are
    # saved to the disk
    print(f'\nTotal # of stories: {len(processed_data)}')
    word_vector, word_idx_dict, idx_word_dict = create_word_vec(processed_data, prefix=prefix)
    print(f'Word Vector Shape: {word_vector.shape}')
    assert word_vector.shape == (len(idx_word_dict.keys()), EMB_SIZE)

    return word_vector, word_idx_dict, idx_word_dict

# USING THE CELL BELOW TO SPEED THINGS UP
# # Full
full_processed_data = pre_process(X, Y)
full_word_vector, full_word_idx_dict, full_idx_word_dict = get_features(full_processed_data, prefix='train')

# Training
train_processed_data = pre_process(X_Train, Y_Train)
# convert sentences into indicies corresponding to the word vector indices
train_data_indices = np.array([sentence_to_idx(sentence, full_word_idx_dict) for sentence in train_processed_data])

# Validation
val_processed_data = pre_process(X_Val, Y_Val)
val_data_indices = np.array([sentence_to_idx(sentence, full_word_idx_dict) for sentence in val_processed_data])

# In[ ]:

if os.path.exists(drive_prefix + 'full_processed_data'):
    print('full_processed_data exists, loading from file...')
    full_processed_data = pickle.load(open(drive_prefix + 'full_processed_data', 'rb'))
else:
    full_processed_data = pre_process(X, Y)
    full_word_vector, full_word_idx_dict, full_idx_word_dict = get_features(full_processed_data, prefix='train')

if os.path.exists(drive_prefix + 'train_processed_data'):
    print('train_processed_data exists, loading from file...')
    train_processed_data = pickle.load(open(drive_prefix + 'train_processed_data', 'rb'))
else:
    train_data_indices = np.array([sentence_to_idx(sentence, full_word_idx_dict) for sentence in train_processed_data])

if os.path.exists(drive_prefix + 'val_processed_data'):
    print('val_processed_data exists, loading from file...')
    val_processed_data = pickle.load(open(drive_prefix + 'val_processed_data', 'rb'))
else:
    val_processed_data = pre_process(X, Y)
    val_data_indices = np.array([sentence_to_idx(sentence, full_word_idx_dict) for sentence in val_processed_data])

# In[ ]:

## Using this to speed things up
def dump(data, name):
    pickle.dump(data, open(drive_prefix + name, 'wb'))

dump(full_processed_data, 'full_processed_data')
dump(train_processed_data, 'train_processed_data')
dump(train_data_indices, 'train_data_indices')
dump(val_processed_data, 'val_processed_data')
dump(val_data_indices, 'val_data_indices')

# In[ ]:

val_data_indices.shape

# ## Model

# In[ ]:

# Create the embedding layer weights based on the pre-trained word vector
def create_pretrained_emb_layer(word_vector):
    # vocab_size, embed_dim = word_vector.shape
    embed_layer = nn.Embedding.from_pretrained(torch.tensor(word_vector).float(), freeze=False)
    return embed_layer

def get_pad_mask(seq, pad_idx):
    return (seq != pad_idx).unsqueeze(-2)

class PositionEmbedding(nn.Module):
    def __init__(self, dropout=DROPOUT_RATE, max_len=500):
        super(PositionEmbedding, self).__init__()

        self.dropout = nn.Dropout(dropout)
        self.pos_embedding = self.compute_pos_embedding(max_len)

    def compute_pos_embedding(self, max_len):

        pos = torch.arange(max_len).unsqueeze(1)
        index = torch.arange(EMB_SIZE).unsqueeze(0)
        angles = 1 / np.power(10000, (2 * (index // 2)) / np.float32(EMB_SIZE))

        pos_to_angle = pos * angles

        # apply sin to even indices in the array; 2i
        pos_to_angle[:, 0::2] = np.sin(pos_to_angle[:, 0::2])

        # apply cos to odd indices in the array; 2i+1
        pos_to_angle[:, 1::2] = np.cos(pos_to_angle[:, 1::2])

        # pos_embedding = pos_to_angle.unsqueeze(0).transpose(0, 1)
        pos_embedding = pos_to_angle.unsqueeze(0)

        return torch.FloatTensor(pos_embedding).to(device)

    def forward(self, x):
        x += self.pos_embedding[:, :x.shape[1]].clone().detach() # adding another feature to the word embedding
        return self.dropout(x)

class PositionwiseFeedForward(nn.Module):
    def __init__(self, d_in, hidden_dim, dropout=DROPOUT_RATE):
        super(PositionwiseFeedForward, self).__init__()
        self.pos1 = nn.Linear(d_in, hidden_dim)
        self.pos2 = nn.Linear(hidden_dim, d_in)
        self.relu = nn.ReLU()
        self.normalization = nn.LayerNorm(d_in, eps=1e-6)
        self.dropout = nn.Dropout(dropout)

    def forward(self, x):

        residual = x
        out = self.pos1(x)
        out = self.relu(out)
        out = self.pos2(out)
        out = self.dropout(out)
        out += residual

        out = self.normalization(out)

        return out

class ScaledDotProductAttention(nn.Module):
    def __init__(self, temperature, attn_dropout=DROPOUT_RATE):
        super(ScaledDotProductAttention, self).__init__()
        self.temperature = temperature
        self.dropout = nn.Dropout(attn_dropout)
        self.softmax = nn.Softmax(dim=-1)

    def forward(self, q, k, v, mask):

        attn = torch.matmul(q / self.temperature, k.transpose(2, 3))

        attn = attn.masked_fill(mask == 0, -1e7)

        attn = self.softmax(attn)

        return torch.matmul(attn, v), self.dropout(attn)

class MultiHeadAttention(nn.Module):
    def __init__(self, n_head, key_dim, value_dim, dropout=DROPOUT_RATE):
        super().__init__()

        self.n_head = n_head
        self.key_dim = key_dim
        self.value_dim = value_dim

        self.query_layer = nn.Linear(EMB_SIZE, n_head * key_dim, bias=False)
        self.key_layer = nn.Linear(EMB_SIZE, n_head * key_dim, bias=False)
        self.value_layer = nn.Linear(EMB_SIZE, n_head * value_dim, bias=False)
        self.fc = nn.Linear(n_head * value_dim, EMB_SIZE, bias=False)

        self.attention = ScaledDotProductAttention(temperature=key_dim ** 0.5).to(device)

        self.dropout = nn.Dropout(dropout)
        self.layer_norm = nn.LayerNorm(EMB_SIZE, eps=1e-6)

    def forward(self, q, k, v, attn_mask):

        key_dim, value_dim, n_head = self.key_dim, self.value_dim, self.n_head
        batch_size, q_len = q.size(0), q.size(1)
        k_len = k.size(1)
        v_len = v.size(1)

        residual = q

        q = self.query_layer(q).view(batch_size, q_len, n_head, key_dim)
        k = self.key_layer(k).view(batch_size, k_len, n_head, key_dim)
        v = self.value_layer(v).view(batch_size, v_len, n_head, value_dim)

        q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)

        attn_mask = attn_mask.unsqueeze(1)

        q, attn = self.attention(q, k, v, mask=attn_mask)

        q = q.transpose(1, 2).contiguous().view(batch_size, q_len, -1)
        q = self.fc(q)
        q = self.dropout(q)
        q += residual

        q = self.layer_norm(q)

        return q, attn

class EncoderLayer(nn.Module):
    def __init__(self, d_inner, n_head, key_dim, value_dim, dropout=DROPOUT_RATE):
        super(EncoderLayer, self).__init__()
        self.slf_attn = MultiHeadAttention(n_head=n_head, key_dim=key_dim, value_dim=value_dim, dropout=dropout).to(device)
        self.pos_ffn = PositionwiseFeedForward(EMB_SIZE, d_inner, dropout=dropout).to(device)

    def forward(self, enc_input, mask=None):
        enc_output, enc_slf_attn = self.slf_attn(enc_input, enc_input, enc_input, attn_mask=mask)
        enc_output = self.pos_ffn(enc_output)
        return enc_output, enc_slf_attn

class DecoderLayer(nn.Module):
    def __init__(self, d_model, d_inner, n_head, key_dim, value_dim, dropout=DROPOUT_RATE):
        super(DecoderLayer, self).__init__()
        self.slf_attn = MultiHeadAttention(n_head, key_dim, value_dim, dropout=dropout).to(device)
        self.enc_attn = MultiHeadAttention(n_head, key_dim, value_dim, dropout=dropout).to(device)
        self.pos_ffn = PositionwiseFeedForward(d_model, d_inner, dropout=dropout).to(device)

    def forward(self, d_in, e_out,y_mask=None, x_mask=None):
        d_out, d_slf_attn = self.slf_attn(d_in, d_in, d_in, attn_mask=y_mask)
        d_out, d_e_attn = self.enc_attn(d_out, e_out, e_out, attn_mask=x_mask)
        d_out = self.pos_ffn(d_out)
        return d_out, d_slf_attn, d_e_attn

class Decoder(nn.Module):
    def __init__(self, n_layers, n_head, key_dim, value_dim,d_model, d_inner, dropout=DROPOUT_RATE):
        super(Decoder, self).__init__()

        self.dropout = nn.Dropout(p=dropout)
        self.layer_stack = nn.ModuleList([DecoderLayer(d_model, d_inner, n_head, key_dim, value_dim, dropout=dropout).to(device) for _ in range(n_layers)])
        self.layer_norm = nn.LayerNorm(d_model, eps=1e-6)

    def forward(self, y, y_mask, e_out, x_mask):
        d_out = self.dropout(y)
        d_out = self.layer_norm(d_out)
        for dec_layer in self.layer_stack:
            d_out, dec_slf_attn, dec_enc_attn = dec_layer(d_out, e_out, y_mask=y_mask, x_mask=x_mask)

        return d_out,

class Encoder(nn.Module):
    def __init__(self, n_layers, n_head, key_dim, value_dim, dropout=DROPOUT_RATE):
        super(Encoder, self).__init__()
        self.dropout = nn.Dropout(dropout)
        self.layers = nn.ModuleList([ EncoderLayer(E_HIDDEN_DIM, n_head, key_dim, value_dim, dropout=dropout).to(device) for _ in range(n_layers)])
        self.normalization = nn.LayerNorm(EMB_SIZE, eps=1e-6)

    def forward(self, x, x_mask):
        e_out = self.dropout(x)
        e_out = self.normalization(e_out)

        for enc_layer in self.layers:
            e_out, enc_slf_attn = enc_layer(e_out, mask=x_mask)

        return e_out,

class Transformer(nn.Module):
    def __init__(self, num_heads, word_vector, word_idx_dict, idx_word_dict):
        super(Transformer, self).__init__()
        self.word_idx_dict = word_idx_dict
        self.pos_embedding = PositionEmbedding(max_len=MAX_ARTICLE_LEN, dropout=DROPOUT_RATE).to(device)
        self.word_embedding = create_pretrained_emb_layer(word_vector).to(device)

        self.encoder = Encoder(n_layers=3, n_head=num_heads, key_dim=KEY_DIM, value_dim=VALUE_DIM, dropout=DROPOUT_RATE).to(device)
        self.decoder = Decoder(n_layers=3, n_head=num_heads, key_dim=KEY_DIM, value_dim=VALUE_DIM, d_model=EMB_SIZE, d_inner=D_HIDDEN_DIM, dropout=DROPOUT_RATE).to(device)

        self.initial_probs = self.get_initial_probs(word_vector.shape[0], word_idx_dict[START_CHAR])

        self.fc_to_logits = nn.Linear(EMB_SIZE, len(idx_word_dict), bias=False)

    def mask_future_seq(self, seq):
        # Our attention is focused only allowed on the past sequence, so we mask
        sz_b, len_s = seq.size()
        subsequent_mask = (1 - torch.triu(torch.ones((1, len_s, len_s), device=seq.device), diagonal=1)).bool()
        return subsequent_mask

    def get_initial_probs(self, vocab_size, initial_token_idx):
        probs = torch.zeros(1, vocab_size)
        probs[0, initial_token_idx] = 1
        return probs.float()

    def forward(self, x, y):
        x_mask = get_pad_mask(x, self.word_idx_dict[PAD_CHAR])
        y_mask = get_pad_mask(y, self.word_idx_dict[PAD_CHAR]) & self.mask_future_seq(y)

        x = self.word_embedding(x)
        x = self.pos_embedding(x)

        y = self.word_embedding(y)
        y = self.pos_embedding(y)

        # x = (batchsize, seqlen, embed dim)
        # y.shape = (batchsize, seqlen)

        # x_mask = (batchsize, 1, sequence length)
        # y_mask = (batchsize, sequence length, sequence length)
        # e_output = (batchsize, sequence length, embedding dimension)
        # d_output = (batchsize, sequence length, embedding dimension)

        e_out, *_ = self.encoder(x, x_mask)
        d_out, *_ = self.decoder(y, y_mask, e_out, x_mask)

        # self.fc_to_logits(dec_output) = (batchsize, seqlen, vocab len)
        logits = self.fc_to_logits(d_out) * 1 # no weight sharing

        return logits.view(-1, logits.size(2))

# In[ ]:

class ScheduledOptim():
    '''A simple wrapper class for learning rate scheduling'''

    def __init__(self, optimizer, init_lr, d_model, n_warmup_steps):
        self._optimizer = optimizer
        self.init_lr = init_lr
        self.d_model = d_model
        self.n_warmup_steps = n_warmup_steps
        self.n_steps = 0

    def step_and_update_lr(self):
        "Step with the inner optimizer"
        self._update_learning_rate()
        self._optimizer.step()

    def zero_grad(self):
        "Zero out the gradients with the inner optimizer"
        self._optimizer.zero_grad()

    def _get_lr_scale(self):
        d_model = self.d_model
        n_steps, n_warmup_steps = self.n_steps, self.n_warmup_steps
        return (d_model ** -0.5) * min(n_steps ** (-0.5), n_steps * n_warmup_steps ** (-1.5))

    def _update_learning_rate(self):
        ''' Learning rate scheduling per step '''

        self.n_steps += 1
        lr = self.init_lr * self._get_lr_scale()

        for param_group in self._optimizer.param_groups:
            param_group['lr'] = lr

# ## Train Functions

# In[ ]:

def _construct_summary(tokens, mapping):
    summary = []
    for token in tokens:
        token = token.item()
        if token in mapping and mapping[token] != PAD_CHAR:
            summary.append(mapping[token])
    return summary

def one_pass(x, y, gold, model, optim, criterion):
    optim.zero_grad()

    y_hat = model.forward(x, y) # (batchsize * seq len, vocab size)

    # NO LABEL SMOOTHING
    # loss = criterion(y_hat, gold)

    # LABEL SMOOTHING
    alpha = 0.1
    n_class = y_hat.size(1)

    y_hat_hot = torch.zeros_like(y_hat).scatter(1, gold.view(-1, 1), 1)
    y_hat_smooth = (1 - alpha) * y_hat_hot + (1 - y_hat_hot) * alpha / (n_class - 1) # y_ls = (1 - α) * y_hot + α / K
    log_prb = F.log_softmax(y_hat, dim=1)

    non_pad_mask = gold.ne(full_word_idx_dict[PAD_CHAR])
    loss = -(y_hat_smooth * log_prb).sum(dim=1)
    loss = loss.masked_select(non_pad_mask).sum()

    pred = y_hat.max(1)[1]

    pred_summary = ' '.join(_construct_summary(pred, full_idx_word_dict)[1:-1])
    true_summary = ' '.join(_construct_summary(y.contiguous().view(-1), full_idx_word_dict)[1:-1])

    loss.backward()
    torch.nn.utils.clip_grad_norm_(model.parameters(), 1)

    # optim.step() #using lr-scheduler outside in train()
    optim.step_and_update_lr() # custom lr scheduler

    return loss, true_summary, pred_summary

## Evaluation One Pass
def one_pass_eval(model, criterion):
    model.eval()
    np.random.shuffle(val_data_indices)
    val_loader = get_batch(val_data_indices)
    eval_loss = 0
    eval_pass_pred = []
    eval_pass_true = []
    for _ in range(NUM_EVAL_BATCHES):
        data = val_loader.__next__()
        x, y = np.array(data[:, 0]), np.array(data[:, 1])
        x = torch.tensor(np.array([x[i] for i in range(len(x))])).long()
        y = torch.tensor(np.array([y[i] for i in range(len(y))])).long()

        x = x.to(device)
        y, gold = map(lambda x: x.to(device), patch_trg(y))
        y = y.to(device)

        with torch.no_grad():
            y_hat = model.forward(x, y) # (batchsize * seq len, vocab size)

            # loss = criterion(y_hat, gold)
            alpha = 0.1
            n_class = y_hat.size(1)

            y_hat_hot = torch.zeros_like(y_hat).scatter(1, gold.view(-1, 1), 1)
            y_hat_smooth = (1 - alpha) * y_hat_hot + (1 - y_hat_hot) * alpha / (n_class - 1) # y_ls = (1 - α) * y_hot + α / K
            log_prb = F.log_softmax(y_hat, dim=1)

            non_pad_mask = gold.ne(full_word_idx_dict[PAD_CHAR])
            loss = -(y_hat_smooth * log_prb).sum(dim=1)
            loss = loss.masked_select(non_pad_mask).sum()
            pred = y_hat.max(1)[1]
            temp_loss = loss.item()/BATCH_SIZE
            eval_loss += temp_loss

    model.train()
    eval_loss /=  NUM_EVAL_BATCHES
    return eval_loss

def train(model: Transformer, optim, criterion, model_name):
    print("Starting to train")
    model.train()
    ep_loss = []
    ep_eval_loss = []
    temp_loss = []
    eps = []
    for ep in range(1, EP + 1):
        np.random.shuffle(train_data_indices)
        train_loader = get_batch(train_data_indices)

        data = train_loader.__next__()
        x, y = np.array(data[:, 0]), np.array(data[:, 1])
        x = torch.tensor(np.array([x[i] for i in range(len(x))])).long()
        y = torch.tensor(np.array([y[i] for i in range(len(y))])).long()

        x = x.to(device)
        y, gold = map(lambda x: x.to(device), patch_trg(y))
        y = y.to(device)

        loss, true_summary, pred_summary = one_pass(x, y, gold, model, optim, criterion)

        temp_loss.append(loss.item() / BATCH_SIZE)

        if ep % PRINT_EVERY_EP == 0 and ep > 0:
            eval_loss = one_pass_eval(model, criterion)
            ep_loss.append(sum(temp_loss)/len(temp_loss))
            ep_eval_loss.append(eval_loss)
            eps.append(ep)
            temp_loss = []
            # print(ep, eval_loss, ep_loss[-1])
            # print(f'\nEP:{ep} | Train Loss: {np.array(ep_loss[-PRINT_EVERY_EP:]).mean()} | Val Loss: {np.array(eval_loss).mean()}')
            # print('Training')
            # print(f'True: {true_summary[:100]}')
            # print(f'Pred: {pred_summary[:100]}')
            # print('Validation')
            # print(f'True: {eval_true_summary[0][:100]}')
            # print(f'Pred: {eval_pred_summary[0][:100]}')

        if ep % SAVE_MODEL_EVERY_EP == 0 and ep > 0:
            torch.save({
            'epoch': ep,
            'model': model.state_dict(),
            'loss': loss
            }, f'{drive_prefix}models/{model_name}/checkpoint_{ep}')

    return model, ep_loss, ep_eval_loss, eps

def predict(model, X):
    model.eval()

    pred_summaries = []
    true_summaries = []

    for i in range(0, X.shape[0]):
      x, y = np.array(X[i:i+1, 0]), np.array(X[i:i+1, 1])
      x = torch.tensor(np.array([x[i] for i in range(len(x))])).long()
      y = torch.tensor(np.array([y[i] for i in range(len(y))])).long()

      x = x.to(device)
      y, gold = map(lambda x: x.to(device), patch_trg(y))
      y = y.to(device)

      with torch.no_grad():
          y_hat = model.forward(x, y)

          pred = y_hat.max(1)[1]
          pred_summary = ' '.join(_construct_summary(pred, full_idx_word_dict)[1:-1])
          true_summary = ' '.join(_construct_summary(y.contiguous().view(-1), full_idx_word_dict)[1:-1])
          pred_summaries.append(pred_summary)
          true_summaries.append(true_summary)

    return pred_summaries, true_summaries

# # Evaluation Metrics

# In[ ]:

def compute_rouge(pred_summaries, true_summaries):
  from rouge_score import rouge_scorer
  scorer = rouge_scorer.RougeScorer(['rouge1', 'rougeL'], use_stemmer=True)
  rogue1_precision = []
  rogue1_recall = []
  rogue1_fmeasure = []
  rogueL_precision = []
  rogueL_recall = []
  rogueL_fmeasure = []
  for i in range(len(pred_summaries)):
    score = scorer.score(pred_summaries[i], true_summaries[i])
    rogue1_precision.append(score['rouge1'].precision)
    rogue1_recall.append(score['rouge1'].recall)
    rogue1_fmeasure.append(score['rouge1'].fmeasure)
    rogueL_precision.append(score['rougeL'].precision)
    rogueL_recall.append(score['rougeL'].recall)
    rogueL_fmeasure.append(score['rougeL'].fmeasure)

  scores = {'rogue1_precision':rogue1_precision, 'rogue1_recall':rogue1_recall, 'rogue1_fmeasure': rogue1_fmeasure, 'rogueL_precision':rogueL_precision, 'rogueL_recall':rogueL_recall, 'rogueL_fmeasure': rogueL_fmeasure }
  return scores

# In[ ]:

def plot_rogue(scores, label_text):
  from matplotlib import pyplot as plt
  import pandas as pd
  import seaborn as sns
  for measure in scores.keys():
    df = pd.DataFrame({"score":scores['rogue1_precision'] , "dummy": range(len(scores['rogue1_precision']))})
    sns.set_style("whitegrid")
    f, (ax_box, ax_hist) = plt.subplots(2, sharex=True, gridspec_kw= {"height_ratios": (0.2, 1)})
    f.set_size_inches(11.7, 8.27)
    mean=df['score'].mean()
    median=df['score'].median()
    mode=df['score'].mode()[0]

    sns.boxplot(df["score"], ax=ax_box)
    ax_box.axvline(mean, color='r', linestyle='--')
    ax_box.axvline(median, color='g', linestyle='-')
    ax_box.axvline(mode, color='b', linestyle='-')

    sns.distplot(df["score"], ax=ax_hist)
    ax_hist.axvline(mean, color='r', linestyle='--')
    ax_hist.axvline(median, color='g', linestyle='-')
    ax_hist.axvline(mode, color='b', linestyle='-')

    plt.legend({'Mean':mean,'Median':median,'Mode':mode})
    plt.title(measure + f'{measure} {label_text}')
    ax_box.set(xlabel='')
    plt.show()

# In[ ]:

def compute_rouge_summary_stats(scores):
  def mean_confidence_interval(data, confidence=0.95):
    import scipy.stats
    a = 1.0 * np.array(data)
    n = len(a)
    m, se = np.mean(a), scipy.stats.sem(a)
    h = se * scipy.stats.t.ppf((1 + confidence) / 2., n-1)
    return m, m-h, m+h
  stats = {}
  for key in scores.keys():
    stats[key] = mean_confidence_interval(scores[key])
  return stats

# In[ ]:

def plot_losses(ep_loss, ep_eval_loss, eps):
  import matplotlib.pyplot as plt
  import matplotlib.ticker as ticker
  plt.plot(eps, ep_loss, label = 'train')
  plt.plot(eps, ep_eval_loss, label = 'val')
  plt.legend()
  plt.xlabel('Epoch')
  plt.ylabel('Train Losses')
  plt.title('Training Loss')
  plt.show()

# ## Run
# - **make sure to create a folder in models/called transformers** (i.e. ese546/models/transformers must exist)

# In[ ]:

# MAIN DRIVER
model_name = 'transformers'
transformer = Transformer(num_heads=NUM_HEADS,
                          word_vector=full_word_vector,
                          word_idx_dict=full_word_idx_dict,
                          idx_word_dict=full_idx_word_dict).to(device)

# model_optim = optim.Adam(transformer.parameters(), lr=LR)

model_optim = ScheduledOptim(
        optim.Adam(transformer.parameters(), betas=(0.9, 0.98), eps=1e-09),
        2.0, EMB_SIZE, 3000)

criterion = nn.CrossEntropyLoss(ignore_index=full_word_idx_dict[PAD_CHAR], reduction='sum')

model, ep_loss, ep_eval_loss, eps = train(transformer, model_optim, criterion, model_name)

# ## TODO
#
# - validation
# - metrics
# - try to preprocess data better (e.g. there are lots of "t" in the label)
# - hyperparameter tweaking

# # Evaluations

# ## Summary of Hyperparamters
#

# In[ ]:

print(f'EMB_SIZE : {EMB_SIZE}')
print(f'BATCH_SIZE : {BATCH_SIZE}')
print(f'NUM_EVAL_BATCHES : {NUM_EVAL_BATCHES}')
print(f'NUM_HEADS : {NUM_HEADS}')
print(f'DROPOUT_RATE : {DROPOUT_RATE}')
print(f'MAX_ARTICLE_LEN : {MAX_ARTICLE_LEN}')
print(f'MAX_LABEL_LEN : {MAX_LABEL_LEN}')
print(f'E_HIDDEN_DIM : {E_HIDDEN_DIM}')
print(f'D_HIDDEN_DIM : {D_HIDDEN_DIM}')
print(f'KEY_DIM : {KEY_DIM}')
print(f'VALUE_DIM : {VALUE_DIM}')
print(f'FORCE_CREATE_DICT : {FORCE_CREATE_DICT}')

# ## Plot training vs validation loss

# In[ ]:

plot_losses(ep_loss, ep_eval_loss, eps)

# ## Compute training vs validation rouge scores

# In[ ]:

pred_summaries_train, true_summaries_train = predict(model, train_data_indices)
pred_summaries_val, true_summaries_val = predict(model, val_data_indices)

# In[ ]:

scores_train =  compute_rouge(pred_summaries_train, true_summaries_train)
scores_val  = compute_rouge(pred_summaries_val, true_summaries_val)

# ##  Rouge scores Training

# In[ ]:

compute_rouge_summary_stats(scores_train)

# ## Rouge Scores Validation

# In[ ]:

compute_rouge_summary_stats(scores_val)

# ## Rouge Scores Histograms (Training)

# In[ ]:

plot_rogue(scores_train, 'Training')

# In[ ]:

plot_rogue(scores_train, '')


from flask import Flask, request, render_template, redirect, url_for
import json
import time
import random
import string
# Flask constructor
app = Flask(__name__, template_folder="Templates")

# A decorator used to tell the application
# which URL is associated function

result = {}

starttime = 0

@app.route('/')
def client():
    with open("time.json") as test:
        d = json.load(test)
    seconds = time.time()*1000 - d["time"]
    if d["time"] != 0 and seconds < 270000:
        with open("output.json") as test_file:
            data = json.load(test_file)
        return data
    if d["time"] != 0 and (seconds >= 270000 and seconds < 300000):
        return {"message": "NULL"}
    if seconds >= 300000:
        output = ''.join(random.choices(
            string.ascii_uppercase+string.digits, k=10))
        result = {"message": str(output)}
        with open("output.json", "w") as c:
            json.dump(result, c)
        t = time.time()*1000
        start = {"time": t}
        with open("time.json", "w") as c:
            json.dump(start, c)
        return result

if __name__ == '__main__':
    app.run()


# -*- coding: utf-8 -*-
import json
import os
import scrapy

from he_or_she.scraping.items import ArticleItem

class NYTimesSpider(scrapy.Spider):
    name = 'nytimes'
    allowed_domains = ['nytimes.com']
    start_urls = ['http://nytimes.com/']
    api_url = 'https://www.nytimes.com/svc/collections/v1/publish/'

    def start_requests(self):
        nytimes_authors_path = os.path.join(os.path.dirname(__file__), 'nytimes_authors.json')
        with open(nytimes_authors_path) as nytimes_authors_fp:
            nytimes_authors = json.load(nytimes_authors_fp)
            for author in nytimes_authors:
                if author['gender'] != 'unclear':
                    query_url = f"{author['url']}?sort=newest&page=0"
                    yield scrapy.Request(f"{self.api_url}{query_url}", meta={'author': author})

    def parse(self, response):
        response_json = json.loads(response.body_as_unicode())
        for page in range(response_json['members']['total_pages']):
            query_url = f"{response.meta['author']['url']}?sort=newest&page={page}"
            yield scrapy.Request(f"{self.api_url}{query_url}", self.parse_article_list, meta=response.meta)

    def parse_article_list(self, response):
        response_json = json.loads(response.body_as_unicode())
        for item in response_json['members']['items']:
            if len(item['authors']) == 1:
                yield ArticleItem(
                    headline=item['headline'],
                    summary=item['summary'],
                    author_gender=response.meta['author']['gender'],
                )


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import os
import shutil
import re
import numpy as np
import matplotlib.pyplot as plt
from tabulate import tabulate

from read_data import read_data_set
from fit_data import emg, fit_emg, get_mean_and_variance

DIR_NAME = "KD"
ACCEPTANCE_RE = re.compile("^\d{2}_\d{3}\d?$")
INITIAL_GUESS_S1 = [1.73491118e+03, 3.53554040e-02, 7.67608013e-02, 1.15105759e+00, 0]
INITIAL_GUESS_S2 = [2.50655373e+02, 1.03182670e+01, 6.82937190e-02, 2.84176352e-01, 0]

analyzed = {"13": {}, "23": {}}

for dir in [name for name in os.listdir(DIR_NAME) if os.path.isdir(os.path.join(DIR_NAME, name))]:
    if ACCEPTANCE_RE.match(dir):
        dir_path = os.path.join(DIR_NAME, dir)
        shutil.rmtree(dir_path + "_fitpng", ignore_errors=True)
        os.mkdir(dir_path + "_fitpng")
        from_to, voltage, header, units, indices, data = read_data_set(dir_path)
        hs1 = []
        hs2 = []
        mus1 = []
        mus2 = []
        sigmas1 = []
        sigmas2 = []
        taus1 = []
        taus2 = []
        cs1 = []
        cs2 = []
        mus_diff = []
        Delta_hs1 = []
        Delta_hs2 = []
        Delta_mus1 = []
        Delta_mus2 = []
        Delta_sigmas1 = []
        Delta_sigmas2 = []
        Delta_taus1 = []
        Delta_taus2 = []
        Delta_cs1 = []
        Delta_cs2 = []
        Delta_mus_diff = []

        if (units[1] == "V"):
            data[:, 1, :] *= 1000
        if (units[2] == "V"):
            data[:, 2, :] *= 1000
        for index, dataset in zip(indices, data):
            try:
                popt1, pcov1 = fit_emg(dataset[0], dataset[1], INITIAL_GUESS_S1)
            except:
                print("could not fit 1!")
                continue
            try:
                INITIAL_GUESS_S2[1] = dataset[0][np.argmax(dataset[2])]
                popt2, pcov2 = fit_emg(dataset[0], dataset[2], INITIAL_GUESS_S2)
            except:
                print("could not fit 2!")
                continue
            hs1.append(popt1[0])
            hs2.append(popt2[0])
            mu1 = popt1[1]
            mu2 = popt2[1]
            mus1.append(popt1[1])
            mus2.append(popt2[1])
            sigmas1.append(popt1[2])
            sigmas2.append(popt2[2])
            Delta_hs1.append(np.sqrt(pcov1[0][0]))
            Delta_hs2.append(np.sqrt(pcov2[0][0]))
            Delta_mus1.append(np.sqrt(pcov1[1][1]))
            Delta_mus2.append(np.sqrt(pcov2[1][1]))
            Delta_sigmas1.append(np.sqrt(pcov1[2][2]))
            Delta_sigmas2.append(np.sqrt(pcov2[2][2]))
            tau1 = popt1[3]
            Delta_mu1 = np.sqrt(pcov1[1][1])
            Delta_mu2 = np.sqrt(pcov2[1][1])
            tau2 = popt2[3]
            taus1.append(tau1)
            taus2.append(tau2)
            Delta_taus1.append(np.sqrt(pcov1[3][3]))
            Delta_taus2.append(np.sqrt(pcov1[3][3]))
            cs1.append(popt1[4])
            cs2.append(popt2[4])
            Delta_cs1.append(np.sqrt(pcov1[4][4]))
            Delta_cs2.append(np.sqrt(pcov2[4][4]))
            if mu1 and mu2:
                mus_diff.append(np.abs(mu2-mu1))
                Delta_mus_diff.append(np.sqrt(Delta_mu1**2+Delta_mu2**2))
            else:
                mus_diff.append(0)
                Delta_mus_diff.append(0)
            # palette = plt.get_cmap('Blues')
            # palette1 = plt.get_cmap('Oranges')
            # plt.plot(dataset[0], emg(dataset[0], *popt1), color=palette(1000))
            # plt.scatter(dataset[0][::10],dataset[1][::10], color=palette(900))
            # # plt.errorbar(dataset[0], dataset[1], 0.01/np.sqrt(3), color=palette(900), fmt='.')
            # plt.plot(dataset[0], emg(dataset[0], *popt2), color=palette1(1000))
            # plt.scatter(dataset[0][::10],dataset[2][::10], color=palette1(900))
            # # plt.errorbar(dataset[0], dataset[2], 0.01/np.sqrt(3), color=palette1(900), fmt='.')
            # plt.savefig(os.path.join(dir_path + "_fitpng", f"{from_to}_{voltage}_{index}.png"))
            # plt.xlabel('Time (µs)')
            # plt.ylabel('Voltage (mV)')
            # plt.grid(True)
            # plt.show()
            # plt.clf()
            print(units, from_to, voltage, index)
        analyzed[from_to][str(voltage)] = np.array([hs1, Delta_hs1, mus1, Delta_mus1, sigmas1, Delta_sigmas1, taus1, Delta_taus1, cs1, Delta_cs1, hs2, Delta_hs2, mus2, Delta_mus2, sigmas2, Delta_sigmas2, taus2, Delta_taus2, cs2, Delta_cs2])

data_dir = "DATA"
shutil.rmtree(data_dir, ignore_errors=True)
os.mkdir(data_dir)
for from_to, byfreq in analyzed.items():
    for voltage, data in byfreq.items():
        np.savetxt(os.path.join(data_dir, f"{from_to}_{voltage}.tsv"), data.T, delimiter="\t", header="h1 [mV]\t Delta h1 [mV] \t mu1 [us] \t Delta_mu1 [us]\tsigma1 [us]\t Delta_sigma1 [us] \t tau1 [us]\t Delta_tau1 [us] \t c1 [mV] \t Delta_c1 [mV]\th2 [mV]\t Delta h2 [mV] \t mu2 [us] \t Delta_mu2 [us]\tsigma2 [us]\t Delta_sigma2 [us] \t tau2 [us]\t Delta_tau2 [us] \t c2 [mV] \t Delta_c2 [mV]")
        f = open(os.path.join(data_dir, f"_tex_{from_to}_{voltage}.tex"), 'w+')
        f.write(tabulate(data.T,
                                         headers=["h1 [mV]", "Delta h1 [mV]", "mu1 [us]", "Delta_mu1 [us]", "sigma1 [us]", "Delta_sigma1 [us]", "tau1 [us]", "Delta_tau1 [us]", "c1 [mV]", "Delta_c1 [mV]", "h2 [mV]", "Delta h2 [mV]", "mu2 [us]", "Delta_mu2 [us]", "sigma2 [us]", "Delta_sigma2 [us]", "tau2 [us]", "Delta_tau2 [us]", "c2 [mV]", "Delta_c2 [mV]"],
                                         tablefmt='latex'))
        f.close()
print("done", analyzed)