# read crimes CSV file
# each entry in CSV has LAT, LONG
# but this will be n^2 complexity
# N: apartments * N: crimes

# optimization after

# or another way
"""
geolocalize all crimes from LAT, LONG to boroughs- identified by Postal code
get total crimes per postal code
this gives a score to each borough

e.g. Lachine - H8T - X number of crimes

1. geo-localize all crime data and add a column
    can probably be done easily in Panda

    reverse geocoding : https://developers.google.com/maps/documentation/geocoding/intro#ReverseGeocoding

    e.g.
    http://maps.googleapis.com/maps/api/geocode/json?sensor=false&latlng=40.7135296,-73.9856844
"""

#  -*- coding: utf-8 -*-

import csv
import json
import pandas as pd
import requests
import pprint

#csvfile = open('crimes.csv', 'r')
df = pd.read_csv('crimes.csv', encoding = "ISO-8859-1")
# df = df[['LAT', 'LONG']]  -> {"LAT":45.5012532105,"LONG":-73.6206786362},{"LAT":45.5242817532,"LONG":-73.5758883043}
#df = map(list, df.values)
#jsonfile = open('crimes.json', 'w')

fieldnames = ("CATEGORIE", "DATE", "QUART", "PDQ", "X", "Y", "LAT", "LONG")

#reader = csv.DictReader( csvfile, fieldnames)
#for row in reader:

# df = df[['LAT', 'LONG']]
# df.to_json("crimes_lat_long.json", orient='values')  # -> [[45.6016777507,-73.5486926833],[45.4449528402,-73.6769260449],[45.6350960394,-73.5028680349],

def reverse_geocode_address(lat, long):
    # address = self.convert_french_accents(address)        # done in parse_apartment_page already

    try:
        # form URL with address in it
        url = " http://maps.googleapis.com/maps/api/geocode/json?sensor=false&latlng={},{}".format(lat, long)

        # request URL
        response = requests.get(url)
        response_json = response.json()

        # Google Map API response
        # https://developers.google.com/maps/documentation/javascript/geocoding#GeocodingResponses

        pprint.pprint(response_json)
        address = response_json['results'][0]['address_components']

        # zipcode is usually the last component, this is efficient and succinct
        # but may not be future-proof
        zipcode = address[len(address) - 1]['long_name']

        # print("{} is at {}, {}".format(zipcode, lat, long))

    except:
        status = response_json['status']
        error_message  = response_json['error_message']
        print(">> failed geocoding for {} {} because of : {}".format(lat, long, status))
        zipcode = "INV"

    return zipcode

crimes_per_borough = {}

for index, row in df.iterrows():
    # print(row['CATEGORIE'] + ':' + str(row['LAT']) + ':' + str(row['LONG']))

    zipcode_result = reverse_geocode_address(row['LAT'], row['LONG'])
    print("{} is at {}, {}".format(zipcode_result, row['LAT'], row['LONG']))

    # get first 3 characters which represent the borough e.g. H2L from H2L3W2
    borough_code = zipcode_result[:3]

    # create borough category if not exist yet, otherwise add 1
    if (borough_code in crimes_per_borough.keys()):
        crimes_per_borough[borough_code] = crimes_per_borough[borough_code] + 1
    else:
        crimes_per_borough[borough_code] = 1

        # 1. reverse-geocode LAT LONG to postal code e.g. H2L
    # 2. sum up each postal code area for each crime

    # pass

pprint.pprint(crimes_per_borough)


from uuid import UUID
from datetime import datetime
from threading import local
from types import GeneratorType

from redis import StrictRedis
from appyratus.utils.time_utils import TimeUtils

class RedisClient(StrictRedis):
    pass

class RedisObject(object):
    def __init__(self, redis, name):
        self._redis = redis
        self._name = name

    @property
    def redis(self):
        return self._redis

    @property
    def name(self):
        return self._name

class HashSet(RedisObject):
    def __contains__(self, key):
        return self.redis.hexists(self.name, key)

    def __iter__(self):
        return self.keys()

    def __getitem__(self, key):
        return self.redis.hget(self.name, key)

    def __setitem__(self, key, value):
        self.redis.hset(self.name, key, value)

    def __delitem__(self, key):
        self.redis.hdel(self.name, key)

    def __len__(self):
        return self.redis.hlen(self.name)

    def increment(self, key, delta=1, pipe=None):
        redis = pipe if pipe is not None else self.redis
        return redis.hincrby(self.name, key, delta)

    def delete(self, key, pipe=None) -> bool:
        redis = pipe if pipe is not None else self.redis
        return bool(redis.hdel(self.name, key))

    def delete_many(self, keys, pipe=None) -> int:
        if isinstance(keys, GeneratorType):
            keys = tuple(keys)
        if keys:
            redis = pipe if pipe is not None else self.redis
            return redis.hdel(self.name, *keys)
        return

    def keys(self, pipe=None):
        redis = pipe if pipe is not None else self.redis
        return (k.decode() for k in redis.hkeys(self.name))

    def values(self, pipe=None):
        redis = pipe if pipe is not None else self.redis
        return (v.decode() for v in redis.hvals(self.name))

    def items(self, pipe=None):
        redis = pipe if pipe is not None else self.redis
        return zip(self.keys(pipe=pipe), self.values(pipe=pipe))

    def update(self, mapping, pipe=None):
        redis = pipe if pipe is not None else self.redis
        if mapping:
            redis.hmset(self.name, mapping)

    def get(self, key, default=None, pipe=None):
        redis = pipe if pipe is not None else self.redis
        return redis.hget(self.name, key) or default

    def get_many(self, keys, pipe=None):
        if isinstance(keys, GeneratorType):
            keys = tuple(keys)
        if keys:
            redis = pipe if pipe is not None else self.redis
            return redis.hmget(self.name, *keys)
        else:
            return []

    def get_all(self):
        return self.redis.hgetall(self.name)

class RangeIndex(RedisObject):

    DELIM = '\0\0'
    DELIM_BYTES = DELIM.encode()

    def upsert(self, _id, value, pipe=None):
        raise NotImplementedError('override in subclass')

    def delete(self, _id, pipe=None):
        raise NotImplementedError('override in subclass')

    def delete_many(self, _ids, pipe=None):
        raise NotImplementedError('override in subclass')

    def search(
        self,
        lower=None,
        upper=None,
        include_lower=True,
        include_upper=False,
        offset=None,
        limit=None,
        pipe=None,
    ):
        raise NotImplementedError('override in subclass')

class NumericIndex(RangeIndex):
    custom_serializers = {
        UUID: lambda x: int(x.hex, 16),
        datetime: lambda x: TimeUtils.to_timestamp(x),
        bool: lambda x: int(x),
    }

    def upsert(self, _id, value, pipe=None):
        redis = pipe if pipe is not None else self.redis
        ser = self.custom_serializers.get(value.__class__)
        value = ser(value) if ser else value

        redis.zrem(self.name, _id)
        redis.zadd(self.name, {_id: value})

    def delete(self, _id, pipe=None):
        redis = pipe if pipe is not None else self.redis
        redis.zrem(self.name, _id)

    def delete_many(self, _ids, pipe=None):
        if isinstance(_ids, GeneratorType):
            _ids = tuple(_ids)
        if _ids:
            redis = pipe if pipe is not None else self.redis
            redis.zrem(self.name, *_ids)

    def search(
        self,
        lower=None,
        upper=None,
        include_lower=True,
        include_upper=False,
        offset=None,
        limit=None,
        pipe=None
    ):
        if lower is not None:
            lower = '({}'.format(lower) if not include_lower else lower
        else:
            lower = '-inf'

        if upper is not None:
            upper = '({}'.format(upper) if not include_upper else upper
        else:
            upper = '+inf'

        if limit is not None and offset is None:
            offset = 0

        redis = pipe if pipe is not None else self.redis

        return [
            v.split(self.DELIM_BYTES)[-1]
            for v in redis.zrangebyscore(
                self.name, lower, upper, start=offset, num=limit
            )
        ]

class StringIndex(RangeIndex):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.lutab = HashSet(self.redis, ':'.join([self.name, 'lutab']))

    def upsert(self, _id, value, pipe=None):
        redis = pipe if pipe is not None else self.redis
        old_key = self.lutab.get(_id)
        new_key = '{}{}{}'.format(value, self.DELIM, _id)

        if old_key is not None:
            redis.zrem(self.name, old_key)

        # TODO: Do this in a pipeline
        redis.zadd(self.name, {new_key: 0.0})
        self.lutab[_id] = new_key

    def delete(self, _id, pipe=None):
        old_key = self.lutab.get(_id)
        if old_key is not None:
            redis = pipe if pipe is not None else self.redis
            self.lutab.delete(old_key)
            redis.zrem(self.name, old_key)

    def delete_many(self, _ids, pipe=None):
        old_keys = self.lutab.get_many(_ids)
        if old_keys:
            redis = pipe if pipe is not None else self.redis
            self.lutab.delete_many(old_keys)
            redis.zrem(self.name, *old_keys)

    def search(
        self,
        lower=None,
        upper=None,
        include_lower=False,
        include_upper=True,
        offset=None,
        limit=None,
        pipe=None,
    ):
        if lower is None:
            lower = '-'
        else:
            lower = '[' + lower + self.DELIM
            if not include_lower:
                lower += '\xff'

        if upper is None:
            upper = '+'
        else:
            upper = '[' + upper + self.DELIM
            if include_upper:
                upper += '\xff'

        if limit is not None and offset is None:
            offset = 0

        redis = pipe if pipe is not None else self.redis
        values = redis.zrangebylex(
            self.name, lower, upper, start=offset, num=limit
        )

        return [
            v.split(self.DELIM_BYTES)[-1] for v in values
        ]


# -*- coding: utf-8 -*-
"""
Created on Sun Jul  8 18:15:35 2021

@author: HP
"""

from numpy.lib.utils import source
from pywebio.platform.flask import webio_view
from pywebio import STATIC_PATH
from pywebio.input import *
from pywebio.output import *
from flask import Flask, send_from_directory
import sklearn
import pickle
import numpy as np
import pandas as pd
import time
model = pickle.load(open('flight-catboost_model.pkl', 'rb'))
app = Flask(__name__)

import argparse
from pywebio import start_server

def predict():

    date_dep = input('Departure Date :', type = DATE,required=True)
    #put_text(date_dep)
    Journey_day = int(pd.to_datetime(date_dep, format = "%Y-%m-%d").day)
    Journey_month = int(pd.to_datetime(date_dep, format ="%Y-%m-%d").month)
    Journey_year = int(pd.to_datetime(date_dep, format ="%Y-%m-%d").year)
    #put_text("Journey Date :" ,Journey_day, Journey_month)

    day = pd.Timestamp(year = Journey_year, month = Journey_month, day = Journey_day).dayofweek
    #put_text(day)

    day_sin = np.sin(2 * np.pi * day/6)
    day_cos = np.cos(2 * np.pi * day/6)
    #put_text('day_sin :', day_sin, 'day_cos :', day_cos)

    dep_time = input('Departure Time :', type = TIME,required=True)
    #put_text(dep_time)
    Dep_hour = int(pd.to_datetime(dep_time, format ="%H:%M").hour)
    Dep_min = int(pd.to_datetime(dep_time, format ="%H:%M").minute)
    #put_text("Departure : ",Dep_hour, Dep_min)

    Total_stops = select('Please select total number of stops :', [0,1,2,3,4])
    #put_text('Total stops :', Total_stops)

    airline = select('Please select the Airline :', ['IndiGo','Air India','Multiple carriers','SpiceJet','Vistara','Air Asia','GoAir','Multiple carriers Premium economy','Vistara Premium economy'])
    if (airline=='Air India'):
            Air_India = 1
            GoAir = 0
            IndiGo = 0
            Multiple_carriers = 0
            Multiple_carriers_Premium_economy = 0
            SpiceJet = 0
            Vistara = 0
            Vistara_Premium_economy = 0
    elif (airline=='GoAir'):
            Air_India = 0
            GoAir = 1
            IndiGo = 0
            Multiple_carriers = 0
            Multiple_carriers_Premium_economy = 0
            SpiceJet = 0
            Vistara = 0
            Vistara_Premium_economy = 0
    elif (airline=='IndiGo'):
            Air_India = 0
            GoAir = 0
            IndiGo = 1
            Multiple_carriers = 0
            Multiple_carriers_Premium_economy = 0
            SpiceJet = 0
            Vistara = 0
            Vistara_Premium_economy = 0
    elif (airline=='Multiple carriers'):
            Air_India = 0
            GoAir = 0
            IndiGo = 0
            Multiple_carriers = 1
            Multiple_carriers_Premium_economy = 0
            SpiceJet = 0
            Vistara = 0
            Vistara_Premium_economy = 0
    elif (airline=='Multiple carriers Premium economy'):
            Air_India = 0
            GoAir = 0
            IndiGo = 0
            Multiple_carriers = 0
            Multiple_carriers_Premium_economy = 1
            SpiceJet = 0
            Vistara = 0
            Vistara_Premium_economy = 0
    elif (airline=='SpiceJet'):
            Air_India = 0
            GoAir = 0
            IndiGo = 0
            Multiple_carriers = 0
            Multiple_carriers_Premium_economy = 0
            SpiceJet = 1
            Vistara = 0
            Vistara_Premium_economy = 0
    elif (airline=='Vistara'):
            Air_India = 0
            GoAir = 0
            IndiGo = 0
            Multiple_carriers = 0
            Multiple_carriers_Premium_economy = 0
            SpiceJet = 0
            Vistara = 1
            Vistara_Premium_economy = 0
    elif (airline=='Vistara Premium economy'):
            Air_India = 0
            GoAir = 0
            IndiGo = 0
            Multiple_carriers = 0
            Multiple_carriers_Premium_economy = 0
            SpiceJet = 0
            Vistara = 0
            Vistara_Premium_economy = 1
    else:
            Air_India = 0
            GoAir = 0
            IndiGo = 0
            Multiple_carriers = 0
            Multiple_carriers_Premium_economy = 0
            SpiceJet = 0
            Vistara = 0
            Vistara_Premium_economy = 0
    #put_text("Selected Airline is :", "Air_India :",Air_India, "GoAir : ", GoAir, "IndiGo :", IndiGo, "Multiple_carriers :", Multiple_carriers, "Multiple_carriers_Premium_economy : ",Multiple_carriers_Premium_economy,"SpiceJet :", SpiceJet, "Vistara:",Vistara, "Vistara_Premium_economy:", Vistara_Premium_economy)

    Source = select('Please select the Source :', ['Chennai','Delhi','Kolkata','Mumbai','Banglore'])
    # Banglore = 0
    if (Source == 'Delhi'):
            s_Delhi = 1
            s_Kolkata = 0
            s_Mumbai = 0
            s_Chennai = 0

    elif (Source == 'Kolkata'):
            s_Delhi = 0
            s_Kolkata = 1
            s_Mumbai = 0
            s_Chennai = 0

    elif (Source == 'Mumbai'):
            s_Delhi = 0
            s_Kolkata = 0
            s_Mumbai = 1
            s_Chennai = 0

    elif (Source == 'Chennai'):
            s_Delhi = 0
            s_Kolkata = 0
            s_Mumbai = 0
            s_Chennai = 1

    else:
            s_Delhi = 0
            s_Kolkata = 0
            s_Mumbai = 0
            s_Chennai = 0
    #put_text("Source :", 'Delhi :',s_Delhi, 'Kolkata :', s_Kolkata,'Mumbai :', s_Mumbai, 'Chennai :', s_Chennai)

    Source = select('Please select the Destination :', ['Cochin','Hyderabad','Delhi','Kolkata','New Delhi','Banglore'])
    # Banglore = 0
    if (Source == 'Delhi'):
            d_Delhi = 1
            d_Kolkata = 0
            d_New_Delhi = 0
            d_Hyderabad = 0
            d_cochin = 0

    elif (Source == 'Kolkata'):
            d_Delhi = 0
            d_Kolkata = 1
            d_New_Delhi = 0
            d_Hyderabad = 0
            d_cochin = 0

    elif (Source == 'New Delhi'):
            d_Delhi = 0
            d_Kolkata = 0
            d_New_Delhi = 1
            d_Hyderabad = 0
            d_cochin = 0

    elif (Source == 'Hyderabad'):
            d_Delhi = 0
            d_Kolkata = 0
            d_New_Delhi = 0
            d_Hyderabad = 1
            d_cochin = 0

    elif (Source == 'Cochin'):
            d_Delhi = 0
            d_Kolkata = 0
            d_New_Delhi = 0
            d_Hyderabad = 0
            d_cochin = 1

    else:
            d_Delhi = 0
            d_Kolkata = 0
            d_New_Delhi = 0
            d_Hyderabad = 0
            d_cochin = 0
    #put_text("Destination :", 'Delhi :',d_Delhi, 'Kolkata :', d_Kolkata,'Hyderabad :', d_Hyderabad, 'Cochin :', d_cochin,' New_Delhi :',d_New_Delhi)
    prediction = model.predict([[Total_stops,Journey_day,Journey_month,Dep_hour,Dep_min,day_sin,day_cos,Air_India,GoAir,IndiGo,Multiple_carriers,Multiple_carriers_Premium_economy,SpiceJet,Vistara,Vistara_Premium_economy,s_Chennai,s_Delhi,s_Kolkata,s_Mumbai,d_cochin,d_Delhi,d_Hyderabad,d_Kolkata,d_New_Delhi]])
    output = round(prediction[0], 2)

    put_processbar('bar')
    for i in range(1, 11):
        set_processbar('bar', i / 10)
        time.sleep(0.1)
    put_markdown("Here is the Prediction!")

    if output < 0:
        put_text("Sorry Invalid Inputs")

    else:
        put_text('Predicted price of the flight is ',output ,'rupees')

app.add_url_rule('/tool', 'webio_view', webio_view(predict),
            methods=['GET', 'POST', 'OPTIONS'])

app.run(host='localhost', port=80, debug=True)

#visit http://localhost/tool to open the PyWebIO application.


import dataclasses
import datetime
import json
import logging
import os
import warnings
from typing import Dict, List

import license_utils
from license_utils import build_licenses
from license_utils.computation import compute_match_scores

def _load_licenses_from_file(cache_file: str):
    if not os.path.exists(cache_file):
        raise RuntimeError(f"License cache file {cache_file} does not exist")
    with open(cache_file, "r") as f:
        licenses_dict = json.load(f)
        if licenses_dict["version"] != license_utils.__version__:
            warnings.warn(
                f"License cache file {cache_file} was created with a different "
                + f"version of spdx-license-matcher (cache version: {licenses_dict['version']}, "
                + f"current version: {license_utils.__version__})."
                + "We thus re-download the licenses and replace the cache."
            )
            raise RuntimeError("Cache file version mismatch")
        return build_licenses.load_licenses_from_dict(licenses_dict["licenses"])

def _store_licenses_to_file(
    licenses: List[build_licenses.SpdxLicense], cache_file: str
):
    licenses_dict = {
        "version": license_utils.__version__,
        "timestamp": datetime.datetime.now().isoformat(),
        "licenses": [dataclasses.asdict(l) for l in licenses],
    }
    os.makedirs(os.path.dirname(cache_file), exist_ok=True)
    with open(cache_file, "w") as f:
        f.write(json.dumps(licenses_dict))

class SpdxLicenseUtils:
    def __init__(self, licenses, cache_file: str | None = None):
        self.licenses = licenses
        self.cache_file = cache_file

    @staticmethod
    async def create(cache_file: str | None = None):
        licenses = None
        if cache_file != None:
            try:
                licenses = _load_licenses_from_file(cache_file)
            except RuntimeError as e:
                warnings.warn(
                    f"Could not load licenses from cache file: {str(e)}. Will re-download licenses and replace cache."
                )
                pass  # We just re-download the licenses
        else:
            logging.info(
                "No cache file specified. We will re-download the licenses every time."
                "This takes a while and requires a internet connection."
            )

        new_instance = SpdxLicenseUtils(licenses, cache_file)

        if licenses == None:
            await new_instance.fetch_licenses()

        return new_instance

    async def fetch_licenses(self):
        self.licenses = await build_licenses.fetch_spdx_licenses(load_text=True)
        if self.cache_file != None:
            _store_licenses_to_file(self.licenses, self.cache_file)

    def match_text_to_id(self, license_text: str) -> Dict[str, float]:
        return compute_match_scores(license_text, self.licenses)

    def find_match(
        self, license_text: str, threshold: float
    ) -> build_licenses.SpdxLicense | None:
        result = compute_match_scores(
            license_text, self.licenses, return_first_threshold=threshold
        )
        if isinstance(result, build_licenses.SpdxLicense):
            return result
        else:
            return None


#!/usr/bin/env python3
import os
import argparse
from pathlib import Path
from tools import colors
from commands.emergen import emergen
from commands.export import exportData
from commands.extract import extractData
from commands.format import formatData
from commands.insert import insertData
from commands.test import test

current_dir = os.path.dirname(os.path.realpath(__file__))
path = Path(current_dir)
storage_dir = str(path.parents[1]) + '/storage/app/public/outputs'

corres_path = str(path) + '/corres'
parser = argparse.ArgumentParser()

parser.add_argument('command', choices=['file', 'emergen', 'export', 'test'],
                    help="Manipulate data from file, get EMERGEN format, or export data from the database.", type=str)

# Import arguments
parser.add_argument('-i', '--input', help="<Required> Input file to be formatted", type=str) #  required=True
parser.add_argument('-d', '--header', help="Line number in the input file containing the column headers", type=int, default=1) # put 12 for glims file, 7 for samplesheet
parser.add_argument('--db', help="Insert the data from the input file into the database", default=False, action='store_true')
parser.add_argument('-n', '--no_table', help="SQL table to do not insert", type=str)
parser.add_argument('--get_inserted', help="get dataframe with data to insert into DB", default=False, action='store_true') # for debog
parser.add_argument('--get_updated', help="get dataframe with data to update into DB", default=False, action='store_true') # for debog

# Emergen arguments
parser.add_argument('-a', '--start_date', help="Validation start date", type=str, default='2021-01-01') # optional
parser.add_argument('-b', '--end_date', help="Validation end date", type=str, default='2023-01-01') # optional
parser.add_argument('-s', '--samples', help="File containing a list of sample numbers", type=str, default='') # optional
parser.add_argument('-o', '--output_emergen', help="Output folder for Emergen data", type=str, default=storage_dir) # optional

# Export arguments
parser.add_argument('-l','--ids', nargs='+', help='<Required> List of data ids to export')
parser.add_argument('-p', '--table', help="Table to export", type=str) # required=True
parser.add_argument('-e', '--output_export', help="Output folder for exported data", type=str, default=storage_dir) # optional

# Other arguments
parser.add_argument('--version', action='version', version='0.0.1')
parser.add_argument('--corres', help="Path to Corres folder", type=str, default=corres_path) # optional

args = parser.parse_args()

if __name__ == "__main__":
    try:
        if args.command == 'file':
            extrac = True
            file = args.input
            header = args.header
            db = args.db
            get_inserted = args.get_inserted
            get_updated = args.get_updated
            no_table = args.no_table
        elif args.command == 'emergen':
            extrac = False
            # If a list of sample numbers is given, the extraction for Emergen will be done according
            # to this list in priority to the validation dates if they are given.
            if args.samples:
                print(colors.bcolors.HEADER + "\n #### GET EMERGEN DATA #### \n" + colors.bcolors.ENDC)
                emergen(args.corres, args.output_emergen, args.samples, args.start_date, args.end_date)
            else:
                print(colors.bcolors.HEADER + "\n #### GET EMERGEN DATA #### \n" + colors.bcolors.ENDC)
                emergen(args.corres, args.output_emergen, '', val_start=args.start_date, val_stop=args.end_date)
        elif args.command == 'export':
            extrac = False
            print(colors.bcolors.HEADER + "\n #### GET DATA FROM DB #### \n" + colors.bcolors.ENDC)
            exportData(args.corres, args.output_export, args.ids, args.table)
        elif args.command == 'test':
            extrac = False
            print(colors.bcolors.HEADER + "\n #### TESTS #### \n" + colors.bcolors.ENDC)
            test(args.corres)

    except:
        raise Exception(colors.bcolors.FAIL + "A problem occurred during the execution of a command: see the errors above. For more information, see https://github.com/CassandreHeritier/db-ngs/tree/master/bdd." + colors.bcolors.ENDC)
    else:
        if extrac:
            print(colors.bcolors.HEADER + "\n #### IMPORT FILE #### \n" + colors.bcolors.ENDC)
            data = extractData(args.corres, file, header)

            print(colors.bcolors.HEADER + "\n #### FORMAT DATA #### \n" + colors.bcolors.ENDC)
            formated_data = formatData(args.corres, data)

            # Insert into database
            if db:
                print(colors.bcolors.HEADER + "\n #### INSERT DATA TO DB #### \n" + colors.bcolors.ENDC)
                insertData(args.corres, formated_data, no_table, get_inserted, get_updated)
                #except:
                #    raise Exception(colors.bcolors.FAIL + "No SQL table has been filled in, please use the --table or -t option followed by the name of the table not to be inserted." + colors.bcolors.ENDC)


import itertools
import pathlib
import re
from typing import List, Tuple

def day14_part1_bitwise(data: List[str]) -> int:
    mask_regex = re.compile(r"mask = (\w+)")
    addr_value_regex = re.compile(r"mem\[(\d+)] = (\d+)")

    memory_layout = dict()

    and_mask, or_mask = -1, -1

    for entry in data:
        addr_value = addr_value_regex.findall(entry)
        if not addr_value:
            mask = mask_regex.findall(entry)[0]
            and_mask, or_mask, _ = get_masks(mask)
            continue

        address, value = int(addr_value[0][0]), int(addr_value[0][1])

        value = (value & and_mask) | or_mask
        memory_layout[address] = value

    return sum(memory_layout.values())

def day14_part2_bitwise(data: List[str]) -> int:
    mask_regex = re.compile(r"mask = (\w+)")
    addr_value_regex = re.compile(r"mem\[(\d+)] = (\d+)")

    memory_layout = dict()

    mask = None
    mask_combinations = None
    or_mask = -1

    for entry in data:
        addr_value = addr_value_regex.findall(entry)
        if not addr_value:
            mask = mask_regex.findall(entry)[0]
            _, or_mask, mask_combinations = get_masks(mask)
            continue

        address, value = int(addr_value[0][0]), int(addr_value[0][1])

        for combination in mask_combinations:
            new_mask = mask.replace("1", " ").replace("0", " ")
            for combo_val in combination:
                new_mask = new_mask.replace("X", combo_val, 1)

            combo_and_mask, combo_or_mask, _ = get_masks(new_mask)

            address = (address & combo_and_mask) | combo_or_mask | or_mask
            memory_layout[address] = value

    return sum(memory_layout.values())

def get_masks(mask: str) -> Tuple[int, int, List[Tuple[int, ...]]]:
    and_mask, or_mask = 2 ** 36 - 1, 0

    for index, char in enumerate(mask):
        if char == "0":
            and_mask &= (2 ** 36 - 1) & ~(1 << (35 - index))
        elif char == "1":
            or_mask |= 1 << (35 - index)

    wildcard_count = sum(char == "X" for char in mask)
    mask_wildcard_combinations = list(itertools.product("01", repeat=wildcard_count))

    return and_mask, or_mask, mask_wildcard_combinations

def get_input_data(file: str) -> List[str]:
    with pathlib.Path(file).open() as f:
        content = f.readlines()

    return content

if __name__ == "__main__":
    print(day14_part1_bitwise(get_input_data("input.txt")))
    print(day14_part2_bitwise(get_input_data("input.txt")))


from itertools import chain

from ..models import Products, ProductDetails, Address, Customers, Dates
from graphos.sources.model import ModelDataSource
from graphos.renderers import gchart
import math
from django.db.models import Q

def getProdNum(prNum):
    object = Products.objects.get(prodNum=prNum)
    return object.prodNum

def getProdName(prNum):
    object = Products.objects.get(prodNum=prNum)
    return object.prodName

def getProdPrice(prNum):
    object = Products.objects.get(prodNum=prNum)
    return object.prodPrice

def getProdStock(prNum):
    object = Products.objects.get(prodNum=prNum)
    return object.prodStock

def getProdGenre(prNum):
    object = ProductDetails.objects.get(prodNum=prNum)
    return object.genre

def getProdType(prNum):
    object = ProductDetails.objects.get(prodNum=prNum)
    return object.type

def getProdPublish(prNum):
    object = ProductDetails.objects.get(prodNum=prNum)
    return object.publisher

def getProdTotalPages(prNum):
    object = ProductDetails.objects.get(prodNum=prNum)
    return object.totalPages

def getProdLanguage(prNum):
    object = ProductDetails.objects.get(prodNum=prNum)
    return object.language

def getProdRating(prNum):
    object = ProductDetails.objects.get(prodNum=prNum)
    return object.rating

def getProdAuthor(prNum):
    object = ProductDetails.objects.get(prodNum=prNum)
    return object.author

def getProdDesc(prNum):
    object = ProductDetails.objects.get(prodNum=prNum)
    return object.desc

def getProdImage(prNum):
    object = ProductDetails.objects.get(prodNum=prNum)
    # print(object)
    return object.imageLink

def getDBResults(query):
    resultsProductName = Products.objects.filter(prodName__icontains=query)
    results = ProductDetails.objects.filter(
        Q(genre__icontains=query) | Q(type__icontains=query) | Q(publisher__icontains=query) | Q(
            language__icontains=query) | Q(author__icontains=query) | Q(desc__icontains=query) | Q(
            pubDatum__icontains=query) | Q(prodNum__in=resultsProductName))
    return results

def queryVerbeterFunctie(query):
  try:
    query = removeUnwanted(query)
    i = 1
    query = ((query[:1].upper())+(query[1:].lower()))
    query = (query.translate("!@#$%^~`&*()_+=-{[]}\|\"\';:?/>.<,"))
    while i <= len(query):
      # Remove unwanted characters.
      if 33 <= ord(query[i-1:i]) <= 47:
        query = queryVerbeterFunctie(query[:i-1] + query[i:])
        print("Removed")
      if 58 <= ord(query[i-1:i]) <= 64:
        query = queryVerbeterFunctie(query[:i-1] + query[i:])
        print("Removed")
      if 91 <= ord(query[i-1:i]) <= 96:
        query = queryVerbeterFunctie(query[:i-1] + query[i:])
        print("Removed")
      if 122 <= ord(query[i-1:i]):
        query = queryVerbeterFunctie(query[:i-1] + query[i:])
        print("Removed")
      ## If first char is a space, remove it.
      if query[0].isspace():
        if query.__len__() > 0:
          query = queryVerbeterFunctie(query[1:])
        else:
          query = "Empty Query"
      ## Replace spider-man with Spiderman
      if query[i-1:i+6].lower() == "spiderman":
          query = "Spider-man"
      ## Replace ironman with Iron Man
      if query[i-1:i+6].lower() == "ironman":
        query = "Iron Man"
      ## Replace captainamerica with Captain America
      if query[i-1:i+13].lower() == "captainamerica":
        query = "Captain America"
      else:
        if query[i-1:i+2].lower() == "cpt":
          query = "Captain"
      ## Replace next letter with an uppercase after a space is found
      if query[i-1:i] == " ":
        query = (query[:i]) + (query[i:i+1].upper()) + (query[i+1:])
      ## Removes "the" from the query, since most comics don't use it anymore
      if query[i-1:i+2].lower() == "the" and i < len(query) - 2 and not query[i-1:i+3].lower() == "they":
        if i == 1:
          query = (query[i+2:])
          ## Make the function recursive because some parts won't work otherwise :')
          query = query[:i] + query[i+2:]
          i = 0;
      ## If search starts with "the", remove it completely
        else:
          query = (query[:i]) + (query[i+2:])
      i += 1

    print(query)
    return query
  except :
    print("Stop Jasper.")
    return query

def removeUnwanted(query):
    # Resolves empty query errors
    if query == "" or len(query) == 0:
        query = "No query found"
    return query

def getPublisherBox(publisherQuery):
    object = ProductDetails.objects.get(publisher=publisherQuery)
    return object.prodNum

def getRatingBox(ratingQuery):
    object = ProductDetails.objects.get(rating=ratingQuery)
    return object.prodNum

def getTypeBox(typeQuery):
    object = ProductDetails.objects.get(type=typeQuery)
    return object.prodNum

def getLanguageBox(languageQuery):
    object = ProductDetails.objects.get(language=languageQuery)
    return object.prodNum

def getPriceBox(priceMin, priceMax):
    object = ProductDetails.objects.raw('SELECT * FROM ProductDetails'
                                        'WHERE prodPrice >= ' + priceMin +
                                        'AND prodPrice <= ' + priceMax)
    return object.prodNum

def getProdData(prNum):
    object = ProductDetails.objects.get(prodNum=prNum)
    return object.pubDatum[0:10]

def getCustomer(id):
    object = Customers.objects.get(customerID=id)
    return object

def getCustomerFName(id):
    customer = getCustomer(id)
    object = customer.name
    return object

def getCustomerLName(id):
    customer = getCustomer(id)
    object = customer.surname
    return object

def getCustomerPhone(id):
    customer = getCustomer(id)
    object = customer.telephone
    return object

def getCustomerAddress(id):
    object = Address.objects.get(customerID=Customers(customerID=id))
    return object

def getStreet(id):
    address = getCustomerAddress(id)
    object = address.address
    return object

def getHouseNumber(id):
    address = getCustomerAddress(id)
    object = address.number
    return object

def getCity(id):
    address = getCustomerAddress(id)
    object = address.city
    return object

def getPostalcode(id):
    address = getCustomerAddress(id)
    object = address.postalcode
    return object

def getVisitsChart():
    queryset = Dates.objects.all()
    data_source = ModelDataSource(queryset, fields=['date', 'customerID'])
    chart = gchart.LineChart(data_source)
    return chart


from statistics import mean
import math
n = int(input().strip())
vals = list(map(int, input().strip().split()))[:n]
s = 0
vals.sort()
m = mean(vals)
for i in vals:
    a = (m-i)**2
    s += a
sd = s/len(vals)
sd = math.sqrt(sd)
print("{:.1f}".format(sd))


# coding=utf-8
'''
Created on Jan 26, 2012

@author: vvitvitskiy
'''
import re
import UserDict
import sap
import string
import command
import fptools
import logger
from collections import namedtuple
import netutils
from sap_db import DbInfo

class IniDocument(UserDict.UserDict):
    r''' Very raw representation of SAP ini file, usually profiles
    Simply a dictionary of keys and values
    '''
    def getAny(self, *names):
        r'@types: tuple[str] -> str or None'
        for name in names:
            if name in self:
                return self.get(name)
        return None

    def findIndexedValues(self, name):
        r''' Find attribute that is indexed like
        attr_name_0=value0
        attr_name_1=value1
        Result will be list [value0, value1]
        @types: str -> list[str]
        @param name: Property name without digit and last underscore
        '''
        index = 0
        values = []
        value = self.get('%s_%s' % (name, index))
        while value:
            values.append(value)
            index += 1
            value = self.get('%s_%s' % (name, index))
        return values

class ProfileParser:
    r''' Base class for profile parsers.
    Each parser for dedicated profile type parses only own data, to take
    full picture - all three parsers has to be applied
    '''

    def __init__(self, sapIniParser):
        r'@types: IniParser'
        assert sapIniParser
        self.__sapIniParser = sapIniParser

    def _getIniParser(self):
        r'@types: -> IniParser'
        return self.__sapIniParser

    def parseAsIniResult(self, content):
        r'@types: str -> IniDocument'
        return self.__sapIniParser.parseValueByNameMapping(content,
                                        valueTransformationFn=string.strip)

    def parseContent(self, content):
        r'@types: str -> Profile'
        document = self._getIniParser().parseValueByNameMapping(content,
                                            None, string.strip)
        return self._parseIniDocument(document)

    def _parseIniDocument(self, document):
        r'''@types: IniDocument -> Profile
        '''
        raise NotImplementedError()

    @staticmethod
    def parseInstance(doc):
        r'@types: IniDocument -> sap.Instance'
        return parse_inst_in_pf(doc)

    @classmethod
    def isApplicable(cls, pfName):
        r'''Determine if parser can be applied to profile with specified name
        @types: PyClass, str -> bool
        '''
        return pfName and cls.PF_NAME_PATTERN.match(pfName) is not None

    @classmethod
    def parseInstanceNameFromPfName(cls, pfName):
        r'@types: PyClass, str -> str or None'
        matchObj = cls.PF_NAME_PATTERN.match(pfName)
        return matchObj and matchObj.group(1)

InstancePfSet = namedtuple('InstancePfSet', ('startPfIni', 'instancePfIni'))

def createPfsIniDoc(defaultPfDoc, startPfDoc, instancePfDoc):
    '''Create ini document that gathers information from all profiles
    so to make lookup in right order
    @types: IniDocument?, IniDocument?, IniDocument? -> IniDocument
    '''
    doc = IniDocument()
    defaultPfDoc and doc.update(defaultPfDoc)
    startPfDoc and doc.update(startPfDoc)
    instancePfDoc and doc.update(instancePfDoc)
    return doc

class AscsInfoPfParser:
    '''
    From doc: http://help.sap.com/saphelp_nw04s/helpdata/en/36/67973c3f5aff39e10000000a114084/content.htm

    Standalone enqueue server is a part of ASCS.
    If you want to use the standalone enqueue server, set the following parameters in the profile files of the enqueue clients (application server instances):
    enque/serverhost = <host name of the enqueue server>
    enque/serverinst = <instance number of the enqueue server>

    From the matters expert's e-mail:
    If "enque/serverinst" is not set then it takes default value $(SAPSYSTEM).

    If enque/serverhost is set, but there is no Standalone enqueue server, SAP system might work, but this case is not supported (by SAP).
    '''
    @staticmethod
    def parse(document):
        instance_nr = (document.get('enque/serverinst')
                       or document.get('SAPSYSTEM'))
        instance_hostname = document.get('enque/serverhost')
        if not instance_hostname:  # there is no standalone enqueue server
            return None, None
        return instance_hostname, instance_nr

class DbInfoPfParser:
    '''
    Based on http://www.sapheaven.com/2012/01/profile-parameters-sap_6540.html
    ADA DB2 DB4 DB6 INF MSS ORA SQL SYB
    '''

    DBINFO_PARAMETERS = (
        (('dbs/db2/ssid', ), ('dbs/db2/hosttcp', 'SAPDBHOST')),
        (('dbs/db4/ssid', ), ('dbs/db4/hosttcp', 'SAPDBHOST')),
        (('dbs/db6/ssid', ), ('dbs/db6/hosttcp', 'SAPDBHOST')),
        (('dbs/mss/dbname',), ('dbs/mss/server',))
    )

    @staticmethod
    def parseAbapSchemaName(iniDocument):
        ''''In case of double stack there is java db instance and ABAP side uses
        own schema
        @types: IniDocument -> str or None
        '''
        return iniDocument.get('dbs/ada/schema')

    @staticmethod
    def parseAbapInstanceDbInfo(iniDocument):
        r'@types: IniDocument -> DbInfo or None'
        dbParameters = DbInfoPfParser.DBINFO_PARAMETERS
        for nameParameters, hostParameters in dbParameters:
            try:
                dbType = iniDocument.getAny('dbms/type')
                name = iniDocument.getAny(*nameParameters)
                if re.match(r'\$\((\w+)\)', name):
                    name = re.search(r'(\w+)', name).group(0)
                hostname = iniDocument.getAny(*hostParameters)
                if name and hostname and dbType:
                    return DbInfo(name, hostname, dbType, None, False)
            except Exception:
                logger.warnException("Database meta-data declaration invalid")

    @staticmethod
    def parseJavaInstanceDbInfo(document):
        r'@types: IniDocument -> DbInfo or None'
        type_ = document.get('j2ee/dbtype')
        name = document.get('j2ee/dbname')
        hostname = (document.get('j2ee/dbhost')
                    or document.get('SAPDBHOST'))
        info = None
        if name and hostname and type_:
            info = DbInfo(name, hostname, type_, None, True)
        return info

    @staticmethod
    def parseDbInfo(iniDocument):
        r'@types: IniDocument -> DbInfo or None'
        dbInfo = DbInfoPfParser.parseJavaInstanceDbInfo(iniDocument)
        if dbInfo:
            schemaName = DbInfoPfParser.parseAbapSchemaName(iniDocument)
            dbInfo = dbInfo._replace(schema=schemaName)
        else:
            dbInfo = DbInfoPfParser.parseAbapInstanceDbInfo(iniDocument)
        return dbInfo

def parse_system_in_pf(doc):
    '''
    Parse system related information from profile
    @types: IniDocument -> sap.System?'''
    sid = doc.get('SAPSYSTEMNAME')
    type_ = doc.get('system/type')
    uuid = doc.get('system/uuid')
    global_host = doc.get('SAPGLOBALHOST')
    if sid:
        return sap.System(sid, globalHost=global_host, type_=type_, uuid_=uuid)

def parse_inst_in_pf(doc):
    '''
    Parse instance related information from profile
    @types: IniDocument -> sap.Instance
    '''
    inst = parseInstanceFromName(doc.get('INSTANCE_NAME'))
    hostname = doc.get('SAPLOCALHOST')
    fqdn = doc.get('SAPLOCALHOSTFULL')
    if not hostname and fqdn and not _has_substitution_variable(fqdn):
        hostname = parse_hostname_from_fqdn(fqdn)
    return sap.Instance(inst.name, inst.number, hostname)

def _has_substitution_variable(value):
    '@types: str -> bool'
    return value.find('$(') > -1

def parse_hostname_from_fqdn(fqdn):
    hostname = fqdn
    if fqdn and fqdn.find('.') > 0:
        hostname = fqdn[0:fqdn.find('.')]
    return hostname

class StartProfileParser(ProfileParser):
    r''' Parser for the SAP START profile. There is one START profile per
    instance. But since SAP NetWeaver 7.3 start profile has been
    removed as separate file and content merged with instance profile '''

    PF_NAME_PATTERN = re.compile(r"START_([A-Za-z]+\d{2})_\w+")

    _Profile = namedtuple('Profile', ('system', 'instance', 'dbInfo'))

    def _parseIniDocument(self, document):
        r'@types: IniDocument -> Profile[System, Instance]'
        system = parse_system_in_pf(document)
        inst = ProfileParser.parseInstance(document)
        db_info = DbInfoPfParser.parseDbInfo(document)
        return self._Profile(system, inst, db_info)

class DefaultProfileParser(ProfileParser):
    r''' Parser for the SAP DEFAULT profile. There is one profile
    per system.'''

    PF_NAME_PATTERN = re.compile('DEFAULT')

    _Profile = namedtuple('Profile', ('system', 'dbHost', 'fqdn', 'abapMsHost',
                                       'jeeScsInstance', 'jeeMsEndpoint',
                                       'jeeDbInfo', 'dbInfo'))

    def _parseScsInstance(self, document):
        r'''@types: IniDocument -> tuple[sap.Instance, Endpoint or None]
        @return: tuple of SCS instance itself and message server endpoint
        '''
        instance = None
        msEndpoint = None
        hostname = document.get('j2ee/scs/host')
        number = document.get('j2ee/scs/system')
        msPort = document.get('j2ee/ms/port')
        if hostname and number:
            instance = sap.Instance('SCS', number, hostname)
            if msPort:
                msEndpoint = netutils.createTcpEndpoint(hostname, msPort)
        return instance, msEndpoint

    def _parseAbapMsEndpoint(self, document):
        r'@types: IniDocument -> Endpoint or None'
        hostname = document.get('rdisp/mshost')
        #TODO: any other parameter for port value ?
        #TODO: what to do with `rdisp/msserv` parameter ?
        port = document.get('rdisp/msserv_internal')
        if hostname and port:
            return netutils.createTcpEndpoint(hostname, port)
        return None

    def _parseIniDocument(self, document):
        r'@types: IniDocument -> Profile'
        system = parse_system_in_pf(document)
        dbHost = document.get('SAPDBHOST')
        fqdn = document.get('SAPFQDN')
        abapMsEndpoint = self._parseAbapMsEndpoint(document)
        scsInstance, jeeMsEndpoint = self._parseScsInstance(document)

        abapDbs = DbInfoPfParser.parseAbapInstanceDbInfo(document)
        abapDb = abapDbs and abapDbs[0] or None

        javaDb = DbInfoPfParser.parseJavaInstanceDbInfo(document)
        return self._Profile(system, dbHost, fqdn, abapMsEndpoint,
                              scsInstance, jeeMsEndpoint,
                              javaDb, abapDb)

    def parse(self, document):
        r'''
        r'@types: IniDocument -> sap.DefaultProfile'
        @deprecated: use `parseContent` instead
        '''
        #TODO: make refactoring to get rid of this method
        pf = self._parseIniDocument(document)
        return sap.DefaultProfile(pf.system)

class InstanceProfileParser(ProfileParser):
    r'''Parser for the SAP instance profile. There is one instance profile per
    instance'''

    PF_NAME_PATTERN = re.compile(r"[A-Za-z]{3}_([A-Za-z]+\d{2})_\w+")

    _Profile = namedtuple('Profile', ('system', 'instance',
                                      'jeeDbDriver', 'jeeEndpoints',
                                      'dbInfo'))

    @staticmethod
    def _parsePortDeclarationToEndpoint(declaration, number, hostname):
        r'''Parse string of format
            PROT=<protocol>, PORT=<number>, TIMEOUT=<number>,
        where PORT may contain $$ substring that has to be replaced with
        instance number where this port is opened

        An example:
        PROT=HTTPS,PORT=443, HOST=hostname, TIMEOUT=3600, EXTBIND=1

        @types: str, str, str -> Endpoint or None
        @param number: instance number
        @param hostname: instance hostname
        '''
        declarationAsIni = declaration.replace(',', '\n')
        iniParser = IniParser()
        iniDoc = iniParser.parseValueByNameMapping(declarationAsIni,
                                                 string.strip, string.strip)
        protocol, port = iniDoc.get('PROT'), iniDoc.get('PORT')
        if not hostname:
            hostname = iniDoc.get('HOST')
        endpoint = None
        if port and hostname:
            # check for the substitution placeholder `$$`
            if port.find('$$') != -1:
                port = port.replace('$$', number)
            # determine port type
            portType = sap.PortTypeEnum.findByName(protocol)
            endpoint = netutils.createTcpEndpoint(hostname, port, portType)
        return endpoint

    def _parseServerPorts(self, document, instanceNumber, hostname):
        r'@types: IniDocument -> tuple[Endpoint]'
        portDeclarations = document.findIndexedValues('icm/server_port')
        parsePortDeclaration = fptools.partiallyApply(
            InstanceProfileParser._parsePortDeclarationToEndpoint, fptools._,
            instanceNumber, hostname)
        return filter(None, map(parsePortDeclaration, portDeclarations))

    def _parseIniDocument(self, document):
        r'@types: IniDocument -> Profile'
        system = parse_system_in_pf(document)
        instance = self.parseInstance(document)
        # determine java side
        # jstartup/release = 700
        # j2ee/dbdriver = D:\sapdb\clients\AGX\runtime\jar\sapdbc.jar

        # ==
        #TODO: can be used to determine whether JEE side is running ?
        # rdisp/j2ee_start = 0

        #TODO: find out the purpose of this port
        # icm/HTTP/j2ee_0 = PREFIX=/,HOST=localhost,CONN=0-500,PORT=5$$00

        #TODO: find out the purpose of this port
        # ms/server_port_0 = PROT=HTTP,PORT=81$$

        jeeDbDriver = document.get('j2ee/dbdriver')
        jeeEndpoints = self._parseServerPorts(document, instance.number,
                                              instance.hostname)
        abapDbs = DbInfoPfParser.parseAbapInstanceDbInfo(document)
        db = abapDbs and abapDbs[0] or None
        return self._Profile(system, instance, jeeDbDriver, jeeEndpoints, db)

    def parse(self, document):
        r''' @types: IniDocument -> sap.InstanceProfile
        @deprecated: use `parseContent` instead
        '''
        pf = self._parseIniDocument(document)
        return sap.InstanceProfile(pf.instance)

class IniParser:
    r''' Generic parser for all ini-like configuration files '''

    @staticmethod
    def parseIniDoc(content):
        r''' Parse content without applying any transformations to parsed keys
        and values except of strip function to value.
        @types: str -> IniDocument
        '''
        return IniParser.parseValueByNameMapping(content,
                                        valueTransformationFn=string.strip)

    @staticmethod
    def parseValueByNameMapping(content, keyTransformationFn=None,
                                               valueTransformationFn=None,
                                               nameValueSeparator='='):
        r'''
        Value is not stripped by default, valueTransformationFn
        may be used for that
        @types: str, (str -> str), (str -> str)  -> IniDocument
        @param keyTransformationFn: transformers for the key before saving
                    value by it. For instance if string.uppercase specified
                    all keys will be in upper case
        @param valueTransformationFn: transformation function applied to each
                    value
        '''
        valueByName = IniDocument()
        for line in content.splitlines():
            line = line.strip()
            if (line
                    # is not commented line
                    and not line.startswith('#')
                    # has mapping of value by name
                    and line.find(nameValueSeparator) != -1):

                name, value = line.split(nameValueSeparator, 1)
                # value part may contain commented substring which
                # has to be truncated
                # if # is not escaped like \#
                matchObj = re.search(r'[^\\](#)', value)
                if matchObj:
                    # found not escaped comment sign
                    value = value[:matchObj.start(1)]
                # apply transformations
                name = name.strip()
                if keyTransformationFn:
                    name = keyTransformationFn(name)
                if valueTransformationFn:
                    value = valueTransformationFn(value)
                valueByName[name] = value
        return valueByName

class IniParserCmdlet(IniParser, command.Cmdlet):
    def __init__(self, keyTransformationFn=None, valueTransformationFn=None):
        self.keyTransformationFn = keyTransformationFn
        self.valueTransformationFn = valueTransformationFn

    def process(self, result):
        return self.parseValueByNameMapping(result,
                        self.keyTransformationFn, self.valueTransformationFn)

def parseIniFile(*args, **kwargs):
    return IniParserCmdlet(*args, **kwargs)

class SapSystemBuilderCmdlet(command.Cmdlet):
    def process(self, result):
        '''
        @types -> UserDict.UserDict -> sap.System
        '''
        assert 'sapsystemname' in result.keys()
        return sap.System(result['sapsystemname'])

def buildSapSystemFromDict(*args, **kwargs):
    return SapSystemBuilderCmdlet(*args, **kwargs)

def getProfilePathFromCommandline(param):
    '''
    @types: str -> str?
    '''
    if not (param or param.strip()):
        logger.debug('No valid parameters on process.')
        return
    # pattern doesn't consider quotes, so there is no spaces
    m = re.match(r'.*\spf\s*=\s*([\w\:\-/\.\\]+)(?:"|$|\s)', param)
    if not m:
        # pattern consider such cases
        # "pf=value" and pf="value"
        m = re.match(r'.*[\s"]pf\s*=[\s"]*([\w\:\-/\.\\\s ]+)(?:"|$)', param)
    return m and m.group(1)

class BaseLayout:
    def __init__(self, pathtools, rootPath, name=None):
        r'''
        @param name: name of entity that holds this layout, instance or system
        @types: file_topology.Path, str, str'''
        assert pathtools and rootPath
        self.__rootPath = pathtools.absolutePath(rootPath)
        self.__name = name
        self.__pt = pathtools

    def getName(self):
        r'@types: -> str?'
        return self.__name

    def _getPathTools(self):
        r'@types: -> file_topology.Path'
        return self.__pt

    def getRootPath(self):
        r'@types: -> str'
        return self.__rootPath

class Layout(BaseLayout):
    r'SAP system layout'

    def getPfDirectory(self):
        r'''Get base directory where instance profiles are saved
        @types: -> str'''
        return self._getPathTools().join(self.getRootPath(), 'SYS', 'profile')

    def getDefaultProfileFilePath(self):
        r'@types: -> str'
        return self._getPathTools().join(self.getPfDirectory(), 'DEFAULT.PFL')

    def composeStartPfPath(self, pfName):
        r''' Compose full path to the start PF based on the instance profile
        name of format '<SID>_<NAME>_<SAPLOCALHOST>'.

        Start profile name has such format 'START_<name>_<SAPLOCALHOST>'
        '@types: str -> str
        '''

        nameWithoutSid = pfName[pfName.find('_'):]
        return self._getPathTools().join(self.getPfDirectory(),
                                        'START%s' % nameWithoutSid)

    def composeInstancePfPath(self, pfName):
        r'@types: str -> str'
        if not pfName:
            raise ValueError("Profile file name is not specified")
        return self._getPathTools().join(self.getPfDirectory(), pfName)

    def composeInstanceDirPath(self, instanceName):
        r''' Compose path to the instance
        @types: str -> str'''
        if not instanceName:
            raise ValueError("Instance name is not specified")
        return self._getPathTools().join(self.getRootPath(), instanceName)

def findSystemBasePath(path, systemName):
    r''' Find root path of system with <systemName> as sub-path of <path>
    @types: str, str -> str?'''
    if not path:
        raise ValueError("Path is not specified")
    if not systemName:
        raise ValueError("SAP System name is not specified")
    index = path.find(systemName)
    return (index != -1
            and path[:index + len(systemName)]
            or None)

def parseInstanceFromName(instanceName):
    r'@types: str -> sap.Instance'
    assert instanceName
    matchObj = re.match(r'\s*([a-z]+?)(\d+)', instanceName, re.I)
    if matchObj:
        return sap.Instance(matchObj.group(1), matchObj.group(2))
    raise ValueError("Wrong instance name")

def parseSapSystemFromInstanceProfileName(profileName):
    r''' Format of profile name is: <SAPSYSTEMNAME>_<INSTANCE_NAME>_<SAPLOCALHOST>
    @types: str -> sap.System
    @raise ValueError: Wrong instance profile name

    @note: potentially relevant only to ABAP instance PF naming
    '''
    if not profileName:
        raise ValueError("Profile name is not specified")
    tokens = profileName.split('_', 2)
    if len(tokens) != 3:
        raise ValueError("Wrong instance profile name")
    systemName = tokens[0]
    instanceName = tokens[1]
    hostname = tokens[2]
    sapSystem = sap.System(systemName)
    instance = parseInstanceFromName(instanceName)
    sapSystem.addInstance(
        sap.Instance(instance.getName(), instance.getNumber(), hostname)
    )
    return sapSystem

# pattern used to parse profile path in the mounted global host directory
# UNIX systems:/usr/sap/<SID>/SYS/profile
# Windows NT: \\<SAPGLOBALHOST>\sapmnt\<SID>\sys\profile\
# For old versions \\SAPGLOBALHOST>\profile.<SID>\
_PF_PATH_AT_GLOBALHOST_PATTERN = re.compile(
       r'''.*?[/\\]+([a-z0-9]{3})      # SYSTEMNAME
       [/\\]+SYS[/\\]+profile
       [/\\]+(\w+)                     # profile file name
       ''',
        re.IGNORECASE | re.VERBOSE)

_PF_PATH_AT_GLOBALHOST_PATTERN_OLD_STYLE = re.compile(
       r'''.*?[/\\]+profile\.([a-z0-9]{3}) # SYSTEMNAME
       [/\\]+(\w+)                     # profile file name
       ''',
        re.IGNORECASE | re.VERBOSE)

_PF_PATH_AT_MOUNTPOINT_PATTERN = re.compile(
       r'''[/\\]+([a-z0-9]{3})      # SYSTEMNAME
       [/\\]+profile
       [/\\]+(\w+)$             # profile file name
       ''', re.IGNORECASE | re.VERBOSE)

def parsePfDetailsFromPath(path):
    r''' Parse profile details from its path
    For example: D:\usr\sap\CWT\SYS\profile\CWT_ASCS00_hostname
                /sapmnt/JSP/profile/JSP_JC01_hostname
    @types: str -> tuple[sap.System?, str?]
    @return: pair of parsed system and profile name
    '''
    if path:
        matchObj = re.match(_PF_PATH_AT_GLOBALHOST_PATTERN, path)
        if matchObj:
            return (sap.System(matchObj.group(1)), matchObj.group(2))
        matchObj = re.search(_PF_PATH_AT_MOUNTPOINT_PATTERN, path)
        if matchObj:
            return (sap.System(matchObj.group(1)), matchObj.group(2))
        matchObj = re.search(_PF_PATH_AT_GLOBALHOST_PATTERN_OLD_STYLE, path)
        if matchObj:
            return (sap.System(matchObj.group(1)), matchObj.group(2))
        #in case system is present in path in an unpredictible manner will try to parse it
        #from filename
        file_name = re.search(r'.*[/\\]+(\w+)$', path)
        if file_name:
            system_name = re.match('([a-z0-9]{3})_.*', path,
                                   re.IGNORECASE | re.VERBOSE)
            if system_name:
                return (sap.System(system_name.group(1)), file_name.group(1))
    return (None, None)

def parseSapSystemFromInstanceBasePath(path, parseHostname=False):
    r''' Parse instance base path and decompose it onto system and instance
    object representation.
    Format of substring participated in search is
    /<SAPSYSTEMNAME>/<INSTANCE_NAME>/<SAPLOCALHOST>

    @types: str, bool -> sap.System
    @param parseHostname: Influence on <SAPLOCALHOST> part parsing.
        When parameter set to True will attempt to parse hostname.
        Parameter has to be used carefully and only in places where there is
        confidence in right pattern format.
    @raise ValueError: Unsupported path format'''
    if not path:
        raise ValueError("Path is not specified")
    mo = re.search(r'''[/\\]([a-z]{3})[/\\]       # SAPSYSTEMNAME
                       ([a-z]+)(\d\d)             # INSTANCE_NAME
                       (?:[/\\]([^/\\]+))?\s*     # SAPLOCALHOST
                    ''', path, re.I | re.VERBOSE)
    if not mo:
        raise ValueError("Unsupported path format")
    sid = mo.group(1)
    number = mo.group(3)
    name = mo.group(2) + number
    hostname = None
    if parseHostname and len(mo.groups()) > 3:
        hostname = mo.group(4).strip()
    return sap.System(sid).addInstance(sap.Instance(name, number, hostname))

def parseInstFromHomeDir(system, path):
    r'''Parse instance details from own home directory
    For instance "/usr/sap/<SID>/JC00/exe" path contains information
    about instance with name "JC00" and number "00"

    @types: System, str -> Instance?'''
    if not path:
        return None
    regexp = r".*?%s[\\/](\w+)" % re.escape(system.getName())
    matchObj = re.match(regexp, path)
    return matchObj and parseInstanceFromName(matchObj.group(1))

def parseSystemAndInstanceDetails(value):
    '''Parse string string of format <SAPLOCALHOST>_<SAPSYSTEMNAME>_<SAPSYSTEM>
    @types: str -> tuple[sap.System, str, str]
    @return: Tuple of SAP System, hostname and instance number respectively
    '''
    if not value:
        raise ValueError("Value to parse is not specified")
    tokens = value.rsplit('_', 2)
    if len(tokens) != 3:
        raise ValueError("Value is not of supported format")
    hostname, sid, number = tokens
    if not sap.isCorrectSapInstanceNumber(number):
        raise ValueError("Instance number is not valid")
    return sap.System(sid), hostname, number

def parseInstNrInMsgServerPort(port):
    r'''Parse instance number from port value which is composed by convnetion

    Considered several message server port values:
    - 3xNN
    - 81NN - HTTP
    - 444NN - HTTPS
    @types: int -> str?'''
    if port and str(port).isdigit():
        matchObj = re.match("(?:3\d(\d\d)|"
                            "81(\d\d)|"
                            "444(\d\d))", str(port))
        return matchObj and filter(None, matchObj.groups())[0]
    return None

def resolveEndpointAddress(resolveAddressFn, endpoint):
    r'@types: (str -> list[str]), Endpoint -> list[Endpoint]'
    endpoints = []
    if sap.createIp(endpoint.getAddress()):
        return [endpoint]
    for ip in resolveAddressFn(endpoint.getAddress()):
        endpoints.append(netutils.updateEndpointAddress(endpoint, ip))
    return endpoints

ROUTE_STR_RE = re.compile("/H/([^/]+)")
def extractDestinationFromRoute(v):
    '''Extract information about destination from route string.
    @types: str -> str?

    Example of such string,
    /H/sap_rout/H/your_rout/H/yourapp/S/sapsrv/P/pass_to_app/H/destination_address

    where destination address is taken from the latest /H/ part

    >>> route = "/H/sap_rout/H/your_rout/H/yourapp/S/sapsrv/P/pass_to_app/H/destination_address"
    >>> extractDestinationFromRoute(route)
    extractDestinationFromRoute
    >>> extractDestinationFromRoute("invalide route")
    None
    >>> extractDestinationFromRoute("/H/194.39.1.3/S/3299/H/hostname)
    hostname
    '''
    xs = ROUTE_STR_RE.findall(v)
    if xs:
        return xs[-1]

def composeGatewayServerPort(inst_nr):
    '''Compose gateway insecure port using patter 33xx based on instance number
    @types: str -> int

    >>> composeGatewayServerPort("00")
    3300
    >>> composeGatewayServerPort("91")
    3391
    '''
    if not sap.isCorrectSapInstanceNumber(inst_nr):
        raise ValueError("Specified instance number '%s' is not correct" % inst_nr)
    return int("33%s" % inst_nr)


from django.db.models import *
from django.shortcuts import render, get_object_or_404
from .models import Movie

def main(request):
    movies = Movie.objects.order_by('-rating', 'name')
    agg = movies.aggregate(Avg('budget'), Min('rating'), Max('rating'), Count('id'))
    context = {
        'movies': movies,
        'agg': agg,
        'total': movies.count()
    }
    return render(request, 'movie_app/index.html', context=context)

def detail(request, slug_movie:str):
    movie = get_object_or_404(Movie, slug=slug_movie)
    context = {
        'movie': movie,
    }
    return render(request, 'movie_app/detail.html', context=context)


"""
    portfolio construction algorithm
"""
import utl, math

def simulate(navs, times, stepval=0.2, maxsteps=None):
    """
        simulate portfolio by input navs with invest @times, input navs:
        [v1, v2, v3, ..., vn]
    or
        [[v1, pr1], [v2, pr2], ..., [vn, prn]]
    where
        v[i] is nav, pri is probability

    :param navs: list, nav random sequence
    :param times: int, invest times
    :param stepval: float, return step value
    :return: list, return probability distribution
    """
    # check input nav data
    if not (isinstance(navs, list) or isinstance(navs, tuple)):
        raise "input navs must be array."

    has_probability = None
    for nav in navs:
        if has_probability is None:
            if (isinstance(nav, list) or isinstance(nav, tuple)):
                has_probability = True
            else:
                has_probability = False
            continue
        if has_probability:
            if not ((isinstance(nav, list) or isinstance(nav, tuple)) and len(nav) == 2):
                raise "element of input navs must be (nav, probability) or nav"
        else:
            if not (isinstance(nav, int) or isinstance(nav, float)):
                raise "element of input navs must be (nav, probability) or nav"

    # transfer equal probability nav set to (nav, probability) set
    if not has_probability:
        tmpnavs, pr = [], 1.0/len(navs)
        for nav in navs:
            tmpnavs.append([nav, pr])
        navs = tmpnavs

    # combine after invested times
    combine_result = utl.math.random.combines(*[navs for i in range(0, times)])

    # generate nav probability distribution set
    prset, maxnav = [], None
    for row in combine_result:
        for elm in row:
            mnavpr = None
            for navpr in elm:
                if mnavpr is None:
                    mnavpr = navpr
                    continue
                mnavpr = utl.math.array.multi(mnavpr, navpr)

            # normalize the probability base on 1
            mnavpr = [mnavpr[0], mnavpr[1]]

            # record the max nav
            if maxnav is None:
                maxnav = mnavpr[0]
            else:
                if mnavpr[0]>maxnav:
                    maxnav = mnavpr[0]

            # add to combine nav probability set
            prset.append(mnavpr)

    # compbine nav probability distribution statistic
    steps = math.ceil(maxnav/stepval)+1
    dists = [[i*stepval, 0.0] for i in range(1, steps+1)]
    for nav, pr in prset:
        pos = math.floor(nav/stepval)
        dists[pos][1] = dists[pos][1] + pr

    # only get the max step nav values
    if maxsteps is not None and maxsteps<steps:
        dists = dists[0:maxsteps]

    return dists, prset

def simulate_to_plot(*funds):
    """
        plot simulate result
    :param funds:
    :return:
    """
    # change the invest times, step value, max steps here
    times, stepval, maxsteps = 5, 0.2, None

    # prepare the plot
    import matplotlib.pyplot as plt
    plt.figure(figsize=(16, 8))
    plt.title("portfolio simulate: invest times(%d), step by(%s), max step(%s)" % (times, str(stepval), str(maxsteps)))
    plt.xlabel("return" )
    plt.ylabel("probability")

    # compute each fund's return probability
    num = 1 # fund number from 1
    for navs in funds:
        result, prset = simulate(navs, times, stepval, maxsteps)
        x, y = utl.math.matrix.subcol(result, 1), utl.math.matrix.subcol(result, 2)
        plt.plot(x, y, label="$Fund%d-%s$" % (num, str(navs)))
        num += 1

    # show the plot
    plt.legend()
    plt.show()

def simulate_to_file(*funds):
    """

    :param navs:
    :return:
    """
    # change the invest times, step value, max steps here
    times, stepval, maxsteps = 5, 0.2, None

    # change the file output directory path
    outdir = "./simulate"

    # create the file output directory
    import os
    if not os.path.exists(outdir):
        os.makedirs(outdir)

    # simulate portfolio, write to file
    num = 1
    for navs in funds:
        result, prset = simulate(navs, times, stepval, maxsteps)

        # write return probability distribution to file
        arrres = ["return-floor, return-ceil, probability"]
        for ret, pr in result:
            arrres.append("%f, %f, %f" % (ret-stepval, ret, pr))
        strres = "\n".join(arrres)
        fpath = "%s/fund-return-range-prob%d" % (outdir, num)
        with open(fpath, "w") as f:
            f.write(strres)
            f.close()

        # write original return probability set to file
        arrres = ["return, probability"]
        for ret, pr in prset:
            arrres.append("%f, %f" % (ret, pr))
        strres = "\n".join(arrres)
        fpath = "%s/fund-return-prob%d" % (outdir, num)
        with open(fpath, "w") as f:
            f.write(strres)
            f.close()

        num += 1

if __name__ == "__main__":
    # input founds, A, B, C, D, ...
    fundA = [0.05, 0.2, 1, 3, 3, 3]
    fundB = [0.8, 0.9, 1.1, 1.1, 1.2, 1.4]
    fundC = [0.95, 1, 1, 1, 1, 1.1]
    fundD = [[0.05, 0.2], [1, 0.2], [1, 0.1], [1, 0.2], [1, 0.1], [2.1, 0.2]]

    # simulate result to plot
    simulate_to_plot(fundA, fundB, fundC, fundD)

    # simulate result to file
    #simulate_to_file(fundA, fundB, fundC, fundD)


import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.model_selection import train_test_split
from datetime import datetime
import random
import time

from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, LSTM, Dropout, BatchNormalization
from tensorflow.keras.callbacks import TensorBoard
from tensorflow.keras.callbacks import ModelCheckpoint
import tensorflow as tf
import os

## --------------------- config -------------------
## define steps to go back to predict, Try steps = [15,50]

steps = 17
# threshold for labelling
alpha = 0.0025

# model values
epochs = 150
batch_size = 32

# labbeling values
smallChange = .025
bigChange = .05

### DATASETS
d_dataset = pd.read_csv('Data/complete_dataset_v9.csv')

## ---------------------- end config ----------------

# select open and high values only (1:2 = position in dataset = open:1)
d_dataset_open = d_dataset.iloc[:, 2:].values

print(d_dataset.describe())
### DATA SCALER
from sklearn.preprocessing import MinMaxScaler, normalize
from sklearn import preprocessing

# scale bitcoin values
min_max_scaler = MinMaxScaler()
minmax_scale = preprocessing.MinMaxScaler().fit(d_dataset_open)
df_minmax = minmax_scale.transform(d_dataset_open)

# print('scaled data: ', df_minmax)
# maxInColumns = np.amax(df_minmax,axis=0)
# print('max values in column', maxInColumns)

## Creating data with timestamps
dataSequence = []
# labelUp = 0
# labelDown = 0
# labelSame = 0

bigDecrease = 0
smallDecrease = 0
staysTheSame = 0
smallIncrease = 0
bigIncrease = 0

while steps < 18:
    timeNow = datetime.now().strftime("%d.%m.%Y_%H.%M.%S")
    NAME = f"{steps}-{str(timeNow)}"
    # take x steps the open value and append them to x_train to predict 61st open price, which is put into y_train
    dataSequence = []
    for i in range(steps, len(df_minmax)):

        # check if followed step is higher [2] same [1] or lower [0]

        today = df_minmax[i-1, 0]
        tomorrow = df_minmax[i, 0]
        # day_after_tomorrow=

        # calculate relative change form tomorrow to today
        change = tomorrow/today

        # 1: big decrease
        if change < 1-bigChange:
            ylabel = 0
            bigDecrease += 1
        # 2 small decrease
        elif change < 1-smallChange:
            ylabel = 1
            smallDecrease += 1
        # 3 stays the same
        elif change > 1-smallChange and change < 1+smallChange:
            ylabel = 2
            staysTheSame += 1
        # 4 small increase
        elif change > 1+smallChange and change < 1+bigChange:
            ylabel = 3
            smallIncrease += 1
        # 5 big decrease
        else:
            ylabel = 4
            bigIncrease += 1

        """ old labelling process
         ylabel = 1
        if (tomorrow - today) > alpha:
            ylabel = 2
            labelUp+=1
        elif (tomorrow - today) < (-alpha):
            ylabel = 0
            labelDown+=1
        else:
            labelSame+=1
        """
        btc_value = df_minmax[i-steps:i, :df_minmax.shape[1]]
        # print('btc value: ', btc_value)

        # add classifications (up down stays the same) as labels
        # dataSequence.append([btc_value, ylabel])

        # add absolute values as labels
        dataSequence.append([btc_value, tomorrow])
    # random data
    random.shuffle(dataSequence)

    print(len(dataSequence))

    # SPLIT DATA SET

    # check our labeling
    # print("big decrease: ", bigDecrease)
    # print("small decrease:",smallDecrease)
    # print("stays the same",staysTheSame)
    # print("small increase: ", smallIncrease)
    # print("big increase: ",bigIncrease)

    ### SPLIT DATASET INTO TRAINING, VALIDATION AND TEST DATA
    # split data into x and y data
    X = []
    y = []

    for seq, target in dataSequence:  # going over our new sequential data
        X.append(seq)  # X is the sequences
        y.append(target)  # y is the targets/labels (bitcoinprice)

    # Split dataset into 70% training, 10% Validation and 20% Test data
    n_cases = len(X)
    n_train = int(round(n_cases*.7))
    n_val = int(round(n_cases*.1))
    n_test = int(round(n_cases*.2))

    y_train = y[:n_train]
    y_val = y[n_train:n_train+n_val]
    y_test = y[n_train+n_val:n_cases]

    X_train = X[:n_train]
    X_val = X[n_train:n_train+n_val]
    X_test = X[n_train+n_val:n_cases]

    ### TRANSFORM DATA INTO NUMPY ARRAY
    # change to numpy array with shape (Amount of rows, features per row)
    X_train, y_train = np.array(X_train), np.array(y_train)
    X_val, y_val = np.array(X_val), np.array(y_val)
    X_test, y_test = np.array(X_test), np.array(y_test)
    # reshape to same architecture
    X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], X_train.shape[2]))
    X_val = np.reshape(X_val, (X_val.shape[0], X_val.shape[1], X_val.shape[2]))
    X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], X_test.shape[2]))

    ### BUILDING THE LSTM
    model = Sequential()
    # 1 Layer
    model.add(
        LSTM(units=256, recurrent_dropout=0.1, return_sequences=True, input_shape=(X_train.shape[1], X_train.shape[2])))
    model.add(Dropout(0.1))
    # add(Dropout(0.2))
    model.add(BatchNormalization())
    # 2 Layer
    model.add(LSTM(units=256, recurrent_dropout=0.1, return_sequences=True))
    model.add(Dropout(0.1))
    # model.add(Dropout(0.2))<
    model.add(BatchNormalization())
    # 3 Layer
    model.add(LSTM(units=256, recurrent_dropout=0.1, return_sequences=True))
    model.add(Dropout(0.1))
    # model.add(Dropout(0.2))
    model.add(BatchNormalization())
    # 4 Layer
    model.add(LSTM(units=256, recurrent_dropout=0.1))
    model.add(Dropout(0.1))
    # model.add(Dropout(0.2))
    model.add(BatchNormalization())
    # 5 Layer
    model.add(Dense(16, activation='sigmoid'))
    model.add(Dropout(0.1))
    # model.add(Dense(16, activation='relu'))
    # model.add(Dropout(0.2))

    # 6 Layer
    # model.add(Dense(5, activation='softmax'))

    # 6 layer
    # add dense layer for absolute value prediction
    model.add(Dense(units=1))

    ### optimizing for classification###
    # try lr=0.1 or 0.01
    opt = tf.keras.optimizers.Adam(lr=0.001, decay=1e-4)
    # Compile the model
    # model.compile(
    #   loss='sparse_categorical_crossentropy',
    #  optimizer=opt,
    # metrics=['accuracy']

    # )
    # model.summary()

    ###optimizing with absolute values
    ##metrics can be exchanged: mean_absolute_percentage_error , mean_absolute_error
    model.compile(
        loss='mean_squared_error',
        optimizer=opt,
        metrics=['mean_absolute_percentage_error', 'mean_absolute_error']
    )

    ### CREATE TENSORBOARD TO DISPLAY THE VALIDATION
    tensorboard = TensorBoard(log_dir="logs\\{}".format(NAME))
    filepath = "RNN_Final-{epoch:02d}-{val_loss:.3f}"  # unique file name that will include the epoch and the validation acc for that epoch
    checkpoint = ModelCheckpoint(
        "models\\{}.model".format(filepath, monitor='mean_absolute_percentage_error, mean_absolute_error', verbose=1,
                                 save_best_only=True, mode='max'))  # saves only the best ones

    ### FIT THE MODEL
    history = model.fit(
        X_train, y_train,
        epochs=epochs,
        batch_size=batch_size,
        validation_data=(X_val, y_val),
        callbacks=[tensorboard, checkpoint],
    )

    """
    # summarize history for loss
    def plot_graphs(history, string):
        plt.plot(history.history[string])
        plt.plot(history.history['val_'+string])
        plt.xlabel("Epochs")
        plt.ylabel(string)
        plt.legend([string, 'val_'+string])
        plt.show()

    plot_graphs(history, "accuracy")
    plot_graphs(history, "loss")

    # summarize history for loss
    plt.plot(history.history["loss"])
    plt.plot(history.history["val_loss"])
    plt.show()

    """

    # Score model
    score = model.evaluate(X_val, y_val, verbose=0)
    # print('Test loss:', score[0])
    # print('Test accuracy:', score[1])

    steps += 1
    # Save model
    model.save("models\\{}".format(NAME))

# predict price
predicted_stock_price = model.predict(X_test)

# inverse Transform the values back
# add zeros to have the same shape
predicted_stock_price_inverseTransform = np.zeros(shape=(len(predicted_stock_price), d_dataset_open.shape[1]))
predicted_stock_price_inverseTransform[:, 0] = predicted_stock_price[:, 0]
predicted_stock_price_inverseTransform = minmax_scale.inverse_transform(predicted_stock_price_inverseTransform)

# X_test_inverseTransform = np.zeros(shape = (len(X_test),d_dataset_open.shape[1]))
# X_test_inverseTransform[:,0] = X_test[:,0,0]
# X_test_inverseTransform = minmax_scale.inverse_transform(X_test_inverseTransform)
# inverse transform the values back
# predicted_stock_price =min_max_scaler.inverse_transform(predicted_stock_price)
# y_test = min_max_scaler.inverse_transform(y_test.reshape(-1,1))

y_test_inverseTransform = np.zeros(shape=(len(y_test), d_dataset_open.shape[1]))
y_test_inverseTransform[:, 0] = y_test[:]
y_test_inverseTransform = minmax_scale.inverse_transform(y_test_inverseTransform)

testValues2 = np.zeros(shape=(len(y_test), 3))

"""
#createArray with test values
testValues = np.zeros(shape = (len(X_test),3))
testValues2 = testValues
# insert actual Values
testValues[:,0] =  X_test_inverseTransform[:,0]
testValues[:,1] =  predicted_stock_price_inverseTransform[:,0]
testValues[:,2] = np.divide(testValues[:,0],np.absolute(np.subtract(X_test_inverseTransform[:,0],predicted_stock_price_inverseTransform[:,0])))
"""

# print('TestWerte',testValues)
# print('Durchschnittsabweichung', testValues[2].mean()/100,'%')

# insert actual Values

# real value
testValues2[:, 0] = y_test_inverseTransform[:, 0]
# predicted value
testValues2[:, 1] = predicted_stock_price_inverseTransform[:, 0]

# relative difference of actual and predicted value
testValues2[:, 2] = np.divide(np.absolute(np.subtract(testValues2[:, 0], testValues2[:, 1])), testValues2[:, 0])

# calculate average relative difference over all predictions
totalAveragePrediction = np.mean(testValues2, axis=0)[2]
print('TestWerte2', testValues2)
print('Durchschnittsabweichung', totalAveragePrediction*100, '%')

# convert array into dataframe
DF = pd.DataFrame(testValues2)
DF.columns = ['Real Value', 'Predicted Value', 'Relative Difference']
# save the dataframe as a csv file
DF.to_csv("Data/Output/trainedData.csv", index=False)


from matplotlib import pyplot as plt
import numpy as np
data = np.load("stats(1).npy")
#data2 = np.load("stats.npy")

test = "More Epochs"
test2 = "GRU"
def save_fig(data,test):

    plt.plot(data[1:,2],label='Loss')
    #plt.plot(data[:,0]+1*data[:,1],data[:,3])

    plt.title("Hyperparameter Tuning: " + test)
    plt.xlabel("Iteration Number (*10)")
    plt.ylabel("Loss/Perplexity")

    plt.plot(data[20:,3],label='Perplexity')
    #plt.plot(data[:,0]+1*data[:,1],data[:,3])
    plt.legend()
    plt.savefig(test+".png")
    plt.close()
save_fig(data,test)
#save_fig(data2,test2)


import cv2
import numpy as np
cam=cv2.VideoCapture(0)
p1_x=(int)(cam.get(cv2.CAP_PROP_FRAME_WIDTH)*3//4)
p1_y=(int)(cam.get(cv2.CAP_PROP_FRAME_HEIGHT)*3.5//4)
p2_x=(int)(cam.get(cv2.CAP_PROP_FRAME_WIDTH)//4)
p2_y=(int)(cam.get(cv2.CAP_PROP_FRAME_HEIGHT)//4)
while True:
    ret, im =cam.read()
    cv2.rectangle(im,pt1=(p2_x,p2_y),pt2=(p1_x,p1_y),color=(0,0,255),thickness=5)
    cv2.imshow('im',im)
    if cv2.waitKey(10) & 0xFF==ord('q'):
        break

cam.release()
cv2.destroyAllWindows()


"""Compatibility tools between Python 2 and Python 3 I/O interfaces.
"""

from __future__ import print_function
from __future__ import unicode_literals

import io
from io import SEEK_SET, SEEK_CUR

from .mode import Mode

class RawWrapper(io.IOBase):
    """Convert a Python 2 style file-like object in to a IO object.
    """

    def __init__(self, f, mode=None, name=None):
        self._f = f
        self.mode = mode or getattr(f, 'mode', None)
        self.name = name
        super(RawWrapper, self).__init__()

    def close(self):
        if not self.closed:
            # Close self first since it will
            # flush itself, so we can't close
            # self._f before that
            super(RawWrapper, self).close()
            self._f.close()

    def fileno(self):
        return self._f.fileno()

    def flush(self):
        return self._f.flush()

    def isatty(self):
        return self._f.isatty()

    def seek(self, offset, whence=SEEK_SET):
        return self._f.seek(offset, whence)

    def readable(self):
        return getattr(
            self._f,
            'readable',
            lambda: Mode(self.mode).reading
        )()

    def writable(self):
        return getattr(
            self._f,
            'writable',
            lambda: Mode(self.mode).writing
        )()

    def seekable(self):
        try:
            return self._f.seekable()
        except AttributeError:
            try:
                self.seek(0, SEEK_CUR)
            except IOError:
                return False
            else:
                return True

    def tell(self):
        return self._f.tell()

    def truncate(self, size=None):
        return self._f.truncate(size)

    def write(self, data):
        count = self._f.write(data)
        return len(data) if count is None else count

    def read(self, n=-1):
        if n == -1:
            return self.readall()
        return self._f.read(n)

    def read1(self, n=-1):
        if hasattr(self._f, 'read1'):
            return self._f.read1(n)
        return self.read(n)

    def readall(self):
        return self._f.read()

    def readinto(self, b):
        try:
            return self._f.readinto(b)
        except AttributeError:
            data = self._f.read(len(b))
            bytes_read = len(data)
            b[:len(data)] = data
            return bytes_read

    def readinto1(self, b):
        try:
            return self._f.readinto1(b)
        except AttributeError:
            data = self._f.read1(len(b))
            bytes_read = len(data)
            b[:len(data)] = data
            return bytes_read

    def readline(self, limit=-1):
        return self._f.readline(limit)

    def readlines(self, hint=-1):
        return self._f.readlines(hint)

    def writelines(self, sequence):
        return self._f.writelines(sequence)

    def __iter__(self):
        return iter(self._f)

def make_stream(name,
                bin_file,
                mode='r',
                buffering=-1,
                encoding=None,
                errors=None,
                newline='',
                line_buffering=False,
                **kwargs):
    """Take a Python 2.x binary file and return an IO Stream.
    """
    reading = 'r' in mode
    writing = 'w' in mode
    appending = 'a' in mode
    binary = 'b' in mode
    if '+' in mode:
        reading = True
        writing = True

    encoding = None if binary else (encoding or 'utf-8')

    io_object = RawWrapper(bin_file, mode=mode, name=name)
    if buffering >= 0:
        if reading and writing:
            io_object = io.BufferedRandom(
                io_object,
                buffering or io.DEFAULT_BUFFER_SIZE
            )
        elif reading:
            io_object = io.BufferedReader(
                io_object,
                buffering or io.DEFAULT_BUFFER_SIZE
            )
        elif writing or appending:
            io_object = io.BufferedWriter(
                io_object,
                buffering or io.DEFAULT_BUFFER_SIZE
            )

    if not binary:
        io_object = io.TextIOWrapper(
            io_object,
            encoding=encoding,
            errors=errors,
            newline=newline,
            line_buffering=line_buffering,
        )

    return io_object

def line_iterator(readable_file, size=None):
    """Iterate over the lines of a file.

    Implementation reads each char individually, which is not very
    efficient.

    Yields:
        str: a single line in the file.

    """
    read = readable_file.read
    line = []
    byte = b'1'
    if size is None or size < 0:
        while byte:
            byte = read(1)
            line.append(byte)
            if byte in b'\n':
                yield b''.join(line)
                del line[:]

    else:
        while byte and size:
            byte = read(1)
            size -= len(byte)
            line.append(byte)
            if byte in b'\n' or not size:
                yield b''.join(line)
                del line[:]


#!/usr/bin/env python

from setuptools import find_packages, setup

__version__ = "0.1.1"

with open("README.rst") as readme_file:
    readme = readme_file.read()

setup(
    name="openflowthrough",
    version=__version__,
    keywords="openflowthrough",
    packages=find_packages(include=["openflowthrough", "openflowthrough.*"]),
    author="Drew Meyers",
    author_email="drewm@mit.edu",
    python_requires=">=3.6",
    classifiers=[
        "License :: OSI Approved :: MIT License",
        "Natural Language :: English",
        "Programming Language :: Python :: 3",
        "Programming Language :: Python :: 3.6",
        "Programming Language :: Python :: 3.7",
        "Programming Language :: Python :: 3.8",
        "Operating System :: OS Independent",
        "Intended Audience :: Science/Research",
        "Topic :: Scientific/Engineering :: Chemistry",
    ],
    description="Software for controlling the Open Flow-through sampling device.",
    long_description=readme,
    url="https://github.com/drewmee/openflowthrough",
    license="MIT license",
    install_requires=["pyFirmata>=1.1.0", "transitions==0.8.8"],
    extras_require={
        "docs": [
            "sphinx>=3.2.0",
            "sphinx-automodapi>=0.12",
            "sphinx-rtd-theme>=0.5.0",
            "msmb_theme>=1.2.0",
            "nbsphinx>=0.7.1",
            "sphinx-copybutton>=0.3.0",
            "black>=20.8b1",
            "isort>=5.4.2",
            "rstcheck>=3.3.1",
        ],
        "develop": ["twine>=3.2.0", "pre-commit>=2.8.2"],
        "tests": ["pytest>=3"],
    },
    test_suite="tests",
    package_data={"openflowthrough": ["json/fsm.json"]},
    include_package_data=True,
    zip_safe=False,
)


from airflow import DAG
from airflow.providers.postgres.operators.postgres import PostgresOperator
# from airflow.operators.postgres_operator import PostgresOperator
from datetime import datetime, timedelta

with DAG(
    'greenplum_dag',
    start_date=datetime(2021, 1, 1),
    max_active_runs=3,
    schedule_interval='@daily',
    default_args={
        "email_on_failure": False,
        "email_on_retry": False,
        "retries": 0,
    },
    catchup=False
) as dag:

    opr_test_query = PostgresOperator(
        task_id='test_query',
        postgres_conn_id='gpdb',
        sql='SELECT * FROM covid_state_data'
    )


import torch
import torchvision.transforms as transforms
import numpy as np
from PIL import Image
from torch.utils.data.dataset import Dataset

class DeblurDataset(Dataset):
    def __init__(self, img_path, args, is_train=True):
        self.img_path = img_path
        self.args = args
        self.is_train = is_train
        self.transform = transforms.Compose([transforms.ToTensor(),
                                             transforms.Normalize([0.5], [0.5])])

    def add_margin(self, pil_img, new_size):
        width, height = pil_img.size
        new_width = new_size[1]
        new_height = new_size[0]

        left = (new_width - width) // 2
        top = (new_height - height) // 2
        result = Image.new(pil_img.mode, (new_width, new_height), 0)
        result.paste(pil_img, (left, top))
        return result

    def __getitem__(self, index):
        """
        Transforms:
        1. Resize image to (fine_size, fine_size)  for  CycleGAN fine_size = 64
        2. random flip image from left to right
        3. PIL.Image(H,W,C) to Tensor(C,H,W)
        4. normalize from [0.0, 1.0] to [-1.0, 1.0]
        """

        line = self.img_path[index]
        img_path, label = line.split('\t')

        img_A = Image.open(img_path + '_blur.png').convert('L')
        img_B = Image.open(img_path + '_sharp.png').convert('L')
        img_name = img_path[-4:]
        img_name.rstrip()

        # Downsample the blurry image to (64, 64)
        img_A = img_A.resize((self.args.fine_size, self.args.fine_size), resample=Image.BILINEAR)

        if self.is_train:
            img_B = img_B.resize((self.args.fine_size, self.args.fine_size), resample=Image.BILINEAR)

        if self.is_train:
            if np.random.random() < 0.5:
                img_A = img_A.transpose(Image.FLIP_LEFT_RIGHT)
                img_B = img_B.transpose(Image.FLIP_LEFT_RIGHT)
            if np.random.random() < 0.5:
                img_A = img_A.transpose(Image.FLIP_TOP_BOTTOM)
                img_B = img_B.transpose(Image.FLIP_TOP_BOTTOM)

        img_A = self.transform(img_A)
        img_B = self.transform(img_B)

        return img_A, img_B, img_name

    def __len__(self):
        return len(self.img_path)

class RealImage(Dataset):
    def __init__(self, img_path, args, is_train=True):
        self.img_path = img_path
        self.args = args
        self.is_train = is_train
        self.transform = transforms.Compose([transforms.ToTensor(),
                                             transforms.Normalize((0.5,), (0.5,))])

    def add_margin(self, pil_img, new_size):
        width, height = pil_img.size
        new_width = new_size[1]
        new_height = new_size[0]

        left = (new_width - width) // 2
        top = (new_height - height) // 2
        result = Image.new(pil_img.mode, (new_width, new_height), 0)
        result.paste(pil_img, (left, top))
        return result

    def __getitem__(self, index):
        img_A = Image.open(self.img_path[index] + '.png').convert('L')
        img_name = self.img_path[index][-4:]
        img_name.rstrip()

        w = int(img_A.size[0])
        h = int(img_A.size[1])

        if w % 4 != 0 or h % 4 != 0:
            new_h = (h // 4 + 1) * 4 if h % 4 != 0 else h
            new_w = (w // 4 + 1) * 4 if w % 4 != 0 else w
            img_A = self.add_margin(img_A, (new_h, new_w))

        img_A = self.transform(img_A)

        return img_A, img_name

    def __len__(self):
        return len(self.img_path)


__all__ = ["TrainBPEModelJob", "ReturnnTrainBpeJob"]

import subprocess as sp
import os
import sys
from typing import Optional

from sisyphus import *

import i6_core.util as util

Path = setup_path(__package__)

class TrainBPEModelJob(Job):
    """
    Create a bpe codes file using the official subword-nmt repo, either installed from pip
    or https://github.com/rsennrich/subword-nmt

    This job is deprecated, to create BPE codes that are compatible with legacy (non-sisyphus) RETURNN setups
    using e.g. language models from Kazuki, please use the `ReturnnTrainBpeJob`.

    Otherwise, please consider using the `sentencepiece` implementation.
    """

    def __init__(
        self,
        text_corpus,
        symbols=1000,
        min_frequency=2,
        dict_input=False,
        total_symbols=False,
        subword_nmt_repo=None,
    ):
        """
        :param Path text_corpus:
        :param int symbols:
        :param int min_frequency:
        :param bool dict_input:
        :param bool total_symbols:
        :param Optional[Path] subword_nmt_repo:
        """
        self.text_corpus = text_corpus
        self.symbols = symbols
        self.min_frequency = min_frequency
        self.dict_input = dict_input
        self.total_symbols = total_symbols

        self.subword_nmt_repo = util.get_subword_nmt_repo(subword_nmt_repo)

        self.out_code_file = self.output_path("codes")

        self.rqmt = {"cpu": 1, "mem": 2, "time": 4}

    def tasks(self):
        yield Task("run", resume="run", rqmt=self.rqmt)

    def run(self):
        train_binary = self.subword_nmt_repo.join_right("subword_nmt/learn_bpe.py")
        args = [
            sys.executable,
            train_binary.get_path(),
            "-o",
            self.out_code_file.get_path(),
            "-s",
            str(self.symbols),
            "--min-frequency",
            str(self.min_frequency),
        ]
        if self.dict_input:
            args += ["--dict-input"]
        if self.total_symbols:
            args += ["--total-symbols"]

        text_corpus = self.text_corpus.get_path()

        with util.uopen(text_corpus, "rb") as f:
            p = sp.Popen(args, stdin=sp.PIPE, stdout=sys.stdout, stderr=sys.stderr)
            while True:
                data = f.read(4096)
                if len(data) > 0:
                    p.stdin.write(data)
                else:
                    break

            p.communicate()
            assert p.returncode == 0

class ReturnnTrainBpeJob(Job):
    """
    Create Bpe codes and vocab files compatible with RETURNN BytePairEncoding
    Repository:
        https://github.com/rwth-i6/subword-nmt

    This job can be used to produce BPE codes compatible to legacy (non-sisyphus) RETURNN setups.

    Outputs:
        - bpe_codes: the codes file to apply BPE to any text
        - bpe_vocab: the index vocab in the form of {"<token>": <index>, ...} that can be used e.g. for RETURNN
            Will contain <s> and </s> pointing to index 0 and the unk_label pointing to index 1
        - bpe_dummy_count_vocab: a text file containing all words, to be used with the `ApplyBPEToTextJob`
            DOES NOT INCLUDE COUNTS, but just set each count to -1. Is used to not cause invalid merges
            when converting text to the BPE form.
        - vocab_size: variable containing the number of indices
    """

    def __init__(
        self,
        text_file: tk.Path,
        bpe_size: int,
        unk_label: str = "UNK",
        subword_nmt_repo: Optional[tk.Path] = None,
    ):
        """
        :param text_file: corpus text file, .gz compressed or uncompressed
        :param int bpe_size: number of BPE merge operations
        :param str unk_label: unknown label
        :param Path|None subword_nmt_repo: subword nmt repository path. see also `CloneGitRepositoryJob`
        """
        self.text_file = text_file
        self.bpe_size = bpe_size
        self.subword_nmt_repo = util.get_subword_nmt_repo(subword_nmt_repo)
        self.unk_label = unk_label

        self.out_bpe_codes = self.output_path("bpe.codes")
        self.out_bpe_vocab = self.output_path("bpe.vocab")
        self.out_bpe_dummy_count_vocab = self.output_path("bpe.dummy_count.vocab")
        self.out_vocab_size = self.output_var("vocab_size")

    def run(self):
        bpe_codes_cmd = [
            sys.executable,
            os.path.join(self.subword_nmt_repo.get_path(), "learn_bpe.py"),
            "--output",
            self.out_bpe_codes.get_path(),
            "--symbols",
            str(self.bpe_size),
        ]

        util.create_executable("create_bpe_codes.sh", bpe_codes_cmd)

        with util.uopen(self.text_file, "rb") as f:
            p = sp.Popen(bpe_codes_cmd, stdin=sp.PIPE, stdout=sys.stdout, stderr=sys.stderr)
            while True:
                data = f.read(4096)
                if len(data) > 0:
                    p.stdin.write(data)
                else:
                    break
            p.communicate()
            assert p.returncode == 0

        bpe_vocab_cmd = [
            sys.executable,
            os.path.join(self.subword_nmt_repo.get_path(), "create-py-vocab.py"),
            "--txt",
            self.text_file.get_path(),
            "--bpe",
            self.out_bpe_codes.get_path(),
            "--unk",
            self.unk_label,
            "--out",
            self.out_bpe_vocab.get_path(),
        ]

        util.create_executable("create_bpe_vocab.sh", bpe_vocab_cmd)
        sp.run(bpe_vocab_cmd, check=True)

        with util.uopen(self.out_bpe_vocab) as f, util.uopen(self.out_bpe_dummy_count_vocab, "wt") as txt_vocab:
            vocab = eval(f.read())
            num_labels = max(list(vocab.values())) + 1  # 0-based index
            self.out_vocab_size.set(num_labels)
            for l in vocab.keys():
                txt_vocab.write(f"{l} -1\n")

    def tasks(self):
        yield Task("run", rqmt={"cpu": 1, "mem": 2, "time": 4})