import datetime
import logging
import os
import sys
from datetime import date, timedelta
from os.path import exists
from pandas.tseries.offsets import *

import numpy as np
import pandas as pd
from numpy.core.arrayprint import _formatArray
from pandas._libs.tslibs import Timedelta, Timestamp

from bsbetl import g
from bsbetl.func_helpers import save_runtime_config
from bsbetl.helpers import append_csv_maxsize_log
from bsbetl.ov_calcs import ov_columns

def ov_dataframe_to_store(df: pd.DataFrame, overview_key: str, stage: int, sharelist_name :str):
    """ save overview dataframe to HDFStore """

    assert sharelist_name in overview_key, "ov_dataframe_to_store must have sharelist_name in its key"

    ov_fn = g.OVERVIEW_STORE_FQ.format(stage)
    data_store = pd.HDFStore(ov_fn)
    # we need a 'natural name' for the key
    ov_key=overview_key.replace('.shl','')
    data_store[ov_key] = df
    data_store.close()
    logging.info(f"saved {ov_fn} under key {ov_key}")

def _1St_results_dataframe_to_store(df: pd.DataFrame, sharelist :str):
    """ save dataframe to HDFStore """

    #df.to_hdf(g._1ST_RESULTS_STORE_FQ,key=g.HDFSTORE_1ST_RESULTS_KEY,mode='w',format='table')

    data_store = pd.HDFStore(g._1ST_RESULTS_STORE_FQ)
    #data_store[g.HDFSTORE_1ST_RESULTS_KEY] = df
    data_store.put(key=g.HDFSTORE_1ST_RESULTS_KEY.format(sharelist.replace('.shl','')),value=df,format='fixed')

    logging.info(f'saved 1St results to _1ST_RESULTS datastore ({df.shape[0]} rows) under key {sharelist}')
    data_store.close()

def _2StPr_results_dataframe_to_store(df: pd.DataFrame, sharelist :str):
    """ save dataframe to HDFStore """

    data_store = pd.HDFStore(g._2STPR_RESULTS_STORE_FQ)
    store_key = g.HDFSTORE_2STPR_RESULTS_KEY.format(sharelist.replace('.shl',''))
    data_store.put(key=store_key,value=df,format='fixed')

    logging.info(f'saved 2StPr results to _2STPR_RESULTS datastore ({df.shape[0]} rows) under key {sharelist}')
    data_store.close()

def _2StVols_results_dataframe_to_store(df: pd.DataFrame, sharelist :str):
    """ save dataframe to HDFStore """

    data_store = pd.HDFStore(g._2STVOLS_RESULTS_STORE_FQ)
    data_store.put(key=g.HDFSTORE_2STVOLS_RESULTS_KEY.format(sharelist.replace('.shl','')),value=df,format='fixed')

    logging.info(f'saved 2StVols results to _2STVOLS_RESULTS datastore ({df.shape[0]} rows) under key {sharelist}')
    data_store.close()

def _combined_1_2_results_dataframe_to_store(df: pd.DataFrame, sharelist :str):
    """ save dataframe to HDFStore """

    store_fn = g._COMBINED_1_2_RESULTS_STORE_FQ
    store_key=g.HDFSTORE_COMBINED_1_2_RESULTS_KEY.format(sharelist.replace('.shl',''))
    logging.info(f'saving combined_1_2 results to {store_fn} under key {store_key} ({df.shape[0]} rows)')

    data_store = pd.HDFStore(store_fn)
    data_store.put(key=store_key, value=df, format='fixed')
    data_store.close()

def _3jP_results_dataframe_to_store(df: pd.DataFrame, sharelist :str, which_part):
    """ save dataframe to HDFStore """

    if which_part=='_3jP_part1':
        data_store = pd.HDFStore(g._3JP_RESULTS_PART1_STORE_FQ)
        filekey=g.HDFSTORE_3JP_RESULTS_PART1_KEY.format(sharelist.replace('.shl',''))

        data_store.put(key=filekey,value=df,format='fixed')
        logging.info(f"saved 3jP part 1 results to {g._3JP_RESULTS_PART1_STORE_FQ} datastore under key '{filekey}' ({df.shape[0]} rows)")
        data_store.close()
    elif which_part=='_3jP_part2':
        data_store = pd.HDFStore(g._3JP_RESULTS_PART2_STORE_FQ)
        filekey=g.HDFSTORE_3JP_RESULTS_PART2_KEY.format(sharelist.replace('.shl',''))

        data_store.put(key=filekey,value=df,format='fixed')
        logging.info(f"saved 3jP part 2 results to {g._3JP_RESULTS_PART2_STORE_FQ} datastore under key '{filekey}' ({df.shape[0]} rows)")
        data_store.close()
    else: # final
        data_store = pd.HDFStore(g._3JP_RESULTS_FINAL_STORE_FQ)
        filekey=g.HDFSTORE_3JP_RESULTS_FINAL_KEY.format(sharelist.replace('.shl',''))

        data_store.put(key=filekey,value=df,format='fixed')
        logging.info(f"saved 3jP final results to {g._3JP_RESULTS_FINAL_FQ} datastore under key '{filekey}' ({df.shape[0]} rows)")
        data_store.close()

def _3V2d_results_dataframe_to_store(df: pd.DataFrame, sharelist :str):
    """ save dataframe to HDFStore """

    data_store = pd.HDFStore(g._3V2D_RESULTS_STORE_FQ)
    filekey=g.HDFSTORE_3V2D_RESULTS_KEY.format(sharelist.replace('.shl',''))

    data_store.put(key=filekey, value=df, format='fixed')

    logging.info(f"saved 3V2d results to {g._3V2D_RESULTS_STORE_FQ} datastore under key '{filekey}' ({df.shape[0]} rows)")
    data_store.close()

def _3nH_results_dataframe_to_store(df: pd.DataFrame, sharelist :str):
    """ save dataframe to HDFStore """

    data_store = pd.HDFStore(g._3NH_RESULTS_STORE_FQ)
    filekey=g.HDFSTORE_3NH_RESULTS_KEY.format(sharelist.replace('.shl',''))

    data_store.put(key=filekey, value=df, format='fixed')

    logging.info(f"saved 3nH results to {g._3NH_RESULTS_STORE_FQ} datastore under key '{filekey}' ({df.shape[0]} rows)")
    data_store.close()

def frt_results_dataframe_to_store(df: pd.DataFrame, sharelist :str):
    """ save dataframe to HDFStore """

    data_store = pd.HDFStore(g._FRT_RESULTS_STORE_FQ)
    filekey=g.HDFSTORE_FRT_RESULTS_KEY.format(sharelist.replace('.shl',''))

    data_store.put(key=filekey, value=df, format='fixed')

    logging.info(f"saved FRT results to {g._FRT_RESULTS_STORE_FQ} datastore under key '{filekey}' ({df.shape[0]} rows)")
    data_store.close()

def share_dataframe_to_store(df: pd.DataFrame, share_num: str, stage: int):
    """ save share dataframe to HDFStore

    NOTE Existing dataframe, if any, gets replaced
    """

    data_store = pd.HDFStore(g.SHARE_STORE_FQ.format(stage))
    # we need a 'natural name' for the key
    natural_key = share_num[-3:] + '_' + share_num[0:-4]

    data_store[natural_key] = df  # alternative 'table' format below
    # df.to_hdf(g.SHARE_STORE_FQ, natural_key, format="table")

    logging.info(f'saved {natural_key} in SHARES_{stage} datastore')
    data_store.close()

def add_calc_columns(df_bh: pd.DataFrame):
    """  Add all extra required columns to df_bh Note: price & volume - assumed present """

    for col in g.AT_COLS[2:]:  # skip price and volume
        arr = np.zeros(len(df_bh.index))
        arr[:] = np.nan
        df_bh[col] = arr

def check_share_num(value: str) -> bool:

    ok = False
    for index, bourse in enumerate(g.BOURSES_LIST):
        if value.endswith(bourse):
            ok = True
            break

    return ok

def get_sharelist_list(sharelist_name_fq: str) -> list:
    """ return a list of (share_name, share_number) tuples """

    sharelist_shares = []
    if exists(sharelist_name_fq):
        with open(sharelist_name_fq, "r", encoding='utf8') as shlf:
            sharelist_line = shlf.readline()  # skip assumed header share_number,sharename
            if sharelist_line.startswith(g.SHARELIST_HEADER):
                sharelist_line = shlf.readline()  # if header, skip
            while sharelist_line:
                share_name = sharelist_line[:31].rstrip()
                share_number = sharelist_line[31:].rstrip()
                if not share_number.endswith('None.ETR'):
                    sharelist_shares.append((share_name, share_number))
                sharelist_line = shlf.readline()  # read next dict line

    return sharelist_shares

def create_results_3jP_sharelist():
    ''' create a standard SW sharelist file from the _3jP_list in runtime config '''

    results3_shl_fq = g.SHARELISTS_FOLDER_FQ + '\\' + 'results_3jP.shl'

    with open(results3_shl_fq,'w', encoding='utf8') as shlf:
        #SHARELIST_HEADER = 'share_name                     number'
        shlf.write(f'{g.SHARELIST_HEADER}\n')
        for number_name_list in g.CONFIG_RUNTIME['_3jP_list']:
            share_number = number_name_list[0]
            share_name = number_name_list[1].ljust(31)
            #NOTE name then number despite other way around in PSMV_list
            shlf.write(f'{share_name}{share_number}\n')

def create_results_V2d_sharelist():
    ''' create a standard SW sharelist file from the _V2d_list in runtime config '''

    shl_fq = g.SHARELISTS_FOLDER_FQ + '\\' + 'results_V2d.shl'

    with open(shl_fq,'w', encoding='utf8') as shlf:
        #SHARELIST_HEADER = 'share_name                     number'
        shlf.write(f'{g.SHARELIST_HEADER}\n')
        for number_name_list in g.CONFIG_RUNTIME['_V2d_list']:
            share_number = number_name_list[0]
            share_name = number_name_list[1].ljust(31)
            #NOTE name then number despite other way around in PSMV_list
            shlf.write(f'{share_name}{share_number}\n')

def create_results_3nH_sharelist():
    ''' create a standard SW sharelist file from the _3nH_list in runtime config '''

    shl_fq = g.SHARELISTS_FOLDER_FQ + '\\' + 'results_3nH.shl'

    with open(shl_fq,'w', encoding='utf8') as shlf:
        #SHARELIST_HEADER = 'share_name                     number'
        shlf.write(f'{g.SHARELIST_HEADER}\n')
        for number_name_list in g.CONFIG_RUNTIME['_3nH_list']:
            share_number = number_name_list[0]
            share_name = number_name_list[1].ljust(31)
            #NOTE name then number despite other way around in PSMV_list
            shlf.write(f'{share_name}{share_number}\n')

def sharenum_in_spinoff_list(share_num :str, list_key :str) ->bool:
    ''' decides whether share_num is present in the config_runtime's  x_list'''
    # list_key eg '_3jP_list', '_V2d_list'
    for number_name_list in g.CONFIG_RUNTIME[list_key]:
        if number_name_list[0] == share_num:
            return True
    return False

def clear_3jP_list():
    ''' invalidate the list of shares for which M_ParticularSumOfMV gets run'''
    g.CONFIG_RUNTIME['_3jP_list']  = []
    save_runtime_config()

def scrub_csv_footers(csv_file_fq :str):
    ''' get rid of any (esp footer) lines in passed in csv file which start with date_time
        NOTE we allow the first line header however
    '''

    scrubbed_lines = []
    # select only the good lines
    lines_scrubbed=0
    with open(csv_file_fq, "r") as inf:
        line_num=0
        for in_line in inf:
            if not in_line.startswith('date_time') or line_num==0:
                scrubbed_lines.append(in_line)
                line_num = line_num+1
            else:
                lines_scrubbed = lines_scrubbed+1

    # write these back to the same csv file
    with open(csv_file_fq, "w") as outf:
        for out_line in scrubbed_lines:
            outf.write(out_line)

    if lines_scrubbed > 0:
        logging.info(f'{lines_scrubbed} unwanted (footer) lines removed from .CSV')

def df_from_csv(share_dest_path: str, share_num: str, share_name: str, start_date: str, end_date :str, stage: int) -> pd.DataFrame:
    """
    Return a dataframe created from the ~.TXT.CSV file for the share with either the full date range or a tail-piece date range,
    as controlled by start_date

    """

    csv_file_fq = f"{share_dest_path}\{share_num}\{share_num}.CSV"
    csv_file_size = os.path.getsize(csv_file_fq)
    if csv_file_size > g.CSV_TRADES_FILESIZE_MAX:
        size_warn = f'{share_name} ({share_num}) trades file {csv_file_fq} is too large! (size {csv_file_size}) Share will be skipped'
        logging.warn(size_warn)
        append_csv_maxsize_log(size_warn)
        return pd.DataFrame()  # an empty DataFrame

    # get rid of unwanted 'date_time,price,volume,vol_cum' footers
    scrub_csv_footers(csv_file_fq)

    # load entire csv file ALL HISTORY
    try:
        df_trades = pd.read_csv(csv_file_fq, index_col='date_time',
                                parse_dates=True, infer_datetime_format=True)
    except:
        # could be eg low memory error
        logging.error(f"Exception in read_csv: {sys.exc_info()[0]}")
        readcsv_warn = f"{share_name} ({share_num}) trades file '{csv_file_fq}' below {g.CSV_TRADES_FILESIZE_MAX} bytes in size, but a pd.read_csv exception still ocurred. Share will be skipped"
        logging.error(readcsv_warn)
        append_csv_maxsize_log(readcsv_warn)
        return pd.DataFrame()  # an empty DataFrame

    if len(df_trades.index) == 0:
        return pd.DataFrame()  # an empty DataFrame

    # we don't need this column
    del df_trades['vol_cum']

    # if we have an end_date, respect it
    if end_date != '':
        start_off_date = start_date.replace('_','-')
        cut_off_date = end_date.replace('_','-')
        #filter on the index
        df_trades = df_trades.loc[start_off_date:cut_off_date].copy()
        logging.debug(f'start_off_date={start_off_date}; Cutting off trades after {cut_off_date}')
        #print(df_trades.tail())
        #exit()

    if stage == 1:

        # before hours trades can occur (mostly from Frankfurt)
        #print(df_trades.index)

        df_early = df_trades.between_time('00:00:00','08:59:59')
        df_early_daily = df_early.resample('D', label='left', origin='start_day').agg(
            {'price': 'mean', 'volume': 'sum'}).pad()
        logging.debug(f'{df_early_daily.shape[0]} days in the period had early trading')

        # after hours trades can occur (mostly from Frankfurt)
        df_late = df_trades.between_time('17:36:00','23:59:59')
        df_late_daily = df_late.resample('D', label='left', origin='start_day').agg(
            {'price': 'mean', 'volume': 'sum'}).pad()
        logging.debug(f'{df_late_daily.shape[0]} days in the period had late trading')

        # now get rid of bands before 09:00:00 and after 17:35 each day
        # (their trades are now captured in df_early and df_late)
        df_trades = df_trades.between_time('09:00:00', '17:35')

        # and append the consolidated early / late trades to
        # the opening / closing minutes of each day
        for idx,row in df_early_daily.iterrows():
            #ensure these go into the 09:00:00 slot
            row.name = idx + pd.offsets.Hour(9)
            df_trades = df_trades.append(row,ignore_index=False)

        for idx,row in df_late_daily.iterrows():
            #ensure these go into the 17:35:00 slot
            row.name = idx + pd.offsets.Minute(17*60+35)
            df_trades = df_trades.append(row,ignore_index=False)

        # compact by resampling for minute intervals
        df = df_trades.resample('1Min', label='left', origin='start_day').agg(
            {'price': 'mean', 'volume': 'sum'}).pad()
        df = df[df.index.dayofweek < 5]

        # Prepare a dataframe which will help us to get rid of no-trading weekdays (ie holidays)
        # see https://stackoverflow.com/questions/44900011/how-to-delete-additional-days-added-by-pandas-resample
        # Lets have a dataframe with individual weekdays only -
        # it will be bereft of weekends and public holidays,
        # (since we explicitly remove weekends & there are no trades on public holidays):
        df_wanted_dates = df_trades.index.floor('D')
        df_wanted_dates = df_wanted_dates[df_wanted_dates.dayofweek < 5]

        # prepare a days only df from the 1 min resmapled trades
        df_dates_1min = df.index.floor('D')
        # and get rid of public holidays (no trade weekdays)
        df_unwanted_dates = df_dates_1min.difference(df_wanted_dates)
        # to leave only the wanted dates' trades
        df = df[~df_dates_1min.isin(df_unwanted_dates)]

    else:
        # resample on business days
        df = df_trades.resample('B', label='left', origin='start_day').agg(
            {'price': 'mean', 'volume': 'sum'}).pad()

        df_wanted_dates = df_trades.index.floor('D')
        df_wanted_dates = df_wanted_dates[df_wanted_dates.dayofweek < 5]

        # and get rid of public holidays (no trade weekdays)
        df_unwanted_dates = df.index.difference(df_wanted_dates)
        # print(df_unwanted_dates)

        # to leave only the wanted dates' trades
        df = df[~df.index.isin(df_unwanted_dates)]

    # if stage == 2:
    #     print(share_num)
    #     print(df.tail(5))

    # recall that when topping up, start_date has been determined beforehand
    # and is a more recent date
    start_date_fields = start_date.split('_')
    start_ts = Timestamp(int(start_date_fields[0]), int(
        start_date_fields[1]), int(start_date_fields[2]))

    # use a filter to filter in only the tailing rows from start_date to end of data
    try:
        tail_filter = df.index.to_series().between(start_ts, df.index[-1])
        # so the df now is either a 'smallish' one or a full-one, based
        # on whether we are topping up (former case) or doing a full process-3
        df = df[tail_filter]
    except IndexError as exc:
        pass

    if stage == 1:
        # NOTE shouldnt be necessary
        # get rid of bands before 09:00:00 and after 17:35 each day
        df = df.between_time('09:00:00', '17:35')

    # if stage == 2:
    #     print(df.head())
    #     exit()

    if stage == 1:
        # ensure it doesn't get bigger than 900 trading days
        # assuming 516 'minute' rows in a day
        rows_per_day = 516
        df = df.tail(g.CALCS_MAX_BUSDAYS * rows_per_day)
    else:
        # assuming 1 row = 1 day
        df = df.tail(g.CALCS_MAX_BUSDAYS)

    return df

def initialize_overview(sharelist_tuples: list, stage: int, sharelist_name :str):
    """ create an overview dataframe with sharenames and numbers and zero filled columns """

    ov_cols = [col for col in ov_columns.OV_STAGE_TO_COLS[stage]]
    ov = pd.DataFrame(columns=ov_cols)
    ov.allows_duplicate_labels = False

    # we add a row for each share to the global Overview dataframe, (global to save excessive passing)
    for share_name, share_number in sharelist_tuples:
        ov = ov.append({'ShareNumber': share_number,
                        'ShareName': share_name,
                        'Status': 'no data'}, ignore_index=True)

    ov.set_index('ShareNumber', drop=True, inplace=True)

    # drop duplicate rows in case sharelist tuples above not kosher (should NOT be required)
    if not ov.index.is_unique:
        ov = ov.loc[~ov.index.duplicated(), :]

    ov = ov.fillna(0.0)
    ov['Lazy'] = True

    # if we're initializing a stage 2 overview, we can already carry forward
    # certain columns as duplicated values already computed in stage 1
    if stage == 2:
        data_store = pd.HDFStore(g.OVERVIEW_STORE_FQ.format(1))
        try:
            # extract dataframe
            df_ov1 = data_store[g.HDFSTORE_OV_KEY.format(sharelist_name.replace('.shl',''),1)]

            # drop duplicate rows in case necessary (should NOT BE!)
            if not df_ov1.index.is_unique:
                df_ov1 = df_ov1.loc[~df_ov1.index.duplicated(), :]

            # grab these columns from the stage1 Ov and lay over the current Ov we're initializing
            # should align on share number
            # for stage1_col in ov_columns.OV_STAGE1_CARRY_FORWARDS:
            #     ov[stage1_col] = df_ov1[stage1_col]

            # grab all columns we need for stage 2 from
            # stage 1 if present and already computed
            # should align on share number
            for stage2_col in ov_columns.OV_COLUMNS:
                if stage2_col in df_ov1.columns:
                    ov[stage2_col] = df_ov1[stage2_col]

        except KeyError as ke:
            logging.error(f'Key Error {ke}')
            return ov

        finally:
            data_store.close()

    return ov

def daterange(start_date, end_date):
    """ generate a sequence of single dates between start and end """

    for n in range(int((end_date - start_date).days)):
        yield start_date + timedelta(n)

def get_start_end_date(df: pd.DataFrame) -> tuple:
    """ returns the start and end dates for passed in df """

    # compute the date range
    starting_year = df.index.year[0]
    starting_month = df.index.month[0]
    starting_day = df.index.day[0]

    ending_year = df.index.year[-1]
    ending_month = df.index.month[-1]
    ending_day = df.index.day[-1]

    # work way thru the df, filtering on successive dates
    start_date = date(starting_year, starting_month, starting_day)
    end_date = date(ending_year, ending_month, ending_day)

    return (start_date, end_date)

def first_trading_row_index(df, cur_date, stage: int):
    ''' create an index which finds the first row of the given day - may fail and return an empty one '''

    year, month, day, hour, minute = get_datetime_indices(df)
    cur_year, cur_month, cur_day = unpack_date(cur_date)
    # obtain the first bus row (mostly its a day's normal first row)
    # create a filtered index of first day rows (will be only 1 or possibly no row)
    if stage == 1:
        first_row_idx = df.iloc[(year == cur_year) & (month == cur_month) & (day == cur_day) & (
            hour == g.FIRST_BUS_HOUR_SLOT) & (minute == g.FIRST_BUS_MIN_SLOT)].index
    else:
        first_row_idx = df.iloc[(year == cur_year) & (
            month == cur_month) & (day == cur_day)].index

    return first_row_idx

def last_trading_row_index(df, cur_date, stage: int):
    ''' create an index which finds the last row of the given day - may fail and return an empty one '''

    year, month, day, hour, minute = get_datetime_indices(df)
    cur_year, cur_month, cur_day = unpack_date(cur_date)
    # obtain the bottom row (mostly its a day's normal end row)
    # create a filtered index of end of day rows (will be only 1 or possibly no row)
    if stage == 1:
        bot_row_idx = df.iloc[(year == cur_year) & (month == cur_month) & (day == cur_day) & (
            hour == g.LAST_BUS_HOUR_SLOT) & (minute == g.LAST_BUS_MIN_SLOT)].index
    else:
        bot_row_idx = df.iloc[(year == cur_year) & (
            month == cur_month) & (day == cur_day)].index

    return bot_row_idx

def get_row_index_from_daily_df(df, cur_date):
    ''' create an index which finds a specific row in a df which only has day rows '''

    year, month, day, hour, minute = get_datetime_indices(df)
    cur_year, cur_month, cur_day = unpack_date(cur_date)

    row_idx = df.iloc[(year == cur_year) & (
        month == cur_month) & (day == cur_day)].index

    return row_idx

def one_day_filter(df, cur_date: date):
    ''' create a filtered index which includes all rows of a entire day '''

    cur_dt64 = to_dt64(cur_date)
    return df.index.to_series().between(cur_dt64, cur_dt64, inclusive=True)

def n_days_forward_filter(df, ref_date: date, days_ahead :int):
    ''' create a filtered index which includes all rows of a day range going forward n days '''

    start_dt64 = to_dt64(ref_date)

    end_dt = ref_date + pd.tseries.offsets.BusinessDay(days_ahead)

    return df.index.to_series().between(start_dt64, end_dt, inclusive=True)

def n_days_back_filter(df, ref_date: date, days_back: int):

    end_dt64 = to_dt64(ref_date)

    start_date = ref_date - pd.tseries.offsets.BusinessDay(days_back)

    return df.index.to_series().between(start_date, end_dt64, inclusive=True)

def last_days_filter(df, days_back: int):
    ''' create a filtered index covering the last num_days_back of the dataframe '''

    last_date = df.index[-1]
    #first_date = last_date - Timedelta(days=num_days_back)
    first_date = last_date - pd.tseries.offsets.BusinessDay(days_back)
    return df.index.to_series().between(first_date, last_date, inclusive=True)

def to_dt64(indate: date):
    ''' takes a plain date and returns a numpy datetime64 '''

    indate_dt = datetime.datetime(indate.year, indate.month, indate.day)
    return np.datetime64(indate_dt)

def get_datetime_indices(df: pd.DataFrame) -> tuple:
    """ pull out separate component indices from the datetime index of the passed in df """

    year = df.index.year
    month = df.index.month
    day = df.index.day
    hour = df.index.hour
    minute = df.index.minute
    return (year, month, day, hour, minute)

def unpack_date(run_date) -> tuple:
    """ return separate year, month day from passed in date """
    return (run_date.year, run_date.month, run_date.day)

# TODO function day_totals below is expensive! ~ 30 msec

def busdays_offset(ref_ts: pd.Timestamp, offset_bus_days: int) -> pd.Timestamp:
    ''' returns offset_days back or forwards from passed in reference timestamp '''

    return ref_ts + pd.tseries.offsets.BusinessDay(n=offset_bus_days)

def day_totals(df: pd.DataFrame, share_num: str, run_date: date, contributing_col: str, totals_col: str):
    """ General purpose function which updates an end-of-day 'totals_col' in dataframe by summing values from the contributing col

        The passed in dataframe is assumed to be datetime-indexed at 5 min sampling over 'business_hours'
    """

    assert totals_col in df.columns, f'{totals_col} not found in dataframe'

    # we need these indexes for the filtering to come
    year, month, day, hour, minute = get_datetime_indices(df)
    # determine start and end dates
    start_date, end_date = get_start_end_date(df)

    run_year, run_month, run_day = unpack_date(run_date)
    # sum  for the day
    day_total = df.iloc[(year == run_year) & (month == run_month) & (
        day == run_day), df.columns.get_loc(contributing_col)].sum()  # [0]

    # create a filtered index of end of day rows (will be only 1 or possibly no row)
    eod_idx = df.iloc[(year == run_year) & (month == run_month) & (
        day == run_day) & (hour == g.LAST_BUS_HOUR_SLOT) & (minute == g.LAST_BUS_MIN_SLOT)].index
    # eod_idx is a DatetimeIndex with 0 (if empty) or 1 entry, the datetime of the YYYY-MM-DD 17:55 5-min slot
    if eod_idx.size > 0:
        # day_volume is a series (with a multi index)
        df.at[eod_idx[0], totals_col] = day_total[0]
    elif run_date == end_date:
        df.at[df.index[-1], totals_col] = day_total[0]
        msg = f'last day {run_date.strftime("%Y-%m-%d")} but the rows did not appear to go all the way to 17:55'
        logging.warn(msg)
    elif run_date.weekday() < 5:  # weekdays are 0->4
        msg = f'df for share {share_num}. Missing data for date {run_date.strftime("%Y-%m-%d")} holiday ? Please INVESTIGATE?'
        logging.warn(msg)

def first_non_zero(df: pd.DataFrame, col: str) -> tuple:
    ''' return the first non zero value and position for passed in dataframe and colum'''
    first_nz = 0
    nonzeroes = df[col].ne(0)  # series of bool values
    for i, truth in enumerate(nonzeroes):
        if truth:
            first_nz = df[col][i]
            return (i, first_nz)

    return (None, None)

def single_day_condition(df: pd.DataFrame, cur_dt: datetime) -> pd.Series:

    cur_dt_str = cur_dt.strftime('%Y-%m-%d')
    left = cur_dt_str+' 09:00:00'
    right = cur_dt_str+' 17:35:00'

    return df.index.to_series().between(left, right)

def between_dates_condition(df: pd.DataFrame, start_dt: datetime, end_dt: datetime) -> pd.Series:
    ''' return a series condition for dates between start and end '''

    start_str = start_dt.strftime('%Y-%m-%d') + ' 09:00:00'
    end_str = end_dt.strftime('%Y-%m-%d') + ' 17:35:00'

    return df.index.to_series().between(start_str, end_str)


import pygame
pygame.font.init()

def mostrar_texto_centrado(screen,texto,center_x,center_y,color,fuente):
    render = fuente.render(texto,True,color)
    rect_text = render.get_rect(center = (center_x,center_y))
    #rect_text_saludar.center = rect_saludar.center
    screen.blit(render,rect_text)

def crear_boton(screen,texto,bg_color,bg_color_hover,rect_boton:pygame.Rect,font_color,fuente = pygame.font.SysFont
                (None,36)):
    if rect_boton.collidepoint(pygame.mouse.get_pos()):
        pygame.draw.rect(screen,bg_color_hover,rect_boton,border_radius=5)
    else:
        pygame.draw.rect(screen,bg_color,rect_boton,border_radius=5)

    mostrar_texto_centrado(screen,texto,*rect_boton.center,font_color,fuente)


'''
Title     : functions_file
Objective : Functions to process and transform TCGA data from https://portal.gdc.cancer.gov/ with R
Created by: Kevin Brinneman for UTSA Electrical & Computer Engineering
Created on: 2022-01-23

This script will recursively iterate over all folder to decompress GZ files.Once decompressed, all RNA_seq files are
opened as .COUNTS format and processed as JSON files. JSON format is preferible over .TXT files to prevent data
corruption.
'''

'''
Directory tree only showing three samples. The same architecture continues for the list showed below.
Directory originally in Data_processing/preprocessed_data/GDC. Files too large to share.

----TCGA
    |----GDC_Data
    |    |----TCGA-ACC
              |----harmonized
                   |----Transcriptome_Profiling
                        |----Gene_Expression_Quantification
                            |----[folders with txt files]
                   |----Clinical
                        |----Clinical_Supplement
                            |----[folders with txt files]
    |    |----TCGA-BLCA
              |----harmonized
                   |----Transcriptome_Profiling
                        |----Gene_Expression_Quantification
                            |----[folders with txt files]
                   |----Clinical
                        |----Clinical_Supplement
                            |----[folders with txt files]
    |    |----TCGA-BRCA
              |----harmonized
                   |----Transcriptome_Profiling
                        |----Gene_Expression_Quantification
                            |----[folders with txt files]
                   |----Clinical
                        |----Clinical_Supplement
                            |----[folders with txt files]

    .
    .
    .

    [           ...          ...        ... TCGA-CESC,TCGA-CHOL,TCGA-COAD,
    TCGA-DLBC, TCGA-ESCA,TCGA-GBM,TCGA-HNSC,TCGA-KICH,TCGA-KIRC,TCGA-KIRP,
    TCGA-LGG, TCGA-LIHC,TCGA-LUAD,TCGA-LUSC,TCGA-MESO,TCGA-OV,TCGA-PAAD,
    TCGA-PCPG, TCGA-PRAD,TCGA-READ,TCGA-SARC,TCGA-SKCM,TCGA-STAD,TCGA-TGCT,
    TCGA-THCA,TCGA-THYM,TCGA-UCEC,TCGA-UCS,TCGA-UVM,TCGA-LAML]
'''

#import libraries
import gzip, pathlib, shutil, json, re

#Where the directory is located
base_dir = "D:/GDCdata"

#Function to open .COUNTS files, proccess them as JSON, and deletion of processed files.

def unpack_all_in_dir(_dir):
    path = pathlib.Path(_dir)
    for gzfilepath in path.rglob('*.gz'):
        with gzip.open(gzfilepath) as f, gzfilepath.with_suffix('').open('wb') as fw:
            shutil.copyfileobj(f, fw)
        os.remove(gzfilepath)

unpack_all_in_dir(base_dir)

#Function used to find clinical-to-rna match. Creates two dictionaries of clinical and rna data
# and is compared to find a match between samples with same ID alphanumeric string

def merge_data_files(_dir):
    path = pathlib.Path(_dir)
    tcga_type = [item for item in path.iterdir() if item.is_dir()] #make the folder a mutable object list
    GDC_data = {}
    for tcga in tcga_type:
        GDC_data[tcga.name] = {}
        for clinical in tcga.glob('Clinical_Supplement'):
            Clinical_Supplement = {}
            for case in clinical.iterdir():
                Clinical_Supplement[case.name] = case
        for rna_cases in tcga.glob('Gene_Expression_Quantification'):
            Gene_Expression_Quantification = {}
            for sample in rna_cases.iterdir():
                Gene_Expression_Quantification[sample.name] = sample
        ({f for f in Clinical_Supplement} & {f for f in Gene_Expression_Quantification})
    print(GDC_data)

#Function to iterate over all folders to save samples as JSON files and convert each .COUNT file
#to JSON format. Each JSON formart file is processed as a dictionary.

def header_check_df (_dir):
    path = pathlib.Path(_dir)
    for headers in path.rglob('*.htseq.counts'):
        object_gene= {}
        feature_rna_string = {}
        with open(headers) as f:
            for line in f:
                gene, count = line.strip().split(None,1)
                feature_rna_string.update({gene: int(count)})
            object_gene[str(re.sub(r'.htseq.counts', '', headers.name))] = feature_rna_string
        file_regexd = re.sub(r'.htseq.counts','',headers.name) + '.json'
        out_file = open(headers.parent.joinpath(file_regexd), 'w')
        json.dump(object_gene, out_file, indent = 4, sort_keys= False)
        out_file.close()

header_check_df(base_dir)


from django.conf.urls import patterns, include, url
from django.conf import settings
from django.contrib import admin
admin.autodiscover()

urlpatterns = patterns('',
    url(r'^admin/', include(admin.site.urls)),
    url(r'^', include('website.urls', namespace='website')),
    url(r'^', include('registration.auth_urls')),
    url(r'^', include('registration.urls', namespace='registration')),
    url(r'^', include('profiles.urls', namespace='profiles')),
    url(r'^books/', include('books.urls', namespace='books')),
    url(r'^search/', include('haystack.urls')),
)

if settings.DEBUG:
    urlpatterns += patterns('',
        url(r'^media/(?P<path>.*)$', 'django.views.static.serve', {
            'document_root': settings.MEDIA_ROOT}),
    )


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Aug 24 14:51:43 2023

@author: hafizzzh
"""
import json
import time
import calendar
import hashlib
import requests
import pandas as pd
import os
from dotenv import load_dotenv

load_dotenv()
df_ev = pd.DataFrame()
getLoop = True

url = "https://gateway.marvel.com/v1/public/events"
name = ""
nameStartsWith = ""
modifiedSince = ""
creators = ""
characters = ""
series = ""
comics = ""
stories = ""
orderBy = "startDate"
limit = 100
offset = 0

current_GMT = time.gmtime()
ts = str(calendar.timegm(current_GMT))
pvkey = os.getenv("MARVEL_PVKEY")
pbkey = os.getenv("MARVEL_PBKEY")
assemble = ts+pvkey+pbkey
juru = hashlib.md5(assemble.encode())

while(getLoop):
    querystring = {"orderBy": str(orderBy),
                   "apikey": str(pbkey),
                   "ts": ts,
                   "hash": juru.hexdigest(),
                   "limit": limit,
                   "offset": offset}

    response = requests.request("GET",
                                url,
                                params=querystring)

    jsondata = json.loads(response.text)

    for i in range(limit):
        try:
            ev = jsondata['data']['results'][i]

            ev_id = ev['id']
            ev_title = ev['title']
            ev_start = ev['start']
            ev_end = ev['end']
            ev_next = ev['next']
            ev_previous = ev['previous']
            ev_modified = ev['modified']

            rev_row = pd.Series([ev_id, ev_title, ev_start, ev_end, ev_next, ev_previous, ev_modified])
            row_df_rev = pd.DataFrame([rev_row], index=[i+offset])

            df_ev = pd.concat([df_ev, row_df_rev])
        except IndexError:
             getLoop = False
             print("All Done. Retrieved items:", df_ev.shape[0])
             break
    print("\t offset", offset, ". limit:", limit, ". items: ", df_ev.shape[0])
    offset = limit+offset

df_ev = df_ev.rename(columns={
    0: "id",
    1: "title",
    2: "start",
    3: "end",
    4: "next",
    5: "previous",
    6: "modified"})
df_ev.to_csv("result_events.csv")


from django.db.models import Sum, F, Case, When
from rest_framework import serializers
from .models import StockTransaction
from users.serializers import UserSerializer
from stocks.serializers import StockSerializer

class StockTransactionsSerializer(serializers.ModelSerializer):

    user = UserSerializer(read_only=True)
    stock = StockSerializer

    class Meta:
        model = StockTransaction
        fields = (
            "user",
            "transaction_type",
            "date",
            "stock",
            "price",
            "quantity",
        )

    def create(self, validated_data):
        request = self.context.get("request")
        transaction = StockTransaction.objects.create(
            **validated_data, shareholder=request.user
        )
        return transaction

    def validate(self, data):
        # get data
        user = self.context.get("request").user
        if self.instance:
            transaction_type = data.get(
                "transaction_type", self.instance.transaction_type
            )
            stock_pk = data.get("stock", self.instance.stock).pk
            quantity = data.get("quantity", self.instance.quantity)
            price = data.get("price", self.instance.price)
        else:
            transaction_type = data.get("transaction_type")
            stock_pk = data.get("stock").pk
            quantity = data.get("quantity")
            price = data.get("price")
        # check negative value
        if quantity < 0 or price < 0:
            raise serializers.ValidationError(
                "Quantity or price should not be negative"
            )
        # check action that sell more than they have
        stocks = user.transactions.filter(stock=stock_pk).annotate(
            Quantity=Case(
                When(transaction_type="buy", then=F("quantity")),
                When(transaction_type="sell", then=-1 * F("quantity")),
            ),
        )
        if stocks.count() != 0:
            result = stocks.aggregate(total_quantity=Sum("Quantity"))
            total_quantity = result["total_quantity"]
        else:
            total_quantity = 0
        if transaction_type == "sell" and total_quantity < quantity:
            raise serializers.ValidationError("You can't sell more than you have")
        return data


import numpy as np
from utils import *
from sklearn.ensemble import RandomForestClassifier

def cv_train_rfcls(x,y,n_estimators,min_samples_leaf,max_feature,num_cross=10):
    cross_train_x, cross_train_y, cross_valid_x, cross_valid_y = cross_validation(x,y,num_cross)
    num_cross = len(cross_train_x)
    max_score = 0
    score_list=[]
    for i in range(num_cross):
        model = RandomForestClassifier(n_estimators=n_estimators, min_samples_leaf=min_samples_leaf, max_features=max_feature, n_jobs=-1)
        model.fit(cross_train_x[i], cross_train_y[i])
        score = model.score(cross_valid_x[i], cross_valid_y[i])
        score_list.append(score)
    print(score_list)
    return sum(score_list)/num_cross

def grid_search_rfcls(x,y,n_e_list,min_sl_list,max_f_list,num_cross=10):
    max_ne ,max_msl, max_mf ,max_score= 0,0,0,0
    for i in n_e_list:
        for j in min_sl_list:
            for k in max_f_list:
                score = cv_train_rfcls(x,y,i,j,k,num_cross)
                print("n_estimators",i," min_samples_leaf=",j," max_feature=",k," score=",score,"\n")
                if score>max_score:
                    max_ne = i
                    max_msl = j
                    max_mf = k
                    max_score = score
    model = RandomForestClassifier(n_estimators=max_ne, min_samples_leaf=max_msl,
                                   max_features=max_mf, n_jobs=-1)
    print(max_ne ,max_msl, max_mf ,max_score)
    model.fit(x,y)
    return model

if __name__ == "__main__":
    train_x = np.load("train_x.npy")
    train_y = np.load("train_y.npy")
    mid = np.median(train_y)
    for i in range(len(train_y)):
        if train_y[i] > mid:
            train_y[i] = 1
        else:
            train_y[i] = 0
    num_cross = 10
    model = grid_search_rfcls(train_x,train_y,[100],[1,2,4,8,16,32],[8,16,32,64,128,256],num_cross=10)


""" Bayesian Determinisitc Policy Gradient evaluated on th
didactic "chain" environment
"""

import tensorflow as tf
from gym import Wrapper
from tensorflow.python.layers.utils import smart_cond
from tensorflow.python.ops.variable_scope import get_local_variable

import chi
from chi import Experiment
from chi import experiment, model
from chi.rl import ReplayMemory

# chi.chi.tf_debug = True
from chi.rl.bdpg import BdpgAgent
from chi.rl.ddpg import DdpgAgent
from chi.rl.util import print_env

@experiment
def bdpg_chains2(self: Experiment, logdir=None, env=1, heads=3, n=50, bootstrap=False, sr=50000):
  from tensorflow.contrib import layers
  import gym
  from gym import spaces
  from gym import wrappers
  import numpy as np
  from tensorflow.contrib.framework import arg_scope

  def gym_make(id) -> gym.Env:
    return gym.make(id)

  chi.set_loglevel('debug')

  if env == 0:
    import gym_mix
    from chi.rl.wrappers import PenalizeAction
    env = gym_mix.envs.ChainEnv(n)
    env = PenalizeAction(env, .001, 1)
  elif env == 1:
    # env = gym.make('Pendulum-v0')
    env = gym.make('MountainCarContinuous-v0')

  if bootstrap:
    class Noise(Wrapper):
      def __init__(self, env):
        super().__init__(env)
        self.n = 3
        self.observation_space = gym.spaces.Box(
          np.concatenate((self.observation_space.low, np.full([self.n], -1))),
          np.concatenate((self.observation_space.high, np.full([self.n], 1))))

      def _reset(self):
        s = super()._reset()
        self.noise = np.random.uniform(-1, 1, [self.n])
        s = np.concatenate([s, self.noise])
        return s

      def _step(self, action):
        s, r, d, i = super()._step(action)
        s = np.concatenate([s, self.noise])
        return s, r, d, i

    env = Noise(env)

  print_env(env)

  def pp(x):
    # v = get_local_variable('noise', [x.shape[0], 100], initializer=tf.random_normal_initializer)
    # y = tf.concat(x, v)
    return x

  def ac(x):
    with tf.name_scope('actor_head'):
      x = layers.fully_connected(x, 50, biases_initializer=layers.xavier_initializer())
      x = layers.fully_connected(x, 50, biases_initializer=layers.xavier_initializer())
      # a = layers.fully_connected(x, env.action_space.shape[0], None, weights_initializer=tf.random_normal_initializer(0, 1e-4))
      a = layers.fully_connected(x, env.action_space.shape[0], None)
      return a

  def cr(x, a):
    with tf.name_scope('critic_head'):
      x = layers.fully_connected(x, 50, biases_initializer=layers.xavier_initializer())
      x = tf.concat([x, a], axis=1)
      x = layers.fully_connected(x, 50, biases_initializer=layers.xavier_initializer())
      # q = layers.fully_connected(x, 1, None, weights_initializer=tf.random_normal_initializer(0, 1e-4))
      q = layers.fully_connected(x, 1, None)
      return tf.squeeze(q, 1)

  if bootstrap:
    agent = DdpgAgent(env, ac, cr, replay_start=sr, noise=lambda a: a)
  else:
    agent = DdpgAgent(env, ac, cr, replay_start=sr)
  threshold = getattr(getattr(env, 'spec', None), 'reward_threshold', None)

  for ep in range(100000):

    R, info = agent.play_episode()

    if ep % 20 == 0:
      head = info.get('head')
      print(f'Return of episode {ep} after timestep {agent.t}: {R} (head = {head}, threshold = {threshold})')

    if ep % 100 == 0 and bootstrap:
      pass

  #
  # @chi.function(logging_policy=lambda _: True)
  # def plot():
  #   # obsp = env.observation_space
  #   # h = obsp.high
  #   # l = obsp.low
  #   # x, y = tf.meshgrid(tf.linspace(l[0], h[0], 100), tf.linspace(l[1], h[1], 100))
  #   # x = tf.reshape(x, [-1])
  #   # y = tf.reshape(y, [-1])
  #   # inp = tf.stack(x, y, axis=1)
  #
  #   x = tf.linspace(0, 30, 100)
  #   x = tf.py_func(env.batch_features, x, tf.float32, stateful=False)
  #   s = pp(x)
  #   a0 = actor(s)
  #   tf.image


""" Class holding data of one or more features
"""
from logging import warning
from typing import List, Tuple, Union
import xarray as xa
import pandas as pd
import numpy as np
import numpy.typing as npt
from proteolizarddata.data import PyTimsDataHandleDDA
from proteolizardalgo.feature_loader_dda import FeatureLoaderDDA

# typing
NDArrayFloat = npt.NDArray[np.float64]
NDArrayInt = npt.NDArray[np.int64]

class AlignedFeatureData:
    """This class manages import, alignment and processing
        of feature data.

    Using FeatureLoaderDDA the features are stored in an aligned fashion,
    meaning from all features 'num_data_points' highest signals are chosen.
    Features that do not have enough signals are discarded.

    Attributes:
        feature_data (xa.Dataset): Feature data is stored in
            xarray format. If several features are aligned 'feature'
            dimensions stores the feature's ids as coordinates.
            Note: If only one feature is used, the dimension 'feature'
            still exists, however length of 'datapoint' dimension is
            depending on the feature's size and not on 'num_data_points'
        accepted_feature_ids (List[int]): List of feature ids, that are
            stored in feature_data.
    Args:
        data_handle (Union[str,PyTimsDataHandleDDA]): Path to experimental data
            or data_handle.
        ids (List[int]): List with feature ids.
        is_parallel (bool, optional): Wether several features are to
            be aligned. If True 'num_data_points' highest signals
            are loaded from features, if False all data points of the
            single feature are loaded. Defaults to True.
        num_data_points (int, optional): Number of data_points to load
            from features to align. Defaults to 10.
    """

    def __init__(
        self,
        data_handle: Union[str, PyTimsDataHandleDDA],
        ids: List[int],
        is_parallel: bool = True,
        num_data_points=10,
    ) -> None:
        self.accepted_feature_ids = []
        charges = []
        feature_data = []
        if isinstance(data_handle, str):
            data_path = data_handle
            data_handle = PyTimsDataHandleDDA(data_path)
        for feature_id in ids:
            feature = FeatureLoaderDDA(data_handle, feature_id)
            if np.isnan(feature.monoisotopic_mz):
                continue
            try:
                # estimate feature hull boundaries with
                # averagine model for isotopic pattern
                # and gaussian model for IMS
                data_tmp = feature.load_hull_data_3d(
                    intensity_min=0,
                    ims_model="gaussian",
                    plot_feature=False,
                )
            except RuntimeError:
                warning(f"RuntimeError with feature {feature_id}")
                continue
            if is_parallel:
                if len(data_tmp) < num_data_points:
                    continue
                feature_data.append(
                    data_tmp.nlargest(num_data_points, columns="Intensity")
                )
            else:
                feature_data.append(data_tmp)
            self.accepted_feature_ids.append(feature_id)
            charges.append(feature.charge)

        if len(self.accepted_feature_ids) == 0:
            raise ValueError("No accepted features. Check chosen features.")
        s, mz, i = self._set_data_parallel(feature_data)
        data_dict = {
            "Charge": ("feature", charges),
            "Scan": (("data_point", "feature"), s),
            "Mz": (("data_point", "feature"), mz),
            "Intensity": (("data_point", "feature"), i),
        }
        coord_dict = {"feature": self.accepted_feature_ids}
        self.feature_data = xa.Dataset(data_vars=data_dict, coords=coord_dict)

    def _set_data_parallel(
        self, feature_data_list: List[pd.DataFrame]
    ) -> Tuple[NDArrayFloat, NDArrayFloat, NDArrayFloat]:
        """Stacking features in axis 1

        Args:
            feature_data_list (List[pd.DataFrame]): Loaded feature dataframes.
        Returns:
            Tuple[NDArrayFloat,NDArrayFloat,NDArrayFloat]: stacked scan, mz and
                intensity data.
        """
        s = np.stack([fd["Scan"] for fd in feature_data_list], axis=1)
        mz = np.stack([fd["Mz"] for fd in feature_data_list], axis=1)
        i = np.stack([fd["Intensity"] for fd in feature_data_list], axis=1).astype(
            "float"
        )

        return (s, mz, i)


import json

from django.contrib import messages
from django.shortcuts import get_object_or_404, redirect
from django.urls import reverse_lazy

from .models import Confirmation, PendingConfirmation
from core.views import SearchAndMenuCreateView

class ConfirmationCreateView(SearchAndMenuCreateView):
    model = Confirmation
    fields = []
    success_url = reverse_lazy("home")

    def dispatch(self, request, *args, **kwargs):
        self.pending_confirmation = get_object_or_404(
            PendingConfirmation,
            slug=kwargs["slug"]
        )

        if self.pending_confirmation.has_expired():
            msg = "Dieser Bestätigungslink ist nicht mehr gültig."
            messages.error(request, msg)
            return redirect("home")

        return super().dispatch(request, *args, **kwargs)

    def form_valid(self, form):
        form.instance.pending_confirmation = self.pending_confirmation
        logdata = {
            k: v
            for k, v in self.request.environ.items()
            if k.startswith("HTTP")
        }
        form.instance.logdata = json.dumps(logdata, indent=2)

        msg = "Vielen Dank!"
        messages.success(self.request, msg)

        return super().form_valid(form)


import dash
import dash_bootstrap_components as dbc
from dash import Input, Output, State, dcc, html, Dash, callback

dash.register_page(__name__, path="/nosotros")

style = {"padding": "1rem 1rem"}

layout = html.Div([
    html.H2("Team 181"),
    html.Hr(),
    html.H4("We're so cool, give us prize!")
], style=style)


import unittest, pytest
from utils import Utils
from pathlib import Path
import os
import shutil
import nltk
nltk.download('wordnet')
from nltk.corpus import wordnet as wn

class TestUtils(unittest.TestCase):

    maxDiff = None

    # == TEST GET RANDOM WORD == #

    def test_get_random_word_returns_a_string_found_in_wordnet_word_list(self):
        # given - an instance of the utils class and wordnet words list
        undertest_class = Utils()
        wordnet = wn.words()

        # when - we call the get random word method of the utils class
        actual_word = undertest_class.get_random_word()

        # then - the returned value is type string, with value found in the wordnet
        self.assertTrue(type(actual_word) == str)
        self.assertTrue(actual_word in wordnet)

    def test_get_random_word_returns_two_different_words_when_called_twice(self):
        # given - an instance of the utils class
        undertest_class = Utils()

        # when - we call the get random word method of the utils class twice
        actual_word1 = undertest_class.get_random_word()
        actual_word2 = undertest_class.get_random_word()

        # then - the words returned are not the same
        self.assertNotEqual(actual_word1, actual_word2)

    # == TEST GET SYNSETS == #
    def test_get_synsets_returns_list_of_synsets_and_number_of_synsets(self):
        # given - an instance of the utils class
        undertest_class = Utils()

        # when - we call the get_synsets method, passing in a common word that can be found in wordnet with multiple synsets
        actual_synsets, actual_num_synsets = undertest_class.get_synsets("class")

        # then - returned values are of the expected types
        self.assertTrue(type(actual_synsets) == list)
        self.assertTrue(type(actual_num_synsets) == int)

    # == TEST GET WORDS FROM SYNSETS == #
    def test_get_words_from_synsets_returns_a_list_of_strings(self):
        # given - an instance of the utils class
        undertest_class = Utils()

        # when - we call the get words from synsets method passing in a list of synsets
        actual_synsets, _ = undertest_class.get_synsets("class")
        actual_words = undertest_class.get_words_from_synsets(actual_synsets)

        # then - a list of strings is returned
        self.assertTrue(type(actual_words) == list)
        self.assertTrue(all([type(i) == str for i in actual_words]))

    # == TEST GET LEMMAS FROM SYNSET == #
    def test_get_lemmas_from_synset_returns_list_of_lemmas(self):
        # given - an instance of the utils class
        undertest_class = Utils()

        # when - we call the get lemmas from synsets method passing in a synset
        actual_synsets, _ = undertest_class.get_synsets("class")
        actual_words = undertest_class.get_lemmas_from_synset(actual_synsets[0])

        # then - a list of strings is returned
        self.assertTrue(type(actual_words) == list)
        self.assertTrue(all([type(i) == nltk.corpus.reader.wordnet.Lemma for i in actual_words]))

    # == TEST VALIDATE RELATED WORDS == #
    def test_validate_related_words_returns_same_words_if_all_valid(self):
        # given - an instance of the undertest utils class, a word and a list of related words that are all considered valid
        undertest_class = Utils()
        word = "example"
        original_list = ["specimen", "sample", "illustration", "guide", "blueprint", "ideal"]

        # when - we call the validate related words method passing in the word and list of words
        actual_list = undertest_class._validate_related_words(word, original_list)

        # then - the same list of related words is returned
        self.assertEqual(actual_list, original_list)

    def test_validate_related_words_removes_multiple_words_with_underscore_between(self):
        # given - an instance of the undertest utils class, a word and a list of related words with some invalid due to underscore
        undertest_class = Utils()
        word = "example"
        original_list = ["specimen", "sample", "illustration", "case_in_point", "role_model", "guide", "blueprint", "ideal"]

        # when - we call the validate related words method passing in the word and list of words
        actual_list = undertest_class._validate_related_words(word, original_list)
        expected_list = ["specimen", "sample", "illustration", "guide", "blueprint", "ideal"]

        # then - the list of related words is returned without the invalid words
        self.assertEqual(actual_list, expected_list)

    def test_validate_related_words_removes_words_with_numbers(self):
        # given - an instance of the undertest utils class, a word and a list of related words with some invalid due to numbers
        undertest_class = Utils()
        word = "example"
        original_list = ["specimen", "100", "sample", "illustration", "eg1","guide", "blueprint", "ideal"]

        # when - we call the validate related words method passing in the word and list of words
        actual_list = undertest_class._validate_related_words(word, original_list)
        expected_list = ["specimen", "sample", "illustration", "guide", "blueprint", "ideal"]

        # then - the list of related words is returned without the invalid words
        self.assertEqual(actual_list, expected_list)

    def test_validate_related_words_removes_words_that_are_very_similar(self):
        # given - an instance of the utils class, a word and a list of related words with some invalid due to extreme similarity to original word
        undertest_class = Utils()
        word = "example"
        original_list = ["examples", "specimen", "sample", "exampled", "illustration", "guide", "blueprint", "ideal"]

        # when - we call the validate related words method passing in the word and list of words
        actual_list = undertest_class._validate_related_words(word, original_list)
        expected_list = ["specimen", "sample", "illustration", "guide", "blueprint", "ideal"]

        # then - the list of related words is returned without the invalid words
        self.assertEqual(actual_list, expected_list)

    def test_validate_related_words_removes_words_that_are_the_same_with_different_caps(self):
        # given - an instance of the utils class, a word and a list of related words with some invalid - differing by caps to original word
        undertest_class = Utils()
        word = "example"
        original_list = [ "Example", "specimen", "sample", "EXAMPLE", "illustration", "guide", "blueprint", "ideal"]

        # when - we call the validate related words method passing in the word and list of words
        actual_list = undertest_class._validate_related_words(word, original_list)
        expected_list = ["specimen", "sample", "illustration", "guide", "blueprint", "ideal"]

        # then - the list of related words is returned without the invalid words
        self.assertEqual(actual_list, expected_list)

if __name__ == "__main__":
    unittest.main()


#!/bin/python3

import os
import argparse
import torch
import sys

def add_padding_to_image(img, padding_width: int):
    # Array of zeros of shape (img + padding_width)
    img_with_padding = torch.zeros((
        img.shape[0] + padding_width * 2,  # Multiply with two because we need padding on all sides
        img.shape[1] + padding_width * 2
    ))

    # Change the inner elements
    # For example, if img.shape = (224, 224), and img_with_padding.shape = (226, 226)
    # keep the pixel wide padding on all sides, but change the other values to be the same as img
    img_with_padding[padding_width:-padding_width, padding_width:-padding_width] = img

    return img_with_padding

def get_padding_width_per_side(kernel_size: int) -> int:
    # Simple integer division
    return kernel_size // 2  # p = [K/2]

def calculate_target_size(Img_Width, Kernel_Width, Stride, P):
    '''you can use this formula [(W−K+2P)/S]+1.
    W is the input volume - in your case 16
    K is the Kernel size - in your case 5
    P is the padding - 0 for valid and 1 is for same
    S is the stride - which you have not provided.'''
    W = Img_Width
    K = Kernel_Width
    S = Stride
    pixels = ((W - K + 2 * P) // S) + 1  # I added padding to input data, so I removed the 2P in this formula
    return pixels

def mac(img, kernel, dt, mac_flag, vec_flag, cast_flag, cast_to):
    if vec_flag == "false":
        temp = torch.zeros(1, dtype=dt)
        for i in range(img.shape[0]):
            for j in range(img.shape[1]):
                a = img[i][j]
                b = kernel[i][j]
                if cast_flag == "true":
                    if cast_to == "FP16":
                        a = a.type(torch.float16)
                        b = b.type(torch.float16)
                    elif cast_to == "FP16ALT":
                        a = a.type(torch.bfloat16)
                        b = b.type(torch.bfloat16)
                if mac_flag == "true":
                    a = a.type(torch.float32)
                    b = b.type(torch.float32)
                    temp = temp.type(torch.float32)
                temp += a * b
                if mac_flag == "true":
                    temp = temp.type(dt)
                # if cast_flag == "true":
                #     temp = temp.type(dt)
        return temp
    else:
        flag = True
        temp = torch.zeros(1, dtype=dt)
        temp1 = torch.zeros(1, dtype=dt)
        for i in range(img.shape[0]):
            for j in range(0, (img.shape[1] & 0xfffffffe), 2):
                a = img[i][j]
                a1 = img[i][j + 1]
                b = kernel[i][j]
                b1 = kernel[i][j + 1]
                if mac_flag == "true":
                    a = a.type(torch.float32)
                    b = b.type(torch.float32)
                    a1 = a1.type(torch.float32)
                    b1 = b1.type(torch.float32)
                    temp = temp.type(torch.float32)
                    temp1 = temp1.type(torch.float32)
                temp += a * b
                temp1 += a1 * b1
                if mac_flag == "true":
                    temp = temp.type(dt)
                    temp1 = temp1.type(dt)
        if img.shape[1] & 0x00000001:
            for i in range(img.shape[0]):
                a = img[i][img.shape[1] - 1]
                b = kernel[i][img.shape[1] - 1]
                if flag:  # temp
                    if mac_flag == "true":
                        a = a.type(torch.float32)
                        b = b.type(torch.float32)
                        temp = temp.type(torch.float32)
                    temp += a * b
                    if mac_flag == "true":
                        temp = temp.type(dt)
                    flag = False
                else:  # temp1
                    if mac_flag == "true":
                        a = a.type(torch.float32)
                        b = b.type(torch.float32)
                        temp1 = temp1.type(torch.float32)
                    temp1 += a * b
                    if mac_flag == "true":
                        temp1 = temp1.type(dt)
                    flag = True
        return temp + temp1

def convolve(img, kernel, out_width, dt, Stride, mac_flag, vec_flag, cast_flag, cast_to):
    out_img = torch.zeros((out_width, out_width), dtype=dt)
    tgt_size = out_img.shape[0]
    # To simplify things
    k = kernel.shape[0]
    if vec_flag == "false":
        # Iterate over the rows
        for i in range(tgt_size):
            # Iterate over the columns
            for j in range(tgt_size):
                # img[i, j] = individual pixel value
                # Get the current matrix
                mat = img[i * Stride:i * Stride + k, j * Stride:j * Stride + k]
                # Apply the convolution - element-wise multiplication and summation of the result
                # Store the result to i-th row and j-th column of our convolved_img array
                out_img[i, j] = mac(mat, kernel, dt, mac_flag, vec_flag, cast_flag, cast_to)
    else:  # based on the vectorized c code
        # Iterate over the columns
        for j in range(tgt_size):
            # Iterate over the rows
            for i in range(tgt_size):
                # img[i, j] = individual pixel value
                # Get the current matrix
                mat = img[i * Stride:i * Stride + k, j * Stride:j * Stride + k]
                # Apply the convolution - element-wise multiplication and summation of the result
                # Store the result to i-th row and j-th column of our convolved_img array
                out_img[i, j] = mac(mat, kernel, dt, mac_flag, vec_flag, cast_flag, cast_to="false")
    return out_img

def relative_absolute_error(true, pred):
    true_mean = torch.mean(true)
    squared_error_num = torch.sum(torch.abs(true - pred))
    squared_error_den = torch.sum(torch.abs(true - true_mean))
    rae_loss = squared_error_num / squared_error_den
    return rae_loss

def mean_squared_error(true, pred):
    squared_error = torch.square(true - pred)
    sum_squared_error = torch.sum(squared_error)
    size = true.size(dim=0) * true.size(dim=1)
    mse_loss = sum_squared_error / size
    return mse_loss

def matrix_init(IN, dt):
    # iterate through rows of IN
    temp = torch.zeros((IN.shape[0], IN.shape[1]), dtype=dt)
    # iterate through rows of IN
    for i in range(IN.shape[0]):
        # iterate through columns of IN
        for j in range(IN.shape[1]):
            temp[i][j] = IN[i][j]
    return temp

def write_matrix(matrix_to_write, name, len, file_pointer, float_type):
    matrix_string = ''
    sz0 = matrix_to_write.size()[0]
    sz1 = matrix_to_write.size()[1]
    if 'Filter_Kern' in name:
        file_pointer.write("DATA_LOCATION FIL_TYPE %s[%s] = {" % (name, len))
    elif 'ref' in name:
        file_pointer.write("PI_L2 OUT_TYPE %s[%s] = {" % (name, len))
    else:
        file_pointer.write("DATA_LOCATION INP_TYPE %s[%s] = {" % (name, len))

    if float_type == torch.float32:
        name = ")"
    elif float_type == torch.float16:
        name = ", dtype=torch.float16)"
    elif float_type == torch.bfloat16:
        name = ", dtype=torch.bfloat16)"
    for i in range(sz0):
        for j in range(sz1):
            matrix_string += str(matrix_to_write[i][j].item()).replace('tensor(', '').replace(name, '')
            matrix_string += ', '
    file_pointer.write("%s" % matrix_string)
    file_pointer.write("};\n")

def select_dtypes(user_dtypes, num_param):
    types_dict = {
        "FP32": torch.float32,
        "FP16": torch.float16,
        "FP16ALT": torch.bfloat16
    }
    dtypes = []
    if len(user_dtypes) == 1:
        for i in range(num_param):
            dtypes.append(types_dict[user_dtypes[0]])
    elif len(user_dtypes) == num_param:
        for i in range(num_param):
            dtypes.append(types_dict[user_dtypes[i]])
    else:
        for i in range(len(user_dtypes)):
            dtypes.append(types_dict[user_dtypes[i]])
        if 'FP32' in user_dtypes:
            for i in range(len(user_dtypes), num_param):
                dtypes.append(types_dict["FP32"])
        elif 'FP16' in user_dtypes:
            for i in range(len(user_dtypes), num_param):
                dtypes.append(types_dict["FP16"])
        else:
            for i in range(len(user_dtypes), num_param):
                dtypes.append(types_dict["FP16ALT"])
    return dtypes

def check_cast(datatypes):
    result = len(set(datatypes)) == 1
    if result : #All Elements in List are Equal
        return "false"
    else: #All Elements in List are Not Equal
        if torch.float32 in datatypes:
            return "false"
        else:
            return "true"

def get_inital_config():
    # get arguments  and data format
    parser = argparse.ArgumentParser()
    parser.add_argument('--IMG_WIDTH')
    parser.add_argument('--FILT_WIN')
    parser.add_argument('--STRIDE', default=1)
    parser.add_argument('--PADDING', default='valid')
    parser.add_argument('--vec_flag', default="false")
    parser.add_argument('--MAC_flag', default="true")
    parser.add_argument('--float_type', default='FP32')
    args = parser.parse_args()

    IMG_WIDTH = int(args.IMG_WIDTH)
    FILT_WIN = int(args.FILT_WIN)
    STRIDE = int(args.STRIDE)
    PADDING = str(args.PADDING)
    mac_flag = str(args.MAC_flag)
    vec_flag = str(args.vec_flag)
    bits = args.float_type.split(",")
    if PADDING == 'same' and STRIDE != 1:
        sys.exit("ValueError: padding='same' is not supported for strided convolutions")
    return IMG_WIDTH, FILT_WIN, STRIDE, PADDING, bits, mac_flag, vec_flag

def save_data_into_hfile(OUT_WIDTH, IMG_WIDTH, FILT_WIN, STRIDE, res, filter_conv, input_conv):
    # Generate header file
    f = open('data.h', 'w')

    f.write('\
#ifndef _INPUT_IMAGE_ \n\
#define _INPUT_IMAGE_\n\
#pragma GCC diagnostic ignored "-Woverflow"\n\n')
    f.write('\
#define OUT_DIM %s\n\
#define OUT_ROW %s\n\
#define OUT_COL %s\n\
#define INP_COL %s\n\
#define STRIDE %s\n\
#define FILT_WIN %s\n\n' % (OUT_WIDTH * OUT_WIDTH, OUT_WIDTH, OUT_WIDTH, IMG_WIDTH, STRIDE, FILT_WIN))
    write_matrix(input_conv, 'In_Img', IMG_WIDTH * IMG_WIDTH, f, input_conv.dtype)
    write_matrix(filter_conv, 'Filter_Kern', FILT_WIN * FILT_WIN, f, filter_conv.dtype)
    write_matrix(res, 'ref', OUT_WIDTH * OUT_WIDTH, f, res.dtype)
    f.write('\
#endif \n')
    f.close()

    f = open('config.h', 'w')

    f.write('\
#define FILT_WIN %s \n\n' % FILT_WIN)
    f.close()

def error_metric(ref, res):

    # calculate manually because metrics doesn't supprt bfloat16
    d = ref - res
    mse_f = torch.mean(d**2)
    mae_f = torch.mean(abs(d))
    rmse_f = torch.sqrt(mse_f)
    r2_f = 1-(torch.sum(d**2)/torch.sum((ref-torch.mean(ref))**2))
    print("Results of metrics:")
    print("MAE:",mae_f.item())
    print("MSE:", mse_f.item())
    print("RMSE:", rmse_f.item())
    print("R-Squared:", r2_f.item())
    rae = relative_absolute_error(ref, res)
    print("RAE is", rae.item())

def main():
    IMG_WIDTH, FILT_WIN, STRIDE, PADDING, bits, mac_flag, vec_flag = get_inital_config()

    # Create reference matrices
    input_ref = torch.rand((IMG_WIDTH, IMG_WIDTH), dtype=torch.float32)
    filter_ref = torch.randn((FILT_WIN, FILT_WIN), dtype=torch.float32) * 4

    OUT_WIDTH = IMG_WIDTH
    if PADDING == 'same':
        pad = get_padding_width_per_side(kernel_size=FILT_WIN)
        input_ref = add_padding_to_image(img=input_ref, padding_width=pad)
        OUT_WIDTH = IMG_WIDTH
        IMG_WIDTH = input_ref.shape[0]

    elif PADDING == 'valid':
        P = 0
        OUT_WIDTH = calculate_target_size(Img_Width=IMG_WIDTH,
Kernel_Width=FILT_WIN, Stride=STRIDE, P=P )
    # calculate reference output
    ref = convolve(img=input_ref, kernel=filter_ref, dt=torch.float32,
                   out_width=OUT_WIDTH, mac_flag="false", Stride=STRIDE, vec_flag="false", cast_flag="false", cast_to="false")

    # set the data types based on the parser input
    datatypes = select_dtypes(bits, 3)
    cast_flag = check_cast(datatypes[0:2])
    cast_to = "FP16"
    input_conv = matrix_init(input_ref, dt=datatypes[0])
    filter_conv = matrix_init(filter_ref, dt=datatypes[1])
    res = convolve(img=input_conv, kernel=filter_conv, dt=datatypes[2],
                   out_width=OUT_WIDTH, Stride=STRIDE, mac_flag=mac_flag, vec_flag=vec_flag, cast_flag=cast_flag, cast_to = cast_to)

    error_metric(ref, res)

    save_data_into_hfile(OUT_WIDTH, IMG_WIDTH, FILT_WIN, STRIDE, res, filter_conv, input_conv)
    print("############################## Done! ###################################")
    return None

if __name__ == "__main__":
    main()
    pass


# This is a brute force approach exploiting all the permutations
# Which still runs in sufficiently short time.
# For a more sophisticated approach: https://blog.dreamshire.com/project-euler-68-solution/

import itertools

gon_dimension = 5
numbers = range(1, gon_dimension * 2 + 1)
solutions = []
for outside_numbers in itertools.combinations(range(1, (gon_dimension * 2) + 1), gon_dimension):
    remaining_numbers = list(set(numbers) - set(outside_numbers))
    for inside_permutation in itertools.permutations(remaining_numbers):
        total = None
        all_the_same = True
        combinations = []
        inside_permutation = list(inside_permutation) + [inside_permutation[0]]
        for idx in range(5):
            combination = [outside_numbers[idx], inside_permutation[idx + 1], inside_permutation[idx]]
            if total is None:
                total = sum(combination)
                combinations.append(combination)
                continue
            if total != sum(combination):
                all_the_same = False
                break
            combinations.append(combination)

        if all_the_same:
            permutation = [combinations[0]] + sorted(combinations[1:], reverse=True)
            permutation = "".join(["".join([str(digit) for digit in comb]) for comb in permutation])
            if len(permutation) == 16:
                solutions.append(int(permutation))

print(f"Maximum 16-digit string for a 'magic' 5-gon ring: {max(solutions)}")


import pytest

from ephios.core.models import Qualification, QualificationCategory, QualificationGrant
from ephios.plugins.qualification_management.importing import (
    DeserializedQualification,
    QualificationChangeManager,
)

SANH_UUID = "b1faab38-2e7c-4507-b753-06d1e653412d"
NOTSAN_UUID = "d114125b-7cf4-49e2-8908-f93e2f95dfb8"

@pytest.fixture
def deserialized_qualifications():
    b = DeserializedQualification(
        {
            "uuid": "247fab6a-8784-4976-a406-985fe47dc683",
            "title": "Deutsches Rettungsschwimmabzeichen Bronze",
            "abbreviation": "DRSA Bronze",
            "includes": [],
            "included_by": ["ef95a854-2eeb-431c-a795-bc291b341d49"],
            "category": {
                "uuid": "cd10e68f-41fe-4ca0-a624-3ab3eb85bd08",
                "title": "Wasserrettung Allgemein",
            },
        }
    )
    s = DeserializedQualification(
        {
            "uuid": "ef95a854-2eeb-431c-a795-bc291b341d49",
            "title": "Deutsches Rettungsschwimmabzeichen Silber",
            "abbreviation": "DRSA Silber",
            "includes": ["247fab6a-8784-4976-a406-985fe47dc683"],
            "included_by": ["b601a18b-cee8-4037-af33-dd7aabeac295"],
            "category": {
                "uuid": "cd10e68f-41fe-4ca0-a624-3ab3eb85bd08",
                "title": "Wasserrettung Allgemein",
            },
        }
    )
    g = DeserializedQualification(
        {
            "uuid": "b601a18b-cee8-4037-af33-dd7aabeac295",
            "title": "Deutsches Rettungsschwimmabzeichen Gold",
            "abbreviation": "DRSA Gold",
            "includes": ["ef95a854-2eeb-431c-a795-bc291b341d49"],
            "included_by": [],
            "category": {
                "uuid": "cd10e68f-41fe-4ca0-a624-3ab3eb85bd08",
                "title": "Wasserrettung Allgemein",
            },
        }
    )
    return b, s, g

@pytest.fixture
def saved_deserialized_qualifications(deserialized_qualifications):
    QualificationChangeManager().add_deserialized_qualifications_to_db(
        *deserialized_qualifications
    ).commit()
    return deserialized_qualifications

@pytest.fixture
def qualification_grant(saved_deserialized_qualifications, volunteer):
    return QualificationGrant.objects.create(
        user=volunteer,
        qualification=Qualification.objects.create(
            title="custom",
            is_imported=False,
            abbreviation="cstm",
            uuid="5543ce30-5593-48b7-aa01-78d4cc54bf22",
            category=QualificationCategory.objects.get(),
        ),
    )


from tensorflow.keras.models import Sequential, load_model as keras_load_model
from tensorflow.keras.layers import Dense, Activation, Dropout, Flatten, Conv2D, MaxPooling2D, BatchNormalization
from tensorflow.keras.optimizers import Adam, SGD
from tensorflow.keras.metrics import categorical_crossentropy
from tensorflow.keras.regularizers import l2
from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, CSVLogger, Callback
from sklearn.metrics import confusion_matrix, accuracy_score, precision_recall_fscore_support, classification_report
from tensorflow.keras.preprocessing.image import ImageDataGenerator

from modules.m2_preprocessing import load_train_valid, load_testing, load_train_valid_tuner, load_testlines
from modules.utils import CLASSES, IMAGE_SIZE, model_metadata, save_model_metadata, \
	update_model_retrained, update_model_status, get_project_name, _load_json, _save_json, show_classification_metrics

from collections import Counter
from scipy.stats import shapiro
from numpy import random

import keras_tuner, json, os, codecs
import tensorflow_addons as tfa
import tensorflow as tf
import numpy as np
import seaborn as sb
import pandas as pd
import scipy.stats as stats

# dictionary for epochs
EPOCHS = {
	'max_dummy': 3,
	'patience_dummy': 3,
	'max': 500,
	'patience': 20,
	# 'patience': 10,
	# observation: 15 epochs is the max interval,
	# gender_2L_nocanny stops learning
}

# dictionary for tuner constants
TUNER = {
	'epochs_dummy': 3,
	'patience_dummy': 3,
	'max_trials_dummy': 3,
	'epochs': 200,
	'patience': 10,
	'max_trials': 10,
	'objective': 'val_accuracy',
	'overwrite': False,
}

class TrainingProcess():
	def __init__(
		self,
		save_path,
		dataset_path,
		nlayer,
		canny,
		category,
		dummy=False,
	):

		# --------- model metadata ---------
		self.dataset_path = dataset_path
		self.project_name = get_project_name(nlayer, canny, category)
		self.save_path = self.init_save_path(save_path)
		self.nlayer = nlayer
		self.canny = canny
		self.category = category
		self.binary = category != 'gender-handedness'
		self.morera = not canny and nlayer == 2
		metadata = model_metadata( # from utils.py
			path=self.save_path,
			nlayer=nlayer,
			canny=canny,
			category=category,
			morera=self.morera)

		# --------- if already done ---------
		if metadata['max_epoch'] > -1:
			raise Exception('Training process for this model is already completed. If you want to redo, delete or rename the project folder first.\n-dev')

		# ----------- set paths ------------
		rootprefix = f'{self.save_path}/{self.project_name}'
		self.metadata = rootprefix + '_metadata.json'
		self.checkpointfile = rootprefix + '_model.h5'
		self.csvloggerfile = rootprefix + '_model.csv'
		self.historyfile = rootprefix + '_history.json'

		# -------- if dummy test run --------
		self.dummy = dummy
		self.dummy_suffix = '_dummy' if self.dummy else ''

		# ------- set number of epochs ------
		self.epochs = EPOCHS['max'+self.dummy_suffix]
		self.patience = EPOCHS['patience'+self.dummy_suffix]

		# ------ if not morera's archi ------
		if not self.morera:
			self.save_path_tuner = self.init_save_path_tuner(self.save_path)
			self.checkpointtuner = self.save_path_tuner + '/latest_trial_model.h5'
			self.istuned = metadata['best_hp'] != 'None'
			self.epochs_tuner = TUNER['epochs'+self.dummy_suffix]
			self.patience_tuner = TUNER['patience'+self.dummy_suffix]
			self.tunername = self.project_name + '_tuner'

		# ----------- load model ------------
		self.model = self.load_model()  # from this class

		# -------- done initializing ----------
		print(f'\nMODEL: {self.project_name}\n(project_name)\n\nInitialized!')

	# creates and returns a save path for the project
	# if already a dir, it just returns the path
	def init_save_path(
		self,
		save_path
	):
		save_path = save_path + '/' + self.project_name

		if not os.path.isdir(save_path):
			os.makedirs(save_path)

		return save_path

	# creates and returns a save path for the tuner
	# if already a dir, it just returns the path
	def init_save_path_tuner(
		self,
		save_path
	):
		# save_path is already output/project_name
		save_path = save_path + '/' + self.project_name + '_tuner_output'

		# save_path will be output/project_name/project_name_tuner_output
		if not os.path.isdir(save_path):
			os.makedirs(save_path)

		return save_path

	# returns the number of trials
	# raises exception if didnt find anything
	def get_current_trials(self):
		max_trials = TUNER['max_trials'+self.dummy_suffix]
		trials = 0
		for trials in range(max_trials):
			trial_dir = 'trial_'
			if trials < 10 and max_trials >= 10:
				trial_dir += '0'

			if not os.path.isdir(f'{self.save_path}/{self.tunername}/{trial_dir}{trials}'):
				return trials
		return max_trials
		raise Exception('Bug. Did not return anything from function get_current_trials. -dev')

	# returns the number of epochs
	# returns 0 if no csv logger
	def get_max_epoch(
		self,
		csvlogger,
	):
		try: # get max epoch from the logger file path
			logs = pd.read_csv(csvlogger) # get logs
			return logs.shape[0]
		# if log not found, must be new
		except FileNotFoundError:
			return 0

	# if existing, returns the h5 file model
	# else, returns the base model, compiled if morera's
	def load_model(self):
		if os.path.isfile(self.checkpointfile):
			return keras_load_model(self.checkpointfile, compile=True) # from keras
		else:
			if self.morera:
				return self.build_model(None)
			else:
				csvloggerfile_tuner = f'{self.save_path_tuner}/latest_trial_log.csv'
				if os.path.isfile(csvloggerfile_tuner) and os.path.isfile(self.checkpointtuner):
					return keras_load_model(self.checkpointtuner, compile=True) # from keras
				return None

	# uses the entire function and compiles the base model if morera's
	# takes the entire function and is passed to the tuner, if tuner
	def build_model(
		self,
		hp,
	):
		input_shape = (IMAGE_SIZE['height'], IMAGE_SIZE['width'], 1)

		# ========== morera's ==========
		learning_rate = 0.001
		kernel_size = (5,5)
		maxpooling_size = (2,2)
		padding = 'same' # zero-padding
		dropout_layer = 0.25
		dropout_dense = 0.5
		units = 512 # last dense layer
		weight_decay = 1E-7

		# constant
		activation_layer = 'relu'
		activation_dense = 'softmax'

		if self.category == 'gender':
			filters = 128
		elif self.category == 'handedness':
			filters = 64
		else:
			filters = 32

		# ========== start of bayesian optimization ==========
		if hp:
			filters = hp.Int('filters', min_value=32, max_value=128, step=32)

			learning_rate = hp.Float(
				'learning_rate',
				min_value=1e-4,
				max_value=1e-2,
				sampling='log',
			)

			dropout_layer = hp.Float(
				'dropout_layer',
				min_value=0.0001,
				max_value=0.5,
				sampling='log',
			)

			dropout_dense = hp.Float(
				'dropout_dense',
				min_value=0.0001,
				max_value=0.5,
				sampling='log',
			)

			weight_decay = hp.Float(
				'weight_decay',
				min_value=1e-8,
				max_value=1e-6,
				sampling='log',
			)

		# Model
		model = Sequential()

		# First block
		model.add(Conv2D(filters, # 32
			kernel_size,
			input_shape=input_shape,
			padding=padding,
		))
		model.add(Activation(activation_layer)) # relu: activates the input of block to block
		model.add(Dropout(dropout_layer)) # reduces number of neurons randomly
		model.add(MaxPooling2D(maxpooling_size)) # summarizes the pixel values which makes the conv smaller essentially

		# Second block
		model.add(Conv2D(filters * 2, # 64
			kernel_size,
			padding=padding,
		))
		model.add(Activation(activation_layer))
		model.add(Dropout(dropout_layer))
		model.add(MaxPooling2D(maxpooling_size))

		# Third block
		# filters *2 is somewhat a default / norm in setting filters
		if self.nlayer == 3:
			model.add(Conv2D(filters * 4, # 128
				kernel_size,
				padding=padding,
			))
			model.add(Activation(activation_layer))
			model.add(Dropout(dropout_layer))
			model.add(MaxPooling2D(maxpooling_size))

		# Fully connected blocks
		model.add(Flatten()) # into 1 dimension for dense layer; prep for output

		# Output Layer
		model.add(Dense(units)) # summarizes the flattened
		model.add(Activation(activation_layer))
		model.add(Dropout(dropout_dense))
		model.add(Dense(2 if self.binary else 4)) # actual output
		model.add(Activation(activation_dense)) # softmax: activates the input of blocks to prediction

		# Instantiate a optimizer
		if self.binary:
			optimizer = tfa.optimizers.SGDW(
				learning_rate=learning_rate,
				weight_decay=weight_decay)
		else:
			optimizer = tfa.optimizers.AdamW(
				learning_rate=learning_rate,
				weight_decay=weight_decay)

		# Instantiate a logistic loss function that expects integer targets
		loss = 'categorical_crossentropy'

		# Instantiate an accuracy metric
		accuracy = 'accuracy'

		model.compile(optimizer=optimizer, loss=loss, metrics=[accuracy])

		return model

	# callback for search and train
	# makes a folder for logs and tensorboard
	def create_callbacks(
		self,
		csvlogger,
		checkpoint,
		patience,
		monitor,
	):
		return [
			CSVLogger(csvlogger, separator=',', append=True),
			ModelCheckpoint(checkpoint, monitor=monitor, verbose=1, save_best_only=True),
			EarlyStopping(monitor=monitor, patience=patience, verbose=1, baseline=None),
		]

	# search best hyperparameters using bayesian optimization tuner
	# will be skipped if morera, or if tuner is done searching
	def search(
		self,
		training_dataset_only
	):
		if self.morera:
			print('\nSearching with Tuner skipped: Morera\'s architecture is being used.')
			return

		# get current number of trials
		trial_count = max(self.get_current_trials(), 1)

		# if theres continuation epoch
		csvloggerfile = f'{self.save_path_tuner}/latest_trial_log.csv'
		cont_epoch = self.get_max_epoch(csvloggerfile)

		# instantiate custom model to tune
		model = MyHyperModel(
			save_path=self.save_path_tuner,
			build_model=self.build_model,
			category=self.category,
			nlayer=self.nlayer,
			binary=self.binary,
			epochs=self.epochs_tuner,
			cont_epoch=cont_epoch,
			model=self.model,
			trial_count=trial_count,
		)

		tuner =  keras_tuner.BayesianOptimization(
			hypermodel=model,
			objective=TUNER['objective'],
			max_trials=TUNER['max_trials'+self.dummy_suffix],
			overwrite=TUNER['overwrite'],
			directory=self.save_path,
			project_name=self.tunername,
		)

		if not self.istuned:
			tuner_train_batch, tuner_valid_batch = load_train_valid_tuner( # from module 2
				path=self.dataset_path,
				category=self.category,
				canny=self.canny,
				dummy=self.dummy,
				training_dataset_only=training_dataset_only,
			)

			tuner.search(
				tuner_train_batch,
				validation_data = tuner_valid_batch,
				verbose = 1,
				callbacks=self.create_callbacks(
					csvlogger=csvloggerfile,
					checkpoint=self.checkpointtuner,
					patience=self.patience_tuner,
					monitor='val_accuracy',
				),
			)

			update_model_status( # from utils.py
				project_path = self.metadata,
				best_hp = tuner.get_best_hyperparameters()[0].values,
			)

		print(f'\nSearching with Tuner finished: {TUNER["max_trials"+self.dummy_suffix]} "max_trial" for the tuner has been reached,')

		# get the top 1 hyperparameters.
		best_hp = tuner.get_best_hyperparameters()[0]

		print('With best hyperparameters:', end=' ')
		print(best_hp.values, end='\n')

		if self.istuned:
			return

		# build the model with the best hp.
		self.model = self.build_model(best_hp)

	# train with best hyperparameters
	# returns the trained model
	def train(
		self,
		training_dataset_only,
	):

		# get last epoch if stopped, then print message
		cont_epoch = self.get_max_epoch(csvlogger=self.csvloggerfile)
		if cont_epoch > 0:
			self.epochs -= cont_epoch
			message = f'Project stopped training at {cont_epoch} epoch/s... '
			message += f'Continuing with {self.epochs} epoch/s left.'
			print(message, end='\n')

		# load dataset
		train_batch, valid_batch = load_train_valid( # from module 2
			path=self.dataset_path,
			category=self.category,
			canny=self.canny,
			dummy=self.dummy,
			training_dataset_only=training_dataset_only,
		)

		# start/continue to train the model
		self.model.fit(
			train_batch,
			validation_data=valid_batch,
			epochs=self.epochs,
			verbose=1,
			callbacks=self.create_callbacks(
				csvlogger=self.csvloggerfile,
				checkpoint=self.checkpointfile,
				patience=self.patience,
				monitor='val_accuracy',
			),
		)

		update_model_status( # from utils.py
			project_path = self.metadata,
			max_epoch = self.get_max_epoch(csvlogger=self.csvloggerfile),
		)

	# function that will start the training process
	# calls search and train function
	def start(
		self,
		training_dataset_only=False,
	):
		self.search(training_dataset_only)
		self.train(training_dataset_only)

class MyHyperModel(keras_tuner.HyperModel):
	def __init__(
		self,
		save_path,
		build_model, 		# function
		category, 			# metadata
		nlayer,
		binary,
		epochs,				# epochs of tuner
		cont_epoch,			# continutation epoch
		model,				# h5 file of trial model
		trial_count,		# current number of trials
	):
		super(MyHyperModel, self).__init__()
		self.save_path = save_path
		self.build_model = build_model
		self.category = category
		self.nlayer = nlayer
		self.binary = binary
		self.epochs = epochs
		self.cont_epoch = cont_epoch
		self.model = model
		self.trial_count = trial_count

	# uses the build of TrainingProcess
	# returns the compiled model
	def build(self, hp):
		model = self.build_model(hp)
		return model

	# custom fit function for this model
	# only used when tuning
	def fit(self, hp, model, *args, **kwargs):
		if self.model is not None:
			model = self.model

		# get last epoch if stopped, then print message
		epochs = self.epochs
		if self.cont_epoch > 0:
			epochs -= self.cont_epoch
			message = f'Project stopped tuning at {self.cont_epoch} epoch/s... '
			message += f'Continuing with {epochs} epoch/s left.'
			print(message, end='\n')

		# fit for one trial
		history = model.fit(
			*args,
			epochs=epochs,
			**kwargs,
		)

		max_trials = TUNER['max_trials']
		trial_suffix = ''
		if self.trial_count < 10 and max_trials >= 10:
			trial_suffix += '0'

		# rename the csv logger to not overwrite it
		os.rename(
			f'{self.save_path}/latest_trial_log.csv',
			f'{self.save_path}/trial_{trial_suffix}{self.trial_count-1}.csv',

		)

		# increase trial count
		self.trial_count += 1

		# reset cont_epoch, since trial is done
		self.cont_epoch = 0

		return history

class TestingProcess():
	def __init__(self,
		save_path,
		dataset_path=None,
		dummy=False,
		show_skipped=False,
		show_results=False,
		return_skipped=False,
	):
		self.save_path_root = save_path
		self.dataset_path = dataset_path
		self.dummy = dummy
		self.show_skipped = show_skipped
		self.show_results = show_results
		self.return_skipped = return_skipped

	def load_model(
		self,
		checkpointfile,
	):
		if os.path.isfile(checkpointfile):
			return keras_load_model(checkpointfile, compile=True) # from keras
		else:
			return None

	def model_summary(self, project_name):
		project_path = f'{self.save_path_root}/{project_name}'
		model = self.load_model(f'{project_path}/{project_name}_model.h5')
		print(model.summary())

	def test(self, project_name):
		if not self.dataset_path:
			raise Exception('Attempting to test without dataset_path. -dev')

		if type(project_name) is not list:
			project_name = [project_name]

		skipped = {}

		for project in project_name:
			# set path
			project_path = f'{self.save_path_root}/{project}'

			# if project does not exist
			if not os.path.isdir(project_path):
				skipped[project] = 'Project directory not found.'
				continue

			# get metadata
			metadata = model_metadata( # from utils.py
				path=project_path,
				project_name=project,
			)
			canny = metadata['canny'] == 'True'
			category = metadata['category']
			binary = category != 'gender-handedness'

			# if already done with results
			if os.path.isfile(f'{project_path}/{project}_results.json'):
				skipped[project] = 'Already been tested. Please check the \"results\" json.'
				continue

			# if not done training
			if metadata['max_epoch'] == -1:
				skipped[project] = 'This model has not finished training yet.'
				continue

			# load model and dataset
			model = self.load_model(f'{project_path}/{project}_model.h5')

			# if model is missing
			if model is None:
				skipped[project] = 'The "_model.h5" file is missing.'
				continue

			# prompt project name
			print(f'\nMODEL: {project}\n(project_name)\n\n')

			# dataset path
			if self.dummy:
				dataset_path = 'data/sample'
			else:
				dataset_path = self.dataset_path

			# load the test lines
			test_x, test_y = load_testlines( # from module 2
				path=dataset_path,
				category=category,
				canny=canny,
				batch_size=1,
				dummy=self.dummy,
			)

			# test the model
			results_data = self.test_category_model(
				model=model,
				test_x=test_x,
				test_y=test_y,
				project_name=project,
				project_path=project_path,
			)

			if self.show_results:
				show_classification_metrics(
					classification_metrics=results_data['classification_metrics'],
					project_name=project,
					category=category,
				)

		if self.show_skipped:
			if skipped != {}:
				print('Skipped:')
				for key, val in skipped.items():
					print(f'{key}: {val}')

		if self.return_skipped:
			return skipped

	def majority_voting_scheme(
		self,
		pred_words_y,
	):
		vote_pred = []
		vote_class = []

		for i, y in enumerate(pred_words_y):
			pred_words_y[i] = y[0]

		#get the highest prediction and its class
		for y in pred_words_y:
			pred_conf = max(y)
			pred_class = y.index(pred_conf)
			# print(f'y: {y}, pred_conf: {pred_conf}, pred_class: {pred_class}')
			vote_pred.append(pred_conf)
			vote_class.append(pred_class)

		# convert array into dictionary
		count = Counter(vote_class)

		# traverse dictionary and check majority element
		size = len(vote_class)

		highest = {
			'class': [],
			'votes': 0,
		}
		for (perclass, votes_perclass) in count.items():
			# print(f'class: {perclass}, votes: {votes_perclass}')
			if votes_perclass > highest['votes']: # if found higher
				highest['class'] = [perclass]
				highest['votes'] = votes_perclass
			elif votes_perclass == highest['votes']: # if tie
				highest['class'].append(perclass)

		if len(highest['class']) == 1:
			return perclass, {'votes/confidence': f'{highest["votes"]}/{size}'}
		else: # if tie
			highest_conf_pred = 0
			highest_conf_class = 0
			for i in range(len(vote_class)):
				if vote_class[i] in highest['class'] and vote_pred[i] > highest_conf_pred:
					highest_conf_pred = vote_pred[i]
					highest_conf_class = vote_class[i]
			return highest_conf_class, {'votes/confidence': highest_conf_pred}

	def predict(
		self,
		model,
		word_imgs,
	):
		pred_words_y = []
		for word in word_imgs:
			pred_words_y.append(model.predict(word).tolist())
		pred_line, pred_metadata = self.majority_voting_scheme(pred_words_y)
		return pred_line, pred_metadata

	def test_category_model(
		self,
		model,
		test_x,
		test_y=None,
		project_name=None,
		project_path=None,
	):
		line_counter =  0
		pred_y_perword = []
		pred_y = []
		pred_y_md = []
		for data_line, data_label in zip(test_x, test_y):
			pred_words_y = []
			# print(f'LABEL: {data_label}')
			for word in data_line:
				pred_words_y.append(model.predict(word).tolist())
			pred_y_perword.append(pred_words_y)
			pred_line, pred_metadata = self.majority_voting_scheme(pred_words_y)
			pred_y.append(pred_line)
			pred_y_md.append(pred_metadata)
			print(f'{line_counter+1} lines predicted.', end='\r')
			line_counter += 1
		# print()
		print('\nDone predicting\n')
		# print('\npred_y:')
		# print(pred_y)

		# performance metrics
		classification_metrics = self.compute_classification_metrics(
			ground=test_y,
			predicted=pred_y,
		)

		# save all as json
		return self.save_model_results(
			test_y=test_y,
			pred_y=pred_y,
			pred_y_md=pred_y_md,
			pred_y_perword=pred_y_perword,
			classification_metrics=classification_metrics,
			project_name=project_name,
			path=project_path,
		)

	def compute_classification_metrics(
		self,
		ground,
		predicted,
	):
		# accuracy
		acc = accuracy_score(ground, predicted)

		# precision, recall, fscore, support
		p, r, f, s = precision_recall_fscore_support( # from sklearn
			ground,
			predicted,
			labels=list(set(ground)),
		)

		# confusion matrix
		conf_matrix = confusion_matrix(ground, predicted)

		# classification report
		report = classification_report(ground, predicted)

		# return metrics
		metrics = {
			'matrix': conf_matrix,
			'accuracy': acc,
			'precision': p,
			'recall': r,
			'fscore': f,
			'support': s,
			'report': report,
		}

		return metrics

	def save_model_results(
		self,
		test_y,
		pred_y,
		pred_y_md,
		pred_y_perword,
		classification_metrics,
		project_name,
		path,
	):
		# PERFORMANCE METRICS
		matrix = []
		for row in classification_metrics['matrix']:
			matrix.append(row.tolist())
		classification_metrics['matrix'] = matrix
		classification_metrics['precision'] = classification_metrics['precision'].tolist()
		classification_metrics['recall'] = classification_metrics['recall'].tolist()
		classification_metrics['fscore'] = classification_metrics['fscore'].tolist()
		classification_metrics['support'] = classification_metrics['support'].tolist()

		# DATA
		data = {
			'name': project_name,
			'test_y': test_y,
			'pred_y': pred_y,
			'pred_y_md': pred_y_md,
			'pred_y_perword': pred_y_perword,
			'classification_metrics': classification_metrics,
		}

		_save_json(f'{path}/{project_name}_results.json', data)

		return data


import pygame
import random
pygame.init()
BLACK = (0, 0, 0)
WHITE = (255, 255, 255)
BLUE = (0, 0, 255)
GREEN = (0, 255, 0)
RED = (255, 0, 0)
ORANGE = (255,165,0)

PI = 3.141592653

size = (400, 500)
screen = pygame.display.set_mode(size)
pygame.display.set_caption("MC Birdie spil")

done= False
clock = pygame.time.Clock()

while not done:
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            done = True

    screen.fill(WHITE)
# Draw on the screen a line from (0,0) to (100,100)
# 5 pixels wide.
    pygame.draw.line(screen, GREEN, [0, 0], [100, 100], 5)
# Draw on the screen several lines from (0,10) to (100,110)
# 5 pixels wide using a loop
    for y_offset in range(0, 100, 10):
        pygame.draw.line(screen, RED, [0, 10 + y_offset], [100, 110 + y_offset], 5)
# Draw a rectangle
    pygame.draw.rect(screen, BLACK, [20, 20, 250, 100], 2)
# Draw an ellipse, using a rectangle as the outside boundaries
    pygame.draw.ellipse(screen, BLACK, [20, 20, 250, 100], 2)
# Draw an arc as part of an ellipse.
# Use radians to determine what angle to draw.
    pygame.draw.arc(screen, BLACK, [20, 220, 250, 200], 0, PI / 2, 2)
    pygame.draw.arc(screen, GREEN, [20, 220, 250, 200], PI / 2, PI, 2)
    pygame.draw.arc(screen, BLUE, [20, 220, 250, 200], PI, 3 * PI / 2, 2)
    pygame.draw.arc(screen, RED, [20, 220, 250, 200], 3 * PI / 2, 2 * PI, 2)
# This draws a triangle using the polygon command
    pygame.draw.polygon(screen, ORANGE, [[100, 100], [0, 200], [200, 200]], 5)
# Select the font to use, size, bold, italics
    font = pygame.font.SysFont('Calibri', 25, True, False)
# Render the text. "True" means anti-aliased text.
# Black is the color. This creates an image of the
# letters, but does not put it on the screen
    text = font.render("FYN FOR THE WIN!!!", True, BLACK)
# Put the image of the text on the screen at 250x250
    screen.blit(text, [250, 250])
    text2 = font.render("SOENDERJYLLAND!!!", True, RED)
    screen.blit(text2, [150, 350])

    pygame.display.flip()
    clock.tick(60)

pygame.quit()


from django.conf.urls import url,include
from django.contrib import admin
from blog import views

urlpatterns = [
    url(r'^$',views.index),#기존에 프로젝트 단에서 설정할 수 있는 경로를 앱마다 따로 url 파일을 생성해 경로를 설정할 수 있음
    url(r'^posts/$','blog.views.post_list'),
    url(r'^posts/(?P<pk>\d+)/$','blog.views.post_detail'),
    url(r'^posts/(?P<post_pk>\d+)/comments/new$','blog.views.comment_new'),
    url(r'^api/v1/', include('blog.api.v1')), #template 파일이 아닌 것을 뷰할 수 있음

]