"""
Input & Output Module
---------------------

*paat.io* provides loading and saving functions to process the raw
acceleration signals.

"""

import os
import logging
import zipfile
from struct import unpack
import tempfile
import sys

import numpy as np
import pandas as pd
from bitstring import Bits

from . import preprocessing

def _unzip_gt3x_file(file, save_location=None, delete_source_file=False):
    """
    Unzip the .gt3x file

    Parameters
    ---------
    file : string
        file location of the .gt3x file
    save_location : string (optional)
        location where the unzipped files should be saved, if not given, files are saved within the same folder as f
    delete_source_file : Boolean (Optional)
        if True, then the original .gt3x file will be deleted after it is unzipped

    Returns
    -------
    log_bin : string
        location where the log.bin file is stored
    info_txt : string
        location where the info.txt file is stored
    """

    # if save location is not given, then save in the same folder
    # as where the file resides with the folder name equal the name of the file
    if save_location is None:
        save_location = os.path.splitext(file)[0]

    # if folder does not exist, create it
    if not os.path.exists(save_location):
        os.makedirs(save_location)

    # check if file already exists
    if not os.path.exists(os.path.join(save_location, 'log.bin')) and not os.path.exists(os.path.join(save_location, 'info.txt')):

        try:
            # unzip the file
            with zipfile.ZipFile(file, 'r') as myzip:

                myzip.extractall(save_location)

        except Exception as msg:
            logging.error('Error unpacked file: %s', msg)
            return None, None

        finally:
            # delete source file if delete_source_file parameter is set to True
            if delete_source_file:
                os.remove(file)
    else:
        logging.debug('file already unpacked: %s', file)

    # create the path locations where the log.bin and info.txt files are stored
    log_bin = os.path.join(save_location, 'log.bin')
    info_txt = os.path.join(save_location, 'info.txt')

    # return location of the files
    return log_bin, info_txt

def _extract_info(info_txt):

    """
    Extract the content from the info.txt file that was unzipped from the raw .gt3x file
    Example:
    {'Battery_Voltage': '3,83', 'Acceleration_Scale': '256.0', 'Download_Date': '635672143370000000', 'Unexpected_Resets': '0', 'Stop_Date': '635664672000000000',
    'Firmware': '1.3.0', 'Acceleration_Min': '-8.0', 'Subject_Name': '90046928', 'Last_Sample_Time': '635664672000000000', 'Sample_Rate': '100',
    'Device_Type': 'wGT3XBT', 'TimeZone': '02:00:00', 'Acceleration_Max': '8.0', 'Serial_Number': 'MOS2C02150348', 'Start_Date': '635658624000000000',
    'Board_Revision': '3'}

    Parameters
    ----------
    info_txt : string
        location of the info.txt file on disk

    Returns
    -------
    info_data : dictionary
        dictionary of key, value pairs as extracted from the info file
    """

    # create empty dictionary
    info_data = {}

    try:
        # open the file
        with open(info_txt, 'r') as file:
            # loop trough each of the lines
            for line in file.readlines():
                # strip away the new lines
                line = line.strip('\r\n')
                # split on semicolon + space (the timezone value has semicolons in it and not using the space would cause issues splitting it)
                key, value = line.split(': ')
                # add to dictionary and replace key values with space in key with underscore
                info_data[key.replace(' ', '_')] = value
    except Exception as msg:
        logging.error('Error extracting data from info.txt file: %s', msg)

    # return dictionary
    return info_data

def _extract_log(log_bin, acceleration_scale, sample_rate, use_scaling=False):
    """
    Extract acceleration data from log.bin file that was unzipped from the raw .gt3x file
    One second of raw activity samples packed into 12-bit values in YXZ order.

    Parameters
    ----------
    log_bin : string
        location of the log.bin file on disk
    acceleration_scale : float
        Scale the resultant by the scale factor (this gives us an acceleration value in g's).
        Device serial numbers starting with NEO and CLE use a scale factor of 341 LSB/g (±6g).
        MOS devices use a 256 LSB/g scale factor (±8g). If a LOG_PARAMETER record is preset,
        then the ACCEL_SCALE value should be used.
    sample_rate : int
        sample rate, i.e. the number of Hz (how many values we obtain per second)
        This value can also be inferred from the payload size when you know how many axis data you have.
        So payload size of 450 bytes = 300 (12 bit sized ints). And when we have 3 axis, we have 300/3 = 100 values per 3-axis per second
    use_scaling: Boolean (optional)
        acceleration data is originally stored as a signed integer. To obtain the acceleration value we need to scale it with the acceleration_scale
        this will give us the decimal numbers. However, this would also increase memory size of the array in which we store the values. We might not want to
        scale if we only want to store the data in a smaller memory size format. Scaling can then be done at a later stage, for instance, when we want to pre-process the data

    Returns
    -------
    log_data : numpy array (time steps * sample_rate, num axes)
        log data contains the raw acceleration values in YXZ order
    log_time : numpy array (time steps, 1)
        log time contains the timestamps of measurements
    """

    # define the size of the payload. This is necessary because we need to define the size of the numpy array before we populate it.
    SIZE = _count_payload_size(log_bin)
    # raw data values are stored in ints, to obtain values in G, we need to scale them by a factor found in the acceleration_scale parameter within the info.txt file. For example, 256.0
    SCALING = 1. / acceleration_scale
    # counter so we can keep track of how many acceleration values we have processed
    COUNTER = 0
    # number of axes, the GTX3 is tri-axial, so we hard code it here.
    NUM_AXES = 3

    # empty dictionary where we can store the 12bit to signed integer values; this saves calculating them all the time
    bit12_to_int = {}

    """
        create empty numpy array where we can store the acceleration data to: dimensions (sample_rate x SIZE,number of axis)
        if we want to scale the values to floats, we need a bigger datatype, it would take more memory. If no scaling is necessary, because we can scale later during preprocessing or so, we can
        proceed with just an int8 datatype array.
        For example, without scaling, 7 days of data equals around 180MB, with scaling the numpy array equals around 1.5GB
    """

    # use int16 as datatype to store the signed integer values, this saves memory when saving the array
    acc_data_type = np.int16
    if use_scaling:
        # use float when we want to store the acceleration data in G, meaning that we the scaling factor to recalculate the signed int into decimal values
        acc_data_type = float
    # create empty array for the acceleration data
    log_data = np.empty((sample_rate * SIZE, NUM_AXES), dtype=acc_data_type)
    # empty numpy array to store the timestamps
    time_data = np.empty((SIZE, 1), dtype=np.uint32)

    # open the log.bin file in binary mode
    with open(log_bin, mode='rb') as file:

        try:

            # keep reading byte by byte
            while True:

                """
                Log Record Format
                Offset (bytes)    Size (bytes)    Name    Description    Part of Record
                0    1    Seperator    An ASCII record separator byte (1Eh) marks the beginning of each log record.    Header
                1    1    Type    A type identifier is used to interpret the payload of the record.    Header
                2    4    Timestamp    The date and time of the data contained in the record are marked to the nearest second in Unix time format.    Header
                6    2    Size    The size of the payload is given in bytes as an little-endian unsigned integer.    Header
                """
                _, payload_type, timestamp, size = unpack("<cbLH", file.read(8))

                # acceleration type 0 is the activity data, we skip all other data but can easily be read with an if statement
                if payload_type == 0:

                    """
                        This is the actual data that varies based on the record *Type* field. It's size is provided in the *Size* field. Please refer to the appropriate section for the record type for the indiviual payload formats.

                        read the payload in bits (the reason we are converting the bytes to bits is that 1 acceleration value is encoded in 12bit, so we can cut out 12 bits at a time from the full payload represented as individual bits)
                        basically the YXZ (3 axis) is 12 bit + 12 bit + 12 bit = 36 bits. When we have 100hz, we have a total of 36 * 100 = 3600 bits. When you look at the size of the payload in bytes, that is for instance 450 bytes, you can
                        see that this is also 450 * 8 = 3600 bits
                    """

                    # read the bytes as bits as a large string
                    payload_bits = Bits(bytes=file.read(size)).bin

                    # extract 12 bits as 1 acceleration value and add them to a list
                    bits_list = []
                    for ii in range(0, len(payload_bits), 12):

                        # extract the 12 bit as a string
                        bitstring = payload_bits[ii:ii + 12]

                        # convert to 12bit two's complement to signed integer value: also store values in dictionary for faster reading if not already present (the Bits function is not as fast as reading it from a dictionary)
                        if bitstring not in bit12_to_int:
                            # convert 12bit to signed integer
                            acc_value = Bits(bin=bitstring).int
                            # add to dictionary so we can read it faster next time we have the same value
                            bit12_to_int[bitstring] = acc_value
                        else:
                            # bitstring previously already converted to signed int, so we can obtain it from the dictionary
                            acc_value = bit12_to_int[bitstring]

                        # add to list
                        bits_list.append(acc_value)

                    # convert list to numpy array and perform scaling if it was set to True: no scaling allows for a smaller numpy array because we can use int8 and not need the float
                    if use_scaling:
                        payload_bits_array = np.array(bits_list).reshape(sample_rate, NUM_AXES) * SCALING
                    else:
                        payload_bits_array = np.array(bits_list).reshape(sample_rate, NUM_AXES)

                    # add payload bits array to overall numpy array
                    np_start = COUNTER * sample_rate
                    np_end = np_start + sample_rate
                    log_data[np_start:np_end] = payload_bits_array

                    # add the time component
                    time_data[COUNTER] = timestamp

                    # increase the counter
                    COUNTER += 1

                else:
                    # skip whatever is not acceleration data
                    # there are different payload types and can easily be read by adding a different payload_type in this section
                    file.seek(size, 1)

                """
                A 1-byte checksum immediately follows the record payload. It is a 1's complement, exclusive-or (XOR) of the log header and payload with an initial value of zero.
                TODO: calculate checksum from payload and header and see if it matches the checksum that we read from the last byte
                """
                _ = unpack("B", file.read(1))

                # stop when all records have been read
                if COUNTER == SIZE:

                    logging.info('Finished processing activity data')
                    break

        except Exception as msg:
            logging.error('Unpacking GTX3 exception: %s', msg)
            return None, None

        # return acceleration data + time data
        return log_data, time_data

def _count_payload_size(log_bin, count_payload=0):
    """
    Count the payload size of the log.bin file. The size of the payload is necessary to know how large
    the numpy array needs to be that will be populated while reading the log.bin file byte by byte. The reasons why
    is that populating an array is much faster than appending new array values to it

    Parameters
    ----------
    log_bin : string
        location of the log.bin file on disk
    count_payload : int (optional)
        the payload type that we want to count. default is 0, which is the acceleration data.

    Returns
    -------
    SIZE : int
        the size (as in count) of the payload
    """

    # define size variable
    SIZE = 0

    # open the log.bin file in binary mode
    with open(log_bin, mode='rb') as file:

        try:
            # keep reading byte by byte
            while True:

                # extract header information
                _, payload_type, _, size = unpack("<cbLH", file.read(8))

                # count payload
                if payload_type == count_payload:

                    # skip the byte content, we don't need to process it here
                    file.seek(size, 1)
                    # increment counter
                    SIZE += 1
                else:
                    # skip other payload types, we don't need to read it here
                    file.seek(size, 1)

                # skip the 1-byte checksum value
                file.seek(1, 1)

        except Exception:
            logging.info('Counted payload size: %s', SIZE)

            # return the value
            return SIZE

def _create_time_array(time_data, hz=100):
    """
    Create a time array by adding the miliseconds range of the sampling frequency
    the standard time array only accounts for full seconds. However, when the sampling
    frequency is 100 hertz for instance, we need to add 100 microseconds to the data

    Parameters
    ----------
    time_data : np.array
        numpy array containing unix timestamps (obtained by reading the raw .gt3x data)
    hz : int (optional)
        sampling frequency of the acceleration data (this is to know how many miliseconds we need to add to the time series)

    Returns
    -------
    time_data : np.array
        numpy array with correct number of time series (so now the sampling frequency is added within the original data which only contained seconds and not miliseconds)

    Raises
    ------
    NotImplementedError
        If the sampling frequency does not fit on milisecond precision
    """

    # check if the sampling frequenzy can fit into equal parts within a 1000ms window
    if 1000 % hz != 0:
        raise NotImplementedError("Creating time array does not support sampling frequencies other than 100hz yet. See https://github.com/Trybnetic/paat/issues/10")

    # calculate the step size of hz in 1s (so 100hz means 100 measurements in 1sec, so if we need to fill 1000ms then we need use a step size of 10)
    step_size = 1000 / hz
    # convert time_data of unix timestamps to numpy array of 64 bit seconds
    time_data = np.asarray(time_data, dtype='datetime64[s]')
    # convert the array to 64 bit milliseconds and add a time delta of a range of ms within a 1000ms window
    time_data = np.asarray(time_data, dtype='datetime64[ms]') + np.asarray(np.arange(0, 1000, step_size), dtype='timedelta64[ms]')
    # flatten the array
    time_data = time_data.flatten()

    return time_data

def _format_time(tstamp):
    tstamp = int(tstamp) // 10000000 + np.datetime64('0001-01-01T00:00:00').astype(int)
    return int(tstamp) * 1000

def _format_meta_data(meta):
    """
    Formats the meta data dict that is extracted from the gt3x file

    Parameters
    ----------
    meta : dict
        the meta data dict from the gt3x file

    Returns
    -------
    new_meta : dict
        an updated meta data dict having the fields of meta in the correct format
    """
    new_meta = {}

    # standard meta data fields of ActiGraph and their conversion functions
    fields_and_formats = {'Serial_Number': str,
                          'Device_Type': str,
                          'Firmware': str,
                          'Battery_Voltage': str,
                          'Sample_Rate': int,
                          'Start_Date': _format_time,
                          'Stop_Date': _format_time,
                          'Last_Sample_Time': _format_time,
                          'TimeZone': str,
                          'Download_Date': _format_time,
                          'Board_Revision': int,
                          'Unexpected_Resets': int,
                          'Acceleration_Scale': float,
                          'Acceleration_Min': float,
                          'Acceleration_Max': float,
                          'Subject_Name': str}

    for field, dtype in fields_and_formats.items():
        try:
            new_meta[field] = dtype(meta[field])
        except KeyError as msg:
            logging.info(msg)

    return new_meta

def _create_time_vector(start, n_samples, hz):
    """
    Creates a time vector starting at point start until start + n_samples/hz
    with a sampling frequency of hz

    Parameters
    ----------
    start : int or np.datetime64
        start point of the time vector at second precision level.
    n_samples : int
        number of samples
    hz : int
        sampling frequency

    Returns
    -------
    time_data : np.array
        a numpy array of np.datetime64 at nanosecond precision

    Raises
    ------
    NotImplementedError
        If the sampling frequency does not fit on milisecond precision
    """

    n_sec = n_samples / hz

    if not n_sec.is_integer():
        logging.error('Actiwave time in seconds not a whole number: %s', n_sec)
        sys.exit()

    # declare how many nanoseconds in one second
    ms_in_sec = 1e3

    # check if the sampling frequenzy can fit into equal parts within a nanosecond window
    if ms_in_sec % hz != 0:
        raise NotImplementedError("Creating time vector does not support sampling frequencies other than 100hz yet. See https://github.com/Trybnetic/paat/issues/10")

    step_size = ms_in_sec / hz

    # convert the array to 64 bit nanoseconds (necessary to fit 128hz for instance) and add a time delta of a range nanoseconds in seconds * lenght_sec, with corresponding stepsize
    time_data = np.asarray(start, dtype='datetime64[ms]') + np.asarray(np.arange(0, ms_in_sec * n_sec, step_size), dtype='timedelta64[ms]')

    return time_data.flatten()

def read_gt3x(file, rescale=True, pandas=True):
    """
    Reads a .gt3x file and returns the tri-axial acceleration values together
    with the corresponding time stamps and all meta data.

    Parameters
    ----------
    file : string
        file location of the .gt3x file
    rescale : boolean (optional)
        boolean indicating whether raw acceleration data should be rescaled to g values
    pandas : boolean (optional)
        boolean indicating whether the data should be returned as a pandas DataFrame

    Returns
    -------
    data : DataFrame
        a DataFrame containg the raw acceleration data
    sample_freq : int
        the sampling frequency in which the data was recorded
    time : np.array (n_samples x 1)
        a numpy array with time stamps for the observations in values
    values : np.array (n_samples x 3)
        a numpy array with the tri-axial acceleration values. If rescale is true, data
        is rescaled to units of g. Note, that this function returns the values in
        the default order of ActiGraph which is ['Y','X','Z']. Depending on further
        use, you might want to adjust that order.
    meta : dict
        a dict containing all meta data produced by ActiGraph

    """
    with tempfile.TemporaryDirectory() as tmpdirname:
        # unzip .gt3x file and get the file location of the binary log.bin (which contains the raw data) and the info.txt which contains the meta-data
        log_bin, info_txt = _unzip_gt3x_file(file=file, save_location=tmpdirname)

        # get meta data from info.txt file
        meta = _extract_info(info_txt)

        meta = _format_meta_data(meta)

        # extract acceleration data from the log file
        values, time_data = _extract_log(log_bin, meta['Acceleration_Scale'], meta['Sample_Rate'], use_scaling=rescale)

        # create time array
        time = _create_time_array(time_data, hz=meta['Sample_Rate'])

        # Add additional keys to meta (Note: they are important to later reconstruct the time vector)
        meta["Number_Of_Samples"] = values.shape[0]
        meta["Start_Time"] = time[0].astype(int)

    if pandas:
        data = pd.DataFrame(values, columns=["Y", "X", "Z"], index=time)
        data = data[["X", "Y", "Z"]]
        return data, meta['Sample_Rate']
    else:
        return time, values, meta


import matplotlib.pyplot as plt
from mplsoccer import VerticalPitch
from scipy.ndimage import gaussian_filter

def juego_de_posicion(df):
    pitch = VerticalPitch(
        pitch_type='statsbomb',
        line_zorder=2
        )

    fig, ax = pitch.draw(figsize=(6.6, 4.125))

    bin_statistic = pitch.bin_statistic_positional(
        df['y'],
        df['x'],
        statistic='count',
        positional='full',
        normalize=True
        )

    pitch.heatmap_positional(
        bin_statistic,
        ax=ax,
        edgecolors='gray',
        cmap='coolwarm'
        )

    labels = pitch.label_heatmap(
        bin_statistic,
        color='black',
        fontsize=9,
        ax=ax,
        ha='center',
        va='center',
        str_format='{:.0%}'
        )
    return fig

def three_zones_positioning(df, vertical_bins=3):
    pitch = VerticalPitch(
        pitch_type='statsbomb',
        line_zorder=2
    )
    fig, ax = pitch.draw(figsize=(6.6, 4.125))

    bin_statistic = pitch.bin_statistic(
        df['y'],
        df['x'],
        statistic='count',
        bins=(vertical_bins, 1),
        normalize=True
        )

    pitch.heatmap(
        bin_statistic,
        ax=ax,
        cmap='coolwarm',
        edgecolor='gray'
    )

    labels = pitch.label_heatmap(
        bin_statistic,
        color='black',
        fontsize=17,
        ax=ax,
        ha='center',
        va='center',
        str_format='{:.0%}')
    return fig

def position_heatmap_grids(df):
    pitch = VerticalPitch(
        pitch_type='statsbomb',
        line_zorder=2,
        line_color='#efefef'
    )

    fig, ax = pitch.draw(figsize=(6.6, 4.125))

    bin_statistic = pitch.bin_statistic(
        df['y'],
        df['x'],
        statistic='count',
        normalize=True,
        bins=(32, 24))

    bin_statistic['statistic'] = gaussian_filter(bin_statistic['statistic'], 1)

    pcm = pitch.heatmap(
        bin_statistic,
        ax=ax,
        cmap='hot',
        edgecolors='#22312b')

    return fig

def convexhull_plot(df):
    pitch = VerticalPitch()

    fig, ax = pitch.draw(figsize=(6.6, 4.125))
    hull = pitch.convexhull(
        df['y'],
        df['x'])

    poly = pitch.polygon(
        hull,
        ax=ax,
        edgecolor='red',
        facecolor='red',
        alpha=0.3)

    scatter = pitch.scatter(
        df['y'],
        df['x'],
        ax=ax,
        edgecolor='black',
        facecolor='red'
        )
    return fig

def get_pos_by_min_plot(df):
    labels = [i for i in range(0,df['minute'].max() + 1)]
    att_points = df[df.zone_played=='attacking']['percentage'].to_list()
    mid_points = df[df.zone_played=='middle']['percentage'].to_list()
    def_points = df[df.zone_played=='defensive']['percentage'].to_list()

    mid_def_points = [x + y for x, y in zip(mid_points, def_points)]

    width = 0.6       # the width of the bars: can also be len(x) sequence

    fig, ax = plt.subplots(figsize=(8,3.125))

    ax.bar(labels, att_points, width,
           label='attacking', bottom = mid_def_points,
           edgecolor = "black", linewidth = 2,
           color='blue')

    ax.bar(labels, mid_points, width,
           label='middle', bottom = def_points,
           edgecolor = "black", linewidth = 2)

    ax.bar(labels, def_points, width,
           label='defensive', edgecolor = "black",
           linewidth = 2, color='red')

    ax.legend(loc='upper right', bbox_to_anchor=(1.23, 1))
    ax.set_xlabel('minute')

    # Labels
    for bar in ax.patches:
        ax.text(bar.get_x() + bar.get_width() / 2,
                bar.get_height() / 2 + bar.get_y(),
                str(bar.get_height())+'%', ha = 'center',
                color = 'w', size = 9)

    return fig


import speech_recognition as sr

def main():
    # Load the audio file
    audio_file = sr.AudioFile("small.wav")

    # Initialize a recognizer object
    recognizer = sr.Recognizer()

    # Extract audio data from the file
    with audio_file as source:
        audio_data = recognizer.record(source)

    # Transcribe the audio data to text
    text = recognizer.recognize_google(audio_data, language="ur-PK")

    # Print the transcribed text
    print(text)

if __name__ == "__main__":
    main()


import vocabulary_funcs
import analysis_funcs

import matplotlib.pyplot as plt

def _tfidf_extract(ui_list, Post_list, mrph_path):
    Tfidf = vocabulary_funcs.TfidfClass(f_mrph=mrph_path)
    f_tfidf, else_tfidf = [], []
    for pi in ui_list:
        # print("   pi", pi)
        Tfidf.add_post_words(pi)
        post = Post_list[pi]
        post_tfidf = Tfidf.ref_tfidf_from_pi(pi)

        if not post.reply_to_id:
            else_tfidf.append(post_tfidf)
        elif Post_list[post.reply_to_id].user_id == "facilitator":
            f_tfidf.append(post_tfidf)
        else:
            else_tfidf.append(post_tfidf)

    return f_tfidf, else_tfidf

def _idf_extract(ui_list, Post_list, mrph_path):
    Tfidf = vocabulary_funcs.TfidfClass(f_mrph=mrph_path)
    f_tfidf, else_tfidf = [], []
    for pi in ui_list:
        # print("   pi", pi)
        Tfidf.add_post_words(pi)
        post = Post_list[pi]
        post_idf = Tfidf.ref_idf_per_pi(pi)

        if not post_idf:
            continue

        if not post.reply_to_id:
            else_tfidf.append(max(post_idf.values()))
        elif Post_list[post.reply_to_id].user_id == "facilitator":
            f_tfidf.append(max(post_idf.values()))
        else:
            else_tfidf.append(max(post_idf.values()))

    return f_tfidf, else_tfidf

def graphing_idf(fs, us, save_f):
    # https://stats.biopapyrus.jp/python/barplot.html
    error_bar_set = dict(
        lw=1,
        capthick=1,
        capsize=2
    )

    fig = plt.figure(figsize=(16, 9))
    print(us)
    print(fs)
    u_xs = list(range(len(us[2])))
    print("usr", u_xs)
    plt.bar(u_xs, us[0], yerr=us[1], label="another", color="steelblue", width=0.4, error_kw=error_bar_set)

    f_xs = list(map(lambda x: x + 0.4, fs[2]))  # 標準偏差が見にくいので
    print("fa", f_xs)
    plt.bar(f_xs, fs[0], yerr=fs[1], label="rep to agent", color="orange", width=0.4, error_kw=error_bar_set)

    u_xs = [x + 0.2 for x in u_xs]
    plt.xticks(u_xs, us[2])
    plt.legend(loc="best")
    # plt.show()
    plt.savefig(str(save_f))

def only_reply_to_facilitator(User_list, Post_list, mrph_path, save_f):
    usrs_idfs = []

    f_ave_list, f_std_list, f_x_list = [], [], []
    u_ave_list, u_std_list = [], []

    usr_name_list = list(User_list.keys())

    usr_name_list.remove("kitkat")
    usr_name_list.remove("faciitator")

    print(usr_name_list, type(usr_name_list))
    for u_idx, u_n in enumerate(usr_name_list):
        print(u_idx, ":", u_n)

    for u_idx, u_n in enumerate(usr_name_list):
        print("[", u_idx, "]", ":", "user name ", u_n)

        f_list, u_list = _idf_extract(User_list[u_n].pi_list, Post_list, mrph_path)

        if len(f_list) != 0:
            print("  ", u_n, "has reply to facilitator!!")
            f_ave, f_std = analysis_funcs.ave_and_var(f_list)
            f_ave_list.append(f_ave)
            f_std_list.append(f_std)
            f_x_list.append(u_idx)
        else:
            print("  ", u_n, "didn't reply to facilitator.")
        # print(u_list)
        u_ave, u_std = analysis_funcs.ave_and_var(u_list)
        u_ave_list.append(u_ave)
        u_std_list.append(u_std)
        # u_name_list[u_idx] = u_n + " (" + str(len(f_list)) + ")"

    graphing_idf([f_ave_list, f_std_list, f_x_list], [u_ave_list, u_std_list, usr_name_list], save_f)


import json
import sys
from configparser import ConfigParser
from enum import Enum
from collections import defaultdict
from pprint import pprint
from .bco_repo import REPO, EventId
import os
from datetime import datetime

class BaseEvent:
    KNOWN_FIELDS = {}
    FIRST_DATA_FIELD = 0

    def __init__(self, index, *fields):
        self.index = index
        self.fields = fields

    @property
    def described_fields(self):
        event_data = REPO.game_event_data.get(self.event_type_id, {})
        for idx, value in enumerate(self.fields):
            data_field_idx = idx - self.FIRST_DATA_FIELD + 1
            if idx in self.KNOWN_FIELDS:
                yield (
                    self.KNOWN_FIELDS[idx],
                    getattr(self, self.KNOWN_FIELDS[idx], self.fields[idx][0]),
                )
            elif (data_field_idx) in event_data:
                yield (event_data[data_field_idx].attr_description, value)
            else:
                yield ("UNKNOWN", value)

    def __repr__(self):
        return f"{self.__class__.__name__}({self.index}, {self.event_type_id}, {self.previous_index}, *{self.fields})"

class Event(BaseEvent):
    KNOWN_FIELDS = {0: "event_type_id", 1: "previous_index"}
    FIRST_DATA_FIELD = 2

    @property
    def event_type_id(self):
        try:
            return EventId(self.fields[0][0])
        except ValueError:
            return self.fields[0][0]

class Setup(BaseEvent):
    KNOWN_FIELDS = {
        2: "player_0_fighter",
        4: "player_0_name",
        7: "player_0_wins",
        8: "player_0_losses",
        9: "player_0_ties",
        10: "player_1_fighter",
        12: "player_1_name",
        15: "player_1_wins",
        16: "player_1_losses",
        17: "player_1_ties",
    }

    @property
    def player_0_name(self):
        return self.fields[4][0]

    @property
    def player_0_fighter(self):
        return REPO.fighter[self.fields[2][0]]

    @property
    def player_0_record(self):
        return (self.fields[7][0], self.fields[8][0], self.fields[9][0])

    @property
    def player_1_name(self):
        return self.fields[12][0]

    @property
    def player_1_fighter(self):
        return REPO.fighter[self.fields[10][0]]

    @property
    def player_1_record(self):
        return (self.fields[15][0], self.fields[16][0], self.fields[17][0])

    def __init__(self, index, *fields):
        self.event_type_id = "GAME_SETUP"
        super(Setup, self).__init__(index, *fields)

EVENT_TYPES = defaultdict(
    lambda: Event,
    {
        # EventId.reveal: Reveal,
        # EventId.ante: Ante,
        # EventId.ante_options: AnteOptions,
        # EventId.pair_options: PairOptions,
        # EventId.discard: Discard
    },
)

class Replay:
    def __init__(self, filename):
        self.filename = filename
        self.parsed = ConfigParser()
        self.parsed.read([filename])

    @property
    def name(self):
        return os.path.basename(self.filename)

    @property
    def match_date(self):
        stat = os.stat(self.filename)
        return datetime.fromtimestamp(stat.st_mtime)

    @property
    def player_0(self):
        setup = next(self.parsed_tuples)
        assert setup.event_type_id == "GAME_SETUP"
        return setup.player_0_name

    @property
    def player_1(self):
        setup = next(self.parsed_tuples)
        assert setup.event_type_id == "GAME_SETUP"
        return setup.player_1_name

    @property
    def fighter_0(self):
        setup = next(self.parsed_tuples)
        assert setup.event_type_id == "GAME_SETUP"
        return setup.player_0_fighter

    @property
    def fighter_1(self):
        setup = next(self.parsed_tuples)
        assert setup.event_type_id == "GAME_SETUP"
        return setup.player_1_fighter

    @property
    def record_0(self):
        setup = next(self.parsed_tuples)
        assert setup.event_type_id == "GAME_SETUP"
        return setup.player_0_record

    @property
    def record_1(self):
        setup = next(self.parsed_tuples)
        assert setup.event_type_id == "GAME_SETUP"
        return setup.player_1_record

    @property
    def winner(self):
        events = list(self.parsed_tuples)
        end_events = (
            event
            for event in events
            if event.event_type_id in (EventId.player_wins, EventId.concede)
        )

        def _winner(event):
            if event.event_type_id == EventId.player_wins:
                winning_index = event.fields[2][0]
            elif event.event_type_id == EventId.concede:
                winning_index = 1 - event.fields[2][0]
            return getattr(self, f"player_{winning_index}")

        winners = set(_winner(event) for event in end_events if len(event.fields) >= 3)

        assert len(winners) <= 1, f"Unexpectedly more than one winner in {self.name}"
        if len(winners) == 1:
            return winners.pop()
        else:
            return None

    def raw_tuple(self, timestamp):
        section = self.parsed[timestamp]
        item_count = int(float(section.get("size").strip('"')))
        return list(self.read_value(timestamp, index) for index in range(item_count))

    def read_value(self, timestamp, index):
        section = self.parsed[timestamp]
        str_value = section[f"{index}_value"].strip('"')
        read_type = section[f"{index}_read_type"].strip('"')
        type = int(float(section.get(f"{index}_type").strip('"')))
        if read_type == "string":
            return (str_value, type)
        elif read_type == "real":
            return (int(float(str_value)), type)

    @property
    def raw_tuples(self):
        yield ("SETUP", self.raw_tuple("GAME_SETUP"))
        section_indexes = sorted(
            (
                section
                for section in self.parsed.sections()
                if section not in ("SIZE", "GAME_SETUP")
            ),
            key=int,
        )
        for section in section_indexes:
            yield (int(section), self.raw_tuple(section))

    def parsed_tuple(self, index, raw_tuple):
        if index == "SETUP":
            return Setup(index, *raw_tuple)
        else:
            try:
                event_type = EventId(raw_tuple[0][0])
            except ValueError:
                event_type = raw_tuple[0][0]
            return EVENT_TYPES[event_type](index, *raw_tuple)

    @property
    def parsed_tuples(self):
        for index, tuple in self.raw_tuples:
            yield self.parsed_tuple(index, tuple)

def parse_replay(filename):
    replayEventDict = {}
    lastevent = 0
    p0char = 0
    p1char = 0
    for line in replayTest:
        if line[0] == "[" and line[-2] == "]":
            try:
                lastevent = int(line[1:-2])
                if lastevent in replayEventDict:
                    print("duplicate event?")
                else:
                    replayEventDict[lastevent] = "{"
            except ValueError:
                lastevent = line[1:-2]
        else:
            if isinstance(lastevent, int):
                eqlidx = line.find("=")
                if eqlidx >= 0:
                    replayEventDict[lastevent] += (
                        '"' + line[0:eqlidx] + '":' + line[eqlidx + 1 : -1] + ",\n"
                    )
            if lastevent == "GAME_SETUP":
                if line.find("10_value") == 0:
                    p1char = int(line[line.find('"') + 1 : line.find(".")])
                if line.find("2_value") == 0:
                    p0char = int(line[line.find('"') + 1 : line.find(".")])
    # print(sorted(replayEventDict))
    test = json.loads(repoFile.read())
    elemDict = {}
    for idx in range(len(test["repo"]["source"])):
        elemDict[int(test["repo"]["source"][idx][0])] = test["repo"]["source"][idx][1]
    p1charName = ""
    p0charName = ""
    for idx in range(len(test["repo"]["fighter"])):
        if int(test["repo"]["fighter"][idx][0]) == p1char:
            p1charName = test["repo"]["fighter"][idx][2]
        if int(test["repo"]["fighter"][idx][0]) == p0char:
            p0charName = test["repo"]["fighter"][idx][2]
    print(p0charName + " VS " + p1charName)

    for idx in sorted(replayEventDict):
        temp = json.loads(replayEventDict[idx][:-2] + "}")
        eventID = int_temp(temp, "0_value")
        if eventID == -812:
            show_fighter(temp, "REVEAL")
            if int_temp(temp, "3_value") == 2:
                styleID = int_temp(temp, "5_value")
                print(styleID, elemDict[styleID])
            else:
                print("FINISHER")
            baseID = int_temp(temp, "4_value")
            print(baseID, elemDict[baseID])
        if eventID == -804:
            show_fighter(temp, "ANTE")
            anteID = int_temp(temp, "3_value")
            print(anteID, elemDict[anteID])
        if eventID == -1201:
            show_fighter(temp, "ANTE OPTIONS")
            anteNum = int(temp["3_value"][: temp["3_value"].find(".")])
            for i in range(anteNum):
                anteID = int(
                    temp[str(i + 4) + "_value"][: temp[str(i + 4) + "_value"].find(".")]
                )
                print(anteID, elemDict[anteID])
        if eventID == -1200:
            print()
            nextNum = 2
            readStep = ReadData(1)
            for i in range(2, int_temp(temp, "size")):
                if i == nextNum:
                    if readStep == ReadData(1):
                        nextNum = i + int_temp(temp, str(i) + "_value") + 1
                        print(p0charName, "Bases")
                        readStep = ReadData(readStep.value + 1)
                    elif readStep == ReadData(2):
                        nextNum = i + int_temp(temp, str(i) + "_value") + 1
                        print(p0charName, "Styles")
                        readStep = ReadData(readStep.value + 1)
                    elif readStep == ReadData(3):
                        nextNum = i + int_temp(temp, str(i) + "_value") + 1
                        print(p0charName, "Finishers")
                        readStep = ReadData(readStep.value + 1)
                    elif readStep == ReadData(4):
                        nextNum = i + int_temp(temp, str(i) + "_value") + 1
                        print(p1charName, "Bases")
                        readStep = ReadData(readStep.value + 1)
                    elif readStep == ReadData(5):
                        nextNum = i + int_temp(temp, str(i) + "_value") + 1
                        print(p1charName, "Styles")
                        readStep = ReadData(readStep.value + 1)


import cv2
import numpy as np

frameWidth = 320
frameHeight = 240
cap = cv2.VideoCapture(0, cv2.CAP_DSHOW) #captureDevice = cv2.VideoCapture(0, cv2.CAP_DSHOW)
cap.set(3, frameWidth)
cap.set(4, frameHeight)
cap.set(10, 150)

colors = [[15, 128, 94, 88, 224, 196]]

def getColor(img, colors):
    imgHSV = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
    lo = np.array(colors[0][:3])
    hi = np.array(colors[0][3:])
    mask = cv2.inRange(imgHSV, lo, hi)
    getContour(mask)
    # cv2.imshow("mask", mask)

def getContour(img):
    contours,hierarchy = cv2.findContours(img,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_NONE)
    for cnt in contours:
        area = cv2.contourArea(cnt)
        if area>500:
            cv2.drawContours(imgRESULT, cnt, -1, (255,0,0), 3)
            peri = cv2.arcLength(cnt, True)
            approx = cv2.approxPolyDP(cnt, 0.02*peri, True)
            x,y,w,h = cv2.boundingRect(approx)

while True:
    _, img = cap.read()
    imgRESULT = img.copy()
    getColor(img, colors)
    cv2.imshow("CAM", imgRESULT)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break


# Backtracking 사용하는 방법

def dfs(l):
    if l == m:
        for i in res:
            print(i, end=" ")
        print()

    else:
        for i in range(1, n+1):
            if ch[i] == 0:
                ch[i] = 1
                res[l] = i
                dfs(l+1)
                ch[i] = 0

n, m = map(int, input().split())
ch = [0]*(n+1)
res = [0]*m
dfs(0)

# Permutations 사용하는 방법

from itertools import permutations

N, M = map(int, input().split())

number = [i+1 for i in range(N)]

per = (list(permutations(number, M)))

for i in per:
    print(*i)


import numpy as np
import pickle as pkl
import tensorflow as tf
import conv_layer_prediction
import dense_layer_prediction
import dense_layer_poly_mult
import mean_pooling_layer_prediction
import max_pooling_layer
import batch_norm
from write_weights import write_weights, read_weights
# from custom_bfv.bfv import BFV
from custom_bfv.bfv_ntt import BFV
import save_mnist_test
from print_outputs import print_1D_output, print_3D_output
import torch
import torchvision
from residual import residual

def load_pickle(filename):
  with open(f'{filename}', 'rb') as f:
    return pkl.load(f)

def main():
  # model_name = 'miniONN_cifar_model'
  model_name = 'resnet18'
  # single_test = load_pickle('cifar_test.pkl')
  # Xtest = single_test[0]
  # Ytest = single_test[1]
  num = 1 / 256
  Xtest = [[[num]*64] * 64]*3
  Xtest = np.array(Xtest)
  Ytest = [[[num]*64] * 64]
  Ytest = np.array(Ytest)
  Ytest = np.reshape(Ytest, (64, 64, 1))
  tf_test(model_name, Xtest, Ytest)
  Xtest = np.reshape(Xtest, (64, 64, 3))
  custom_test(model_name, Xtest)
  return

P_2_SCALE = 16

def scale_to_int(arr):
  scale = 2**P_2_SCALE
  rescaled = arr * scale
  # rescaled = np.round(rescaled)
  return rescaled.astype(int)

def ReLU(arr):
  ret = arr.copy()

  for i, image in enumerate(ret):
    for j, row in enumerate(image):
      for k, val in enumerate(row):
        if val < 0:
          ret[i][j][k] = 0
  # ret = scale_down(ret, 2**P_2_SCALE)
  return ret

def scale_down(arr, scale):
  ret = arr.copy()

  for i, image in enumerate(ret):
    for j, row in enumerate(image):
      for k, val in enumerate(row):
        ret[i][j][k] = val // scale
  return ret

def wrapper_conv_layer(input_layer, layer_path, pad=0, enc_scheme=None, stride=1, as_ints=False):
  layer = input_layer.copy()
  if pad:
    layer = conv_layer_prediction.pad_images( layer, pad )
  output = conv_layer_prediction.conv_layer_prediction( layer, read_weights(layer_path), stride, enc_scheme )
  output = np.array(output)
  if (as_ints):
    output = scale_down(output, 2**P_2_SCALE)
  layer_name = layer_path.split('/')[-1].split('.')[0]
  print(f'{layer_name} Done')
  return output

def custom_test(model_name, Xtest):

  as_ints = True

  # Load and reshape the test image
  # Xtest = scale_to_int(Xtest)
  image_height, image_width, image_channels = Xtest.shape
  images = np.zeros( (image_channels, image_height, image_width))

  if (as_ints):
    Xtest = scale_to_int(Xtest)
    images = np.zeros( (image_channels, image_height, image_width), dtype=int )
    model_name = 'int_' + model_name

  for wi, w in enumerate(Xtest):
    for hi, h in enumerate(w):
      for ci, c in enumerate(h):
        images[ci][wi][hi] = c
  Xtest = images

  directory = f'./model_weights/{model_name}'

  enc_scheme = BFV(q = 2**38, t = 2**25, n = 2**10)

  OUTPUT_PRINT = lambda output: print(output.reshape( np.prod(output.shape) ).astype(int)[:100])

  def basic_block(inp, layer, first_stride=1, as_ints=False):
    output = wrapper_conv_layer(inp, f'{directory}/layer{layer}.conv1.weight.txt', pad=1, enc_scheme=enc_scheme, stride=first_stride, as_ints=as_ints)
    output = batch_norm.batch_main(output, f'{directory}/layer{layer}.bn1.weight.txt', f'{directory}/layer{layer}.bn1.bias.txt', f'{directory}/layer{layer}.bn1.running_mean.txt', f'{directory}/layer{layer}.bn1.running_var.txt', as_ints=as_ints)
    output = ReLU(output)
    output = wrapper_conv_layer(output, f'{directory}/layer{layer}.conv2.weight.txt', pad=1, enc_scheme=enc_scheme, stride=1, as_ints=as_ints)
    output = batch_norm.batch_main(output, f'{directory}/layer{layer}.bn2.weight.txt', f'{directory}/layer{layer}.bn2.bias.txt', f'{directory}/layer{layer}.bn2.running_mean.txt', f'{directory}/layer{layer}.bn2.running_var.txt', as_ints=as_ints)
    # print(inp.shape, output.shape)
    print(f'inp.shape: {inp.shape}')
    print(f'output.shape: {output.shape}')
    print()
    # output = ReLU(output)
    return output

  def downsample(inp, layer, as_ints=False):
    output = wrapper_conv_layer(inp, f'{directory}/layer{layer}.downsample.0.weight.txt', pad=0, enc_scheme=enc_scheme, stride=2, as_ints=as_ints)
    output = batch_norm.batch_main(output, f'{directory}/layer{layer}.downsample.1.weight.txt', f'{directory}/layer{layer}.downsample.1.bias.txt', f'{directory}/layer{layer}.downsample.1.running_mean.txt', f'{directory}/layer{layer}.downsample.1.running_var.txt', as_ints=as_ints)
    return output

  def _layer(inp, n, _downsample=False, first_stride=1, as_ints=False):
    save_inp = inp.copy()
    if _downsample:
      save_inp = downsample(inp, n, as_ints=as_ints)
    output = basic_block(inp, n, first_stride, as_ints=as_ints)
    output = residual(save_inp, output)
    output = ReLU(output)
    return output

  #test = [[[1]*16]*16]*64
  #test = np.array(test)
  # begin
  output = wrapper_conv_layer(Xtest, f'{directory}/conv1.weight.txt', pad=3, enc_scheme=enc_scheme, stride=2, as_ints=as_ints)
  # OUTPUT_PRINT(output)
  # return
  #print(output.shape)
  output = batch_norm.batch_main(output, f'{directory}/bn1.weight.txt', f'{directory}/bn1.bias.txt', f'{directory}/bn1.running_mean.txt', f'{directory}/bn1.running_var.txt', as_ints=as_ints)
  output = ReLU(output)
  # pad before max pool
  output = conv_layer_prediction.pad_images(output, 1)
  output = max_pooling_layer.max_pooling_layer(output, (3, 3), stride=2)
  # layer 1.0
  output = _layer(output, "1.0", as_ints=as_ints)
  # layer 1.1
  output = _layer(output, "1.1", as_ints=as_ints)
  # layer 2.0
  output = _layer(output, "2.0", _downsample=True, first_stride=2, as_ints=as_ints)
  # layer 2.1
  output = _layer(output, "2.1", as_ints=as_ints)
  # layer 3.0
  output = _layer(output, "3.0", _downsample=True, first_stride=2, as_ints=as_ints)
  # layer 3.1
  output = _layer(output, "3.1", as_ints=as_ints)
  # layer 4.0
  output = _layer(output, "4.0", _downsample=True, first_stride=2, as_ints=as_ints)
  # layer 4.1
  output = _layer(output, "4.1", as_ints=as_ints)
  print()
  # print((output/256))

  # adaptive avg pooling ?
  output = mean_pooling_layer_prediction.adaptive_mean_pooling_layer(output, output_shape=(1, 1))
  # print()
  # print((output/(2**P_2_SCALE))[:50])
  # return output

  filters, width, height = output.shape
  output = output.reshape(width*height*filters)

  dense_kernel = read_weights(f"{directory}/fc.weight.txt")
  dense_bias = read_weights(f"{directory}/fc.bias.txt")
  # output = dense_layer_prediction.dense_layer( output, dense_kernel)
  dense_output = [0] * 1000

  '''
  split = 4000
  for i in range(0, width*height*filters, split):
    temp = dense_layer_prediction.dense_layer( output[i:min(i+split, width*height*filters - i)], dense_kernel[:][i:min(i+split, width*height*filters - i)])
    for j in range(len(temp)):
      dense_output[j] += temp[j]

  for i in range(len(dense_output)):
    dense_output[i] += dense_bias[i]
  '''

  dense_output = dense_layer_prediction.dense_layer( output, dense_kernel, dense_bias)

  if (as_ints):
    dense_output = np.array(dense_output) / 2**P_2_SCALE
    dense_output = np.array(dense_output) / 2**P_2_SCALE
    # dense_output = dense_output.astype(int)
    pass
  output = dense_output

  print(f'Prediction: [', end=' ')
  for pred in output[:100]:
    pred = float(pred)
    print(f'{pred:.7f}', end=' ')
  print(']')
  return output

  max_scale = max(output)
  norm_scale = (2**P_2_SCALE)**2
  for ind, val in enumerate(output):
    output[ind] = val // norm_scale
    pass

  output = dense_layer_prediction.softmax( output )
  print(f'Prediction: [', end=' ')
  for pred in output:
    pred = float(pred)
    print(f'{pred:.7f}', end=' ')
  print(']')
  return output

def tf_test(model_name, Xtest, Ytest):
  # model_name = 'small_model'
  # model = tf.keras.models.load_model(f'./models/{model_name}.h5')
  model = torchvision.models.resnet18()
  model.load_state_dict(torch.load('./models/resnet18.pth'))

  Xtest = np.expand_dims(Xtest, 0)
  Xtest = torch.Tensor(Xtest)
  # print(f'Xtest: {Xtest}')
  # print(f'Xtest shape: {Xtest.shape}')
  # print(f'Ytest: {Ytest}')
  # print(f'Ytest shape: {Ytest.shape}')

  # Ypred = model.predict( Xtest )
  model.eval()

  # test = [[[[1]*16]*16]*64]
  # test = torch.Tensor(test)
  # temp = model.conv1(Xtest)
  # temp = model.bn1(temp)
  # temp = model.relu(temp)
  # temp = model.maxpool(temp)
  # temp = model.layer1(temp)
  # temp = model.layer2(temp)
  # temp = model.layer3(temp)
  # temp = model.layer4(temp)
  # temp = model.avgpool(temp)
  # print(temp[0][:50])
  # return temp

  #temp = model.layer2(temp)
  #temp = model.layer3(temp)
  #temp = model.layer4(temp)

  #output = model.avgpool(temp)
  #print(output.shape)
  #print(output[0])
  #return

  #conv_output_image = Ypred.permute(0, 2, 3, 1).detach().numpy()
  # print(conv_output_image)
  # print(output)

  with torch.no_grad():
    Ypred = model(Xtest)
  print(f'Prediction: [', end=' ')
  for pred in Ypred[0][:100]:
    pred = float(pred)
    print(f'{pred:.7f}', end=' ')
  print(']')
  # print(f'Acutal: {Ytest}')
  return Ypred[0]

def test_many_mnist_examples():
  num_tests = 20
  same_pred = 0
  total_diff = 0
  diff_arr = []
  for i in range(num_tests):
    save_mnist_test.main(i)
    print(f'Test {i+1}')
    print('---'*8)
    model_name = '4_layer_mnist_model'
    single_test = load_pickle('single_test.pkl')
    Xtest = single_test[0]
    Ytest = single_test[1]
    tf_output  = tf_test(model_name, Xtest, Ytest)
    our_output = custom_test(model_name, Xtest, Ytest)
    if ( np.argmax(tf_output) == np.argmax(our_output) ):
      same_pred += 1
    diff = sum(map(abs, [t-o for t,o in zip(tf_output, our_output) ]))
    diff_arr.append( diff )
    total_diff += diff
    print(f'diff: {diff}')
    print('\n'*2)
  print(f'total diff:   {total_diff}')
  print(f'average diff: {total_diff / num_tests}')
  print(f'standard dev: {np.std(diff_arr)}')
  print(f'same pred:    {same_pred} / {num_tests}')

if __name__ == '__main__':
  # test_many_mnist_examples()
  main()


#!/usr/bin/python3

import numpy as np
from scipy import interpolate

def FirstInperlotation(filename, a0, b0, index):
    #Load the impedance file
    Impedance = np.loadtxt(filename)

    #Gets the component for different poisson's ratio.
    K25 = np.transpose(Impedance[:, index[ 0: 5]])
    K35 = np.transpose(Impedance[:, index[ 5:10]])
    K45 = np.transpose(Impedance[:, index[10:15]])

    # Data axes
    x0 = Impedance[:, 0]
    y0 = np.array([0.1, 0.25, 0.50, 1.0, 2.0])

    #Create interpolator
    Fun_25 = interpolate.interp2d(x0, y0, K25, kind='linear')
    Fun_35 = interpolate.interp2d(x0, y0, K35, kind='linear')
    Fun_45 = interpolate.interp2d(x0, y0, K45, kind='linear')

    #Interpolate the values
    z0 = Fun_25(a0, b0)
    z1 = Fun_35(a0, b0)
    z2 = Fun_45(a0, b0)

    z = np.array([z0, z1, z2])

    return z

def SecondInterpolation(b, nu):
    #Solves a linear system.
    A = np.array([[1.0, 0.25, 0.25**2], [1.0, 0.35, 0.35**2], [1.0, 0.45, 0.45**2]]);
    x = np.linalg.solve(A, b)

    #Interpolate for Foisson's ratio.
    z = x[0] + x[1]*nu + x[2]*nu**2

    return z

def ComputeImpedance(filename, a0, b0, nu):
    #Computes real part
    index = range(1,30,2)
    zp    = FirstInperlotation(filename, a0, b0, index)
    kij   = SecondInterpolation(zp, nu)

    #Computes imaginary part
    index = range(2,31,2)
    zp    = FirstInperlotation(filename, a0, b0, index)
    cij   = SecondInterpolation(zp, nu)

    Impedance = np.array([kij, cij])

    return Impedance

#INPUTS
hb = 15.0
Tb = 0.50
B  = 40.0
D  = 27.0
Vs = 250
f0 = 2.0
nu = 0.4275

#DIMENSIONLESS VECTOR
PI = np.array([B/Vs/Tb, hb/B, D/B, nu, f0*B/Vs])
print(PI)

#DIMENIONLESS FREQUENCY
a0 = 1.111 #model.predict(PI)

kxx = ComputeImpedance("Normalized_Impedace_Functions/Kxx.txt", a0, PI[2], PI[3])
kxt = ComputeImpedance("Normalized_Impedace_Functions/Kxt.txt", a0, PI[2], PI[3])
ktt = ComputeImpedance("Normalized_Impedace_Functions/Ktt.txt", a0, PI[2], PI[3])

print(kxx)
print(kxt)
print(ktt)

#[[ 0.72240007]
# [ 1.70080473]]
#[[ 0.13824467]
# [ 0.48635204]]
#[[ 1.28825378]
# [ 1.02496305]]


from __future__ import annotations

__all__ = ['SequentialExecutionEngine']

import logging
import time
import traceback
from typing import Iterable, List, cast
from typing_extensions import Literal

import nni
from nni.runtime.trial_command_channel import (
    set_default_trial_command_channel, get_default_trial_command_channel, TrialCommandChannel
)
from nni.nas.space import ExecutableModelSpace, ModelStatus
from nni.typehint import ParameterRecord, Parameters, TrialMetric

from .engine import ExecutionEngine
from .event import FinalMetricEvent, IntermediateMetricEvent, TrainingEndEvent

_logger = logging.getLogger(__name__)

class SequentialTrialCommandChannel(TrialCommandChannel):

    def __init__(self, engine: SequentialExecutionEngine, model: ExecutableModelSpace):
        self.engine = engine
        self.model = model

    def receive_parameter(self) -> ParameterRecord:
        return ParameterRecord(
            parameter_id=0,
            parameters=cast(Parameters, self.model),
        )

    def send_metric(
        self,
        type: Literal['PERIODICAL', 'FINAL'],  # pylint: disable=redefined-builtin
        parameter_id: int | None,
        trial_job_id: str,
        sequence: int,
        value: TrialMetric,
    ) -> None:
        if type == 'PERIODICAL':
            self.engine.dispatch_model_event(IntermediateMetricEvent(self.model, value))
        elif type == 'FINAL':
            self.engine.dispatch_model_event(FinalMetricEvent(self.model, value))
        else:
            raise ValueError(f'Unknown metric type: {type}')

class SequentialExecutionEngine(ExecutionEngine):
    """
    The execution engine will run every model in the current process.
    If multiple models have been submitted, they will be queued and run sequentially.

    Keyboard interrupt will terminate the currently running model and raise to let the main process know.
    """

    def __init__(self, max_model_count: int | None = None,
                 max_duration: float | None = None,
                 continue_on_failure: bool = False) -> None:
        super().__init__()

        self.max_model_count = max_model_count
        self.max_duration = max_duration
        self.continue_on_failure = continue_on_failure

        self._history: List[ExecutableModelSpace] = []
        self._model_count = 0
        self._total_duration = 0

    def _run_single_model(self, model: ExecutableModelSpace) -> None:
        model.status = ModelStatus.Training
        start_time = time.time()
        _prev_channel = get_default_trial_command_channel()
        try:
            # Reset the channel to overwrite get_next_parameter() and report_xxx_result()
            _channel = SequentialTrialCommandChannel(self, model)
            set_default_trial_command_channel(_channel)
            # Set the current parameter
            parameters = nni.get_next_parameter()
            assert parameters is model
            # Run training.
            model.execute()
            # Training success.
            status = ModelStatus.Trained
            duration = time.time() - start_time
            self._total_duration += duration
            _logger.debug('Execution time of model %d: %.2f seconds (total %.2f)',
                          self._model_count, duration, self._total_duration)
        except KeyboardInterrupt:
            # Training interrupted.
            duration = time.time() - start_time
            self._total_duration += duration
            _logger.error('Model %d is interrupted. Exiting gracefully...', self._model_count)
            status = ModelStatus.Interrupted
            raise
        except:
            # Training failed.
            _logger.error('Model %d fails to be executed.', self._model_count)
            duration = time.time() - start_time
            self._total_duration += duration
            status = ModelStatus.Failed
            if self.continue_on_failure:
                _logger.error(traceback.format_exc())
                _logger.error('Continue on failure. Skipping to next model.')
            else:
                raise
        finally:
            # Restore the trial command channel.
            set_default_trial_command_channel(_prev_channel)

            # Sometimes, callbacks could do heavy things here, e.g., retry the model.
            # So the callback should only be done at the very very end.
            # And we don't catch exceptions happen inside.
            self.dispatch_model_event(TrainingEndEvent(model, status))  # pylint: disable=used-before-assignment
            _logger.debug('Training end callbacks of model %d are done.', self._model_count)

    def submit_models(self, *models: ExecutableModelSpace) -> None:
        for model in models:
            if not model.status.frozen() or model.status.completed():
                raise RuntimeError(f'Model must be frozen before submitting, but got {model}')

            self._model_count += 1

            if self.max_model_count is not None and self._model_count > self.max_model_count:
                _logger.error('Maximum number of models reached (%d > %d). Models cannot be submitted anymore.',
                              self._model_count, self.max_model_count)
            if self.max_duration is not None and self._total_duration > self.max_duration:
                _logger.error('Maximum duration reached (%f > %f). Models cannot be submitted anymore.',
                              self._total_duration, self.max_duration)

            self._history.append(model)

            _logger.debug('Running model %d: %s', self._model_count, model)

            self._run_single_model(model)

    def list_models(self, status: ModelStatus | None = None) -> Iterable[ExecutableModelSpace]:
        if status is not None:
            return [m for m in self._history if m.status == status]
        return self._history

    def idle_worker_available(self) -> bool:
        """Return true because this engine will run models sequentially and never invokes this method when running the model."""
        return True

    def budget_available(self) -> bool:
        return (self.max_model_count is None or self._model_count < self.max_model_count) \
            and (self.max_duration is None or self._total_duration < self.max_duration)

    def shutdown(self) -> None:
        _logger.debug('Shutting down sequential engine.')

    def state_dict(self) -> dict:
        return {
            'model_count': self._model_count,
            'total_duration': self._total_duration,
        }

    def load_state_dict(self, state_dict: dict) -> None:
        if state_dict['model_count'] > 0:
            _logger.warning('Loading state for SequentialExecutionEngine does not recover previous submitted model history.')
        self._model_count = state_dict['model_count']
        self._total_duration = state_dict['total_duration']


import collections
class Solution:
    def intersect(self, nums1, nums2):
        d = collections.defaultdict(int)
        for i in nums1:
            d[i] += 1

        answer = []
        for i in nums2:
            if d[i] >= 1:
                answer.append(i)
                d[i] -= 1
        return answer

if __name__ == '__main__':
    nums1 = [1,2,2,1]
    nums2 = [2,2]
    expected_output = [2,2]
    test = Solution()
    assert(test.intersect(nums1, nums2) == expected_output)


from dataclasses import dataclass, field
from typing import Any, Dict, Optional, Sequence, Tuple

from torch import Tensor

from flash.core.serve.types.base import BaseType

@dataclass(unsafe_hash=True)
class Repeated(BaseType):
    """Allow repeated specification of some dtype.

    Attributes
    ----------
    dtype:
        Data type of the repeated object.
    max_len:
        Optional parameter specifying if there is a maximum length of the
        repeated elements (`int > 0`). If `max_len=None`, there can be any
         number of repeated elements. By default: `None`.

    """

    dtype: BaseType
    max_len: Optional[int] = field(default=None)

    @property
    def type_hints(self):
        """Fetch hints from ``dtype`` attr and make it available for ``EndpointProtocol``."""
        _type_hints = getattr(self, "_type_hints", None)
        if not _type_hints:
            _type_hints = {
                "output_args": self.dtype.type_hints["output_args"],
                "input_args": self.dtype.type_hints["input_args"],
            }
            setattr(self, "_type_hints", _type_hints)
        return _type_hints

    def __post_init__(self):
        if not isinstance(self.dtype, BaseType):
            raise TypeError(f"dtype argument must inherit from {BaseType}")
        if isinstance(self.dtype, type(self)):
            raise TypeError(f"cannot specify {type(self)} as dtype of {type(self)}")

        if self.max_len is not None:
            if not isinstance(self.max_len, int):
                raise TypeError(f"`max_len` must be {int}, not {type(self.max_len)}")
            if self.max_len <= 0:
                raise ValueError(f"`max_len={self.max_len}` is not >= 1.")

    def deserialize(self, *args: Dict) -> Tuple[Tensor, ...]:
        if (self.max_len is not None) and (len(args) > self.max_len):
            raise ValueError(f"len(arg)={len(args)} > self.max_len={self.max_len}")
        return tuple(self.dtype.deserialize(**item) for item in args)

    def packed_deserialize(self, args):
        """Arguments are positional arguments for deserialize, unlike other datatypes."""
        return self.deserialize(*args)

    def serialize(self, args: Sequence) -> Tuple[Any, ...]:
        if (self.max_len is not None) and (len(args) > self.max_len):
            raise ValueError(f"len(arg)={len(args)} > self.max_len={self.max_len}")
        return tuple(self.dtype.serialize(item) for item in args)


import matplotlib.pyplot as plt
import matplotlib
import numpy as np
import seaborn as sns
sns.set()

font_size=18
font = {'size' : font_size}
matplotlib.rc('font', **font)
matplotlib.rcParams['lines.markersize'] = matplotlib.rcParams['lines.markersize'] + 8

# Define the means and variance of the athletes plot
alignment = np.array([90, 43, 11.8, 0.16, 0.01, 57.3, 4.9])
# alignment = np.array([7, 5, 4, 2, 1, 6, 3])

alpha = 0.0

xnli = np.array([-2.1, -3.8, -5.7, -19.2, -27.7, -5.7, -9.2])
# xnli = np.array([7, 6, 4, 2, 1, 5, 3])

# ner = np.array([-2.3, -4.1, -14.3, -51.5, -46.3, -14.2, -19.1])
# ner = np.array([7, 6, 4, 2, 1, 5, 3])

# pos = np.array([-0.5, -0.7, -1.5, -42.7, -59, -2, -2.5])
# pos = np.array([7, 6, 5, 2, 1, 4, 3])

# Set plot limits
# xmin = 5.5
# xmax=7.5
fig = plt.figure()
ax = fig.add_subplot(111)
# ax.set_xlim(xmin,xmax)
# ax.set_ylim(y-0.1,y+1)

# Plot the data
plt.scatter(xnli - alpha, alignment - alpha, color='r', marker='*', label='XNLI')
# plt.scatter(ner, alignment, color='g', marker='*', label='NER')
# plt.scatter(pos + alpha, alignment + alpha, color='b', marker='*', label='POS')

# Show the plot
plt.xlabel(r'$\mathbf{\Delta_{(BZ-BS)}}$', fontsize=font_size)
plt.ylabel("Alignment", fontsize=font_size, labelpad=8)
plt.tight_layout()
plt.xticks(fontsize=font_size-2)
plt.yticks(fontsize=font_size-2)

plt.text(-27, 65, 'Spearman\'s\n' + r'$\rho = 0.94$, $p < .005$', fontsize = font_size-3, color='darkblue')
# plt.text(-50, 65, 'Spearman\'s correlation\n' + r'$\rho = 0.93$, $p < .005$', fontsize = font_size-2)
# plt.text(-60, 65, 'Spearman\'s correlation\n' + r'$\rho = 0.89$, $p < .01$', fontsize = font_size-2)

plt.legend(prop={'size': font_size-2})

# Add annotations for different points
delta=-0.5
alpha=0.7
dict_design = dict(facecolor='black', shrink=0.05, width=2, headwidth=8, alpha=0.15)
ax.annotate('Parallel', xy=(-2.1 + delta, 90), xytext=(-9, 88), xycoords='data', textcoords='data', fontsize=font_size-6, arrowprops=dict_design, alpha=alpha)
ax.annotate('Non-parallel (Same)', xy=(-3.8 + delta, 43), xytext=(-15, 42), xycoords='data', textcoords='data', fontsize=font_size-6, arrowprops=dict_design, alpha=alpha)
ax.annotate('Non-parallel (Diff)', xy=(-5.7 + delta, 11.8), xytext=(-15, 25), xycoords='data', textcoords='data', fontsize=font_size-6, arrowprops=dict_design, alpha=alpha)
ax.annotate(r'$\mathbf{\mathcal{T}}_{trans}}$' + r'$\circ$' + r'$\mathbf{\mathcal{T}}_{inv}}$', xy=(-19.2 + delta, 0.16), xytext=(-25, 10), xycoords='data', textcoords='data', fontsize=font_size-4, arrowprops=dict_design, alpha=alpha)
ax.annotate(r'$\mathbf{\mathcal{T}}_{trans}}$' + r'$\circ$' + r'$\mathbf{\mathcal{T}}_{perm}}$', xy=(-27.7 + delta, 0.01), xytext=(-28, 25), xycoords='data', textcoords='data', fontsize=font_size-4, arrowprops=dict_design, alpha=alpha)
ax.annotate(r'$\mathbf{\mathcal{T}}_{trans}}$' + r'$\circ$' + r'$\mathbf{\mathcal{T}}_{syn}}$', xy=(-5.7 + delta, 57.3), xytext=(-15, 55), xycoords='data', textcoords='data', fontsize=font_size-4, arrowprops=dict_design, alpha=alpha)
ax.annotate('Non-parallel (50%)', xy=(-9.2 + delta, 4.9), xytext=(-19, 13), xycoords='data', textcoords='data', fontsize=font_size-6, arrowprops=dict_design, alpha=alpha)

# plt.savefig('../images/correlation.png')
plt.savefig('../images/XNLI_correlation_annotated.pdf')


import copy
import inspect
import time
from typing import Callable, Generator, Optional, Sequence, Union

from src.typing import TimerData

__all__ = ['Timer', 'MultiLevelTimer']

class Timer:
    """Context manager for timing a block of code.

    Attributes:
    :param name: (str) Timer label when printing.
    :param as_ms: (bool) If `True`, store time as `milliseconds`, otherwise `seconds`.
    :param sync_gpu: (bool) If `True`, ensure that GPU is synced on Timer enter and exit.
    :param precision: (int) Number of decimal places to print.

    Example:
    ```
        with Timer('MyTimer') as t:
            time.sleep(1)
        elapsed = t.elapsed
        print(t)

        ===>
        MyTimer: 1.003 s
    ```
    """
    def __init__(self, name: str = 'Timer', as_ms: bool = False, sync_gpu: bool = False, precision: int = 6) -> None:
        self.name: str = name
        self.as_ms: bool = as_ms
        self.sync_gpu: bool = sync_gpu
        self.precision: int = precision

        self._sf: int = 1000 if self.as_ms else 1
        self._units: str = 'ms' if self.as_ms else 's'
        self._sync_fn: Optional[Callable] = None

        self._start: Optional[float] = None
        self._end: Optional[float] = None

        if self.sync_gpu:  # Removes 'torch' dependency unless we want to sync gpu
            import torch
            self._sync_fn = torch.cuda.synchronize

    def __repr__(self) -> str:
        """Convert class constructor into string representation."""
        sig = inspect.signature(self.__init__)
        kwargs = {key: getattr(self, key) for key in sig.parameters if hasattr(self, key)}
        s = ', '.join(f'{k}={v}' for k, v in kwargs.items())
        return f'{self.__class__.__qualname__}({s})'

    def __str__(self) -> str:
        """Convert into string representation."""
        return f'{self.name}: {self.elapsed} {self._units}'

    def __enter__(self) -> 'Timer':
        """Start timer and sync GPU."""
        if self.sync_gpu:
            self._sync_fn()
        self._start = time.perf_counter()
        return self

    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
        """End timer and sync GPU."""
        if self.sync_gpu:
            self._sync_fn()
        self._end = time.perf_counter()

    @property
    def elapsed(self) -> float:
        """Time taken between enter and exit."""
        assert self._start, '`Timer` has not begun'
        assert self._end, '`Timer` has not finished'
        time_taken = self._sf * (self._end - self._start)
        return round(time_taken, self.precision)

class MultiLevelTimer:
    """Context manager Timer capable of being nested across multiple levels.

    NOTE: We use the *instance* of this class as a context manager, not the class itself (see examples).

    Timers are stored as a dict, mapping labels to (depth, start, end, elapsed).
    In order to allow for the nesting of these timers, we keep track of what timers are active (effectively, a stack).
    On __exit__ we pop the most recent label and end that timer.

    Attributes:
    :param name: (str) Global Timer name.
    :param as_ms: (bool) If `True`, store time as `'milliseconds`', otherwise `seconds`.
    :param sync_gpu: (bool) If `True`, ensure that GPU is synced on Timer enter and exit.
    :param precision: (int) Number of decimal places to print.

    Examples:
    ```
    timer = MultiLevelTimer(name='MyTimer', as_ms=True, precision=4)

    with timer('OuterLevel'):
        time.sleep(2)
        with timer('InnerLevel'):
            time.sleep(1)

    print(timer)

    ==>
    MyTimer
        OuterLevel: 3002.3414 ms
            InnerLevel: 1000.7601 ms
    ```

    Levels can also be named automatically
    ```
    timer = MultiLevelTimer(name='MyTimer')

    with timer:
        time.sleep(2)

    print(timer)

    ==>
    MyTimer
        Level1: 2.002093 s
    ```
    """
    def __init__(self, name: str = 'Timer', as_ms: bool = False, sync_gpu: bool = False, precision: int = 6) -> None:
        self.name: str = name
        self.as_ms: bool = as_ms
        self.sync_gpu: bool = sync_gpu
        self.precision: int = precision

        self.depth: int = 0  # Nesting level of the current active timer.
        self._sf: int = 1000 if self.as_ms else 1
        self._units: str = 'ms' if self.as_ms else 's'
        self._sync_fn: Optional[Callable] = None

        self._label: Optional[str] = None  # Tag for the current active timer.
        self._active: list[str] = []  # Stack of active timers.
        self._data: dict[str, TimerData] = {}

        if self.sync_gpu:
            # Removes 'torch' dependency unless we want to sync gpu
            import torch
            self._sync_fn = torch.cuda.synchronize

    def __repr__(self) -> str:
        """Convert class constructor into string representation."""
        sig = inspect.signature(self.__init__)
        kwargs = {key: getattr(self, key) for key in sig.parameters if hasattr(self, key)}
        s = ', '.join(f'{k}={v}' for k, v in kwargs.items())
        return f'{self.__class__.__qualname__}({s})'

    def __str__(self) -> str:
        """Convert into string representation."""
        s = [self.name]
        s += ['\t'*v['depth'] + f'{k}: {v["elapsed"]} {self._units}' for k, v in self]
        return '\n'.join(s)

    def __getitem__(self, label: str) -> TimerData:
        """Return timer info for the given label."""
        return self._data[label]

    def __iter__(self) -> Generator[tuple[str, TimerData], None, None]:
        """Iterate through all timers as (`label`, `timer`)"""
        for k in self._data:
            yield k, self[k]

    def __call__(self, label: str) -> 'MultiLevelTimer':
        """Required to call a `Timer` instance in a context manager and create a new label."""
        self._label = label
        return self

    def __enter__(self) -> 'MultiLevelTimer':
        """Context manager entry point."""
        self.depth += 1

        label, self._label = self._label, None  # Reset
        label = label or f'Level{self.depth}'  # Autogenerated label
        if label in self._data:
            raise KeyError(f"Duplicate Timer key: {label}")

        if self.sync_gpu: self._sync_fn()

        self._active.append(label)
        self._data[label] = {
            'depth': self.depth,
            'start': time.perf_counter(),  # Start timer
            'end': None,
            'elapsed': None,
        }
        return self

    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
        """Context manager exit point."""
        assert self._active, "What are you doing here??"
        label = self._active.pop()
        timer = self._data[label]

        if self.sync_gpu: self._sync_fn()

        timer['end'] = time.perf_counter()
        timer['elapsed'] = round(self._sf * (timer['end'] - timer['start']), self.precision)
        self.depth -= 1

    def reset(self) -> None:
        """Delete all existing `Timer` data."""
        if self._active: raise RuntimeError(f"Attempt to reset Timer while active: {self._active}")
        self._data = {}

    def copy(self) -> 'MultiLevelTimer':
        """Return a deep copy of the timer."""
        return copy.deepcopy(self)

    def to_dict(self, key: str = 'elapsed') -> dict:
        """Return a dict containing only the data for the specified key."""
        return {label: data[key] for label, data in self}

    @staticmethod
    def mean_elapsed(timers: Sequence['MultiLevelTimer']) -> Union[Sequence, dict[str, float]]:
        """Return the average elapsed time for each label in a list of timers."""
        if not timers: return timers

        data = {}
        for t in timers:
            for k, v in t:
                if k in data: data[k].append(v['elapsed'])
                else:         data[k] = [v['elapsed']]

        data = {k: sum(v)/len(v) for k, v in data.items()}
        return data


# -*- coding: utf-8 -*-
"""
-------------------------------------------------
# @Project  :BiLSTM_CRF
# @File     :BiLSTM_CRF
# @Date     :2021/7/17 11:30
# @Author   :huangjie
# @Email    :728155808@qq.com
# @Software :PyCharm
-------------------------------------------------
"""
import torch
import torch.nn as nn
from torchcrf import CRF

class BiLSTM_CRF(nn.Module):
    def __init__(self,con):
        super(BiLSTM_CRF, self).__init__()
        self.con = con
        self.embedding = nn.Embedding(self.con.vocab_size,self.con.word_dim)
        self.bilstm = nn.LSTM(self.con.word_dim,self.con.units,num_layers=1,bidirectional=True,batch_first=True)
        self.linear = nn.Linear(self.con.units * 2 ,self.con.num_rel)
        self.crf = CRF(self.con.num_rel,batch_first=True)

    def forward(self,token_ids,mask=None,tag_ids=None):
        embed = self.embedding(token_ids)
        lstm,(c,h) = self.bilstm(embed)
        logits = self.linear(lstm)
        if tag_ids is not None:
            logits = -1.0 * self.crf(emissions=logits,tags = tag_ids,mask = mask)
        return logits


import json
from json import encoder
encoder.FLOAT_REPR = lambda o: format(o, '.2f')

class Measures:

    def __init__(self, cursor, sql_fields):

        self.data = []

        for l in cursor:

            row = {}

            k = 0
            for i in sql_fields:
                row[i] = l[k]
                k += 1

            self.data.append(row)

        cursor.connection.commit()

    def length(self):
        return len(self.data)

    def _prefilter_data(self, window_width=1):

        d = None
        date_start = date_end = None

        avg = {}
        post_filter_dict = {}

        keys = self.data[0].keys()
        keys.remove('time')

        for k in keys:
            avg[k] = 0
            post_filter_dict['%s_DATA' % k.upper()] = []

        i = 0
        n = 0
        for row in self.data:

            if n == 0:
                date_start = row['time']
                n = 1

            i += 1

            for k in keys:
                avg[k] = avg[k]*(i-1)/i + row[k]/i

            if window_width == 1 or i == window_width//2:
                date_end = d = row['time']

            if i < window_width:
                continue
            else:
                i = 0
                for k in keys:
                    post_filter_dict['%s_DATA' % k.upper()].append(round(avg[k],2))
                    avg[k] = 0

        post_filter_dict['START_DATE'] = date_start
        post_filter_dict['END_DATE'] = date_end

        return post_filter_dict

    def serialize_json(self, filename=None, window_width=1):

        assert self.length() > 0, 'Measure set is empty, cannot render'

        filtered_data = self._prefilter_data(window_width)

        obj_to_serial = filtered_data.copy()

        obj_to_serial['START_DATE'] = obj_to_serial['START_DATE'].isoformat()
        obj_to_serial['END_DATE']   = obj_to_serial['END_DATE'].isoformat()

        obj_to_serial['TIME_STEP'] = 1000 * 60 * 5 * window_width

        if filename:
            with open(filename, 'w') as fd:
                json.dump(obj_to_serial, fd)
        else:
            return json.dumps(obj_to_serial)

    def render_template(self, filename, window_width=1):

        assert self.length() > 0, 'Measure set is empty, cannot render'

        try:
            import pystache
        except:
            raise ImportError('MeteoPyris needs the pystache package to render templates')

        filtered_data = self._prefilter_data(window_width)
        ds = filtered_data['START_DATE']
        de = filtered_data['END_DATE']
        template_context = {
            'START_DATE' : 'Date.UTC(%d, %d, %d, %d ,%d, 0)' % (ds.year, ds.month-1, ds.day, ds.hour, ds.minute),
            'END_DATE'   : 'Date.UTC(%d, %d, %d, %d ,%d, 0)' % (de.year, de.month-1, de.day, de.hour, de.minute),
            'TIME_STEP'  : '%d' % (1000 * 60 * 5 * window_width)
            }
        for k in filtered_data.keys():
            if k == 'START_DATE' or k == 'END_DATE':
                continue
            template_context[k] = ','.join(map(str, filtered_data[k]))

        r = ''
        renderer = pystache.Renderer(file_encoding='utf-8', string_encoding='utf-8')
        with open(filename, 'r') as template:
            for line in template:
                r += renderer.render(line, template_context)
        return r


from datetime import datetime, timedelta
import logging

import aiohttp
import asyncio

async def request(url: str):
    async with aiohttp.ClientSession() as session:
        try:
            async with session.get(url) as resp:
                if resp.status == 200:
                    r = await resp.json()
                    return r
                logging.error(f"Error status: {resp.status} for {url}")
                return None
        except aiohttp.ClientConnectionError as err:
            logging.error(f"Connection error: {str(err)}")
            return None

async def get_exchange(currency_code, valid_date):
    result = await request(f"https://api.privatbank.ua/p24api/exchange_rates?date={valid_date}")
    if result:
        rates = result.get('exchangeRate')
        exc, = list(filter(lambda element: element["currency"] == currency_code, rates))
        return f"{currency_code}: buy: {exc['purchaseRateNB']}, sale: {exc['saleRateNB']}. Date: {valid_date}"
    return "Failed to retrieve data"

def get_valid_date():
    while True:
        try:
            input_date_str = input("Enter a date (dd.mm.yyyy): ")
            input_date = datetime.strptime(input_date_str, "%d.%m.%Y")

            current_date = datetime.now()
            max_valid_date = current_date + timedelta(days=10)

            if input_date <= max_valid_date:
                return input_date.strftime("%d.%m.%Y")
            else:
                print("The date should be within the next 10 days.")
        except ValueError:
            print("Invalid date format. Please use dd.mm.yyyy.")

if __name__ == "__main__":
    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
    currency_code = input("Currency code: ")
    valid_date = get_valid_date()
    result = asyncio.run(get_exchange(currency_code, valid_date))
    print(result)


# -*- coding: utf-8 -*-
import scrapy
from scrapy import log
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
from rcbi.items import Part

import re

WAREHOUSE_RE=re.compile("\( ?(RU|USA|US|AR|AUS|AU|EU|UK|NL) [Ww]arehouse ?\)")
MANUFACTURERS = ["Turnigy", "Walkera", "ZTW", "Zippy", "Rhino", "Hobbywing", "OrangeRx", "HobbyKing", "Goteck", "Futaba", "Flycolor", "Corona"]
CORRECT = {"ZIPPY": "Zippy", "TURNIGY": "Turnigy", "OrangeRX": "OrangeRx", "Hobbyking": "HobbyKing", u"Hobbyking™": "HobbyKing", u"HobbyKing™": "HobbyKing", u"Hobbyking® ™": "HobbyKing", u"HobbyKing® ™" : "HobbyKing", u"Hobbyking®" : "HobbyKing", u"HobbyKing®" : "HobbyKing", "Hobby King": "HobbyKing", u"Durafly™": "Durafly"}
MANUFACTURERS.extend(CORRECT)
MANUFACTURERS.sort(key=lambda x: len(x), reverse=True)
NEW_PREFIX = {}
class HobbyKingSpider(CrawlSpider):
  name = "hobbyking"
  allowed_domains = ["hobbyking.com"]
  start_urls = ["http://www.hobbyking.com/"]

  rules = (
    Rule(LinkExtractor(restrict_css=["#nav", ".paging"])),
    Rule(LinkExtractor(restrict_xpaths="//*[@id=\"tableContent\"]/div/table[2]/tr/td[2]/a")),
    Rule(LinkExtractor( restrict_xpaths=["//*[@id=\"tableContent\"]/div/table[2]/tr/td/table/tr[2]/td[2]/table/tr/td[2]/div[1]", "//*[@id=\"tableContent\"]/div/table[2]/tr/td/table/tr[2]/td[2]/table/tr/td[2]/div[2]/div[5]/div[3]"]), callback='parse_item'),
  )

  def parse_item(self, response):
    item = Part()
    item["site"] = self.name
    product_name = response.css("#productDescription")
    if not product_name:
      return

    variant = {}
    item["variants"] = [variant]

    variant["url"] = response.url
    item["name"] = product_name[0].xpath("text()").extract()[0]
    suffix = WAREHOUSE_RE.search(item["name"])
    if suffix != None:
      item["name"] = item["name"][:suffix.start()].strip()

    headers = response.xpath("//*[@class=\"data_tbl\"]/table/tr/td/table/tr[2]/td[2]/table/tr/td[1]/span/table/tr/td[1]/text()")
    details = response.xpath("//*[@class=\"data_tbl\"]/table/tr/td/table/tr[2]/td[2]/table/tr/td[1]/span/table/tr/td[2]/text()")
    manufacturer = None
    for i, header in enumerate(headers):
      header = header.extract()
      if len(header) == 0:
        continue
      data = details[i].extract()
      if header == "Weight (g)":
        item["weight"] = data

    price = response.css("#price_lb")
    if price:
      variant["price"] = price[0].xpath("text()").extract()[0].strip()

    stock = response.css("#pstock2")
    if stock:
      q = stock[0].xpath("text()").extract()[0]
      if q == "BK":
        variant["stock_state"] = "backordered"
      elif q == "0":
        variant["stock_state"] = "out_of_stock"
        variant["stock_text"] = q
      else:
        variant["stock_state"] = "in_stock"
        variant["stock_text"] = q

    sku = response.css("#desc_sku")
    if sku:
      item["sku"] = sku[0].xpath("text()").extract()[0]

    variant["location"] = response.xpath("//*[@id=\"details\"]/div[1]/div[2]/div/table/tbody/tr[1]/td[2]/text()").extract()[0].strip()

    if not manufacturer:
      for m in MANUFACTURERS:
        if item["name"].startswith(m):
          item["name"] = item["name"][len(m):].strip("- ")
          item["manufacturer"] = m
          break
    else:
      item["manufacturer"] = manufacturer

    if "manufacturer" in item:
      m = item["manufacturer"]
      if m in NEW_PREFIX:
        item["name"] = NEW_PREFIX[m] + " " + item["name"]
      if m in CORRECT:
        item["manufacturer"] = CORRECT[m]
        m = CORRECT[m]
      if item["name"].startswith(m):
        item["name"] = item["name"][len(m):].strip("- ")
    return item


import os
from pathlib import Path
import argparse

from natsort import natsorted
import numpy as np
import torch

from models.i3d_inception import i3d_model
from models.c3d import c3d_model
from utils import from_frames_to_clips, load_rgb_batch, split_clip_indices_into_batches, \
    extract_frames_from_video

def extract_features(model, feature, device, clip_length, temp_path, batch_size, n_crops):
    # rgb files
    rgb_files = natsorted([i for i in os.listdir(temp_path)])

    # creating clips
    frame_indices = from_frames_to_clips(rgb_files, clip_length)

    # divide clips into batches (for memory optimization)
    # k = true values in the last batch
    frame_indices, k = split_clip_indices_into_batches(frame_indices, batch_size)

    # iterate N-crop augmentation
    n_batches = frame_indices.shape[0]
    n_clips_inside_a_batch = frame_indices.shape[1]

    if feature == "I3D":
        full_features = torch.zeros((n_batches, n_crops, n_clips_inside_a_batch, 1024, 1, 1, 1), device=device)
    else:
        full_features = torch.zeros((n_batches, n_crops, n_clips_inside_a_batch, 4096), device=device)

    for batch_id in range(n_batches):

        # B=16 x T=16 x CROP=10 x CH=3 x H x W
        batch_data = load_rgb_batch(temp_path, rgb_files, frame_indices[batch_id], feature, device, n_crops)

        # iterate 10-crop augmentation
        n_crops = batch_data.shape[2]
        for i in range(n_crops):
            b_data = batch_data[:, :, i, :, :, :]  # select the i-crop
            b_data = b_data.transpose(1, 2)  # B=16 x CH=3 x T=16 x H x W

            with torch.no_grad():
                inp = {'frames': b_data}
                features = model(inp)

            v = features  # single clip features extracted
            v = v / torch.linalg.norm(v)  # single clip features normalized (L2)

            if feature == "I3D":
                full_features[batch_id, i, :, :, :, :, :] = v
            else:
                full_features[batch_id, i, :, :] = v

    full_features = full_features.cpu().numpy()

    # from fixed tensor to variable length list
    out = [[] for i in range(n_crops)]
    for batch_id in range(n_batches):
        for i in range(n_crops):
            if batch_id == n_batches - 1:
                # ignore padded values in the last batch
                out[i].append(full_features[batch_id, i, :k, :])
            else:
                out[i].append(full_features[batch_id, i, :, :])

    out = [np.concatenate(i, axis=0) for i in out]
    out = [np.expand_dims(i, axis=0) for i in out]
    out = np.concatenate(out, axis=0)

    if feature == "I3D":
        out = out[:, :, :, 0, 0, 0]

    out = np.array(out).transpose([1, 0, 2])
    return out

def generate(dataset_path, output_path, feature, clip_size, batch_size, video_format_type, n_crops, save_single_crops):
    Path(output_path).mkdir(parents=True, exist_ok=True)
    temp_path = output_path + "/temp/"
    root_dir = Path(dataset_path)
    videos = [str(f) for f in root_dir.glob('**/*{}'.format(video_format_type))]

    # set up the model
    torch.backends.cudnn.benchmark = True
    torch.cuda.empty_cache()
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    if feature == "I3D":
        # Inception-v1 / kinetics-400
        model = i3d_model(nb_classes=400, pretrainedpath='pretrained/rgb_imagenet.pt')
    else:
        model = c3d_model(nb_classes=487, pretrainedpath='pretrained/c3d.pickle', feature_layer=6)

    model.to(device)  # Put model to GPU
    model.train(False)  # Set model to evaluate mode

    # create temp folder
    Path(temp_path).mkdir(parents=True, exist_ok=True)

    k = 1
    for video in videos:

        # remove files into temp folder
        for f in Path(temp_path).glob('**/*'):
            os.remove(f)

        # splitting the video into frames and saving them to an output folder
        extract_frames_from_video(video, temp_path)

        # extract features
        features = extract_features(model, feature, device, clip_size, temp_path, batch_size, n_crops)

        # save features
        video_name = video.split("/")[-1].split(video_format_type)[0]

        if save_single_crops:
            for i in range(n_crops):
                feature_filename = output_path + "/" + video_name + '__' + str(i)
                np.save(feature_filename, features[:, i, :])
                # print('features of dimension {} saved as {}'.format(features[:, i, :].shape, feature_filename))
        else:
            feature_filename = output_path + "/" + video_name
            np.save(feature_filename, features)
            # print('features of dimension {} saved as {}'.format(features.shape, feature_filename))

        print('{} / {} - features extracted from video {}'.format(k, len(videos), video))
        k += 1

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--dataset_path', type=str,
                        default="/media/ubuntu/30E046F6E046C1B8/prova/datasets/XD-Violence/test")
    parser.add_argument('--output_path', type=str, default="output")
    parser.add_argument('--feature', type=str, default="C3D")
    parser.add_argument('--clip_length', type=int, default=16)
    parser.add_argument('--batch_size', type=int, default=16)
    parser.add_argument('--video_format_type', type=str, default='.mp4')
    parser.add_argument('--n_crops', type=int, default=5)
    parser.add_argument('--save_single_crops', type=bool, default=True)
    args = parser.parse_args()
    generate(args.dataset_path, str(args.output_path), args.feature, args.clip_length, args.batch_size,
             args.video_format_type, args.n_crops, args.save_single_crops)

# "/media/ubuntu/TrekStor/TESI_ANOMALY_DETECTION/Datasets/videos/ShanghaiTech_new_split/test"
# python3 extract_features_from_videos.py --dataset_path=/home/ubuntu/Downloads/UCF-Crime/train --output_path=output
# python3 extract_features_from_videos.py --dataset_path=ShanghaiTech_new_split/train/ --output_path=output


import pytest
from os import path as p
from wwfs.board import Board
from wwfs.word import Word
from wwfs.tiles import TileBag

_wwfs = p.dirname(p.abspath(__file__))
test_data = p.join(_wwfs, '..', 'wwfs', 'data')

@pytest.fixture
def mytilebag():
    return TileBag(p.join(test_data, 'tiles.csv'))

@pytest.fixture
def myword():
    word = Word("TASTY", coord=(5, 5), direction=0)
    layout = p.join(test_data, 'board.csv')
    board = Board(layout)
    word.squares = board.get_square_xy(word, word.x, word.y, word.direction)
    for sq in word.squares:
        sq.tile_used = True
    return word

def test_tilebag_instansiate(mytilebag):
    """Can instansiate Word object."""
    assert isinstance(mytilebag, TileBag)
    assert mytilebag.remaining == 104
    assert mytilebag.played == []

def test_update(myword, mytilebag):
    """Remove tiles from tilebag aftern playing word."""
    mytilebag.update(myword)
    assert mytilebag.played == list('TASTY')
    assert mytilebag.remaining == (104 - len(myword))


from collections import deque, defaultdict

class Offer:
    def __init__(self, type: str, name: str, quantity: int):
        self.type = type
        self.name = name
        self.quantity = quantity

def matchOffers(offers: [Offer]):
    buzzQueue = deque([[offer.name, offer.quantity] for offer in offers if offer.type == 'Buzz'])
    fizzQueue = deque([[offer.name, offer.quantity] for offer in offers if offer.type == 'Fizz'])
    match = defaultdict(int)
    while buzzQueue and fizzQueue:
        n1, q1 = fizzQueue[0]
        n2, q2 = buzzQueue[0]
        if q1 < q2:
            buzzQueue[0][1] -= q1
            fizzQueue.popleft()
        elif q1 > q2:
            fizzQueue[0][1] -= q2
            buzzQueue.popleft()
        else:
            buzzQueue.popleft()
            fizzQueue.popleft()
        match[n1, n2] += min(q1, q2)
    for (n1, n2), q in match.items():
        print("FizzBuzz! " + str(q) + " " + n1 + ":" + n2)

offers = [Offer('Fizz', 'Alice', 5), Offer('Buzz', 'Bob', 3), Offer('Buzz', 'Catherine', 4), Offer('Fizz', 'Alice', 4),
          Offer('Buzz', 'Bob', 2)]
matchOffers(offers)


'''
Created on Oct 21, 2010

@author: joeykblack
'''
from google.appengine.api import mail

def sendResults(resultset):
    giftsuser = resultset.giftsuser

    if mail.is_email_valid(giftsuser.email):
        message = mail.EmailMessage()
        message.sender = 'Chuck Coker <chuck.coker@meclabs.com>'
        message.to = giftsuser.firstname + ' ' + giftsuser.lastname + ' <' + giftsuser.email + '>'
        message.subject = 'Your Spiritual Gifts Results'
        message.body = 'You can access your results at ' + resultset.url

        message.send()


from pathlib import Path

from mlf_core.create.domains.mlflow_creator import MlflowCreator
from mlf_core.create.domains.package_creator import PackageCreator
from mlf_core.custom_cli.questionary import mlf_core_questionary_or_dot_mlf_core

def choose_domain(path: Path, domain: str or None, dot_mlf_core: dict = None):  # type: ignore
    """
    Prompts the user for the template domain.
    Creates the .mlf_core file.
    Prompts the user whether or not to create a Github repository

    :param path: The path, the project should be created at
    :param domain: Template domain
    :param dot_mlf_core: Dictionary created from the .mlf_core.yml file. None if no .mlf_core.yml file was used.
    """
    if not domain:
        domain = mlf_core_questionary_or_dot_mlf_core(
            function="select",
            question="Choose the project's domain",
            choices=["mlflow", "package"],
            default="mlflow",
            dot_mlf_core=dot_mlf_core,
            to_get_property="domain",
        )

    switcher = {"mlflow": MlflowCreator, "package": PackageCreator}

    creator_obj = switcher.get(domain.lower())()  # type: ignore
    creator_obj.create_template(path, dot_mlf_core)  # type: ignore


from easydict import EasyDict as edict

BATCH_SIZE = 1
POINT_FEATURES = ['x', 'y', 'z', 'intensity', 'timestamp']

data_cfg = edict({
    'POINT_CLOUD_RANGE': [-51.2, -51.2, -5.0, 51.2, 51.2, 3.0],
    'VOXEL_SIZE': [0.2, 0.2, 8.0],
    'CLASSES': ['car','truck', 'construction_vehicle', 'bus', 'trailer',
                'barrier', 'motorcycle', 'bicycle', 'pedestrian', 'traffic_cone'],
    'NUM_POINT_FEATURES': len(POINT_FEATURES)
})

model_cfg = edict({
    'NAME': 'CenterPoint',

    'VFE':
        {
            'NAME': 'DynamicPillarVFE',
            'WITH_DISTANCE': False,
            'USE_ABSOLUTE_XYZ': True,
            'USE_NORM': True,
            'NUM_FILTERS': [64, 64],
            'NUM_RAW_POINT_FEATURES': 5
        },

    'MAP_TO_BEV':
        {
            'NAME': 'PointPillarScatter',
            'NUM_BEV_FEATURES': 64,
            'BATCH_SIZE': BATCH_SIZE
        },

    'BACKBONE_2D':
        {
            'NAME': 'BaseBEVBackbone',
            'LAYER_NUMS': [3, 5, 5],
            'LAYER_STRIDES': [2, 2, 2],
            'NUM_FILTERS': [64, 128, 256],
            'UPSAMPLE_STRIDES': [0.5, 1, 2],
            'NUM_UPSAMPLE_FILTERS': [128, 128, 128]
        },

    'DENSE_HEAD':
        {
            'NAME': 'CenterHead',
            'BATCH_SIZE': BATCH_SIZE,
            'CLASS_NAMES_EACH_HEAD': [
                ['car'], # 0
                ['truck', 'construction_vehicle'],  # 1, 2
                ['bus', 'trailer'],  # 3, 4
                ['barrier'],  # 5
                ['motorcycle', 'bicycle'],  # 6, 7
                ['pedestrian', 'traffic_cone'],  # 8, 9
            ],
            'SHARED_CONV_CHANNEL': 64,
            'USE_BIAS_BEFORE_NORM': True,
            'NUM_CONV_IN_ONE_HEAD': 2,
            'FEATURE_MAP_STRIDE': 4,
            'POST_PROCESSING':{
                'SCORE_THRESH': 0.3,
                'NMS_IOU_THRESH': 0.3,
                'NMS_PRE_MAXSIZE': 200,
                'NMS_POST_MAXSIZE': 30,
                'POST_CENTER_LIMIT_RANGE': [-61.2, -61.2, -10.0, 61.2, 61.2, 10.0],
                'MAX_OBJ_PER_SAMPLE': 500,
            }
        }
})


import matplotlib.pyplot as plt

class Graph:

   def __init__(self,function=lambda x: x):
      self.function = function

   def plot(self, fptr):
      x = [i for i in range(0,100,1)]
      y = [self.function(x_i) for x_i in x]
      #print(x)
      #print(y)
      plt.clf()
      plt.plot(x,y)
      plt.savefig(fptr)
      plt.show()


# Monitors the source files for any new insertions new excel files for loops only. Once it finds new excel file database operations.py does the insertion
from os import listdir
from os.path import isfile, join, isdir
import json
import os

currentPath = os.path.dirname(os.path.realpath(__file__))

def newfiledetector(targetdir, targetYear):
    returningList = []

    # CAUTION! the JSON file must be under json format: {}
    with open(currentPath + '/logs/log.json', 'a+') as data_file:
        PreviousFiles = json.load(data_file)

    targetTypes = [targetdir + '\\' + f for f in listdir(targetdir) if isdir(join(targetdir, f))]

    # onetype is the data type: Class, Volume, Speed
    for onetype in targetTypes:

        # Only select Class for now.
        if onetype.split('\\')[-1:][0] != 'Class':
            continue

        # In case of new data types add in
        if onetype not in PreviousFiles:
            PreviousFiles[onetype] = {}

        # targetYear is the specific Year
        yeardir = onetype + '\\' + targetYear

        # In case of new year
        if yeardir not in PreviousFiles[onetype]:
            PreviousFiles[onetype][yeardir] = {}

        StationIDs = [yeardir + '\\' + f for f in listdir(yeardir) if isdir(join(yeardir, f))]

        # onestation is the StationID
        for onestation in StationIDs:

            # In case of new stations add in
            if onestation not in PreviousFiles[onetype][yeardir]:
                PreviousFiles[onetype][yeardir][onestation] = []

            sourcefiles = [onestation + '\\' + f for f in listdir(onestation) if isfile(join(onestation, f))]

            set_PreviousFiles = set(PreviousFiles[onetype][yeardir][onestation])
            set_CurrentFiles = set(sourcefiles)

            PreviousFiles[onetype][yeardir][onestation] = list(set_CurrentFiles | set_PreviousFiles)
            returningList.extend(list(set_CurrentFiles - set_PreviousFiles))

    with open(currentPath + '/logs/log.json', 'w') as data_file:
        json.dump(PreviousFiles, data_file)

    return returningList

    # Commet out
    # import datetime
    # print newfiledetector('C:\\ProgramData\\Peek Traffic\\Viper\\Output1\\Binned Data Reports', datetime.datetime.now().strftime('%Y'))