text:
# Copyright (c) 2022, Leonardo Lamanna
# All rights reserved.
# This source code is licensed under the MIT-style license found in the
# LICENSE file in the root directory of this source tree.

import argparse
import datetime
import re
from collections import defaultdict
from pathlib import Path
import shutil

import pandas as pd
import os

from scipy import interpolate

import Configuration
import numpy as np
import matplotlib.pyplot as plt

pd.set_option('display.max_rows', None)
pd.set_option('display.max_columns', None)
pd.set_option('display.width', None)

def preprocess(data):

    filtered = [str.replace("\t\t", " ").replace("|","").split() for str in data
                if str.strip() != "" and all([el.isnumeric()
                for el in str.replace("\t\t", " ").replace(".","").replace("|","").split()])]

    filtered = np.array(filtered)

    times = [float(str) for str in filtered[:, 0]]
    iterations = [float(str) for str in filtered[:, 1]]
    prec_recall = [float(str) for str in filtered[:, 2]]
    prec_precision = [float(str) for str in filtered[:, 3]]
    eff_recall = [float(str) for str in filtered[:, 4]]
    eff_precision = [float(str) for str in filtered[:, 5]]
    overall_recall = [float(str) for str in filtered[:, 6]]
    overall_precision = [float(str) for str in filtered[:, 7]]

    return times, iterations, prec_recall, prec_precision, eff_recall, eff_precision, overall_recall, overall_precision

def generate_domain_dataframes(uncert_neg_effects = False):

    # TEST_DIR = "Tests"
    # RESULTS_DIR = "Results"

    if uncert_neg_effects:
        RESULTS_DIR = "Results_uncert_neg"
    else:
        RESULTS_DIR = "Results_cert"

    # RESULTS_DIR = os.path.join("Analysis", RESULTS_DIR)
    RESULTS_DIR = os.path.join("/".join(Configuration.ROOT_TEST_DIR.split('/')[:-1]), RESULTS_DIR)

    if os.path.exists(RESULTS_DIR):
        shutil.rmtree(RESULTS_DIR)

    os.mkdir(RESULTS_DIR)

    # for domain in os.listdir(os.path.join("Analysis", TEST_DIR)):
    for domain in [d for d in os.listdir(Configuration.ROOT_TEST_DIR) if not d.lower().startswith("results")]:

        checked_log = False
        checked_facts = False
        checked_domain = False
        extended_dataframe = False

        # instances_dir = "{}{}".format(Configuration.ROOT_TEST_DIR, domain)
        instances_dir = os.path.join(Configuration.ROOT_TEST_DIR, domain)

        df = pd.DataFrame({'Instance': [],
                           'Ground actions': [],
                            'Computation time': [],
                           'Executed actions': [],
                            'Success actions': [],
                           'Failed actions': [],
                           'Real precs': [],
                           'Learn precs': [],
                           'Real pos': [],
                           'Learn pos': [],
                           'Real neg': [],
                           'Learn neg': [],
                           'Ins_pre': [],
                           'Del_pre': [],
                           'Ins_pos': [],
                           'Del_pos': [],
                           'Ins_neg': [],
                           'Del_neg': [],
                           'Precs recall': [],
                           'Pos recall': [],
                           'Neg recall': [],
                           'Precs precision': [],
                           'Pos precision': [],
                           'Neg precision': [],
                           'Overall recall': [],
                           'Overall precision': []
                           })

        all_benchmark_dirs = []
        for benchmark_subdir, dirs, files in os.walk(instances_dir):
            all_benchmark_dirs = dirs
            break

        max_goal = 1
        for dir in sorted(all_benchmark_dirs, key = lambda x: int(x.split("_")[0])):
            for subdir, dirs, files in os.walk("{}/{}".format(instances_dir, dir)):

                for file in files:

                    if file.endswith("log"):
                        instance_name = file[:file.index("_" + file.split("_")[-1])].strip()
                        benchmark_dir = dir
                        plan_goal = 0
                        total_actions = None
                        executed_actions = 0
                        failed_actions = 0
                        success_actions = 0

                        # precs_precision = None
                        # precs_recall = None
                        # eff_precision = None
                        # eff_recall = None
                        # overall_precision = None
                        # overall_recall = None
                        #
                        # computation_time = None

                        with open(os.path.join(subdir, file), "r") as f:
                            data = [el.replace("|","") for el in f.read().split('\n') if el.strip() != ""]

                            # computation_time = data[-1].split()[0]
                            # precs_recall = data[-1].split()[2]
                            # precs_precision = data[-1].split()[3]
                            # eff_recall = data[-1].split()[4]
                            # eff_precision = data[-1].split()[5]
                            # overall_recall = data[-1].split()[6]
                            # overall_precision = data[-1].split()[7]

                            all_metrics = [str.replace("\t\t", " ").replace("|", "").split() for str in data
                                        if str.strip() != "" and all([el.isnumeric()
                                                                      for el in str.replace("\t\t", " ").replace(".", "").replace(
                                                                          "|", "").split()])]

                            #################### Consider certain domain metrics ################
                            if not uncert_neg_effects:
                                all_metrics = all_metrics[:-1]
                            #####################################################################

                            computation_time = all_metrics[-1][0]
                            real_precs = all_metrics[-1][2]
                            learn_precs = all_metrics[-1][3]
                            real_pos = all_metrics[-1][4]
                            learn_pos = all_metrics[-1][5]
                            real_neg = all_metrics[-1][6]
                            learn_neg = all_metrics[-1][7]
                            ins_pre = all_metrics[-1][8]
                            del_pre = all_metrics[-1][9]
                            ins_pos = all_metrics[-1][10]
                            del_pos = all_metrics[-1][11]
                            ins_neg = all_metrics[-1][12]
                            del_neg = all_metrics[-1][13]
                            precs_recall = all_metrics[-1][14]
                            pos_recall = all_metrics[-1][15]
                            neg_recall = all_metrics[-1][16]
                            precs_precision = all_metrics[-1][17]
                            pos_precision = all_metrics[-1][18]
                            neg_precision = all_metrics[-1][19]
                            overall_recall = all_metrics[-1][20]
                            overall_precision = all_metrics[-1][21]
                            inst_objs = defaultdict(int)

                            for i in range(len(data)):
                                line = data[i]

                                if line.strip().lower().startswith("not successfully executed"):
                                    failed_actions += 1
                                    executed_actions += 1

                                elif line.strip().lower().startswith("successfully executed"):
                                    success_actions += 1
                                    executed_actions += 1

                                elif line.strip().lower().startswith("objects list"):
                                    for j in range(i+1, len(data)):
                                        if data[j].lower().strip().startswith("time") and ":" not in data[j].lower():
                                            break
                                        inst_objs[data[j].lower().split(":")[0]] = int(data[j].lower().split(":")[1])

                        # Total actions
                        with open(os.path.join(subdir, file), "r") as f:
                            data = f.read().split('\n')

                            for i in range(len(data)):
                                line = data[i]

                                if line.strip().find("Total actions") != -1:
                                    total_actions = line.split(":")[1].strip()
                                    break

                        checked_log = True

                # if len(files) > 0:
                if checked_log:
                    evaluate = {
                                'Instance': instance_name,
                                'Ground actions': int(total_actions),
                                'Computation time': float(computation_time),
                                'Executed actions': int(executed_actions),
                                'Success actions': int(success_actions),
                                'Failed actions': int(failed_actions),
                               'Real precs': int(real_precs),
                               'Learn precs': int(learn_precs),
                               'Real pos': int(real_pos),
                               'Learn pos': int(learn_pos),
                               'Real neg': int(real_neg),
                               'Learn neg': int(learn_neg),
                               'Ins_pre': int(ins_pre),
                               'Del_pre': int(del_pre),
                               'Ins_pos': int(ins_pos),
                               'Del_pos': int(del_pos),
                               'Ins_neg': int(ins_neg),
                               'Del_neg': int(del_neg),
                               'Precs recall': float(precs_recall),
                               'Pos recall': float(pos_recall),
                               'Neg recall': float(neg_recall),
                               'Precs precision': float(precs_precision),
                               'Pos precision': float(pos_precision),
                               'Neg precision': float(neg_precision),
                                'Overall recall': float(overall_recall),
                                'Overall precision': float(overall_precision)
                                }
                    checked_log = False

                    if not extended_dataframe:
                        for obj_type in inst_objs.keys():
                            extended_dataframe = True
                            df[obj_type] = -1
                            df["Total objs"] = -1

                    total_objs = 0
                    for obj_type in inst_objs.keys():
                        evaluate[obj_type] = inst_objs[obj_type]
                        total_objs += inst_objs[obj_type]

                    evaluate["Total objs"] = total_objs

                    df = df.append(evaluate, ignore_index=True)
                        # print(data)

        ordered_instances = None

        ordered_instances = [el.replace(".pddl", "") for el in
                             sorted(os.listdir(benchmark_subdir), key = lambda x: int(x.split("_")[0]))]

        df['Instance'] = pd.Categorical(df.Instance, categories=ordered_instances, ordered=True)

        df.sort_values("Instance", inplace=True)

        # Create a Pandas Excel writer using XlsxWriter as the engine.
        writer = pd.ExcelWriter(os.path.join(RESULTS_DIR,"{}_action_model_eval.xls".format(domain)))

        summary = ['Instance',  'Computation time', 'Executed actions', 'Overall recall', 'Overall precision']

        df.to_excel(writer, sheet_name='Total', index=False)
        df[summary].to_excel(writer, sheet_name='Summary', index=False)
        df[list(inst_objs.keys()) + ["Total objs"]].to_excel(writer, sheet_name='Objects', index=False)

        writer.close()

        # print("minimum number of objects of {} : {}".format(domain, min(df["Total objs"])))
        # print("maximum number of objects of {} : {}".format(domain, max(df["Total objs"])))

        # print("minimum number of ground actions of {} : {}".format(domain, min(df["Ground actions"])))
        # print("maximum number of ground actions of {} : {}".format(domain, max(df["Ground actions"])))

def generate_fama_average_results():

    all_domains_time = defaultdict(list)
    all_domains_recall = defaultdict(list)
    all_domains_precision = defaultdict(list)

    for execution_result in os.listdir("Results_FAMA"):

        df_path = os.path.join("Results_FAMA", execution_result)

        df = pd.read_excel(df_path)

        for index, row in df.iterrows():

            domain = row['domain']
            time = float(row['time'])
            recall = float(row['recall'])
            precision = float(row['precision'])

            all_domains_time[domain].append(time)
            all_domains_recall[domain].append(recall)
            all_domains_precision[domain].append(precision)

    # Create summary dataframe over all domains
    df = pd.DataFrame({'Domain': [],
                       'Avg time': [],
                       'Avg recall': [],
                       'Avg precision': []
                       })

    for domain in all_domains_time.keys():

        results_exec = len(all_domains_time[domain])
        # avg_time = sum(all_domains_time[domain])/results_exec
        # avg_precision = sum(all_domains_precision[domain])/results_exec
        # avg_recall = sum(all_domains_recall[domain])/results_exec
        med_time = sorted(all_domains_time[domain])[2]
        med_precision = sorted(all_domains_precision[domain])[2]
        med_recall = sorted(all_domains_recall[domain])[2]

        evaluate = {'Domain': domain,
                       'Avg time': float(med_time),
                       'Avg recall': float(med_recall),
                       'Avg precision': float(med_precision)
        }

        df = df.append(evaluate, ignore_index=True)

    # Create a Pandas Excel writer using XlsxWriter as the engine.
    writer = pd.ExcelWriter('Results_FAMA/average_results.xlsx', engine='xlsxwriter')

    df.to_excel(writer, index=False, float_format = "%0.2f")

    writer.close()

def generate_domain_summary(uncert_neg_effects = False):

    # RESULTS_DIR = ""

    if uncert_neg_effects:
        RESULTS_DIR = "Results_uncert_neg"
    else:
        RESULTS_DIR = "Results_cert"

    # RESULTS_DIR = os.path.join("Analysis", RESULTS_DIR)
    RESULTS_DIR = os.path.join("/".join(Configuration.ROOT_TEST_DIR.split('/')[:-1]), RESULTS_DIR)

    if os.path.exists(os.path.join(RESULTS_DIR, "overall_summary.xls")):
        os.remove(os.path.join(RESULTS_DIR, "overall_summary.xls"))

    all_domains = os.listdir(RESULTS_DIR)

    all_domains_total_objs = defaultdict(list)
    all_domains_executed_actions = defaultdict(list)
    all_domains_success_actions = defaultdict(list)
    all_domains_failed_actions = defaultdict(list)
    all_domains_time = defaultdict(list)
    all_domains_ground_actions = defaultdict(list)

    all_domains_recall_prec = defaultdict(list)
    all_domains_precision_prec = defaultdict(list)

    all_domains_recall_pos = defaultdict(list)
    all_domains_precision_pos = defaultdict(list)

    all_domains_recall_neg = defaultdict(list)
    all_domains_precision_neg = defaultdict(list)

    all_domains_recall = defaultdict(list)
    all_domains_precision = defaultdict(list)

    all_domains_correctness = defaultdict(list)
    all_domains_integrity = defaultdict(list)

    for domain in [el for el in sorted(all_domains) if not el.startswith("overall")]:

        df_path = os.path.join(RESULTS_DIR, domain)

        df = pd.read_excel(df_path, sheet_name="Total")

        domain_total_objs = []
        domain_executed_actions = []
        domain_success_actions = []
        domain_failed_actions = []
        domain_ground_actions = []
        domain_time = []
        domain_recall_prec = []
        domain_precision_prec = []
        domain_recall_pos = []
        domain_precision_pos = []
        domain_recall_neg = []
        domain_precision_neg = []
        domain_recall = []
        domain_precision = []
        domain_correctness = []
        domain_integrity = []

        max_recall = 0
        max_precision = 0

        for index, row in df.iterrows():

            if row['Overall recall'] > max_recall and row['Overall precision'] > max_precision:
            # if (row['Overall recall'] > max_recall and row['Overall precision'] > max_precision)\
            #         or Configuration.RANDOM_WALK:

                max_recall = float(row['Overall recall'])
                max_precision = float(row['Overall precision'])

                max_recall_precs = float(row['Precs recall'])
                max_precision_precs = float(row['Precs precision'])

                max_recall_pos = float(row['Pos recall'])
                max_precision_pos = float(row['Pos precision'])

                max_recall_neg = float(row['Neg recall'])
                max_precision_neg = float(row['Neg precision'])

                domain_executed_actions.append(row['Executed actions'])
                domain_success_actions.append(row['Success actions'])
                domain_failed_actions.append(row['Failed actions'])
                domain_ground_actions.append(row['Ground actions'])
                domain_time.append(row['Computation time'])
                domain_total_objs.append(row['Total objs'])

                domain_recall_prec.append(row['Precs recall'])
                domain_precision_prec.append(row['Precs precision'])
                domain_recall_pos.append(row['Pos recall'])
                domain_precision_pos.append(row['Pos precision'])
                domain_recall_neg.append(row['Neg recall'])
                domain_precision_neg.append(row['Neg precision'])
                domain_recall.append(row['Overall recall'])
                domain_precision.append(row['Overall precision'])

                # Compute correctness
                size_m = row['Learn precs'] + row['Learn pos'] + row['Learn neg']
                del_pos = row['Del_pos']
                ins_pre = row['Ins_pre']
                ins_neg = row['Ins_neg']
                correctness = (size_m - del_pos)/(size_m + ins_pre + ins_neg)
                domain_correctness.append(correctness)

                # Compute integrity
                size_m = row['Learn precs'] + row['Learn pos'] + row['Learn neg']
                del_pre = row['Del_pre']
                del_neg = row['Del_neg']
                ins_pos = row['Ins_pos']
                integrity = (size_m - del_pre - del_neg)/(size_m + ins_pos)
                domain_integrity.append(integrity)

            elif row['Overall recall'] > max_recall:
            # elif row['Overall recall'] > max_recall or Configuration.RANDOM_WALK:

                max_recall = float(row['Overall recall'])

                domain_executed_actions.append(row['Executed actions'])
                domain_success_actions.append(row['Success actions'])
                domain_failed_actions.append(row['Failed actions'])
                domain_ground_actions.append(row['Ground actions'])
                domain_time.append(row['Computation time'])
                domain_total_objs.append(row['Total objs'])

                domain_recall_prec.append(row['Precs recall'])
                domain_precision_prec.append(row['Precs precision'])
                domain_recall_pos.append(row['Pos recall'])
                domain_precision_pos.append(row['Pos precision'])
                domain_recall_neg.append(row['Neg recall'])
                domain_precision_neg.append(row['Neg precision'])
                domain_recall.append(row['Overall recall'])
                domain_precision.append(row['Overall precision'])

                # Compute correctness
                size_m = row['Learn precs'] + row['Learn pos'] + row['Learn neg']
                del_pos = row['Del_pos']
                ins_pre = row['Ins_pre']
                ins_neg = row['Ins_neg']
                correctness = (size_m - del_pos)/(size_m + ins_pre + ins_neg)
                domain_correctness.append(correctness)

                # Compute integrity
                size_m = row['Learn precs'] + row['Learn pos'] + row['Learn neg']
                del_pre = row['Del_pre']
                del_neg = row['Del_neg']
                ins_pos = row['Ins_pos']
                integrity = (size_m - del_pre - del_neg)/(size_m + ins_pos)
                domain_integrity.append(integrity)

            elif row['Overall precision'] > max_precision:
            # elif row['Overall precision'] > max_precision or Configuration.RANDOM_WALK:

                max_precision = float(row['Overall precision'])

                domain_executed_actions.append(row['Executed actions'])
                domain_success_actions.append(row['Success actions'])
                domain_failed_actions.append(row['Failed actions'])
                domain_ground_actions.append(row['Ground actions'])
                domain_time.append(row['Computation time'])
                domain_total_objs.append(row['Total objs'])

                domain_recall_prec.append(row['Precs recall'])
                domain_precision_prec.append(row['Precs precision'])
                domain_recall_pos.append(row['Pos recall'])
                domain_precision_pos.append(row['Pos precision'])
                domain_recall_neg.append(row['Neg recall'])
                domain_precision_neg.append(row['Neg precision'])
                domain_recall.append(row['Overall recall'])
                domain_precision.append(row['Overall precision'])

                # Compute correctness
                size_m = row['Learn precs'] + row['Learn pos'] + row['Learn neg']
                del_pos = row['Del_pos']
                ins_pre = row['Ins_pre']
                ins_neg = row['Ins_neg']
                correctness = (size_m - del_pos)/(size_m + ins_pre + ins_neg)
                domain_correctness.append(correctness)

                # Compute integrity
                size_m = row['Learn precs'] + row['Learn pos'] + row['Learn neg']
                del_pre = row['Del_pre']
                del_neg = row['Del_neg']
                ins_pos = row['Ins_pos']
                integrity = (size_m - del_pre - del_neg)/(size_m + ins_pos)
                domain_integrity.append(integrity)

        all_domains_executed_actions[domain].extend(domain_executed_actions)
        all_domains_success_actions[domain].extend(domain_success_actions)
        all_domains_failed_actions[domain].extend(domain_failed_actions)
        all_domains_total_objs[domain].extend(domain_total_objs)
        all_domains_ground_actions[domain].extend(domain_ground_actions)
        all_domains_time[domain].extend(domain_time)

        all_domains_recall_prec[domain].extend(domain_recall_prec)
        all_domains_precision_prec[domain].extend(domain_precision_prec)

        all_domains_recall_pos[domain].extend(domain_recall_pos)
        all_domains_precision_pos[domain].extend(domain_precision_pos)

        all_domains_recall_neg[domain].extend(domain_recall_neg)
        all_domains_precision_neg[domain].extend(domain_precision_neg)

        all_domains_recall[domain].extend(domain_recall)
        all_domains_precision[domain].extend(domain_precision)

        all_domains_correctness[domain].extend(domain_correctness)
        all_domains_integrity[domain].extend(domain_integrity)

    # Create summary dataframe over all domains
    df = pd.DataFrame({'Domain': [],
                       'Instances': [],
                       'Avg time': [],
                       'Avg total objs': [],
                       'Avg ground actions': [],
                       'Avg executed actions': [],
                       'Avg success actions': [],
                       'Avg failed actions': [],
                       'Precs recall': [],
                       'Precs precision': [],
                       'Pos recall': [],
                       'Pos precision': [],
                       'Neg recall': [],
                       'Neg precision': [],
                       'Overall recall': [],
                       'Overall precision': [],
                       'Final correctness': [],
                       'Final integrity': []
                       })

    for domain in [el for el in all_domains if not el.startswith("overall")]:

        instances = len(all_domains_time[domain])
        avg_time = sum(all_domains_time[domain])/instances
        avg_total_objs = sum(all_domains_total_objs[domain])/instances
        avg_ground_actions = sum(all_domains_ground_actions[domain])/instances
        avg_executed_actions = sum(all_domains_executed_actions[domain])/instances
        avg_success_actions = sum(all_domains_success_actions[domain])/instances
        avg_failed_actions = sum(all_domains_failed_actions[domain])/instances

        final_recall_prec = all_domains_recall_prec[domain][-1]
        final_precision_prec = all_domains_precision_prec[domain][-1]

        final_recall_pos = all_domains_recall_pos[domain][-1]
        final_precision_pos = all_domains_precision_pos[domain][-1]

        final_recall_neg = all_domains_recall_neg[domain][-1]
        final_precision_neg = all_domains_precision_neg[domain][-1]

        final_recall = all_domains_recall[domain][-1]
        final_precision = all_domains_precision[domain][-1]

        final_correctness = all_domains_correctness[domain][-1]
        final_integrity = all_domains_integrity[domain][-1]

        evaluate = {'Domain': domain.split("_")[0],
                    'Instances': instances,
                    'Avg time': avg_time,
                    'Avg total objs': avg_total_objs,
                    'Avg ground actions': avg_ground_actions,
                    'Avg executed actions': avg_executed_actions,
                    'Avg success actions': avg_success_actions,
                    'Avg failed actions': avg_failed_actions,
                    'Precs recall': final_recall_prec,
                    'Precs precision': final_precision_prec,
                    'Pos recall': final_recall_pos,
                    'Pos precision': final_precision_pos,
                    'Neg recall': final_recall_neg,
                    'Neg precision': final_precision_neg,
                    'Overall recall': final_recall,
                    'Overall precision': final_precision,
                    'Final correctness': final_correctness,
                    'Final integrity': final_integrity
        }

        df = df.append(evaluate, ignore_index=True)

    # Create a Pandas Excel writer using XlsxWriter as the engine
    # writer = pd.ExcelWriter(os.path.join(RESULTS_DIR,"overall_summary.xlsx"), engine='xlsxwriter')
    # writer = pd.ExcelWriter(os.path.join(RESULTS_DIR,"overall_summary.xls"), engine='openpyxl')
    writer = pd.ExcelWriter(os.path.join(RESULTS_DIR,"overall_summary.xls"))
    # writer = pd.ExcelWriter(os.path.join(RESULTS_DIR,"overall_summary.xlsx"))
    # writer = pd.ExcelWriter(os.path.join(RESULTS_DIR,"overall_summary.xls"), engine='xlswriter')
    df.to_excel(writer, sheet_name='Summary', index=False, float_format = "%0.2f")
    writer.close()

def generate_comparison_latex_table(labels, file_name, caption, header):
    # folder = "Results"
    file_path = os.path.join("comparison_summary_uncertain.xlsx")

    with open(file_name + ".tex", "w") as f:
        df = pd.read_excel(file_path)
        df_restricted = df[labels]
        f.write(df_restricted.to_latex(index=False, escape=False,
                                       label="tab:{}".format(file_name),
                                       caption= caption,
                                       header = header))
                                       # .format(df_restricted.shape[0])))

        # f.write(df_restricted.to_latex(index=False))

def generate_comparison_latex_table_fama(labels, file_name, caption, header):
    # folder = "Results"
    file_path = os.path.join("comparison_fama.xlsx")

    with open(file_name + ".tex", "w") as f:
        df = pd.read_excel(file_path)
        df_restricted = df[labels]
        f.write(df_restricted.to_latex(index=False, escape=False,
                                       label="tab:{}".format(file_name),
                                       caption= caption,
                                       header = header))
                                       # .format(df_restricted.shape[0])))

        # f.write(df_restricted.to_latex(index=False))

def generate_summary_latex_table(labels, file_name, caption, header):
    folder = "Results"
    file_path = os.path.join(folder, "overall_summary.xlsx")

    with open(file_name + ".tex", "w") as f:
        df = pd.read_excel(file_path, sheet_name="Summary")
        df_restricted = df[labels]
        f.write(df_restricted.to_latex(index=False, escape=False,
                                       label="tab:{}".format(file_name),
                                       caption= caption,
                                       header = header))
                                       # .format(df_restricted.shape[0])))

        # f.write(df_restricted.to_latex(index=False))

if __name__ == "__main__":

# Generate domain dataframes and summary
    generate_domain_dataframes()

    generate_domain_summary()
#
#
#
# Generate summary (latex table)
    # labels = ["Domain", "Instances", "Precs precision",  "Precs recall","Pos precision", "Pos recall",
    #           "Neg precision", "Neg recall", "Overall precision", "Overall recall"]
    # header = ["Domain", "$I$", "$P_{\\prec}$", "$R_{\\prec}$", "$P_{\\eff^{+}}$", "$R_{\\eff^{+}}$", "$P_{\\eff^{-}}$",
    #           "$R_{\\eff^{-}}$", "$P$", "$R$"]
    # caption = "For each domain:statistics on final metrics of the last instance grouped by " \
    #           "preconditions, positive effects and negative ones."
    # file_name = "overall_summary_uncertain_nostripsass"
    # generate_summary_latex_table(labels, file_name, caption, header)
#
#
#
#
#
# Generate comparison summary uncertain negative effects (latex table)
    labels = ["Domain", "Neg precision A", "Neg recall A", "Overall precision A", "Overall recall A",
              "Neg precision B", "Neg recall B", "Overall precision B", "Overall recall B"]
    header = ["Domain", "$P_{\\eff^{-}}$", "$R_{\\eff^{-}}$", "$P$", "$R$",
              "$P_{\\eff^{-}}$", "$R_{\\eff^{-}}$", "$P$", "$R$"]
    caption = "For each domain:statistics on final metrics of the last instance grouped by " \
              "negative effects."
    file_name = "comparison_summary_uncertain"
    generate_comparison_latex_table(labels, file_name, caption, header)
#
#
# Generate FAMA average results
#     generate_fama_average_results()
#
# Generate comparison with FAMA (latex table)
#     labels = ["Domain", "Tot time", "Overall precision", "Overall recall", "FAMA tot time",
#               "FAMA precision", "FAMA recall", "Delta act"]
#     header = ["Domain", "$t$", "$P$", "$R$", "$t$", "$P$", "$R$", "$\delta_{A}$"]
#     caption = "Comparison among OLAM and FAMA with full observability. FAMA is run with all plan traces " \
#               "provided in \protect\cite{aineto_AIJ2019}. MODEL WITH UNCERTAIN NEGATIVE EFFECTS AND STRIPS ASSUMPTION."
#     file_name = "comparison_fama"
#     generate_comparison_latex_table_fama(labels, file_name, caption, header)
#
#
#
#
# Generate correctness and integrity (latex table)
    # labels = ["Domain", "Instances", "Avg time",  "Final correctness","Final integrity"]
    # header = ["Domain", "$I$", "$CPU_{time}$", "$C$", "$I$"]
    # caption = "For each domain: evaluation on average CPU time, correctness and integrity of the last instance."
    # file_name = "overall_corrint_certain"
    # generate_summary_latex_table(labels, file_name, caption, header)

max_stars_repo_name: LamannaLeonardo/OLAM

max_issues_repo_name: LamannaLeonardo/OLAM

max_forks_repo_name: LamannaLeonardo/OLAM


text:
# Diffusion Maps Framework implementation as part of MSc Data Science Project of student
# Napoleon Koskinas at University of Southampton, MSc Data Science course

# Script 3: Principal Component Analysis

import os, math
import string
import openpyxl

import numpy as np
import pandas as pd

import matplotlib
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec

from sklearn import preprocessing
from sklearn.decomposition import PCA

import datetime

matplotlib.style.use('ggplot')
matplotlib.rcParams['legend.scatterpoints'] = 1

datasource = './data/normalised/sqrtData.xlsx'
dtsource = './data/datetimes.xlsx'

def main():

	# Read data, get names of the 6 sheets, for different types of bacteria
	xlData = pd.ExcelFile(datasource)
	sheetNames = xlData.sheet_names
	dtExcel = pd.ExcelFile(dtsource)

	# writer = pd.ExcelWriter('./data/pcaData.xlsx')
	no = 321

	fig = plt.figure()

	for bactName in sheetNames:
		# no += licycle.next()
		worksheet = xlData.parse(bactName)
		# Read time-date from file
		dtDf = dtExcel.parse(bactName)
		dt = pd.DataFrame(dtDf).as_matrix()

		# Keep only the actual timeseries data, last 30 columns, and transpose
		X = pd.DataFrame(worksheet.ix[:,:29])
		XTransposed = X.as_matrix().transpose()

		# Perform PCA on Data
		pca = PCA(n_components = 2, whiten = True)
		XTransposedPca = pca.fit_transform(XTransposed)
		scatterplot2(XTransposedPca, bactName, no, fig, dt)

		no+=1

	plt.tight_layout()
	plt.show()
	# 	pd.DataFrame(XTransposedPca).to_excel(writer, bactName)
	# writer.save()

def  scatterplot(X):

	markers = ['d', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', '^', '^', '^', '^', '^', '^', '^', '^', '^', '^', '+']
	colors = ['purple', 'b', 'aqua', 'lightseagreen', 'green', 'lightgreen', 'orangered', 'magenta', 'orchid', 'purple', 'darkblue', 'b', 'g', 'lightgreen', 'y', 'orange', 'r', 'magenta', 'purple', 'b', 'aqua', 'lightseagreen', 'g', 'lightgreen', 'orangered', 'magenta', 'orchid', 'purple', 'darkblue', 'b']

	for m, c, i, j in zip(markers, colors, X[:,0], X[:,1]):
	    plt.scatter(i, j, s = 50, marker=m, c=c)

	plt.show()

def  scatterplot2(X, bactName, no, fig, datetimes):

	# Create a custom set of markers with custom colours to reproduce results
	 # of previous research and compare after dimensionality reduction
	markers = ['d', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', \
	'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', '^', '^', '^', '^', '^', '^', \
	'^', '^', '^', '^', '+']

	colors = ['purple', 'b', 'aqua', 'lightseagreen', 'green', 'lightgreen',\
	 'orangered', 'magenta', 'orchid', 'purple', 'darkblue', 'b', 'g', 'lightgreen',\
	  'y', 'orange', 'r', 'magenta', 'purple', 'b', 'aqua', 'lightseagreen', 'g', \
	  'lightgreen', 'orangered', 'magenta', 'orchid', 'purple', 'darkblue', 'b']

	ax = fig.add_subplot(no)
	ax.set_title(bactName)
	points = []
	names = []

	for name_arr in datetimes.tolist():
		names.append(name_arr[0])

	for m, c, i, j in zip(markers, colors, X[:,0], X[:,1]):
		points.append(ax.scatter(i, j, s = 50, marker=m, c=c))

	fig.legend(points, names, 'lower center', ncol=5)

	plt.xlabel('Principal Component 1')
	plt.ylabel('Principal Component 2')
	# plt.tight_layout()

	return fig

if __name__ == '__main__':
	main()

ext: py

max_stars_repo_name: koskinap/diffusion-maps

max_issues_repo_name: koskinap/diffusion-maps

max_forks_repo_name: koskinap/diffusion-maps

reason: import numpy


text:
import torch
import numpy as np
from tqdm import tqdm

from utils.utils_heads import (
    ElectraClassificationHeadIdentityPooler,
    BertClassificationHeadIdentityPooler,
    ElectraNERHeadIdentityPooler,
)
from utils.utils_inference import (
    is_custom_head,
    unpad_features,
    pad_scores
)
from ue4nlp.mahalanobis_distance import (
    mahalanobis_distance,
    mahalanobis_distance_relative,
    mahalanobis_distance_marginal,
    compute_centroids,
    compute_covariance
)

import logging
import time

log = logging.getLogger()

class UeEstimatorLMahalanobis:
    def __init__(self, cls, ue_args, config, train_dataset):
        self.cls = cls
        self.ue_args = ue_args
        self.config = config
        self.train_dataset = train_dataset

    def __call__(self, X, y):
        return self._predict_with_fitted_cov(X, y)

    def fit_ue(self, X, y=None, X_test=None):
        cls = self.cls
        model = self.cls._auto_model

        log.info("****************Start fitting covariance and centroids **************")

        if y is None:
            y = self._exctract_labels(X)

        self._replace_model_head()
        X_features, X_hidden_states = self._exctract_features(X)

        self.class_cond_centroids = self._fit_centroids(X_features, y)
        self.class_cond_covariance = self._fit_covariance(X_features, y)

        self.class_cond_centroids_hiddens = []
        self.class_cond_covariance_hiddens = []

        for X_hidden_state in X_hidden_states:
            self.class_cond_centroids_hiddens.append(self._fit_centroids(X_hidden_state, y))
            self.class_cond_covariance_hiddens.append(self._fit_covariance(X_hidden_state, y, centroids=self.class_cond_centroids_hiddens[-1]))

        log.info("**************Done.**********************")

    def _fit_covariance(self, X, y, class_cond=True, centroids=None):
        if class_cond:
            if centroids is None:
                centroids = self.class_cond_centroids
            return compute_covariance(centroids, X, y, class_cond)
        if centroids is None:
            centroids = self.train_centroid
        return compute_covariance(centroids, X, y, class_cond)

    def _fit_centroids(self, X, y, class_cond=True):
        return compute_centroids(X, y, class_cond)

    def _replace_model_head(self):
        log.info("Change classifier to Identity Pooler")
        cls = self.cls
        model = self.cls._auto_model

        if is_custom_head(model):
            model.classifier = ElectraClassificationHeadIdentityPooler(model.classifier)
        else:
            model.classifier = BertClassificationHeadIdentityPooler(model.classifier)

    def _exctract_labels(self, X):
        return np.asarray([example["label"] for example in X])

    def _exctract_features(self, X):
        cls = self.cls
        model = self.cls._auto_model

        try:
            X = X.remove_columns("label")
        except:
            X.dataset = X.dataset.remove_columns("label")

        features = self.cls.predict(
            X, apply_softmax=False, return_preds=False
        )

        X_hidden_states = [np.tanh(state) for state in features[1][1]]
        X_features = features[0]

        return X_features, X_hidden_states

    def _predict_with_fitted_cov(self, X, y):
        cls = self.cls
        model = self.cls._auto_model

        log.info("****************Compute MD with fitted covariance and centroids **************")

        start = time.time()
        if y is None:
            y = self._exctract_labels(X)
        X_features, X_hidden_states = self._exctract_features(X)
        end = time.time()

        eval_results = {}

        md_n, inf_time = mahalanobis_distance(None, None, X_features,
                                             self.class_cond_centroids, self.class_cond_covariance)
        sum_inf_time = inf_time + (end - start)

        md_l = []
        for i, X_hidden_state in enumerate(X_hidden_states):
            md_i, inf_time = mahalanobis_distance(None, None, X_hidden_state,
                                                  self.class_cond_centroids_hiddens[i],
                                                  self.class_cond_covariance_hiddens[i])
            md_l.append(md_i.copy())
            sum_inf_time += inf_time

        md_l = np.asarray(md_l)
        eval_results["mahalanobis_distance"] = (md_n + md_l.sum(axis=0)).tolist()
        eval_results["ue_time"] = sum_inf_time
        log.info(f"UE time: {sum_inf_time}")

        log.info("**************Done.**********************")
        return eval_results


text:
import argparse
import ray
from ray import tune
from my_env import ContentCaching
import pickle
import time
import numpy as np
import time
import matplotlib.pyplot as plt
from caching import customExperimentClass

#import torch
#torch.set_deterministic(True)

if __name__ == "__main__":

	parser = argparse.ArgumentParser()
	parser.add_argument("--run", type=str, default="ppo")
	parser.add_argument("--epochs", type=int, default= 50)#50)
	parser.add_argument("--stop-timesteps", type=int, default=99999900)
	parser.add_argument("--stop-reward", type=float, default=0.000001)
	parser.add_argument("--para", type=str, default="rc")
	parser.add_argument("--ttl_var", type=int, default=3)
	parser.add_argument("--cpu", type=int, default= 8)
	parser.add_argument("--gpu", type=float, default= 0)
	parser.add_argument("--lr", type=float, nargs="+", default=[1e-2])
	parser.add_argument("--activation", nargs="+", default= ["relu"])
	parser.add_argument('-l','--layer', type=int, nargs='+', required=True, action='append', help='layer list')
	parser.add_argument("--num_gpus_per_worker", type=float, default= 0)
	parser.add_argument("--num_workers", type=int, default= 0)

	args = parser.parse_args()

	y=0
	parameters = [[y, 8, 8, 4] , [8, y, 8, 4], [8, 8, y, 4], [8, 8, 8, y]]

	pdf_plot = ["rc", "co", "cu", "k"] #["R_c", "C_o", "C_u", "k"]
	if args.para == "rc":
		para = 0
	if args.para == "co":
		para = 1
	if args.para == "cu":
		para = 2
	if args.para == "k":
		para = 3

	lst=0

	algo_reward_test = []
	algo_unused_shared = []
	algo_unused_own = []
	algo_unsatisfied_shared =[]
	algo_unsatisfied_own = []

	max_algo_unused_shared = []
	max_algo_unused_own = []
	max_algo_unsatisfied_shared =[]
	max_algo_unsatisfied_own = []

	mean_algo_unused_shared = []
	mean_algo_unused_own = []
	mean_algo_unsatisfied_shared =[]
	mean_algo_unsatisfied_own = []

	variable = [1,2,4,6,8,10,12,14,16,18,20,25,30,35,40,45,50,55,60] #[1,10,20,60,150,400,700,1000] #

	ray.shutdown()
	ray.init()
	for x in range(len(variable)):

		parameters[para][para]= variable[x]

		all_unused_shared = []
		all_unused_own = []
		all_unsatisfied_shared = []
		all_unsatisfied_own = []

		for cpt in range(1,11):

			print("calcul of : "+pdf_plot[para], " for the value : ", variable[x]  )

			if args.run == "ppo" or args.run == "ddpg" or args.run == "appo" or args.run == "td3" or args.run == "a3c":
				# Class instance
				exper = customExperimentClass(args.ttl_var, cpt, parameters[para], \
											fcnet_hidd_lst =args.layer, fcnet_act_lst =args.activation, lr_lst =args.lr, stop_iters=args.epochs, num_gpus=args.gpu, num_gpus_per_worker=args.num_gpus_per_worker, num_workers=args.num_workers)
				checkpoint_path, results, lr, fc_hid, fc_act = exper.train(args.run)
				# Load saved and Test loaded
				reward, unused_shared ,unused_own, unsatisfied_shared, unsatisfied_own  = exper.test(args.run, checkpoint_path, lr, fc_hid, fc_act)

			if args.run == "random" :
				# Class instance
				exper = customExperimentClass(args.ttl_var, cpt, parameters[para], \
											fcnet_hidd_lst =args.layer, fcnet_act_lst =args.activation, lr_lst =args.lr, stop_iters=args.epochs, num_gpus=args.gpu, num_gpus_per_worker=args.num_gpus_per_worker, num_workers=args.num_workers)
				# Load saved and Test loaded
				reward, unused_shared ,unused_own, unsatisfied_shared, unsatisfied_own  = exper.test_random(args.run)

			#all_reward_test.append(reward_test)
			all_unused_shared.append(unused_shared)
			all_unused_own.append(unused_own)
			all_unsatisfied_shared.append(unsatisfied_shared)
			all_unsatisfied_own.append(unsatisfied_own)

		mean_all_unused_shared = [(a + b + c + d + e + f + g + h + i + j) / 10 for a,b,c,d,e,f,g,h,i,j  in zip(all_unused_shared[0], all_unused_shared[1], \
			all_unused_shared[2], all_unused_shared[3], all_unused_shared[4],all_unused_shared[5], all_unused_shared[6], all_unused_shared[7], all_unused_shared[8], all_unused_shared[9])]

		mean_all_unused_own = [(a + b + c + d + e + f + g + h + i + j) / 10 for a,b,c,d,e,f,g,h,i,j  in zip(all_unused_own[0], all_unused_own[1], all_unused_own[2],\
			all_unused_own[3],all_unused_own[4], all_unused_own[5], all_unused_own[6], all_unused_own[7],all_unused_own[8], all_unused_own[9])]

		mean_all_unsatisfied_shared = [(a + b + c + d + e + f + g + h + i + j) / 10 for a,b,c,d,e,f,g,h,i,j  in zip(all_unsatisfied_shared[0], \
			all_unsatisfied_shared[1], all_unsatisfied_shared[2], all_unsatisfied_shared[3],all_unsatisfied_shared[4], all_unsatisfied_shared[5], all_unsatisfied_shared[6],all_unsatisfied_shared[7], all_unsatisfied_shared[8], all_unsatisfied_shared[9])]

		mean_all_unsatisfied_own = [(a + b + c + d + e + f + g + h + i + j) / 10 for a,b,c,d,e,f,g,h,i,j  in zip(all_unsatisfied_own[0], all_unsatisfied_own[1],\
			all_unsatisfied_own[2], all_unsatisfied_own[3], all_unsatisfied_own[4], all_unsatisfied_own[5], all_unsatisfied_own[6], all_unsatisfied_own[7], all_unsatisfied_own[8], all_unsatisfied_own[9])]

		algo_unused_shared.append(np.mean(mean_all_unused_shared))
		algo_unused_own.append(np.mean(mean_all_unused_own))
		algo_unsatisfied_shared.append(np.mean(mean_all_unsatisfied_shared))
		algo_unsatisfied_own.append(np.mean(mean_all_unsatisfied_own))

		print("mean_all_unused_shared[0] ", mean_all_unused_shared[0])

		all_unused_shared = [max(a,b,c,d,e,f,g,h,i,j)  for a,b,c,d,e,f,g,h,i,j  in zip(all_unused_shared[0], all_unused_shared[1], \
			all_unused_shared[2], all_unused_shared[3], all_unused_shared[4],all_unused_shared[5], all_unused_shared[6], all_unused_shared[7], all_unused_shared[8], all_unused_shared[9])]

		all_unused_own = [max(a,b,c,d,e,f,g,h,i,j) for a,b,c,d,e,f,g,h,i,j  in zip(all_unused_own[0], all_unused_own[1], all_unused_own[2],\
			all_unused_own[3],all_unused_own[4], all_unused_own[5], all_unused_own[6], all_unused_own[7],all_unused_own[8], all_unused_own[9])]

		all_unsatisfied_shared = [max(a,b,c,d,e,f,g,h,i,j) for a,b,c,d,e,f,g,h,i,j  in zip(all_unsatisfied_shared[0], \
			all_unsatisfied_shared[1], all_unsatisfied_shared[2], all_unsatisfied_shared[3],all_unsatisfied_shared[4], all_unsatisfied_shared[5], all_unsatisfied_shared[6],all_unsatisfied_shared[7], all_unsatisfied_shared[8], all_unsatisfied_shared[9])]

		all_unsatisfied_own = [max(a,b,c,d,e,f,g,h,i,j) for a,b,c,d,e,f,g,h,i,j  in zip(all_unsatisfied_own[0], all_unsatisfied_own[1],\
			all_unsatisfied_own[2], all_unsatisfied_own[3], all_unsatisfied_own[4], all_unsatisfied_own[5], all_unsatisfied_own[6], all_unsatisfied_own[7], all_unsatisfied_own[8], all_unsatisfied_own[9])]

		max_algo_unused_shared.append(np.mean(all_unused_shared))
		max_algo_unused_own.append(np.mean(all_unused_own))
		max_algo_unsatisfied_shared.append(np.mean(all_unsatisfied_shared))
		max_algo_unsatisfied_own.append(np.mean(all_unsatisfied_own))

	times = [1,2,4,6,8,10,12,14,16,18,20,25,30,35,40,45,50,55,60]

	plt.plot(times , algo_unused_shared, color='orange', linestyle='dotted', marker='x' ,label=args.run+'_$Unused_{g}$') #  unused shared  'ppo_$Unused$'
	plt.plot(times , algo_unused_own, color='purple', linestyle='-', marker='+' ,label=args.run+'_$Unused_{o}$') # unused own
	plt.plot(times , max_algo_unused_shared, color='red', linestyle='dashed', marker='v' ,label='max_'+args.run+'_$Unused_{g}$') #  unused shared  'ppo_$Unused$'
	plt.plot(times , max_algo_unused_own, color='blue', linestyle='--', marker='D' ,label='max_'+args.run+'_$Unused_{o}$') # unused own

	plt.ylabel('Unused caching resources', size= 8 ) #'$U_{nused}$' #Reward
	plt.xlabel('$'+pdf_plot[para]+'$', size= 10) #'$'+pdf_plot[para]+'$'

	plt.xticks(size = 7)
	plt.yticks(size = 7)

	# Add a legend
	plt.legend()

	# save file .pdf
	plt.savefig('plot/unused_'+pdf_plot[para]+'_'+args.run+'.pdf')

	#to stock data
	our_file = [algo_unused_shared,algo_unused_own,max_algo_unused_shared,max_algo_unused_own]
	with open('model/unused_'+pdf_plot[para]+'_'+args.run+'.data', 'wb') as filehandle:
	#  # store the data as binary data stream
		pickle.dump(our_file, filehandle)

	#plt.show()
	plt.close()
	print("End")

	#plot only the last one
	plt.plot(times , algo_unsatisfied_shared, color='orange', linestyle='dotted', marker='x' ,label=args.run+'_$Unsatisfied_{g}$') #  unused shared
	plt.plot(times , algo_unsatisfied_own, color='purple', linestyle='-', marker='+' ,label=args.run+'_$Unsatisfied_{o}$') # unused own
	plt.plot(times , max_algo_unsatisfied_shared, color='red', linestyle='dashed', marker='v' ,label='max_'+args.run+'_$Unused_{g}$') #  unused shared  'ppo_$Unused$'
	plt.plot(times , max_algo_unsatisfied_own, color='blue', linestyle='--', marker='D' ,label='max_'+args.run+'_$Unused_{o}$') # unused own

	plt.ylabel('Unsatisfied caching demands', size= 8 )
	plt.xlabel('$'+pdf_plot[para]+'$', size= 10)

	plt.xticks(size = 7)
	plt.yticks(size = 7)

	# Add a legend
	plt.legend()

	# save file .pdf
	#"""
	plt.savefig('plot/unsatisfied_'+pdf_plot[para]+'_'+args.run+'.pdf')
	#to stock data
	our_file = [algo_unsatisfied_shared, algo_unsatisfied_own,max_algo_unsatisfied_shared, max_algo_unsatisfied_own]
	with open('model/unsatisfied_'+pdf_plot[para]+'_'+args.run+'.data', 'wb') as filehandle:
	  # store the data as binary data stream
		pickle.dump(our_file, filehandle)

	#plt.show()
	plt.close()
	print("End")

max_stars_repo_name: anaskn/ray

max_issues_repo_name: anaskn/ray

max_forks_repo_name: anaskn/ray


text:
#!/usr/bin/env python
# coding: utf-8

# # Stock Price Prediction using Echo State Networks
#
# ## Introduction
#
# Time Series Prediction is one important regression task, which can be solved using several machine learning techniques.
#
# In this notebook, we briefly introduce one very basic example for time series prediction:
#
# Stock Price Prediction
#
# Disclaimer: We are signal processing experts, not financial advisors. Do not use any of the models presented herein to steer your investments.
#
# At first, we need to import all required packages

# In[1]:

import numpy as np
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import mean_squared_error
from sklearn.metrics import make_scorer
from sklearn.model_selection import TimeSeriesSplit
from sklearn.model_selection import GridSearchCV

from matplotlib import pyplot as plt
plt.rcParams["font.family"] = "Times New Roman"
plt.rcParams["font.size"] = 10

from pyrcn.echo_state_network import ESNRegressor
from pyrcn.linear_model import IncrementalRegression
from pyrcn.base import InputToNode, NodeToNode

# ## Loading and visualizing stock prices
#
# A good ressource for stock prices is Yahoo Finance ([https://finance.yahoo.com/](https://finance.yahoo.com/)), where a
# lot of financial data can be downloaded as csv files.
#
# We have already downloaded several stock prices, which are all stored in the directory "dataset".
#
# Now, we are working with the gold price in USD (Gold Aug 20(GC=F)) between 2000-02-28 and 2020-05-28.

# In[2]:

X = np.genfromtxt(fname="./examples/dataset/GC=F.csv", usecols=4, skip_header=1,delimiter=",")

unit_impulse = np.zeros(shape=(100, 1), dtype=int)
unit_impulse[5] = 1

plt.figure()
plt.plot(X)
plt.xlim([0, 100])
plt.ylim([0, 1])
plt.xlabel('n')
plt.ylabel('X[n]')
plt.grid()
plt.show()

# Analyzing the dataset, we can analyze the structure. The first column is the date, the second column is the open value,
# e.g. the first value of a day.
# The third column is the highest value of a specific day, the fourth column the lowest and "Close" is the final value of
# a day. The "Adj Close" is a corrected  final value, I DON 'T YET KNOW ANYTHING ABOUT VOLUME!!!
#
# Here, we just use the "Close" value, the final value of one day.

# In[3]:

X = X[~np.isnan(X)].reshape(-1, 1)

# As one can see, the stock price trends upwards for the first few thousand days. Then goes down at around $n=3300$ before climbing again after around $n = 4900$.

# In[4]:

plt.figure(figsize=(4, 2.5))
plt.plot(X)
plt.xlabel("Timestamp")
plt.xlim([0, len(X)])
plt.ylabel("Price")
plt.grid()
plt.tight_layout()
plt.show()

# We pre-processed the dataset by removing undefined values, namely, weekends and public holidays. The remaining values were normalized to be in a range of $[0 1]$.

# In[5]:

train_len = 3000
future_len = 1

scaler = MinMaxScaler(feature_range=(-1, 1)).fit(X=X)

# Echo State Network preparation

# In[7]:

base_input_to_nodes = InputToNode(hidden_layer_size=100, activation='identity', k_in=1, input_scaling=0.6, bias_scaling=0.0)
base_nodes_to_nodes = NodeToNode(hidden_layer_size=100, spectral_radius=0.9, leakage=1.0, bias_scaling=0.0, k_rec=10)

esn = ESNRegressor(input_to_node=base_input_to_nodes,
                   node_to_node=base_nodes_to_nodes,
                   regressor=IncrementalRegression(alpha=1e-8), random_state=10)

# Training and Prediction.

# In[8]:

X_train = scaler.transform(X[0:train_len])
y_train = scaler.transform(X[1:train_len+1])
X_test = scaler.transform(X[train_len+1:-1])
y_test = scaler.transform(X[train_len+1+future_len:])

fig = plt.figure()
plt.plot(scaler.transform(X.reshape(-1, 1)))
plt.xlabel("n")
plt.xlim([0, len(X)])
plt.ylabel("u[n]")
plt.grid()
fig.set_size_inches(4, 2.5)
plt.savefig('input_data.pdf', bbox_inches = 'tight', pad_inches = 0)

param_grid = {'input_to_node__hidden_layer_size': [100],
              'input_to_node__input_scaling': [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0,  1.1, 1.2, 1.3, 1.4, 1.5],
              'input_to_node__bias_scaling': [0.0],
              'input_to_node__activation': ['identity'],
              'input_to_node__random_state': [42],
              'node_to_node__hidden_layer_size': [100],
              'node_to_node__leakage': [1.0],
              'node_to_node__spectral_radius': [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5],
              'node_to_node__bias_scaling': [0.0],
              'node_to_node__bi_directional': [False],
              'node_to_node__continuation': [True],
              'node_to_node__activation': ['tanh'],
              'node_to_node__wash_out': [0],
              'node_to_node__random_state': [42],
              'regressor__alpha': [1e-5],
              'random_state': [42] }

scorer = make_scorer(score_func=mean_squared_error, greater_is_better=False)

ts_split = TimeSeriesSplit()
grid_search = GridSearchCV(ESNRegressor(), cv=ts_split, param_grid=param_grid, scoring=scorer, n_jobs=-1).fit(X=X_train, y=y_train)

print(grid_search.best_params_)

esn = grid_search.best_estimator_
# esn.set_params(**{'node_to_node__leakage': 0.1})
# esn.fit(X=X_train, y=y_train)
esn.predict(X=unit_impulse)
fig = plt.figure()
im = plt.imshow(np.abs(esn._node_to_node._hidden_layer_state[:, 1:].T),vmin=0, vmax=1)
plt.xlim([0, 100])
plt.ylim([0, esn._node_to_node._hidden_layer_state[:, 1:].shape[1] - 1])
plt.xlabel('n')
plt.ylabel('R[n]')
plt.colorbar(im)
plt.grid()
fig.set_size_inches(4, 2.5)
plt.savefig('InputScaling_SpectralRadius.pdf', bbox_inches = 'tight', pad_inches = 0)

# esn.fit(X=X_train, y=y_train.ravel())
y_train_pred = esn.predict(X=X_train)
y_test_pred = esn.predict(X=X_test)

train_err = mean_squared_error(y_true=y_train, y_pred=y_train_pred)
test_err = mean_squared_error(y_true=y_test, y_pred=y_test_pred)

print("Train MSE:\t{0}".format(train_err))
print("Test MSE:\t{0}".format(test_err))

# We see that the ESN even captures the downward trend in the test set, although it has not seen any longer downward movement during the training.

# In[9]:

plt.figure()
plt.plot(scaler.inverse_transform(y_test), label='Target')
plt.plot(scaler.inverse_transform(y_test_pred.reshape(-1, 1)), label='Predicted')
plt.xlabel("Timestamp")
plt.ylabel("Price")
plt.legend()
plt.show()

# Disclaimer: We are signal processing experts, not financial advisors. Do not use any of the models presented herein to steer your investments.

max_stars_repo_name: TUD-STKS/PyRCN

max_issues_repo_name: TUD-STKS/PyRCN

max_forks_repo_name: TUD-STKS/PyRCN


text:
# coding: utf-8
# Copyright (c) Pymatgen Development Team.
# Distributed under the terms of the MIT License.

from pymatgen.core.structure import Structure, Lattice
from pymatgen.io.vasp.outputs import Xdatcar, Vasprun
from monty.json import MSONable
from fnmatch import fnmatch
import numpy as np
import warnings
import os

"""
This module provides classes used to define a MD trajectory.
"""

__author__ = "Eric Sivonxay, Shyam Dwaraknath"
__version__ = "0.0"
__date__ = "Jan 25, 2019"

class Trajectory(MSONable):
    """
    Trajectory object that stores structural information related to a MD simulation.
    Provides basic functions such as slicing trajectory or obtaining displacements.
    """
    def __init__(self, lattice, species, frac_coords, time_step=2, site_properties=None, constant_lattice=True,
                 coords_are_displacement=False, base_positions=None):
        """
        Create a trajectory object

        Args:
            lattice: The lattice as any 2D array. Each row should correspond to a lattice
                vector. E.g., [[10,0,0], [20,10,0], [0,0,30]] specifies a
                lattice with lattice vectors [10,0,0], [20,10,0] and [0,0,30].
            species: List of species on each site. Can take in flexible input,
                including:

                i.  A sequence of element / specie specified either as string
                    symbols, e.g. ["Li", "Fe2+", "P", ...] or atomic numbers,
                    e.g., (3, 56, ...) or actual Element or Specie objects.

                ii. List of dict of elements/species and occupancies, e.g.,
                    [{"Fe" : 0.5, "Mn":0.5}, ...]. This allows the setup of
                    disordered structures.
            frac_coords (MxNx3 array): list of fractional coordinates of
                each species
            time_step (int, float): Timestep of simulation in femtoseconds. Defaults to 2fs.
            site_properties (list): Properties associated with the sites as a list of
                dicts of sequences, e.g., [{"magmom":[5,5,5,5]}, {"magmom":[5,5,5,5]}]. The sequences
                have to be the same length as the atomic species and fractional_coords. Number of supplied
                dicts should match number of frames in trajectory
                Defaults to None for no properties.
            constant_lattice (bool): Whether the lattice changes during the simulation, such as in an NPT MD simulation.
            coords_are_displacement (bool): Whether supplied coordinates are given in displacements (True) or
                positions (False)
            base_positions (Nx3 array): The starting positions of all atoms in trajectory. Used to reconstruct positions
                when converting from displacements to positions. Only needs to be specified if
                coords_are_displacement=True. Defaults to first index of frac_coords if coords_are_displacement=False.
        """
        # To support from_dict and as_dict
        if isinstance(frac_coords, list):
            frac_coords = np.array(frac_coords)

        if isinstance(lattice, list):
            lattice = np.array(lattice)

        self.frac_coords = frac_coords
        if coords_are_displacement:
            if base_positions is None:
                warnings.warn(
                    "Without providing an array of starting positions, the positions for each time step will not be available")
            self.base_positions = base_positions
        else:
            self.base_positions = frac_coords[0]
        self.coords_are_displacement = coords_are_displacement

        if not constant_lattice and np.shape(lattice) == (3, 3):
            self.lattice = [lattice for i in range(self.frac_coords.shape[0])]
        else:
            self.lattice = lattice
        self.constant_lattice = constant_lattice
        self.species = species
        self.site_properties = site_properties
        self.time_step = time_step

    def get_structure(self, i):
        """
        Returns structure at specified index
        Args:
            i (int): Index of structure

        Returns:
            (Structure) pymatgen structure object
        """
        return self[i]

    def to_positions(self):
        """
        Converts fractional coordinates of trajectory into positions
        """
        if self.coords_are_displacement:
            cumulative_displacements = np.cumsum(self.frac_coords, axis=0)
            positions = self.base_positions + cumulative_displacements
            self.frac_coords = positions
            self.coords_are_displacement = False
        return

    def to_displacements(self):
        """
        Converts position coordinates of trajectory into displacements between consecutive frames
        """
        if not self.coords_are_displacement:
            displacements = np.subtract(self.frac_coords, np.roll(self.frac_coords, 1, axis=0))
            displacements[0] = np.zeros(np.shape(self.frac_coords[0]))
            # Deal with PBC
            displacements = [np.subtract(item, np.round(item)) for item in displacements]

            self.frac_coords = displacements
            self.coords_are_displacement = True
        return

    def extend(self, trajectory):
        """
        Concatenate another trajectory

        Args:
            trajectory (Trajectory): Trajectory to add

        """
        if self.time_step != trajectory.time_step:
            raise ValueError('Trajectory not extended: Time steps of trajectories is incompatible')

        if len(self.species) != len(trajectory.species) and self.species != trajectory.species:
            raise ValueError('Trajectory not extended: species in trajectory do not match')

        self.to_positions()
        trajectory.to_positions()

        self.frac_coords = np.concatenate((self.frac_coords, trajectory.frac_coords), axis=0)
        self.lattice, self.constant_lattice = self._combine_attribute(self.lattice, trajectory.lattice,
                                                                      self.frac_coords.shape[0],
                                                                      trajectory.frac_coords.shape[0])
        self.site_properties = self._combine_attribute(self.site_properties, trajectory.site_properties,
                                                       self.frac_coords.shape[0], trajectory.frac_coords.shape[0])

    def __iter__(self):
        for i in range(self.frac_coords.shape[0]):
            yield self[i]

    def __len__(self):
        return np.shape(self.frac_coords)[0]

    def __getitem__(self, frames):
        """
        Gets a subset of the trajectory if a slice is given, if an int is given, return a structure

        Args:
            frames (int, slice): int or slice of trajectory to return

        Return:
            (Trajectory, Structure) Subset of trajectory
        """
        if isinstance(frames, int) and frames < self.frac_coords.shape[0]:
            lattice = self.lattice if self.constant_lattice else self.lattice[frames]
            site_properties = self.site_properties[frames] if self.site_properties else None
            return Structure(Lattice(lattice), self.species, self.frac_coords[frames], site_properties=site_properties,
                             to_unit_cell=True)

        if isinstance(frames, slice):
            frames = np.arange(frames.start, frames.stop, frames.step)
        elif not (isinstance(frames, list) or isinstance(frames, np.ndarray)):
            try:
                frames = np.asarray(frames)
            except:
                raise Exception('Given accessor is not of type int, slice, tuple, list, or array')

        if (isinstance(frames, list) or isinstance(frames, np.ndarray)) and \
                (np.asarray([frames]) < self.frac_coords.shape[0]).all():
            if self.constant_lattice:
                lattice = self.lattice
            else:
                lattice = self.lattice[frames, :]
            return Trajectory(lattice, self.species, self.frac_coords[frames, :], self.time_step,
                              self.site_properties)
        else:
            warnings.warn('Some or all selected frames exceed trajectory length')
        return

    def copy(self):
        return Trajectory(self.lattice, self.species, self.frac_coords, self.time_step, self.site_properties,
                          self.constant_lattice, self.coords_are_displacement, self.base_positions)

    @classmethod
    def from_structures(cls, structures, constant_lattice=True, **kwargs):
        """
        Convenience constructor to obtain trajectory from a list of structures.
        Note: Assumes no atoms removed during simulation

        Args:
            structures (list): list of pymatgen Structure objects.
            constant_lattice (bool): Whether the lattice changes during the simulation, such as in an NPT MD
                simulation. True results in

        Returns:
            (Trajectory)

        """
        frac_coords = [structure.frac_coords for structure in structures]
        if constant_lattice:
            lattice = structures[0].lattice.matrix
        else:
            lattice = [structure.lattice.matrix for structure in structures]
        site_properties = [structure.site_properties for structure in structures]
        return cls(lattice, structures[0].species, frac_coords, site_properties=site_properties,
                   constant_lattice=constant_lattice, **kwargs)

    @classmethod
    def from_file(cls, filename, constant_lattice=True, **kwargs):
        """
        Convenience constructor to obtain trajectory from XDATCAR or vasprun.xml file

        Args:
            filename (str): The filename to read from.
            constant_lattice (bool): Whether the lattice changes during the simulation, such as in an NPT MD
                simulation. True results in

        Returns:
            (Trajectory)
        """
        # TODO: Support other filetypes

        fname = os.path.basename(filename)
        if fnmatch(fname, "*XDATCAR*"):
            structures = Xdatcar(filename).structures
        elif fnmatch(fname, "vasprun*.xml*"):
            structures = Vasprun(filename).structures
        else:
            raise ValueError("Unsupported file")

        return cls.from_structures(structures, constant_lattice=constant_lattice, **kwargs)

    def as_dict(self):
        d = {"@module": self.__class__.__module__,
             "@class": self.__class__.__name__,
             "species": self.species, "time_step": self.time_step,
             "site_properties": self.site_properties,
             "constant_lattice": self.constant_lattice,
             "coords_are_displacement": self.coords_are_displacement,
             "base_positions": self.base_positions}
        d["lattice"] = self.lattice.tolist()
        d["frac_coords"] = self.frac_coords.tolist()

        return d

    @staticmethod
    def _combine_attribute(attr_1, attr_2, len_1, len_2):
        """
        Helper function to combine trajectory properties such as site_properties or lattice
        """
        if isinstance(attr_1, list) or isinstance(attr_2, list):
            attribute = np.concatenate((attr_1, attr_2), axis=0)
            attribute_changes = True
        else:
            if isinstance(attr_1, list) and isinstance(attr_2, list) and np.allclose(attr_1, attr_2):
                attribute = attr_1
                attribute_changes = False
            else:
                attribute = [attr_1.copy()] * len_1 if type(attr_1) != list else attr_1.copy()
                attribute.extend([attr_2.copy()] * len_2 if type(attr_2 != list) else attr_2.copy())
                attribute_changes = True
        return attribute, attribute_changes

max_stars_repo_name: anjlip/pymatgen

max_issues_repo_name: darnoceloc/pymatgen

max_forks_repo_name: darnoceloc/pymatgen


text:
import numpy as np
import random
import pika
import sys, os
sys.path.append(os.path.join(sys.path[0], '../vim/'))
import vim_params as vp

try:
    import cPickle
except BaseException:
    print('cPickle not found, use _pickle instead ...')
    import _pickle as cPickle

class BalanceDataGenerator(object):
    def __init__(self, batch_size, type, te_max_iter=100):
        assert type in ['train', 'test']
        self._batch_size_ = batch_size
        self._type_ = type
        self._te_max_iter_ = te_max_iter

    def generate(self, xs, ys):
        batch_size = self._batch_size_
        x = xs[0]
        y = ys[0]
        (n_samples, n_labs) = y.shape
        n_each = batch_size // n_labs

        index_list = []
        for i1 in xrange(n_labs):
            index_list.append(np.where(y[:, i1] == 1)[0])

        for i1 in xrange(n_labs):
            np.random.shuffle(index_list[i1])

        pointer_list = [0] * n_labs
        len_list = [len(e) for e in index_list]
        iter = 0
        while True:
            if (self._type_) == 'test' and (iter == self._te_max_iter_):
                break
            iter += 1
            batch_x = []
            batch_y = []
            for i1 in xrange(n_labs):
                if pointer_list[i1] >= len_list[i1]:
                    pointer_list[i1] = 0
                    np.random.shuffle(index_list[i1])

                batch_idx = index_list[i1][pointer_list[i1] : min(pointer_list[i1] + n_each, len_list[i1])]
                batch_x.append(x[batch_idx])
                batch_y.append(y[batch_idx])
                pointer_list[i1] += n_each
            batch_x = np.concatenate(batch_x, axis=0)
            batch_y = np.concatenate(batch_y, axis=0)
            yield batch_x, batch_y

class RatioDataGenerator(object):
    def __init__(self, batch_size, type, te_max_iter=100, verbose=1):
        assert type in ['train', 'test']
        self._batch_size_ = batch_size
        self._type_ = type
        self._te_max_iter_ = te_max_iter
        self._verbose_ = verbose

    def _get_lb_list(self, n_samples_list):
        lb_list = []
        for idx in xrange(len(n_samples_list)):
            n_samples = n_samples_list[idx]
            if n_samples < 1000:
                lb_list += [idx]
            elif n_samples < 2000:
                lb_list += [idx] * 2
            elif n_samples < 3000:
                lb_list += [idx] * 3
            elif n_samples < 4000:
                lb_list += [idx] * 4
            else:
                lb_list += [idx] * 5
        return lb_list

    def generate(self, xs, ys):
        batch_size = self._batch_size_
        x = xs[0]
        y = ys[0]
        (n_samples, n_labs) = y.shape

        n_samples_list = np.sum(y, axis=0)
        lb_list = self._get_lb_list(n_samples_list)

        if self._verbose_ == 1:
            print("n_samples_list: %s" % (n_samples_list,))
            print("lb_list: %s" % (lb_list,))
            print("len(lb_list): %d" % len(lb_list))

        index_list = []
        for i1 in xrange(n_labs):
            index_list.append(np.where(y[:, i1] == 1)[0])

        for i1 in xrange(n_labs):
            np.random.shuffle(index_list[i1])

        queue = []
        pointer_list = [0] * n_labs
        len_list = [len(e) for e in index_list]
        iter = 0
        while True:
            if (self._type_) == 'test' and (iter == self._te_max_iter_):
                break
            iter += 1
            batch_x = []
            batch_y = []

            while len(queue) < batch_size:
                random.shuffle(lb_list)
                queue += lb_list

            batch_idx = queue[0 : batch_size]
            queue[0 : batch_size] = []

            n_per_class_list = [batch_idx.count(idx) for idx in xrange(n_labs)]

            for i1 in xrange(n_labs):
                if pointer_list[i1] >= len_list[i1]:
                    pointer_list[i1] = 0
                    np.random.shuffle(index_list[i1])

                per_class_batch_idx = index_list[i1][pointer_list[i1] : min(pointer_list[i1] + n_per_class_list[i1], len_list[i1])]
                batch_x.append(x[per_class_batch_idx])
                batch_y.append(y[per_class_batch_idx])
                pointer_list[i1] += n_per_class_list[i1]
            batch_x = np.concatenate(batch_x, axis=0)
            batch_y = np.concatenate(batch_y, axis=0)
            yield batch_x, batch_y

class QueueDataGenerator(object):
    def __init__(self, batch_size, type, te_max_iter=100, verbose=1):
        assert type in ['train', 'test']
        self._batch_size_ = batch_size    # get batch from outside, so we do not need batch size.
        self._type_ = type
        self._te_max_iter_ = te_max_iter
        self._verbose_ = verbose

        credentials = pika.PlainCredentials('myuser', 'mypassword')
        self._connection_ = pika.BlockingConnection(
            pika.ConnectionParameters('ice-P910',5672,'myvhost', credentials))
        self._channel_ = self._connection_.channel()
        self._channel_.basic_qos(prefetch_count=1)   # 消息未处理完前不要发送信息的消息

    def __del__(self):
        print('__del__ close connection')
        self._connection_.close()

    def generate(self, xs=None, ys=None):
        channel = self._channel_
        batch_size = self._batch_size_
        while True:
            method_frame, header_frame, body = channel.basic_get(queue=vp.logmel_q)      # consumer
            if method_frame:
                # print(method_frame)
                batch_x, batch_y, batch_seq_len = cPickle.loads(body)   # print(batch_x.shape)      # print(batch_y.shape)
                channel.basic_ack(method_frame.delivery_tag)

                if len(batch_x) != batch_size:
                    continue
                yield batch_x, batch_y


text:
#!/usr/bin/env python

from causalprob import CausalProb

import numpy as np
import jax
import jax.numpy as jnp
from jax import vmap, jit, value_and_grad
from jax.tree_util import tree_flatten

import time

def train(model, x: jnp.array, y: jnp.array, o: jnp.array, theta0: jnp.array, n_samples: int = 1000, n_epoch: int = 100,
          batch_size: int = 100, loss_type: str = 'neg-evidence', step_size: float = 1e-4) -> tuple:
    """
    It trains the parameters of the model, given data `x`, `y`, `o`. Starting from an initial solution `theta0`, the
    algorithm uses the Adam optimizer to minimize with respect to the negative evidence (default) or the negative
    averaged log-likelihood.
    Currently, the number of observations for each observed variable must be the same.

    Parameters
    ----------
    x: jnp.array
        Observation of treatment X.
    y: jnp.array
        Observation of outcome Y.
    o: dict
        Observations of observed variables O.
    theta0: dict
        Initial solution of model parameters.
    n_samples: int
        Number of samples to approximate the loss function and its gradient.
    n_epoch: int
        Number of epochs to run the optimization for.
    batch_size: int
        Data batch size at every epoch.
    loss_type: str
        Type of loss function. It can either be:
            - `neg-evidence`: the evidence density function, namely -p(x,y,o);
            - `neg-avg-log-evidence`: the average logarithmic-evidence, namely -E_{u_L}[p(x,y,o|u_L)]
    step_size: float
        Initial step size in Adam optimizer.

    Returns
    -------
    theta, losses: tuple
        theta: jnp.array
            Estimated parameters minimizing the loss function.
        losses: list
            History of loss function evaluations at each iteration of the optimization.
    """
    allowed_losses = ['neg-evidence', 'neg-avg-log-evidence']
    if loss_type not in allowed_losses:
        raise Exception('loss_type={} not recognized. Please select within {}.'.format(loss_type, allowed_losses))

    oy = {**o, 'Y': y}
    all_n_obs = set()
    for v in oy.values():
        if v.ndim > 1:
            all_n_obs.add(v.shape[0])
    if len(all_n_obs) > 1:
        raise Exception('Each observed variable must have the same number of observations.')

    cp = CausalProb(model=model)

    # loss function
    # @jit
    # def loss(_theta: dict):
    #     u_prior = {k: u(n_samples, _theta) for k, u in cp.draw_u.items()}
    #
    #     def _loss(xj, oyj):
    #         u, v = cp.fill(u_prior, {**oyj, 'X': xj}, _theta, cp.draw_u.keys())
    #
    #         def __loss(i: int):
    #             ui = {k: _u[i] if _u.ndim > 1 else _u for k, _u in u.items()}
    #             vi = {k: _v[i] if _v.ndim > 1 else _v for k, _v in v.items()}
    #             llkd = cp.llkd(ui, xj, oyj, _theta, vi)
    #             return jnp.exp(llkd) if loss_type == 'neg-evidence' else llkd
    #         return -jnp.mean(jnp.vectorize(__loss)(range(n_samples)))
    #
    #     if x.ndim > 1:
    #         return jnp.sum(vmap(_loss, (0, {k: 0 for k in oy}))(x, oy))
    #     return _loss(x, oy)

    @jit
    def loss_fn(_theta: dict, x_batch, oy_batch):
        u_prior = {k: u(n_samples, _theta) for k, u in cp.draw_u.items()}

        def _loss_fn(xj, oyj):
            u, v = cp.fill(u_prior, {**oyj, 'X': xj}, _theta, cp.draw_u.keys())
            llkd = cp.llkd(u, xj, oyj, _theta, v)
            return -jnp.mean(llkd)

        if x.ndim > 1:
            return jnp.sum(vmap(_loss_fn, (0, {k: 0 for k in oy}))(x_batch, oy_batch))
        return _loss_fn(x, oy)

    # gradient of loss function
    def dloss_dtheta(_theta: jnp.array):
        u_prior = {k: u(n_samples, _theta) for k, u in cp.draw_u.items()}

        def _dloss_dtheta(xj, oyj):
            u, v = cp.fill(u_prior, {**oyj, 'X': xj}, _theta, cp.draw_u.keys())

            def __dloss_dtheta(i: int):
                ui = {k: _u[i] if _u.ndim > 1 else _u for k, _u in u.items()}

                # prior gradient
                dlpu_dtheta = {key: 0. for key in _theta}
                for rv in ui:
                    if rv != 'X' and rv not in oyj:
                        dlpu_dtheta = {key: dlpu_dtheta[key] + cp.dlpu_dtheta(rv, key, ui, _theta) for key in _theta}

                # likelihood gradient
                dllkd_dtheta = {key: cp.dllkd_dtheta(key, ui, xj, oyj, _theta) for key in _theta}

                # REINFORCE estimator (if prior does not depend on parameters, this is standard gradient direction)
                if loss_type == 'neg-evidence':
                    lkd = jnp.exp(cp.llkd(ui, xj, oyj, _theta))
                    return {key: (dlpu_dtheta[key] + dllkd_dtheta[key]) * lkd for key in _theta}
                else:
                    llkd = cp.llkd(ui, xj, oyj, _theta)
                    return {key: dlpu_dtheta[key] * llkd + dllkd_dtheta[key] for key in _theta}

            return {key: -jnp.mean(g, 0) for key, g in jnp.vectorize(__dloss_dtheta)(range(n_samples)).items()}

        if x.ndim > 1:
            return {key: jnp.sum(g, 0) for key, g in vmap(_dloss_dtheta, (0, {k: 0 for k in oy}))(x, oy).items()}
        return _dloss_dtheta(x, oy)

    # batching
    n_batches = int(np.ceil(x.shape[0] / batch_size))
    rand_perm = np.random.permutation(x.shape[0])
    x_batches = np.array_split(x[rand_perm], n_batches, axis=0)
    oy_batches = {k: np.array_split(v[rand_perm], n_batches, axis=0) for k, v in oy.items()}

    # run optimization
    from jax.experimental import optimizers
    opt_init, opt_update, get_params = optimizers.adam(step_size=step_size)

    losses = []

    @jit
    def update(i, _opt_state, _x_batch, _oy_batch):
        params = get_params(_opt_state)

        # g = dloss_dtheta(params)
        loss, g = value_and_grad(lambda _theta: loss_fn(_theta, _x_batch, _oy_batch))(params)
        return opt_update(i, g, _opt_state), loss

    opt_state = opt_init(theta0)
    for epoch in range(n_epoch):
        epoch_losses = []

        start_time = time.time()
        for i in range(n_batches):
            x_batch = x_batches[i]
            oy_batch = {k: _oy_batches[i] for k, _oy_batches in oy_batches.items()}

            opt_state, epoch_loss = update(i, opt_state, x_batch, oy_batch)
            epoch_losses.append(epoch_loss)
        epoch_time = time.time() - start_time

        losses.append(np.mean(epoch_losses))
        print("Epoch: {} | time: {:0.2f}s | loss: {:0.8f}".format(epoch + 1, epoch_time, losses[-1]))

    return get_params(opt_state), losses

def train(model, x_train: jnp.array, y_train: jnp.array, o_train: dict, x_pred: jnp.array, o_pred: dict,
          theta0: jnp.array, lam: float = 0.1, reg_loss: str = 'l1-bias', n_samples: int = 1000,
          n_epoch: int = 100, batch_size: int = 100, step_size: float = 1e-4) -> tuple:
    """
    It trains the parameters of the model, given data `x`, `y`, `o`. Starting from an initial solution `theta0`, the
    algorithm uses the Adam optimizer to minimize with respect to the negative evidence (default) or the negative
    averaged log-likelihood.
    Currently, the number of observations for each observed variable must be the same.

    Parameters
    ----------
    x_train: jnp.array
        Observation of treatment X for training.
    y_train: jnp.array
        Observation of outcome Y for training.
    o_train: dict
        Observations of observed variables O for training.
    x_pred: jnp.array
        Observations of observed variables X for prediction of Y and causal effect and bias estimation.
    o_pred: dict
        Observations of observed variables O for prediction of Y and causal effect and bias estimation.
    theta0: dict
        Initial solution of model parameters.
    lam: float
        It controls the regularization amount.
    reg_loss: str
        Type of regularization function. Allowed: 'l1-bias' (default), 'l2-bias', 'l1' and 'l2'.
    n_samples: int
        Number of samples to approximate the loss function and its gradient.
    n_epoch: int
        Number of epochs to run the optimization for.
    batch_size: int
        Data batch size at every epoch.
    step_size: float
        Initial step size in Adam optimizer.

    Returns
    -------
    theta, losses: tuple
        theta: jnp.array
            Estimated parameters minimizing the loss function.
        losses: list
            History of loss function evaluations.
        training_losses: list
            History of training loss function evaluations.
        reg_losses: list
            History of reg_loss function evaluations.
    """
    allowed_reg_losses = ['l1-bias', 'l2-bias', 'l1', 'l2']
    if reg_loss not in allowed_reg_losses:
        raise Exception('reg_loss={} not recognized. Please select within {}.'.format(reg_loss, allowed_reg_losses))

    all_n_obs = {1 if y_train.ndim <= 1 else y_train.shape[0]}
    for v in o_train.values():
        if v.ndim > 1:
            all_n_obs.add(v.shape[0])
    if len(all_n_obs) > 1:
        raise Exception('Each observed variable must have the same number of observations.')

    cp = CausalProb(model=model)

    @jit
    def training_loss_fn(theta: dict, x_train_batch: jnp.array, y_train_batch: jnp.array, o_train_batch: dict):
        u_prior = {k: u(n_samples, theta) for k, u in cp.draw_u.items()}

        def _training_loss_fn(xj, yj, oj):
            oyj = {**oj, 'Y': yj}
            u, v = cp.fill(u_prior, {**oyj, 'X': xj}, theta, cp.draw_u.keys())
            return -jnp.log(jnp.mean(jnp.exp(cp.llkd(u, xj, oyj, theta, v))))

        if x_train.ndim > 1:
            return jnp.mean(vmap(_training_loss_fn, (0, 0, {k: 0 for k in o_train_batch}))(x_train_batch, y_train_batch, o_train_batch))
        return _training_loss_fn(x_train, y_train, o_train)

    @jit
    def reg_loss_fn(theta: dict, x_pred: jnp.array, o_pred: dict):
        if reg_loss == 'l1-bias':
            return jnp.mean(jnp.abs(cp.causal_bias(x_pred, o_pred, theta, n_samples=n_samples)))
        if reg_loss == 'l2-bias':
            return jnp.mean(cp.causal_bias(x_pred, o_pred, theta, n_samples=n_samples) ** 2)
        if reg_loss in ['l1', 'l2']:
            flat_theta = jnp.concatenate([_theta.flatten() for _theta in tree_flatten(theta)[0]])
            if reg_loss == 'l1':
                return jnp.mean(jnp.abs(flat_theta))
            if reg_loss == 'l2':
                return jnp.mean(flat_theta ** 2)

    # batching
    n_batches = int(np.ceil(x_train.shape[0] / batch_size))
    rand_perm = np.random.permutation(x_train.shape[0])
    x_batches = np.array_split(x_train[rand_perm], n_batches, axis=0)
    y_batches = np.array_split(y_train[rand_perm], n_batches, axis=0)
    o_batches = {k: np.array_split(v[rand_perm], n_batches, axis=0) for k, v in o_train.items()}

    # run optimization
    from jax.experimental import optimizers
    opt_init, opt_update, get_params = optimizers.adam(step_size=step_size)

    losses = []
    training_losses = []
    reg_losses = []
    strf = "Epoch: {} | time: {:0.2f}s | loss: {:e} | training loss: {:e} | {:s} loss: {:e}"

    @jit
    def update(i, opt_state, x_batch, y_batch, o_batch, x_pred, o_pred):
        params = get_params(opt_state)

        training_loss, training_g = value_and_grad(lambda theta: training_loss_fn(theta, x_batch, y_batch, o_batch))(params)
        if lam > 0:
            reg_loss, reg_loss_g = value_and_grad(lambda theta: reg_loss_fn(theta, x_pred, o_pred))(params)
            g = jax.tree_multimap(lambda x,y: x+lam*y, training_g, reg_loss_g)
        else:
            reg_loss = reg_loss_fn(params, x_pred, o_pred)
            g = training_g
        return opt_update(i, g, opt_state), training_loss, reg_loss

    opt_state = opt_init(theta0)
    for epoch in range(n_epoch):
        epoch_training_losses = []
        epoch_regs = []

        start_time = time.time()
        for i in range(n_batches):
            x_batch = x_batches[i]
            y_batch = y_batches[i]
            o_batch = {k: _o_batches[i] for k, _o_batches in o_batches.items()}

            opt_state, epoch_training_loss, epoch_reg = update(i, opt_state, x_batch, y_batch, o_batch, x_pred, o_pred)
            epoch_training_losses.append(epoch_training_loss)
            epoch_regs.append(epoch_reg)
        epoch_time = time.time() - start_time

        training_losses.append(np.mean(epoch_training_losses))
        reg_losses.append(np.mean(epoch_regs))
        losses.append(training_losses[-1] + lam * reg_losses[-1])
        print(strf.format(epoch + 1, epoch_time, losses[-1], training_losses[-1], reg_loss, reg_losses[-1]))

    return get_params(opt_state), losses, training_losses, reg_losses

max_stars_repo_name: jimimvp/CausalProb

max_issues_repo_name: jimimvp/CausalProb

max_forks_repo_name: jimimvp/CausalProb

reason: import numpy,import jax,from jax


text:
import random
import numpy as np
import torch
import os
import shutil

def init_seeds(seed=0):
    random.seed(seed)
    np.random.seed(seed)

    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)

    os.environ['PYTHONHASHSEED'] = str(seed)

def init_dirs(path):
    if os.path.exists(path):
        shutil.rmtree(path)

    os.mkdir(path)
    os.mkdir(os.path.join(path, 'log'))
    os.mkdir(os.path.join(path, 'run'))
    os.mkdir(os.path.join(path, 'weights'))
    os.mkdir(os.path.join(path, 'images'))

max_stars_repo_name: Peterisfar/ObjectSegment

max_issues_repo_name: Peterisfar/ObjectSegment

max_forks_repo_name: Peterisfar/ObjectSegment


text:
import cv2
import numpy as np

lowerBlue = np.array([100, 10, 10])
upperBlue = np.array([250, 200,200])

lowerBlack = np.array([0, 0, 0])
upperBlack = np.array([100, 100, 100])

compressionRatio = 200

def segmentary(image):
    blueLineMask = cv2.inRange(image, lowerBlue, upperBlue)
    image = cv2.blur(image,(5,5))
    blackLineMask = cv2.inRange(image, lowerBlack, upperBlack)

    return blueLineMask, blackLineMask

def readImageAndResize(filePath):
    image = cv2.imread(filePath)
    image = cv2.resize(image, (compressionRatio, compressionRatio))

    return image

max_stars_repo_name: daviromao/parkinson-classifier

max_issues_repo_name: daviromao/parkinson-classifier

max_forks_repo_name: daviromao/parkinson-classifier