"""
Analyze the results of the repetated experiments:
    - read simulations results from csv files,
    - calculate kpis,
    - plot relevant figures,
    - export results to LaTeX tables, images, and csv files.
"""
# Import packages
import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
# Import local functions
from src.analysis import write_table_to_tex
from src.analysis import get_percentage_features_changed
from src.analysis import get_computation_times_table
from src.analysis import get_small_computation_times_table
from src.analysis import get_forest_depth_computation_times_table
from src.analysis import get_distance_table
from src.analysis import get_x_as_list
from src.analysis import percentage_relevant_feature_change
from src.analysis import get_symmetric_context_table

# Set global plotting style
plt.style.use('seaborn-v0_8-paper')
plt.rcParams['lines.linewidth'] = 4
plt.rcParams["figure.figsize"] = (18, 6)
plt.rcParams.update({"axes.grid": True,
                     "grid.color": "grey"})
plt.rc('text', usetex=True)
plt.rc('font', family='serif')
plt.rcParams["font.family"] = "Times New Roman"
plt.rcParams.update({'font.size': 12})
sns.set_palette('colorblind')
sns.set_context('paper')
SUBPLT_SIZE = (8, 2)

# ------ Read simulations results ------
# Import result files and concatenate them into a single Dataframe
experimentList = ['newsvendor', 'shipment', 'path', 'cvar-path']
nList = [50, 100, 200]
dirname = os.path.dirname(__file__)
outputPath = os.path.join(dirname, "output",)
filepaths = [outputPath+"/"
             + f for f in os.listdir(outputPath) if f.endswith('.csv')]

# -------- Computation times --------
# -> Table 1 and 5 in paper
# Read and concatenate results into one single dataframe
resultsFiles = [p for p in filepaths if 'synth_results' in p]
resultDf = pd.concat(map(pd.read_csv, resultsFiles))
# -- Long table used in appendix --
# Create and formate table
compTimesTable = get_computation_times_table(
    resultDf, experimentList, nList)
# Write and export table to tex file
write_table_to_tex(compTimesTable, 'computation_times', outputPath)
# -- Short table used in paper body --
smallCompTimesTable = get_small_computation_times_table(
    resultDf, experimentList, nList)
# Write and export table to tex file
write_table_to_tex(smallCompTimesTable, 'small_computation_times', outputPath)

# -------- Percentage of features changed --------
# Read list of explanations
x_init = get_x_as_list('x_init', resultDf)
x_rel = get_x_as_list('x_rel', resultDf)
x_abs = get_x_as_list('x_abs', resultDf)
# Add column to dataframe with distance
perc_feat_change_rel = []
perc_feat_change_abs = []
for i in range(len(resultDf.index)):
    if len(x_rel[i]) == 0:
        perc_feat_change_rel.append(-1)
    else:
        perc_feat_change_rel.append(
            (100/len(x_init[i])) * np.linalg.norm(x_init[i] - x_rel[i],
                                                  ord=0))
    if len(x_abs[i]) == 0:
        perc_feat_change_abs.append(-1)
    else:
        perc_feat_change_abs.append(
            (100/len(x_init[i])) * np.linalg.norm(x_init[i] - x_abs[i],
                                                  ord=0))
# Store in result df
resultDf['perc_feat_change_rel'] = perc_feat_change_rel
resultDf['perc_feat_change_abs'] = perc_feat_change_abs
featChangeTable = get_percentage_features_changed(
    resultDf, experimentList, nList)
write_table_to_tex(featChangeTable, 'percentage_feature_changed', outputPath)

# -------- Symmetric contexts --------
# -> Table 4 in paper
symmTable = get_symmetric_context_table(resultDf, experimentList, nList)
write_table_to_tex(symmTable, 'percentage_symmetric', outputPath)

# -------- Counterfactual distance --------
# -> Table 7 in paper
# Read list of explanations
x_init = get_x_as_list('x_init', resultDf)
x_rel = get_x_as_list('x_rel', resultDf)
x_abs = get_x_as_list('x_abs', resultDf)
# Add column to dataframe with distance
dist_rel = []
dist_abs = []
for i in range(len(resultDf.index)):
    if len(x_rel[i]) == 0:
        dist_rel.append(-1)
    else:
        dist_rel.append((1/len(x_init[i]))
                        * np.linalg.norm(x_init[i] - x_rel[i], ord=1))
    if len(x_abs[i]) == 0:
        dist_abs.append(-1)
    else:
        dist_abs.append((1/len(x_init[i]))
                        * np.linalg.norm(x_init[i] - x_abs[i], ord=1))
# Store in result df
resultDf['distance_relative'] = dist_rel
resultDf['distance_absolute'] = dist_abs
# Create and formate table
distanceTable = get_distance_table(resultDf, experimentList, nList)
# Write and export table to tex file
write_table_to_tex(distanceTable, 'explanation_distance', outputPath)

# -- Direction of explanations --
# -> Figure 6
# Measure the correlation between the explanations
# and the alternative contexts
directionDf = resultDf[(resultDf['nbTrees'] == 100)
                       + (resultDf['nbNeighbors'] == 10)]
directionDf = directionDf[directionDf['experiment'] == 'newsvendor']
corr_rel = []
corr_abs = []
n_rows = len(directionDf.index)
x_init = get_x_as_list('x_init', directionDf)
x_rel = get_x_as_list('x_rel', directionDf)
x_abs = get_x_as_list('x_abs', directionDf)
x_alt = get_x_as_list('x_alt', directionDf)
# Measure counterfactual change
for i in range(n_rows):
    # Calculate direction of explanations
    true_direction = x_alt[i] - x_init[i]
    rel_direction = x_rel[i] - x_init[i]
    # Normalize directions
    true_direction = true_direction / np.linalg.norm(true_direction)
    rel_direction = rel_direction / np.linalg.norm(rel_direction)
    # Calculate dot product and append to lists
    corr_rel.append(np.dot(true_direction, rel_direction))
    if len(x_abs[i]) == 0:
        corr_abs.append(None)
    else:
        abs_direction = x_abs[i] - x_init[i]
        abs_direction = abs_direction / np.linalg.norm(abs_direction)
        corr_abs.append(np.dot(true_direction, abs_direction))
directionDf['Relative explanations'] = corr_rel
directionDf['Absolute explanations'] = corr_abs
directionDf = directionDf.rename(
    columns={'corr_rel': 'Relative explanations'})
directionDf = directionDf.rename(
    columns={'corr_abs': 'Absolute explanations'})
# Rename fields for plotting
directionDf.loc[directionDf.experiment
                == 'newsvendor', 'experiment'] = 'Newsvendor'
directionDf.loc[directionDf.method == 'rf', 'method'] = 'RF'
directionDf.loc[directionDf.method == 'knn', 'method'] = 'k-NN'
explanationList = ['Relative explanations', 'Absolute explanations']

PRT_SIZE = (4.5, 2.5)
yMins = [-1, 0.0]
fig, axs = plt.subplots(nrows=2, ncols=1, sharex=True, figsize=PRT_SIZE)
for i in range(len(explanationList)):
    sns.boxplot(ax=axs[i], x="nbSamples", y=explanationList[i],
                hue="method", data=directionDf, showmeans=True,
                linewidth=2,
                meanprops={"marker": "o", "markerfacecolor": "white",
                           "markeredgecolor": "black",
                           "markersize": "7"})
    axs[i].set_title(explanationList[i])
    handles, labels = axs[i].get_legend_handles_labels()
    axs[i].legend(handles=handles[0:], labels=labels[0:], loc='lower right')
    axs[i].set_ylim(ymin=yMins[i], ymax=1.0)
    axs[i].set_ylabel(r"Correlation")
    axs[i].grid(visible=True, which='both', color='gray', linewidth=0.1)
    axs[i].set_axisbelow(True)
axs[0].get_legend().remove()
axs[0].set_xlabel(None)
axs[1].set_xlabel(r"Sample size $n$")
fig.tight_layout()
plt.savefig(outputPath+'/figs/newsvendor_corr.pdf', bbox_inches='tight')

# ----- Problem complexity -----
# -> Figure 3 and 4
# Read and concatenate results into one single dataframe
complexFiles = [p for p in filepaths if 'complex' in p]
complexDf = pd.concat(map(pd.read_csv, complexFiles))
# Replace 0.0 by NaN
complexDf[['solvetime_relative', 'solvetime_absolute']] = complexDf[[
    'solvetime_relative', 'solvetime_absolute']].replace(0, np.nan)
# Transform runtime to log seconds
complexDf['solvetime_relative'] = np.log10(complexDf['solvetime_relative'])
complexDf['solvetime_absolute'] = np.log10(complexDf['solvetime_absolute'])
# Rename columns and fields
complexDf = complexDf.rename(columns={'solvetime_relative': 'Relative time'})
complexDf = complexDf.rename(columns={'solvetime_absolute': 'Absolute time'})
complexDf.loc[complexDf.experiment
              == 'newsvendor', 'experiment'] = 'Newsvendor'
complexDf.loc[complexDf.experiment == 'path', 'experiment'] = 'Shortest path'
complexDf.loc[complexDf.method == 'rf', 'method'] = 'RF'
complexDf.loc[complexDf.method == 'knn', 'method'] = 'k-NN'

PRT_SIZE2 = (4.5, 3)
explanationList = ['Relative', 'Absolute']
problemList = ['Newsvendor', 'Shortest path']
for j in range(len(problemList)):
    fig, axs = plt.subplots(nrows=2, ncols=1, sharex=True, figsize=PRT_SIZE2)
    tempDf = complexDf[complexDf['experiment'] == problemList[j]]
    for i in range(len(explanationList)):
        sns.boxplot(ax=axs[i], x="problemSens", y=explanationList[i]+" time",
                    hue="method", data=tempDf, showmeans=True,
                    linewidth=2,
                    meanprops={"marker": "o", "markerfacecolor": "white",
                               "markeredgecolor": "black",
                               "markersize": "7"})
        axs[i].set_title(explanationList[i]+' explanations')
        handles, labels = axs[i].get_legend_handles_labels()
        axs[i].legend(handles=handles[0:], labels=labels[0:],
                      ncol=2, loc='upper left')
        axs[i].set_ylabel(r"Time [in $s$]")
        axs[i].set_yticks(np.arange(-1, 4))
        axs[i].set_yticklabels(10.0**np.arange(-1, 4))
        minor_yticks = np.log10(
            np.concatenate((np.arange(1, 10) * 1,
                            np.arange(1, 10) * 0.1,
                            np.arange(1, 10) * 10,
                            np.arange(1, 10) * 100)).astype(float))
        axs[i].set_yticks(minor_yticks, minor=True)
        axs[i].grid(visible=True, which='major', color='gray', linewidth=0.1)
        axs[i].set_axisbelow(True)
        if j == 0:
            axs[i].set_ylim(ymax=1.5)
        else:
            axs[i].set_ylim(ymax=3.2)
    axs[1].get_legend().remove()
    axs[0].set_xlabel(None)
    axs[1].set_xlabel(r"Problem complexity")
    fig.tight_layout()
    plt.savefig(outputPath+'/figs/'+problemList[j]
                + '_problem_cplx.pdf', bbox_inches='tight')

# ----- Spurious features -----
# -> Figure 9 and 10
spuriousFiles = [p for p in filepaths if 'spurious' in p]
spuriousDf = pd.concat((pd.read_csv(f) for f in spuriousFiles),
                       ignore_index=True)
nList = [50, 100, 200, 400, 600, 800]
NB_ROWS = 100 * len(nList) * 1  # * forest size variations
assert len(spuriousDf.index) == NB_ROWS
# Read init points and explanations
x_init = get_x_as_list('x_init', spuriousDf)
x_rel = get_x_as_list('x_rel', spuriousDf)
x_abs = get_x_as_list('x_abs', spuriousDf)
assert len(x_init) == NB_ROWS
assert len(x_rel) == NB_ROWS
assert len(x_abs) == NB_ROWS
# Calculate percentage of relevant feature change and add to df
spuriousDf = percentage_relevant_feature_change(
    spuriousDf, x_init, x_rel, x_abs)
# Rename columns and fields
spuriousDf = spuriousDf.rename(
    columns={'relative_percentage': 'Relative explanations'})
spuriousDf = spuriousDf.rename(
    columns={'absolute_percentage': 'Absolute explanations'})

explanationList = ['Relative explanations', 'Absolute explanations']
fig, axs = plt.subplots(nrows=1, ncols=2, sharey=True, figsize=SUBPLT_SIZE)
for i in range(len(explanationList)):
    sns.violinplot(ax=axs[i], x="nbSamples", y=explanationList[i],
                   hue='method',
                   cut=0, data=spuriousDf, showmeans=True,
                   linewidth=2)
    axs[i].set_title(explanationList[i])
    axs[i].set_ylabel(None)
    fig.supylabel(r"Relevant change [in\%]")
    axs[i].set_xlabel(r"Sample size $n$")
    axs[i].grid(visible=True, which='both', color='gray', linewidth=0.1)
    axs[i].set_axisbelow(True)
    axs[i].get_legend().remove()
    axs[i].set_ylim(ymin=0.0, ymax=100)
fig.tight_layout()
plt.savefig(outputPath+'/figs/spurious_perc.pdf', bbox_inches='tight')

# -- Direction of explanations --
corr_rel = []
corr_abs = []
n_rows = len(spuriousDf.index)
x_alt = get_x_as_list('x_alt', spuriousDf)
# Measure counterfactual change
for i in range(n_rows):
    # Calculate direction of explanations
    true_direction = x_alt[i] - x_init[i]
    rel_direction = x_rel[i] - x_init[i]
    # Normalize directions
    true_direction = true_direction / np.linalg.norm(true_direction)
    rel_direction = rel_direction / np.linalg.norm(rel_direction)
    # Calculate dot product and append to lists
    corr_rel.append(np.dot(true_direction, rel_direction))
    if len(x_abs[i]) == 0:
        corr_abs.append(None)
    else:
        abs_direction = x_abs[i] - x_init[i]
        abs_direction = abs_direction / np.linalg.norm(abs_direction)
        corr_abs.append(np.dot(true_direction, abs_direction))
spuriousDf['Relative correlation'] = corr_rel
spuriousDf['Absolute correlation'] = corr_abs

explanationList = ['Relative correlation', 'Absolute correlation']
fig, axs = plt.subplots(nrows=1, ncols=2, sharey=True, figsize=SUBPLT_SIZE)
for i in range(len(explanationList)):
    sns.violinplot(ax=axs[i], x="nbSamples", y=explanationList[i],
                   hue='method',
                   cut=0, data=spuriousDf, showmeans=True,
                   linewidth=2)
    axs[i].set_title(explanationList[i])
    axs[i].set_ylabel(None)
    fig.supylabel(r"Correlation")
    axs[i].set_xlabel(r"Sample size $n$")
    axs[i].grid(visible=True, which='both', color='gray', linewidth=0.1)
    axs[i].set_axisbelow(True)
    axs[i].set_ylim(ymin=-1.0, ymax=1.0)
    axs[i].get_legend().remove()
fig.tight_layout()
plt.savefig(outputPath+'/figs/spurious_corr.pdf', bbox_inches='tight')

# ----- Sensitivity to forest depth -----
# -> Table 6
nList = [50, 100, 200]
# Read and concatenate results into one single dataframe
depthPath = outputPath+"/forest_depth/"
filepaths = [depthPath+"/"
             + f for f in os.listdir(depthPath) if f.endswith('.csv')]
resultsFiles = [p for p in filepaths if 'synth_results' in p]
resultDf = pd.concat(map(pd.read_csv, resultsFiles))
# Create and formate table
compTimesTable = get_forest_depth_computation_times_table(
    resultDf, experimentList, nList)
# Write and export table to tex file
write_table_to_tex(compTimesTable, 'depth_comp_times', outputPath)

# ----- Sensitivity to number of features and sample size -----
#           EXPECTED COSTS SHORTEST PATH
# -> Export csv data used to create Figure 2 in tikz
# Read and concatenate results into one single dataframe
featPath = outputPath+"/path_feat/"
filepaths = [featPath + f for f in os.listdir(featPath) if f.endswith('.csv')]
featuresFiles = [p for p in filepaths if 'cvar' not in p]
featuresDf = pd.concat((pd.read_csv(f) for f in featuresFiles),
                       ignore_index=True)
nList = [100, 500, 1000, 5000]
d_xList = np.sort(featuresDf['d_x'].unique())
# Get average computations times and std
avgRelTime = np.zeros((len(nList), len(d_xList)))
avgAbsTime = np.zeros((len(nList), len(d_xList)))
stdRelTime = np.zeros((len(nList), len(d_xList)))
stdAbsTime = np.zeros((len(nList), len(d_xList)))
for i in range(len(nList)):
    tempDf = featuresDf[featuresDf['nbSamples'] == nList[i]]
    for j in range(len(d_xList)):
        tempDf2 = tempDf[tempDf['d_x'] == d_xList[j]]
        relTimes = np.array(tempDf2['solvetime_relative'])
        absTimes = np.array(tempDf2['solvetime_absolute'])
        avgRelTime[i, j] = np.mean(relTimes[relTimes != 0.])
        avgAbsTime[i, j] = np.mean(absTimes[absTimes != 0.])
        stdRelTime[i, j] = np.std(relTimes[relTimes != 0.])
        stdAbsTime[i, j] = np.std(absTimes[absTimes != 0.])
# Export to csv file
dfToExport = pd.DataFrame({'d_x': d_xList})
for i, n in enumerate(nList):
    dfToExport[str(n)+'_avgRelTime'] = avgRelTime[i, :]
    dfToExport[str(n)+'_avgRelTime_plus_std'] = (
        avgRelTime[i, :]+stdRelTime[i, :])
    dfToExport[str(n)+'_avgRelTime_minus_std'] = (
        avgRelTime[i, :]-stdRelTime[i, :])
    dfToExport[str(n)+'_avgBsTime'] = avgAbsTime[i, :]
    dfToExport[str(n)+'_avgBsTime_plus_std'] = (
        avgAbsTime[i, :]+stdAbsTime[i, :])
    dfToExport[str(n)+'_avgBsTime_minus_std'] = (
        avgAbsTime[i, :]-stdAbsTime[i, :])
folderName = os.path.join(outputPath, "csv")
os.makedirs(folderName, exist_ok=True)
dfToExport.to_csv(folderName+'/path_feat_sens.csv', index=False)

# Percentage of features changed by explanations for fixed n and varying d_x
percFeatChangRel = np.zeros(len(d_xList))
percFeatChangAbs = np.zeros(len(d_xList))
nbFeatChangRel = np.zeros(len(d_xList))
nbFeatChangAbs = np.zeros(len(d_xList))
tempDf = featuresDf[featuresDf['nbSamples'] == 500]
for j in range(len(d_xList)):
    tempDf2 = tempDf[tempDf['d_x'] == d_xList[j]]
    x_init = get_x_as_list('x_init', tempDf2)
    x_rel = get_x_as_list('x_rel', tempDf2)
    x_abs = get_x_as_list('x_abs', tempDf2)
    rel_nb_change = []
    abs_nb_change = []
    rel_change = []
    abs_change = []
    for i in range(len(x_init)):
        if len(x_rel[i]) > 0:
            change = np.linalg.norm(x_init[i] - x_rel[i], ord=0)
            rel_nb_change.append(change)
            rel_change.append((100/len(x_init[i])) * change)
        if len(x_abs[i]) > 0:
            change = np.linalg.norm(x_init[i] - x_abs[i], ord=0)
            abs_nb_change.append(change)
            abs_change.append((100/len(x_init[i])) * change)
    rel_change = np.array(rel_change)
    abs_change = np.array(abs_change)
    rel_change = rel_change[rel_change != 0.0]
    abs_change = abs_change[abs_change != 0.0]
    rel_nb_change = np.array(rel_nb_change)
    abs_nb_change = np.array(abs_nb_change)
    rel_nb_change = rel_nb_change[rel_nb_change != 0.0]
    abs_nb_change = abs_nb_change[abs_nb_change != 0.0]
    percFeatChangRel[j] = np.mean(rel_change)
    percFeatChangAbs[j] = np.mean(abs_change)
    nbFeatChangRel[j] = np.mean(rel_nb_change)
    nbFeatChangAbs[j] = np.mean(abs_nb_change)
# Export to csv file
dfToExport = pd.DataFrame({'d_x': d_xList})
dfToExport['nbFeatChangRel'] = nbFeatChangRel
dfToExport['nbFeatChangAbs'] = nbFeatChangAbs
dfToExport['percFeatChangRel'] = percFeatChangRel
dfToExport['percFeatChangAbs'] = percFeatChangAbs
folderName = os.path.join(outputPath, "csv")
os.makedirs(folderName, exist_ok=True)
dfToExport.to_csv(folderName+'/perc_feat_change.csv', index=False)

# ----- Sensitivity to number of features and sample size -----
#           CVaR SHORTEST PATH
# -> Export csv data used to create Figure 2 in tikz
# Read and concatenate results into one single dataframe
featPath = outputPath+"/path_feat/"
filepaths = [featPath + f for f in os.listdir(featPath) if f.endswith('.csv')]
featuresFiles = [p for p in filepaths if 'cvar' in p]
featuresDf = pd.concat((pd.read_csv(f) for f in featuresFiles),
                       ignore_index=True)
nList = [100, 500, 1000, 5000]
d_xList = np.sort(featuresDf['d_x'].unique())
# Get average computations times and std
avgRelTime = np.zeros((len(nList), len(d_xList)))
avgAbsTime = np.zeros((len(nList), len(d_xList)))
stdRelTime = np.zeros((len(nList), len(d_xList)))
stdAbsTime = np.zeros((len(nList), len(d_xList)))
for i in range(len(nList)):
    tempDf = featuresDf[featuresDf['nbSamples'] == nList[i]]
    for j in range(len(d_xList)):
        tempDf2 = tempDf[tempDf['d_x'] == d_xList[j]]
        relTimes = np.array(tempDf2['solvetime_relative'])
        absTimes = np.array(tempDf2['solvetime_absolute'])
        avgRelTime[i, j] = np.mean(relTimes[relTimes != 0.])
        avgAbsTime[i, j] = np.mean(absTimes[absTimes != 0.])
        stdRelTime[i, j] = np.std(relTimes[relTimes != 0.])
        stdAbsTime[i, j] = np.std(absTimes[absTimes != 0.])
# Export to csv file
dfToExport = pd.DataFrame({'d_x': d_xList})
for i, n in enumerate(nList):
    dfToExport[str(n)+'_avgRelTime'] = avgRelTime[i, :]
    dfToExport[str(n)+'_avgRelTime_plus_std'] = (
        avgRelTime[i, :]+stdRelTime[i, :])
    dfToExport[str(n)+'_avgRelTime_minus_std'] = (
        avgRelTime[i, :]-stdRelTime[i, :])
    dfToExport[str(n)+'_avgBsTime'] = avgAbsTime[i, :]
    dfToExport[str(n)+'_avgBsTime_plus_std'] = (
        avgAbsTime[i, :]+stdAbsTime[i, :])
    dfToExport[str(n)+'_avgBsTime_minus_std'] = (
        avgAbsTime[i, :]-stdAbsTime[i, :])
folderName = os.path.join(outputPath, "csv")
os.makedirs(folderName, exist_ok=True)
dfToExport.to_csv(folderName+'/cvar_path_feat_sens.csv', index=False)

# ----- Value of dual reformulation for linear decision problems -----
# -> Figure 8
# Read and concatenate results into one single dataframe
filepaths = [outputPath+"/"
             + f for f in os.listdir(outputPath) if f.endswith('.csv')]
resultsFiles = [p for p in filepaths if 'dual' in p]
resultDf = pd.concat(map(pd.read_csv, resultsFiles))
# Rename methods
methodList = ['Random forest prescriptors', 'Nearest-neighbor prescriptors']
resultDf.loc[resultDf.method == 'rf', 'method'] = methodList[0]
resultDf.loc[resultDf.method == 'knn', 'method'] = methodList[1]
# Remove symmetric contexts
resultDf = resultDf.drop(resultDf[resultDf['solvetime_absolute'] <= 0.].index)

# Extract dataframes with and without dual formulation
dualDf = resultDf[resultDf.useDual]
iterativeDf = resultDf[resultDf.useDual == False]
# Calculate relative times
dualTimes = np.array(dualDf.solvetime_absolute)
iterativeTimes = np.array(iterativeDf.solvetime_absolute)
iterativeDf['relativeTimes'] = np.log10(iterativeTimes / dualTimes)

fig, axs = plt.subplots(nrows=1, ncols=2, figsize=SUBPLT_SIZE)
for i in range(len(methodList)):
    tempDf = iterativeDf[iterativeDf['method'] == methodList[i]]
    sns.boxplot(ax=axs[i], x="nbSamples", y='relativeTimes',
                hue='useDual', data=tempDf, showmeans=True,
                linewidth=2,
                meanprops={"marker": "o", "markerfacecolor": "white",
                           "markeredgecolor": "black",
                           "markersize": "7"})
    axs[i].set_title(methodList[i])
    axs[i].set_ylabel(None)
    fig.supylabel(r"Time ratio")
    axs[i].set_xlabel(r"Sample size $n$")
    axs[i].grid(visible=True, which='both', color='gray', linewidth=0.1)
    axs[i].set_axisbelow(True)
    axs[i].get_legend().remove()
    # Set y-axis to log scale
    axs[i].set_yticks(np.arange(-1, 2))
    axs[i].set_yticklabels(10.0**np.arange(-1, 2))
    minor_yticks = np.log10(
        np.concatenate((np.arange(1, 10) * 1,
                        np.arange(1, 10) * 0.1)).astype(np.float))
fig.tight_layout()
plt.savefig(outputPath+'/figs/dual_vs_iterative.pdf', bbox_inches='tight')


# -*- coding: utf-8 -*-

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
# from __future__ import unicode_literals

import MeCab

from typing import Any, List, Text

from rasa_nlu.components import Component
from rasa_nlu.config import RasaNLUModelConfig
from rasa_nlu.tokenizers import Tokenizer, Token
from rasa_nlu.training_data import Message
from rasa_nlu.training_data import TrainingData

class WordDividor():
    INDEX_CATEGORY = 0
    INDEX_ROOT_FORM = 6
    TARGET_CATEGORIES = ["名詞", "動詞",  "形容詞", "副詞", "連体詞", "感動詞"]

    def __init__(self, dictionary="mecabrc"):
        self.dictionary = dictionary
        self.tagger = MeCab.Tagger(self.dictionary)

    def extract_words(self, text):
        self.TARGET_CATEGORIES = ["名詞", "動詞",  "形容詞", "副詞", "連体詞", "感動詞"]
        if not text:
            return []

        words = []
        if type(text) != str:
            text = u''.join((text)).encode('utf-8')
        node = self.tagger.parseToNode(str(text))
        while node:
            features = node.feature.split(',')
            #print features
            if features[self.INDEX_CATEGORY] in self.TARGET_CATEGORIES:
                if features[self.INDEX_ROOT_FORM] == "*":
                    words.append(node.surface)
                else:
                    words.append(features[self.INDEX_ROOT_FORM])

            node = node.next
        for eachword in words:
            print('Word ==> {}'.format(eachword))
        return words

class My_Mecab(Tokenizer, Component):
    name = "My_Mecab"

    provides = ["tokens"]

    # INDEX_CATEGORY = 0
    # INDEX_ROOT_FORM = 6
    # TARGET_CATEGORIES = ["名詞", "動詞",  "形容詞", "副詞", "連体詞", "感動詞"]

    # dictionary = "mecabrc"

    # def __init__(self, dictionary="mecabrc"):

    def train(self, training_data, config, **kwargs):
        # type: (TrainingData, RasaNLUModelConfig, **Any) -> None

        for example in training_data.training_examples:
            example.set("tokens", self.tokenize(example.text))

    def process(self, message, **kwargs):
        # type: (Message, **Any) -> None

        message.set("tokens", self.tokenize(message.text))

    def tokenize(self, text):
        indivi = WordDividor()
        data = indivi.extract_words(text)
        print(data)
        return data

        # self.dictionary = "mecabrc"
        # self.tagger = MeCab.Tagger(self.dictionary)
        #
        # if not text:
        #     return []
        #
        # words = []
        # if type(text) != str:
        #     text = u''.join((text)).encode('utf-8')
        # node = self.tagger.parseToNode(str(text))
        # while node:
        #     features = node.feature.split(',')
        #     #print features
        #     if features[self.INDEX_CATEGORY] in self.TARGET_CATEGORIES:
        #         if features[self.INDEX_ROOT_FORM] == "*":
        #             words.append(node.surface)
        #         else:
        #             words.append(features[self.INDEX_ROOT_FORM])
        #
        #     node = node.next
        # # for eachword in words:
        # #     print('Word ==> {}'.format(eachword))
        # return words

call = WordDividor()
print (call.extract_words("This is Me モバイルドメインリストの最新版が欲しい"))


from tkinter import *
import editor

# initialize artist function
def initArtist(root):
    global mode, penMode, clearButton, sizeScale

    mode = StringVar()
    mode.set("Pen")
    penMode = OptionMenu(root, mode, "Pen", "Line", "Circle", "Square", "Triangle", "Eraser", command = lineMode)
    penMode.config(width = 15)
    penMode.place(x = 210, y = 360)

    clearButton = Button(root, text = "Clear")
    clearButton.place(x = 350, y = 360)

    sizeScale = Scale(root, from_ = 1, to = 50, orient = HORIZONTAL, length = 200)
    sizeScale.place(x = 150, y = 390)

# stamping function (click canvas)
def stamp(event, toolkit, canvas):
    global x, y, line
    x, y = event.x, event.y
    # set pen color and size
    penColor = ""
    penSize = sizeScale.get()
    if (editor.selected):
        penColor = toolkit.itemcget(editor.currShape, "fill")

    # draws based on mode selected
    if (penColor != ""):
        if (mode.get() == "Eraser" or mode.get() == "Pen"):
            draw(event, toolkit, canvas)
        elif (mode.get() == "Circle"):
            canvas.create_oval(x-penSize, y-penSize, x+penSize, y+penSize, fill = penColor, outline = penColor)
        elif (mode.get() == "Square"):
            canvas.create_rectangle(x-penSize, y-penSize, x+penSize, y+penSize, fill = penColor, outline = penColor)
        elif (mode.get() == "Triangle"):
            canvas.create_polygon(x, y-penSize, x-penSize, y+penSize, x+penSize, y+penSize, fill = penColor, outline = penColor)
        elif (mode.get() == "Line"):
            if (line[0] == True):
                canvas.create_line(x, y, line[1], line[2], fill = penColor, width = penSize, capstyle = "round")
                line = [False]
            else:
                canvas.create_line(x, y, event.x, event.y, fill = penColor, width = penSize, capstyle = "round")
                line = [True, x, y]

# drawing function
def draw(event, toolkit, canvas):
    global x, y
    # set pen color and size
    penColor = ""
    penSize = sizeScale.get()
    if (editor.selected and mode.get() == "Pen"):
        penColor = toolkit.itemcget(editor.currShape, "fill")

    if (mode.get() == "Eraser"):
        penColor = "#ffffff"
        penSize = 25

    if (penColor != ""):
        canvas.create_line(x, y, event.x, event.y, fill = penColor, width = penSize, capstyle = "round")
    x, y = event.x, event.y

# line status function
def lineMode(event):
    global line
    # line holds boolean that tracks whether to draw line as well as previous x and y values
    line = [False]


import openpyxl

ROW_NUM = 652
COL_NUM = 6

file_name ='bien2.xlsx'

wb = openpyxl.load_workbook('src/'+file_name)
sheet_list = wb.get_sheet_names()
for sheet_name in sheet_list:
    sheet = wb.get_sheet_by_name(sheet_name)

    des_wb = openpyxl.Workbook()
    des_sheet = des_wb.active

    dr = 1

    for r in range(2, ROW_NUM+1):
        symbol = sheet.cell(row=r, column=1).value
        for c in range(2, COL_NUM+1):
            value = sheet.cell(row=r, column=c).value
            year = sheet.cell(row=1, column=c).value

            des_sheet.cell(row=dr, column=1).value = symbol
            des_sheet.cell(row=dr, column=2).value = year
            des_sheet.cell(row=dr, column=3).value = value

            dr+=1

    des_wb.save('dx/'+sheet_name+'.xlsx')

    print(sheet_name)


"""
    ████░████░███░███░██░██░████░
    ░██░░██░░░██░█░██░██░██░███░░
    ░██░░████░██░░░██░█████░█████
    ═════════════════════════════════════════
    ████░████░░██░██░██░███░░██░█████░██░░░██
    ██░░░███░░░████░░██░██░█░██░██░██░░██░██░
    ████░█████░██░██░██░██░░███░█████░░░███░░
    ═════════════════════════════════════════
    Litsenziya: LLC © N.OA.ZL.QW (qaysi tupoy modul uchun litsenziya oladi? beradi?)
    Taqdim qilingan manzil: https://telegram.me/umodules
    ═════════════════════════════════════════
    GeekTG yoki FTG oʻrnatish qoʻllanmasi: https://t.me/TGraphUz/1620
"""
# meta developer: @umodules
# Uzbekistan presents in World top coders team
__version__ = (1, 0, 0)
from .. import loader, utils
import asyncio
import requests
from telethon.tl.types import DocumentAttributeFilename

def register(cb):
 cb(TelegraphMod())

class TelegraphMod(loader.Module):
	"""telegra.ph hostiga rasm/video/gif yuklash"""
	strings = {
               "name": "Telegraph",
               "reply": "🥷 Mediaga javob tariqasida..."}

	def __init__(self):
		self.name = self.strings['name']

	async def ph_qollanmacmd(self, message):
		"""qoʻllanma"""

		qollanma = """
🥷 Qoʻllanma: Telegraph host
├╴╴╴╴╴╴╴╴╴╴
├ 👾 - Kerakli, yuklamoqchi boʻlgan rasm/gif/video topasiz, aniqlaysiz. Va shu media'ga javoban .ph buyrugʻini yozib yuborasiz.
├ 👾 - Buyruqdan keyin darrov "Nega qotib qoldi?" degan savolga berilmang.
├ 👾 - Modul tezligi media hajmiga bogʻliq.

🥷 Qisqacha namuna:
├╴╴╴╴╴╴╴╴╴╴
└ 👾 Videoni koʻrish"""
		await message.edit(qollanma)
		return

	async def phcmd(self, message):
			"""javob tariqasida: rasm/video/gif"""
			if message.is_reply:
				reply_message = await message.get_reply_message()
				data = await check_media(reply_message)
				if isinstance(data, bool):
					await message.edit(self.strings("reply", message))
					return
			else:
				await message.edit(self.strings("reply", message))
				return

			file = await message.client.download_media(data, bytes)
			path = requests.post('https://te.legra.ph/upload', files={'file': ('file', file, None)}).json()
			try:
				link = 'https://te.legra.ph'+path[0]['src']
			except KeyError:
				link = path["error"]
			await message.edit("🥷 Havola tayyor.\n├╴╴╴╴╴╴╴╴╴╴\n├👾 Yuklangan host: https://te.legra.ph\n├ 👾 Yuklangan manzilga havola:\n└ 👾 "+link+"")

async def check_media(reply_message):
	if reply_message and reply_message.media:
		if reply_message.photo:
			data = reply_message.photo
		elif reply_message.document:
			if DocumentAttributeFilename(file_name='AnimatedSticker.tgs') in reply_message.media.document.attributes:
				return False
			if reply_message.audio or reply_message.voice:
				return False
			data = reply_message.media.document
		else:
			return False
	else:
		return False
	if not data or data is None:
		return False
	else:
		return data


from setuptools import setup, find_packages
import os

moduleDirectory = os.path.dirname(os.path.realpath(__file__))
exec(open(moduleDirectory + "/psat_server_web/__version__.py").read())

def readme():
    with open(moduleDirectory + '/README.md') as f:
        return f.read()

setup(
    name="psat-server-web",
    description='Pan-STARRS and ATLAS web interface.',
    long_description=readme(),
    long_description_content_type="text/markdown",
    version=__version__,
    author='genghisken',
    author_email='ken.w.smith@gmail.com',
    license='MIT',
    url='https://github.com/genghisken/psat-server-web',
    packages=find_packages(),
    classifiers=[
          'Development Status :: 4 - Beta',
          'License :: OSI Approved :: MIT License',
          'Programming Language :: Python :: 3.6',
          'Topic :: Utilities',
    ],
    install_requires=[
          'numpy',
          'mysqlclient',
          'django',
          'django_tables2',
          'pyyaml',
          'docopt',
          'python-dotenv',
          'gkhtm',
          'gkutils>=0.2.22',
          'requests',
          'lasair',
          'dustmaps',
      ],
    python_requires='>=3.6',
    include_package_data=True,
    zip_safe=False
)

# Add 'djangorestframework' when the code is ready.


import numpy as np
from sklearn.datasets import load_iris
from sklearn.cross_validation import KFold

def fit_model(features, labels):
    '''Learn a simple threshold model'''

    # initialize the best accuracy to the worst possible value
    best_acc = -1.0;

    # loop over all the features
    for fi in range(features.shape[1]):
        # test every possible threshold value for feature fi
        thresholds = features[:, fi].copy()
        thresholds.sort()

        for t in thresholds:
            # determine predictions using t as a threshold
            predictions = (features[:, fi] > t)

            # accuracy is the fraction of predictions that match the provided labels
            acc = (predictions == labels).mean()

            # test whether reversing the test achieves a better threshold
            rev_acc = (predictions == ~labels).mean()
            if rev_acc > acc:
                acc = rev_acc
                reverse = True
            else:
                reverse = False

            # if this is better the previous best, then swap previous with current best
            if acc > best_acc:
                best_acc = acc
                best_feature = fi
                best_threshold = t
                best_reverse = reverse

    return best_threshold, best_feature, best_reverse

def predict(model, features):
    '''Apply a learned model'''
    threshold, feature, reverse = model
    if reverse:
        return features[:, feature] <= threshold
    else:
        return features[:, feature] > threshold

def accuracy(features, labels, model):
    '''Compute the accuracy of the model'''
    predictions = predict(model, features)
    return np.mean(predictions == labels)

data = load_iris()
features = data.data
labels = data.target_names[data.target]

species = ['setosa', 'versicolor', 'virginica']
folds = 10
kf = KFold(features.shape[0], n_folds=folds, shuffle=True)
print('{0:10s} {1:5s} {2:^5s} {3:^30s} {4:^12s} {5:^12s}'.format(
            'species',
            'fold#',
            'theshold',
            'feature',
            'training acc',
            'testing acc'
        ))
for iris in species:
    i = 1
    for training, testing in kf:
        cur_labels = (labels == iris)
        model = fit_model(features[training], cur_labels[training])
        best_acc_training = accuracy(features[training], cur_labels[training], model)
        best_acc_testing = accuracy(features[testing], cur_labels[testing], model)

        best_t, best_fi, best_rev = model
        print('{0:10s} {1:5d} {2:8} {3:^20s} (index {4:d}) {5:12.1%} {6:12.1%}'.format(
            iris,
            i,
            best_t,
            data.feature_names[best_fi],
            best_fi,
            best_acc_training,
            best_acc_testing
        ))

        i = i+1

    print()


# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html

import sqlite3

class OutercrawlPipeline(object):
    def open_spider(self, spider):
        self.conn = sqlite3.connect('outerdata.sqlite')
        self.cur = self.conn.cursor()
        self.cur.execute('create table if not exists outerdata(store varchar(20), current_time datetime,\
         name varchar(100), price varchar(20), link varchar(100),fee varchar(20), sell varchar(20))')
        # pass

    def close_apider(self, spider):
        self.conn.commit()
        self.conn.close()
        # pass

    def process_item(self, item, spider):
        col = ','.join(item.keys())
        placeholders = ','.join(len(item) * '?')
        sql = 'insert into outerdata({}) values({})'
        self.cur.execute(sql.format(col, placeholders), item.values())
        return item


#!python3

from flask import Flask, render_template, request, session, redirect, url_for, escape

app = Flask(__name__)

@app.route('/', methods=['GET', 'POST'])
def index():
    if request.method == 'POST' and 'wage' in request.form:
        session['wage'] = float(request.form.get('wage'))
        return redirect(url_for('pay_calc'))
    return render_template("index.html")

@app.route('/pay', methods=['GET', 'POST'])
def pay_calc():
    pay = ''
    if request.method == 'POST' and 'hours' in request.form and 'wage' in session:
        hours = float(request.form.get('hours'))
        pay = calc_wage(session['wage'], hours)
        #pay = (session['wage'] * hours)
    return render_template("pay_calc.html",
                            pay=pay)

def calc_wage(wage, hours):
    return (wage * hours)

app.secret_key = "Test_Secret_Key"
app.run()


from django.shortcuts import render, redirect
from django.http import HttpResponse
from django.contrib.auth.decorators import login_required
from .models import Product, Order, Supplier
from .forms import ProductForm, OrderForm, SupplierForm
from django.contrib import messages
from django.contrib.auth.models import User

# Create your views here.
@login_required(login_url='user-login')
def index(request):
    orders = Order.objects.all()
    products = Product.objects.all()
    clients = Supplier.objects.all()
    orders_count = orders.count()
    product_count = products.count()
    clients_count = clients.count()
    workers_count = User.objects.all().count()
    if request.method == 'POST':
        form = OrderForm(request.POST)
        if form.is_valid():
            instance = form.save(commit=False)
            instance.staff = request.user
            instance.save()
            return redirect('dashboard-index')
    else:
        form = OrderForm()
    context = {
        'orders': orders,
        'form': form,
        'products': products,
        'product_count': product_count,
        'workers_count': workers_count,
        'orders_count': orders_count,
        'clients_count': clients_count,
    }
    return render(request, 'dashboard/index.html', context)

@login_required(login_url='user-login')
def staff(request):
    workers = User.objects.all()
    workers_count = workers.count()
    orders_count = Order.objects.all().count()
    clients_count = Supplier.objects.all().count()
    product_count = Product.objects.all().count()
    context = {
        'workers': workers,
        'workers_count': workers_count,
        'orders_count': orders_count,
        'clients_count': clients_count,
        'product_count': product_count
    }
    return render(request, 'dashboard/staff.html', context)

@login_required(login_url='user-login')
def staff_delete(request, pk):
    workers = User.objects.get(id=pk)
    if request.method == 'POST':
        workers.delete()
        return redirect('dashboard-staff')
    context = {
        'workers': workers
    }
    return render(request, 'dashboard/staff_delete.html', context)

def staff_detail(request, pk):
    workers = User.objects.get(id=pk)
    context = {
        'workers': workers,
    }
    return render(request, 'dashboard/staff_detail.html', context)

@login_required(login_url='user-login')
def product(request):
    items = Product.objects.all()
    product_count = items.count()
    workers_count = User.objects.all().count()
    orders_count = Order.objects.all().count()
    clients_count = Supplier.objects.all().count()
    if request.method == 'POST':
        form = ProductForm(request.POST)
        if form.is_valid():
            form.save()
            product_name = form.cleaned_data.get('name')
            messages.success(request, f'{product_name} has been added')
            return redirect('dashboard-product')
    else:
        form = ProductForm()
    context = {
        'items': items,
        'form': form,
        'workers_count': workers_count,
        'orders_count': orders_count,
        'clients_count': clients_count,
        'product_count': product_count,
    }
    return render(request, 'dashboard/product.html', context)

@login_required(login_url='user-login')
def product_detail(request, pk):
    items = Product.objects.get(id=pk)
    context = {
        'items': items,
    }
    return render(request, 'dashboard/staff_detail.html', context)

@login_required(login_url='user-login')
def product_delete(request, pk):
    item = Product.objects.get(id=pk)
    if request.method == 'POST':
        item.delete()
        return redirect('dashboard-product')
    context = {
        'item': item
    }
    return render(request, 'dashboard/product_delete.html', context)

@login_required(login_url='user-login')
def product_update(request, pk):
    item = Product.objects.get(id=pk)
    if request.method == 'POST':
        form = ProductForm(request.POST, instance=item)
        if form.is_valid():
            form.save()
            return redirect('dashboard-product')
    else:
        form = ProductForm(instance=item)
    context = {
        'form': form,
    }
    return render(request, 'dashboard/product_update.html', context)

@login_required(login_url='user-login')
def supplier(request):
    clients = Supplier.objects.all()
    clients_count = clients.count()
    workers_count = User.objects.all().count()
    orders_count = Order.objects.all().count()
    product_count = Product.objects.all().count()
    if request.method == 'POST':
        form = SupplierForm(request.POST)
        if form.is_valid():
            form.save()
            return redirect('dashboard-supplier')
    else:
        form = SupplierForm()
    context = {
        'clients': clients,
        'form': form,
        'workers_count': workers_count,
        'orders_count': orders_count,
        'clients_count': clients_count,
        'product_count': product_count,

    }
    return render(request, 'dashboard/supplier.html', context)

@login_required(login_url='user-login')
def supplier_delete(request, pk):
    clients = Supplier.objects.get(id=pk)
    if request.method == 'POST':
        clients.delete()
        return redirect('dashboard-supplier')
    context = {
        'clients': clients
    }
    return render(request, 'dashboard/supplier_delete.html', context)

@login_required(login_url='user-login')
def supplier_detail(request, pk):
    clients = Supplier.objects.get(id=pk)
    context = {
        'clients': clients,
    }
    return render(request, 'dashboard/supplier_detail.html', context)

@login_required(login_url='user-login')
def order(request):
    orders = Order.objects.all()
    orders_count = orders.count()
    workers_count = User.objects.all().count()
    clients_count = Supplier.objects.all().count()
    product_count = Product.objects.all().count()
    context = {
        'orders': orders,
        'workers_count': workers_count,
        'orders_count': orders_count,
        'clients_count': clients_count,
        'product_count': product_count,
    }
    return render(request, 'dashboard/order.html', context)

@login_required(login_url='user-login')
def order_detail(request, pk):
    orders = Order.objects.get(id=pk)
    context = {
        'orders': orders,
    }
    return render(request, 'dashboard/order_detail.html', context)
    print(orders)
    return render(request, 'dashboard/order_detail.html', context)

@login_required(login_url='user-login')
def order_delete(request, pk):
    orders = Order.objects.get(id=pk)
    if request.method == 'POST':
        orders.delete()
        return redirect('dashboard-order')
    context = {
        'orders': orders
    }
    return render(request, 'dashboard/order_delete.html', context)


import os
import glob
from osgeo import gdal
from tqdm import tqdm

files = glob.glob('../data/Sentinel-1/*/geocoded_subset_dat/*/*.tif')
out_data_dir = '../data/Sentinel-1_analysis_ready/'

for i in tqdm(range(len(files))):
#for i in range(1):
    '''
    load data information and initial output data directory
    '''
    lee_data_file = files[i]
    info = lee_data_file.split('/')
    place = info[3]
    time = info[5][:4]+'_'+ info[5][-2:]
    out_data_file_name ='/'+ place+'_'+time+'.tif'
    place_folder = out_data_dir+place
    if not os.path.exists(place_folder):
        os.mkdir(place_folder)

    '''
    load lee filtered data
    '''
    fid = gdal.Open(lee_data_file)
    lee = fid.ReadAsArray()
    row = fid.RasterYSize
    col = fid.RasterXSize
    bnd = fid.RasterCount
    proj = fid.GetProjection()
    geoInfo = fid.GetGeoTransform()
    del fid

    '''
    load unfiltered data
    '''
    unfil_data_file = lee_data_file.replace('geocoded_subset_dat','geocoded_subset_unfilt_dat')
    unfil_data_file = unfil_data_file.replace('Deb_Spk_TC','Deb_TC')
    fid = gdal.Open(unfil_data_file)
    unfil = fid.ReadAsArray()
    del fid

    '''
    save the analysis ready data
    '''
    driver = gdal.GetDriverByName('GTiff')
    f = driver.Create(place_folder+out_data_file_name, col, row, 8, gdal.GDT_Float32)
    f.SetProjection(proj)
    f.SetGeoTransform(geoInfo)
    for idxBnd in range(4):
        outBand = f.GetRasterBand(idxBnd+1)
        outBand.WriteArray(lee[idxBnd,:,:])
        outBand.FlushCache()
        del(outBand)

    for idxBnd in range(4):
        outBand = f.GetRasterBand(idxBnd+5)
        outBand.WriteArray(unfil[idxBnd,:,:])
        outBand.FlushCache()
        del(outBand)


import sys
from io import StringIO
import unittest

class TestClass(unittest.TestCase):
    maxDiff = None
    def assertIO(self, input, output):
        stdout, stdin = sys.stdout, sys.stdin
        sys.stdout, sys.stdin = StringIO(), StringIO(input)
        resolve()
        sys.stdout.seek(0)
        out = sys.stdout.read()[:-1]
        sys.stdout, sys.stdin = stdout, stdin
        self.assertEqual(out, output)

    def test_Sample_Input_1(self):
        input = """1"""
        output = """Aoki"""
        self.assertIO(input, output)

    def test_Sample_Input_2(self):
        input = """5"""
        output = """Takahashi"""
        self.assertIO(input, output)

    def test_Sample_Input_3(self):
        input = """7"""
        output = """Aoki"""
        self.assertIO(input, output)

    def test_Sample_Input_4(self):
        input = """10"""
        output = """Takahashi"""
        self.assertIO(input, output)

    def test_Sample_Input_5(self):
        input = """123456789123456789"""
        output = """Aoki"""
        self.assertIO(input, output)

def resolve():
  # DP っぽい
  # 一回の操作でパターン数は 2 倍に増える。
  # N <= 10**18 なので最大 60 回程度の操作しかしない。
  # i の時に手番をとったら負けるゾーンがある。
  # => 遷移先に i を選べる場合はそこに放り込めば勝ち。
  # => i に遷移できるならば勝ち
  # 遷移を考えるのが厄介。

  N = int(input())+1
  win = False
  while N > 1:
    win = not win
    N = (N+1)//2 if win else (N)//2

  print("Takahashi" if not win else "Aoki")

import sys
if sys.argv[-1] == './Main.py':
  resolve()

if __name__ == "__main__":
  unittest.main()


from typing import List, Optional

import graphene
from graphql import GraphQLError

from ...permission.enums import ProductPermissions
from ...permission.utils import has_one_of_permissions
from ...product.models import ALL_PRODUCTS_PERMISSIONS
from ..channel import ChannelContext
from ..channel.utils import get_default_channel_slug_or_graphql_error
from ..core import ResolveInfo
from ..core.connection import create_connection_slice, filter_connection_queryset
from ..core.descriptions import ADDED_IN_310
from ..core.enums import ReportingPeriod
from ..core.fields import ConnectionField, FilterConnectionField, PermissionsField
from ..core.tracing import traced_resolver
from ..core.types import NonNullList
from ..core.utils import from_global_id_or_error
from ..core.validators import validate_one_of_args_is_in_query
from ..translations.mutations import (
    CategoryTranslate,
    CollectionTranslate,
    ProductTranslate,
    ProductVariantTranslate,
)
from ..utils import get_user_or_app_from_context
from .bulk_mutations import (
    CategoryBulkDelete,
    CollectionBulkDelete,
    ProductBulkDelete,
    ProductMediaBulkDelete,
    ProductTypeBulkDelete,
    ProductVariantBulkCreate,
    ProductVariantBulkDelete,
    ProductVariantBulkUpdate,
    ProductVariantStocksCreate,
    ProductVariantStocksDelete,
    ProductVariantStocksUpdate,
)
from .filters import (
    CategoryFilterInput,
    CollectionFilterInput,
    ProductFilterInput,
    ProductTypeFilterInput,
    ProductVariantFilterInput,
)
from .mutations import (
    CategoryCreate,
    CategoryDelete,
    CategoryUpdate,
    CollectionAddProducts,
    CollectionCreate,
    CollectionDelete,
    CollectionRemoveProducts,
    CollectionReorderProducts,
    CollectionUpdate,
    ProductCreate,
    ProductDelete,
    ProductMediaCreate,
    ProductMediaDelete,
    ProductMediaReorder,
    ProductMediaUpdate,
    ProductTypeCreate,
    ProductTypeDelete,
    ProductTypeUpdate,
    ProductUpdate,
    ProductVariantCreate,
    ProductVariantDelete,
    ProductVariantPreorderDeactivate,
    ProductVariantReorder,
    ProductVariantSetDefault,
    ProductVariantUpdate,
    VariantMediaAssign,
    VariantMediaUnassign,
)
from .mutations.attributes import (
    ProductAttributeAssign,
    ProductAttributeAssignmentUpdate,
    ProductAttributeUnassign,
    ProductReorderAttributeValues,
    ProductTypeReorderAttributes,
    ProductVariantReorderAttributeValues,
)
from .mutations.channels import (
    CollectionChannelListingUpdate,
    ProductChannelListingUpdate,
    ProductVariantChannelListingUpdate,
)
from .mutations.digital_contents import (
    DigitalContentCreate,
    DigitalContentDelete,
    DigitalContentUpdate,
    DigitalContentUrlCreate,
)
from .resolvers import (
    resolve_categories,
    resolve_category_by_id,
    resolve_category_by_slug,
    resolve_collection_by_id,
    resolve_collection_by_slug,
    resolve_collections,
    resolve_digital_content_by_id,
    resolve_digital_contents,
    resolve_product,
    resolve_product_type_by_id,
    resolve_product_types,
    resolve_product_variants,
    resolve_products,
    resolve_report_product_sales,
    resolve_variant,
)
from .sorters import (
    CategorySortingInput,
    CollectionSortingInput,
    ProductOrder,
    ProductOrderField,
    ProductTypeSortingInput,
    ProductVariantSortingInput,
)
from .types import (
    Category,
    CategoryCountableConnection,
    Collection,
    CollectionCountableConnection,
    DigitalContent,
    DigitalContentCountableConnection,
    Product,
    ProductCountableConnection,
    ProductType,
    ProductTypeCountableConnection,
    ProductVariant,
    ProductVariantCountableConnection,
)

def search_string_in_kwargs(kwargs: dict) -> bool:
    return bool(kwargs.get("filter", {}).get("search", "").strip())

def sort_field_from_kwargs(kwargs: dict) -> Optional[List[str]]:
    return kwargs.get("sort_by", {}).get("field") or None

class ProductQueries(graphene.ObjectType):
    digital_content = PermissionsField(
        DigitalContent,
        description="Look up digital content by ID.",
        id=graphene.Argument(
            graphene.ID, description="ID of the digital content.", required=True
        ),
        permissions=[
            ProductPermissions.MANAGE_PRODUCTS,
        ],
    )
    digital_contents = ConnectionField(
        DigitalContentCountableConnection,
        description="List of digital content.",
        permissions=[
            ProductPermissions.MANAGE_PRODUCTS,
        ],
    )
    categories = FilterConnectionField(
        CategoryCountableConnection,
        filter=CategoryFilterInput(description="Filtering options for categories."),
        sort_by=CategorySortingInput(description="Sort categories."),
        level=graphene.Argument(
            graphene.Int,
            description="Filter categories by the nesting level in the category tree.",
        ),
        description="List of the shop's categories.",
    )
    category = graphene.Field(
        Category,
        id=graphene.Argument(graphene.ID, description="ID of the category."),
        slug=graphene.Argument(graphene.String, description="Slug of the category"),
        description="Look up a category by ID or slug.",
    )
    collection = graphene.Field(
        Collection,
        id=graphene.Argument(
            graphene.ID,
            description="ID of the collection.",
        ),
        slug=graphene.Argument(graphene.String, description="Slug of the category"),
        channel=graphene.String(
            description="Slug of a channel for which the data should be returned."
        ),
        description=(
            "Look up a collection by ID. Requires one of the following permissions to "
            "include the unpublished items: "
            f"{', '.join([p.name for p in ALL_PRODUCTS_PERMISSIONS])}."
        ),
    )
    collections = FilterConnectionField(
        CollectionCountableConnection,
        filter=CollectionFilterInput(description="Filtering options for collections."),
        sort_by=CollectionSortingInput(description="Sort collections."),
        description=(
            "List of the shop's collections. Requires one of the following permissions "
            "to include the unpublished items: "
            f"{', '.join([p.name for p in ALL_PRODUCTS_PERMISSIONS])}."
        ),
        channel=graphene.String(
            description="Slug of a channel for which the data should be returned."
        ),
    )
    product = graphene.Field(
        Product,
        id=graphene.Argument(
            graphene.ID,
            description="ID of the product.",
        ),
        slug=graphene.Argument(graphene.String, description="Slug of the product."),
        external_reference=graphene.Argument(
            graphene.String, description=f"External ID of the product. {ADDED_IN_310}"
        ),
        channel=graphene.String(
            description="Slug of a channel for which the data should be returned."
        ),
        description=(
            "Look up a product by ID. Requires one of the following permissions to "
            "include the unpublished items: "
            f"{', '.join([p.name for p in ALL_PRODUCTS_PERMISSIONS])}."
        ),
    )
    products = FilterConnectionField(
        ProductCountableConnection,
        filter=ProductFilterInput(description="Filtering options for products."),
        sort_by=ProductOrder(description="Sort products."),
        channel=graphene.String(
            description="Slug of a channel for which the data should be returned."
        ),
        description=(
            "List of the shop's products. Requires one of the following permissions to "
            "include the unpublished items: "
            f"{', '.join([p.name for p in ALL_PRODUCTS_PERMISSIONS])}."
        ),
    )
    product_type = graphene.Field(
        ProductType,
        id=graphene.Argument(
            graphene.ID, description="ID of the product type.", required=True
        ),
        description="Look up a product type by ID.",
    )
    product_types = FilterConnectionField(
        ProductTypeCountableConnection,
        filter=ProductTypeFilterInput(
            description="Filtering options for product types."
        ),
        sort_by=ProductTypeSortingInput(description="Sort product types."),
        description="List of the shop's product types.",
    )
    product_variant = graphene.Field(
        ProductVariant,
        id=graphene.Argument(
            graphene.ID,
            description="ID of the product variant.",
        ),
        sku=graphene.Argument(
            graphene.String, description="Sku of the product variant."
        ),
        external_reference=graphene.Argument(
            graphene.String, description=f"External ID of the product. {ADDED_IN_310}"
        ),
        channel=graphene.String(
            description="Slug of a channel for which the data should be returned."
        ),
        description=(
            "Look up a product variant by ID or SKU. Requires one of the following "
            "permissions to include the unpublished items: "
            f"{', '.join([p.name for p in ALL_PRODUCTS_PERMISSIONS])}."
        ),
    )
    product_variants = FilterConnectionField(
        ProductVariantCountableConnection,
        ids=NonNullList(
            graphene.ID, description="Filter product variants by given IDs."
        ),
        channel=graphene.String(
            description="Slug of a channel for which the data should be returned."
        ),
        filter=ProductVariantFilterInput(
            description="Filtering options for product variant."
        ),
        sort_by=ProductVariantSortingInput(description="Sort products variants."),
        description=(
            "List of product variants. Requires one of the following permissions to "
            "include the unpublished items: "
            f"{', '.join([p.name for p in ALL_PRODUCTS_PERMISSIONS])}."
        ),
    )
    report_product_sales = ConnectionField(
        ProductVariantCountableConnection,
        period=graphene.Argument(
            ReportingPeriod, required=True, description="Span of time."
        ),
        channel=graphene.String(
            description="Slug of a channel for which the data should be returned.",
            required=True,
        ),
        description="List of top selling products.",
        permissions=[
            ProductPermissions.MANAGE_PRODUCTS,
        ],
    )

    @staticmethod
    def resolve_categories(_root, info: ResolveInfo, *, level=None, **kwargs):
        qs = resolve_categories(info, level=level)
        qs = filter_connection_queryset(qs, kwargs)
        return create_connection_slice(qs, info, kwargs, CategoryCountableConnection)

    @staticmethod
    @traced_resolver
    def resolve_category(_root, _info: ResolveInfo, *, id=None, slug=None, **kwargs):
        validate_one_of_args_is_in_query("id", id, "slug", slug)
        if id:
            _, id = from_global_id_or_error(id, Category)
            return resolve_category_by_id(id)
        if slug:
            return resolve_category_by_slug(slug=slug)

    @staticmethod
    @traced_resolver
    def resolve_collection(
        _root, info: ResolveInfo, *, id=None, slug=None, channel=None
    ):
        validate_one_of_args_is_in_query("id", id, "slug", slug)
        requestor = get_user_or_app_from_context(info.context)

        has_required_permissions = has_one_of_permissions(
            requestor, ALL_PRODUCTS_PERMISSIONS
        )
        if channel is None and not has_required_permissions:
            channel = get_default_channel_slug_or_graphql_error()
        if id:
            _, id = from_global_id_or_error(id, Collection)
            collection = resolve_collection_by_id(info, id, channel, requestor)
        else:
            collection = resolve_collection_by_slug(
                info, slug=slug, channel_slug=channel, requestor=requestor
            )
        return (
            ChannelContext(node=collection, channel_slug=channel)
            if collection
            else None
        )

    @staticmethod
    def resolve_collections(_root, info: ResolveInfo, *, channel=None, **kwargs):
        requestor = get_user_or_app_from_context(info.context)
        has_required_permissions = has_one_of_permissions(
            requestor, ALL_PRODUCTS_PERMISSIONS
        )
        if channel is None and not has_required_permissions:
            channel = get_default_channel_slug_or_graphql_error()
        qs = resolve_collections(info, channel)
        kwargs["channel"] = channel
        qs = filter_connection_queryset(qs, kwargs)
        return create_connection_slice(qs, info, kwargs, CollectionCountableConnection)

    @staticmethod
    def resolve_digital_content(_root, _info: ResolveInfo, *, id):
        _, id = from_global_id_or_error(id, DigitalContent)
        return resolve_digital_content_by_id(id)

    @staticmethod
    def resolve_digital_contents(_root, info: ResolveInfo, **kwargs):
        qs = resolve_digital_contents(info)
        return create_connection_slice(
            qs, info, kwargs, DigitalContentCountableConnection
        )

    @staticmethod
    @traced_resolver
    def resolve_product(
        _root,
        info: ResolveInfo,
        *,
        id=None,
        slug=None,
        external_reference=None,
        channel=None,
    ):
        validate_one_of_args_is_in_query(
            "id", id, "slug", slug, "external_reference", external_reference
        )
        requestor = get_user_or_app_from_context(info.context)

        has_required_permissions = has_one_of_permissions(
            requestor, ALL_PRODUCTS_PERMISSIONS
        )

        if channel is None and not has_required_permissions:
            channel = get_default_channel_slug_or_graphql_error()

        product = resolve_product(
            info,
            id=id,
            slug=slug,
            external_reference=external_reference,
            channel_slug=channel,
            requestor=requestor,
        )

        return ChannelContext(node=product, channel_slug=channel) if product else None

    @staticmethod
    @traced_resolver
    def resolve_products(_root, info: ResolveInfo, *, channel=None, **kwargs):
        if sort_field_from_kwargs(kwargs) == ProductOrderField.RANK:
            # sort by RANK can be used only with search filter
            if not search_string_in_kwargs(kwargs):
                raise GraphQLError(
                    "Sorting by RANK is available only when using a search filter."
                )
        if search_string_in_kwargs(kwargs) and not sort_field_from_kwargs(kwargs):
            # default to sorting by RANK if search is used
            # and no explicit sorting is requested
            product_type = info.schema.get_type("ProductOrder")
            kwargs["sort_by"] = product_type.create_container(
                {"direction": "-", "field": ["search_rank", "id"]}
            )

        requestor = get_user_or_app_from_context(info.context)
        has_required_permissions = has_one_of_permissions(
            requestor, ALL_PRODUCTS_PERMISSIONS
        )
        if channel is None and not has_required_permissions:
            channel = get_default_channel_slug_or_graphql_error()
        qs = resolve_products(info, requestor, channel_slug=channel)
        kwargs["channel"] = channel
        qs = filter_connection_queryset(qs, kwargs)
        return create_connection_slice(qs, info, kwargs, ProductCountableConnection)

    @staticmethod
    def resolve_product_type(_root, _info: ResolveInfo, *, id):
        _, id = from_global_id_or_error(id, ProductType)
        return resolve_product_type_by_id(id)

    @staticmethod
    def resolve_product_types(_root, info: ResolveInfo, **kwargs):
        qs = resolve_product_types(info)
        qs = filter_connection_queryset(qs, kwargs)
        return create_connection_slice(qs, info, kwargs, ProductTypeCountableConnection)

    @staticmethod
    @traced_resolver
    def resolve_product_variant(
        _root,
        info: ResolveInfo,
        *,
        id=None,
        sku=None,
        external_reference=None,
        channel=None,
    ):
        validate_one_of_args_is_in_query(
            "id", id, "sku", sku, "external_reference", external_reference
        )
        requestor = get_user_or_app_from_context(info.context)
        has_required_permissions = has_one_of_permissions(
            requestor, ALL_PRODUCTS_PERMISSIONS
        )

        if channel is None and not has_required_permissions:
            channel = get_default_channel_slug_or_graphql_error()

        variant = resolve_variant(
            info,
            id,
            sku,
            external_reference,
            channel_slug=channel,
            requestor=requestor,
            requestor_has_access_to_all=has_required_permissions,
        )

        return ChannelContext(node=variant, channel_slug=channel) if variant else None

    @staticmethod
    def resolve_product_variants(
        _root, info: ResolveInfo, *, ids=None, channel=None, **kwargs
    ):
        requestor = get_user_or_app_from_context(info.context)
        has_required_permissions = has_one_of_permissions(
            requestor, ALL_PRODUCTS_PERMISSIONS
        )
        if channel is None and not has_required_permissions:
            channel = get_default_channel_slug_or_graphql_error()
        qs = resolve_product_variants(
            info,
            ids=ids,
            channel_slug=channel,
            requestor_has_access_to_all=has_required_permissions,
            requestor=requestor,
        )
        kwargs["channel"] = qs.channel_slug
        qs = filter_connection_queryset(qs, kwargs)
        return create_connection_slice(
            qs, info, kwargs, ProductVariantCountableConnection
        )

    @staticmethod
    @traced_resolver
    def resolve_report_product_sales(
        _root, info: ResolveInfo, *, period, channel, **kwargs
    ):
        qs = resolve_report_product_sales(period, channel_slug=channel)
        kwargs["channel"] = qs.channel_slug
        return create_connection_slice(
            qs, info, kwargs, ProductVariantCountableConnection
        )

class ProductMutations(graphene.ObjectType):
    product_attribute_assign = ProductAttributeAssign.Field()
    product_attribute_assignment_update = ProductAttributeAssignmentUpdate.Field()
    product_attribute_unassign = ProductAttributeUnassign.Field()

    category_create = CategoryCreate.Field()
    category_delete = CategoryDelete.Field()
    category_bulk_delete = CategoryBulkDelete.Field()
    category_update = CategoryUpdate.Field()
    category_translate = CategoryTranslate.Field()

    collection_add_products = CollectionAddProducts.Field()
    collection_create = CollectionCreate.Field()
    collection_delete = CollectionDelete.Field()
    collection_reorder_products = CollectionReorderProducts.Field()
    collection_bulk_delete = CollectionBulkDelete.Field()
    collection_remove_products = CollectionRemoveProducts.Field()
    collection_update = CollectionUpdate.Field()
    collection_translate = CollectionTranslate.Field()
    collection_channel_listing_update = CollectionChannelListingUpdate.Field()

    product_create = ProductCreate.Field()
    product_delete = ProductDelete.Field()
    product_bulk_delete = ProductBulkDelete.Field()
    product_update = ProductUpdate.Field()
    product_translate = ProductTranslate.Field()

    product_channel_listing_update = ProductChannelListingUpdate.Field()

    product_media_create = ProductMediaCreate.Field()
    product_variant_reorder = ProductVariantReorder.Field()
    product_media_delete = ProductMediaDelete.Field()
    product_media_bulk_delete = ProductMediaBulkDelete.Field()
    product_media_reorder = ProductMediaReorder.Field()
    product_media_update = ProductMediaUpdate.Field()

    product_type_create = ProductTypeCreate.Field()
    product_type_delete = ProductTypeDelete.Field()
    product_type_bulk_delete = ProductTypeBulkDelete.Field()
    product_type_update = ProductTypeUpdate.Field()
    product_type_reorder_attributes = ProductTypeReorderAttributes.Field()
    product_reorder_attribute_values = ProductReorderAttributeValues.Field()

    digital_content_create = DigitalContentCreate.Field()
    digital_content_delete = DigitalContentDelete.Field()
    digital_content_update = DigitalContentUpdate.Field()

    digital_content_url_create = DigitalContentUrlCreate.Field()

    product_variant_create = ProductVariantCreate.Field()
    product_variant_delete = ProductVariantDelete.Field()
    product_variant_bulk_create = ProductVariantBulkCreate.Field()
    product_variant_bulk_update = ProductVariantBulkUpdate.Field()
    product_variant_bulk_delete = ProductVariantBulkDelete.Field()
    product_variant_stocks_create = ProductVariantStocksCreate.Field()
    product_variant_stocks_delete = ProductVariantStocksDelete.Field()
    product_variant_stocks_update = ProductVariantStocksUpdate.Field()
    product_variant_update = ProductVariantUpdate.Field()
    product_variant_set_default = ProductVariantSetDefault.Field()
    product_variant_translate = ProductVariantTranslate.Field()
    product_variant_channel_listing_update = ProductVariantChannelListingUpdate.Field()
    product_variant_reorder_attribute_values = (
        ProductVariantReorderAttributeValues.Field()
    )
    product_variant_preorder_deactivate = ProductVariantPreorderDeactivate.Field()

    variant_media_assign = VariantMediaAssign.Field()
    variant_media_unassign = VariantMediaUnassign.Field()


import os

from fastapi import FastAPI
from fastapi.responses import FileResponse

app = FastAPI()

path = "/home/nathalia/Documents/python/intro-fastapi"

@app.get("/")
def index():
    return {"message": "Hello World"}

@app.get("/dog", responses={200: {"description": "A picture of a dog."}})
def dog():
    file_path = os.path.join(path, "files/dog.jpg")
    if os.path.exists(file_path):
        return FileResponse(
            file_path,
            media_type="image/jpeg",
            filename="dog.jpg"
        )

    return {"error": "File not found!"}


import datetime
import random

from sklearn.model_selection import train_test_split
import pandas as pd
from svm.accuracy import *
from svm.linear import LinearSVM

start = datetime.datetime.now()
f = pd.read_csv('dataset_breastcancer.csv')
f.columns=["id", "diagnosis", "radius_2ean", "texture_2ean", "peri2eter_2ean", "area_2ean", "s2oothness_2ean", "co2pactness_2ean", "concavity_2ean", "concave points_2ean", "sy22etry_2ean", "fractal_di2ension_2ean", "radius_se, ", "texture_se, "	, "peri2eter_se, ", "	area_se, ", "	s2oothness_se, ", "	co2pactness_se, ", "	concavity_se, ", "	concave points_se, ", "	sy22etry_se, ", "	fractal_di2ension_se, ", "	radius_worst, ", "	texture_worst, ", "	peri2eter_worst, ", "	area_worst, ", "	s2oothness_worst, ", "	co2pactness_worst, ", "	concavity_worst, ", "	concave points_worst, ", "	sy22etry_worst, ", "	fractal_di2ension_worst"]
file = f[[ "radius_2ean", "texture_2ean", "peri2eter_2ean", "area_2ean", "s2oothness_2ean", "co2pactness_2ean", "concavity_2ean", "concave points_2ean", "sy22etry_2ean", "fractal_di2ension_2ean", "radius_se, ", "texture_se, "	, "peri2eter_se, ", "	area_se, ", "	s2oothness_se, ", "	co2pactness_se, ", "	concavity_se, ", "	concave points_se, ", "	sy22etry_se, ", "	fractal_di2ension_se, ", "	radius_worst, ", "	texture_worst, ", "	peri2eter_worst, ", "	area_worst, ", "	s2oothness_worst, ", "	co2pactness_worst, ", "	concavity_worst, ", "	concave points_worst, ", "	sy22etry_worst, ", "	fractal_di2ension_worst"]]
file = np.array(file)
y = f[["diagnosis"]]

''' 打亂column(Pi)
l1 = random.sample(range(0, 30), 30)
file = file[:, l1]
print(l1)
'''

X_train1, X_test1, y_train1, y_test1 = train_test_split(file, y, test_size=0.3)

y_train1 = np.array(y_train1)
y_test1 = np.array(y_test1)

for i in range(y_train1.shape[0]):
    if(y_train1[i] == 2):
        y_train1[i] = -1
    if(y_train1[i] == 4):
        y_train1[i] = 1
for i in range(y_test1.shape[0]):
    if (y_test1[i] == 2):
        y_test1[i] = -1
    if (y_test1[i] == 4):
        y_test1[i] = 1

k=0
x_train = np.zeros((15,X_train1.shape[0],2))
for j in range(15):
    for i in range(X_train1.shape[0]): #(0~499, 500)
        x_train[j][i] = X_train1[i][k:k+2]
    k+=2

k=0
x_test = np.zeros((15,X_test1.shape[0],2))  #ALL is 15 parties (2 featrues in each)
for j in range(15):
    for i in range(X_test1.shape[0]): #(500~558, 59)
        x_test[j][i] = X_test1[i][k:k+2]
    k+=2

Xtrain = x_train[0]   #(500,30)
for i in range(1, x_train.shape[0]):  #range(1, x_train.shape[0])
    Xtrain = np.hstack([Xtrain, x_train[i]])

Xtest = x_test[0]     #(59,30)
for i in range(1, 15):  #range(1, x_test.shape[0])
    Xtest = np.hstack([Xtest, x_test[i]])

#print(x_train[:3].shape)
print("Dataset: Breast cancer")

clf = LinearSVM()
clf.fit(X=x_train[:15], y=y_train1)

acc_test = accuracy(clf, X=Xtest, y=y_test1)
print("Accuracy (on test): "+str(acc_test))
end = datetime.datetime.now()
#print("All time： ", end - start)


import torch
from torch import nn, optim
from torch.autograd import Variable
import torch.nn.functional as F
from assessor.base_model import BaseModel
import os

class FFNN(nn.Module):
    def __init__(self, input_size, output_size, hidden_size):
        super(FFNN, self).__init__()
        self.linear = nn.Linear(input_size, hidden_size)
        self.relu = F.relu
        self.linear2 = nn.Linear(hidden_size, output_size)

    def forward(self, x):
        y = self.linear(x)
        y = self.relu(y)
        y = self.linear2(y)
        return y

class NNModel(BaseModel):
    def __init__(self, config):
        input_size = config.get('input_size', 1)
        output_size = config.get('output_size', 1)
        lr = config.get('lr', 0.001)
        hidden_size = config.get('hidden_size', 100)

        self.loss = nn.SmoothL1Loss()
        self.model = FFNN(input_size, output_size, hidden_size)
        self.optimizer = optim.Adam(self.model.parameters(), lr=lr)
        self.epochs = config.get('epochs', 100)

    def _torch_var(self, value):
        return Variable(torch.Tensor(value))

    def train(self, x_inputs, y_labels, x_val=None, y_val=None):
        for epoch in range(self.epochs):
            y_pred = self.model(self._torch_var(x_inputs))
            loss = self.loss(y_pred, self._torch_var(y_labels))

            val_loss = 0
            if x_val and y_val:
                y_pred_val = self.model(self._torch_var(x_val))
                val_loss = self.loss(y_pred_val, self._torch_var(y_val))

            print(f'Epoch {epoch}, Train_Loss: {loss}, Val_Loss: {val_loss}')

            self.optimizer.zero_grad()
            loss.backward()
            self.optimizer.step()

    def eval(self, x):
        y_pred = self.model(self._torch_var(x))
        formatted_y = y_pred.data.numpy()
        return formatted_y

    def save(self, filepath):
        model_path = os.path.join(filepath, 'nn_model.pth')
        torch.save(self.model.state_dict(), model_path)

    def load(self, filepath):
        model_path = os.path.join(filepath, 'nn_model.pth')
        self.model.load_state_dict(torch.load(model_path))


# lsh.py (question 2)
# Implements both locality sensitive hashing (LSH) and a brute-force nearest neighbor
# approach on a dataset of 39,861 e-mails.
#---------------------------------------------------------------------------------------
# Il Shan Ng
# April 7, 2017

import numpy as np
import heapq as hq
from collections import defaultdict
import sys
import random
import math
import time
import argparse

#####################################################################################
# specify path to and name of data file
file_name = 'docword.enron.txt'
#####################################################################################

def character_matrix(D):
	"""
	Reads in the first D documents of the data and creates a character matrix,
	implemented as a set of words for each document.
	"""
	# start timer
	start_time = time.time()
	# initialize list of D empty sets
	chr_mat = [set() for _ in range(D)]
	# open file containing data
	with open(file_name) as file:
		# skip first three lines
		for _ in range(3):
			next(file)
		while True:
			# read and split line on white space, casting fragments into integers
			line = [int(i) for i in str.split(file.readline())]
			# if reached end of data set, return list of sets
			if not line:
				return chr_mat
			# if document ID is within first D documents
			if line[0] <= D:
				# add word ID to the set for the current document
				chr_mat[line[0]-1].add(line[1])
			# otherwise, return list of sets
			else:
				return (chr_mat, time.time()-start_time)

def jaccard(doc1, doc2):
	"""
	Takes in two documents, in the form of two sets, and returns the Jaccard
	similarity between them.
	"""
	return (len(doc1 & doc2)/len(doc1 | doc2))

def signature_matrix(chr_mat, N, D):
	"""
	Takes in a character matrix and creates a signature matrix with N rows using
	the min-hash algorithm.
	"""
	# start timer
	start_time = time.time()
	# set random seed for debugging purposes
	random.seed(1)
	# compute number of words in the character matrix
	words = set.union(*chr_mat)
	W = len(words)
	# initialize empty list to store N randomly generated hash functions
	hash_functions = []
	# for each hash function
	for n in range(N):
		# generate random b between 0 and W-1
		b = random.randint(0, W-1)
		# generate random a between 0 and W-1, which is coprime with W
		gcd = 2
		while gcd != 1:
			a = random.randint(0, W-1)
			gcd = math.gcd(a, W)
		# add (a,b) tuple to list of hash functions
		hash_functions.append((a,b))
	# initialize signature matrix with inf in all entries
	sig_mat = np.empty(shape=(N, len(chr_mat)), dtype=int)
	sig_mat.fill(99999999)
	# for each row of the character matrix
	for w in range(W):
		# compute hash values for row number
		hash_values = np.array([(coeff[0]*w + coeff[1])%W for coeff in hash_functions])
		# obtain word ID corresponding to row
		wordID = words.pop()
		# for each document
		for d in range(D):
			# if the word ID is in the document
			if wordID in chr_mat[d]:
				# compare the existing column for the document to the vector of
				# computed hash values, and choose the element-wise minimum
				sig_mat[:,d] = np.minimum(sig_mat[:,d], hash_values)
	# return signature matrix along with time taken
	return (sig_mat, time.time() - start_time)

def estimated_jaccard(doc1, doc2):
	"""
	Takes in two documents, in the form of two columns of the signature matrix, and
	returns the estimated Jaccard similarity between them.
	"""
	return sum(doc1 == doc2)/len(doc1)

def knn_brute_force(chr_mat, K, D):
	"""
	Takes in a character matrix, in the form of a list of sets, and finds the K closest
	neighbors to each document, measured by Jaccard similarity. Computes the average
	Jaccard similarities for those K neighbors with respect to each document, and returns
	the average over all documents.
	"""
	# start timer
	start_time = time.time()
	# initialize numpy array to store average Jaccard similarities
	avg_jaccard_sim = np.empty(D, dtype=float)
	# for each document
	for d in range(D):
		# initialize empty priority queue to store K closest neighbors
		queue = []
		# loop over all other documents
		for j in list(range(d)) + list(range(d+1, D)):
			# compute the Jaccard similarity between the current pair and add to queue
			hq.heappush(queue, (jaccard(chr_mat[d], chr_mat[j]), j))
			# if more than K items in queue
			if len(queue) > K:
				# remove document with lowest Jaccard similarity
				hq.heappop(queue)
		# initialize numpy array to store Jaccard similarities
		jaccard_sim = np.empty(K, dtype=float)
		# iterate through all K neighbors in queue
		for k in range(K):
			# obtain Jaccard similarity and add it to the array
			jaccard_sim[k] = hq.heappop(queue)[0]
		# compute the average Jaccard similarity and add to array at top
		avg_jaccard_sim[d] = np.mean(jaccard_sim)
		# print progress to screen
		if D >= 10:
			if (d+1) % (D//10) == 0:
				print('Average Jaccard similarity for document ', d+1, ': ',
					'%.4f' % avg_jaccard_sim[d], sep='')
				sys.stdout.flush()
	# return average of averages along with time taken
	return(np.mean(avg_jaccard_sim), time.time() - start_time)

def knn_lsh(sig_mat, K, D, R):
	"""
	Takes in a signature matrix, in the form of a numpy 2D array, and finds the K
	closest neighbors to each document using banding and locality sensitive hashing.
	Computes the average estimated Jaccard similarities for those K neighbors with
	respect to each document, and returns the average over all documents.
	"""
	# start timer
	start_time = time.time()
	# set random seed for debugging purposes
	random.seed(1)
	# determine number of bands (drop remainder rows that do not form a band)
	B = sig_mat.shape[0]//R
	# initialize empty list of dictionaries, one for each band
	dict_list = [defaultdict(set) for b in range(B)]
	# for each band
	for b in range(B):
		# iterate over all documents
		for d in range(D):
			# use the vector of R integers, converted to a tuple, as key
			key = tuple(sig_mat[b*R:(b+1)*R, d])
			# add key-documentID pair to corresponding dictionary
			dict_list[b][key].add(d)
	# initialize numpy array to store average estimated Jaccard similarities
	avg_jaccard_sim = np.empty(D, dtype=float)
	# initialize count for number of documents whose neighbors were padded at random
	num_padded = 0
	# for each document
	for d in range(D):
		# initialize empty set of candidates
		candidates = set()
		# iterate over all bands
		for b in range(B):
			# compute the key of the document for the current dictionary
			key = tuple(sig_mat[b*R:(b+1)*R, d])
			# add all document IDs in the corresponding bucket to set of candidates
			candidates = candidates.union(dict_list[b][key])
		# remove ID of current document from set of candidates
		candidates.remove(d)
		# initialize numpy array to store Jaccard similarities
		jaccard_sim = np.empty(K, dtype=float)
		# if number of candidates <= K for current document
		if len(candidates) < K:
			# sample randomly from the other non-candidate documents
			docs = random.sample((set(range(d))|set(range(d+1, D)))-candidates, K-len(candidates))
			# pad set of candidates with these document IDs and recast as list
			candidates = list(candidates.union(set(docs)))
			num_padded += 1
			# then iterate through all K neighbors
			for k in range(K):
				# and compute the estimated Jaccard similarity and add it to array
				neighbor = candidates[k]
				jaccard_sim[k] = estimated_jaccard(sig_mat[:,d], sig_mat[:,neighbor])
		# otherwise
		else:
			# initialize empty priority queue to store K closest neighbors
			queue = []
			# loop over all candidates
			for j in candidates:
				# compute the Jaccard similarity between the current pair and enqueue
				hq.heappush(queue, (estimated_jaccard(sig_mat[:,d], sig_mat[:,j]), j))
				# if more than K items in queue
				if len(queue) > K:
					# remove document with lowest Jaccard similarity
					hq.heappop(queue)
			# iterate through all K neighbors
			for k in range(K):
				# obtain Jaccard similarity and add it to the array
				jaccard_sim[k] = hq.heappop(queue)[0]
		# compute the average Jaccard similarity and add to array at top
		avg_jaccard_sim[d] = np.mean(jaccard_sim)
	 	# print progress to screen
		if D >= 10:
			if (d+1) % (D//10) == 0:
		 		print('Average estimated Jaccard similarity for document ', d+1, ': ',
		 			'%.4f' % avg_jaccard_sim[d], sep='')
		 		sys.stdout.flush()
	# return average of averages along with time taken and number of documents whose
	# neighbors were chosen randomly
	return(np.mean(avg_jaccard_sim), time.time() - start_time, num_padded)

def main():
	# # obtain command line arguments
	# parser = argparse.ArgumentParser()
	# parser.add_argument('--D', type=int,
	# 	help='Integer number of documents to read into program')
	# parser.add_argument('--doc1', type=int, default=1,
	# 	help='Document ID of first document')
	# parser.add_argument('--doc2', type=int, default=2,
	# 	help='Document ID of second document')
	# parser.add_argument('--N', type=int, default=1000,
	# 	help='Number of rows of signature matrix')
	# parser.add_argument('--K', type=int,
	# 	help='Number of closest neighbors for the knn algorithm')
	# parser.add_argument('--R', type=int,
	# 	help='Number of rows in each band')
	# flags, unparsed = parser.parse_known_args()

	# get user input for number of documents D
	D = int(input('Input number of documents to read: '))
	# create character matrix stored as a list of sets
	chr_mat, time_taken0 = character_matrix(D)
	print('Created character matrix (%.4f seconds).' % float(time_taken0))
	# get user input for two documents whose Jaccard similarity is to be computed
	doc1 = int(input('Input first document ID: '))
	doc2 = int(input('Input second document ID: '))
	# compute Jaccard similarity between requested documents
	print('Actual Jaccard similarity for documents %s and %s: %.4f' %
		(doc1, doc2, jaccard(chr_mat[doc1-1], chr_mat[doc2-1])))
	# get user input for number of rows of signature matrix N
	N = int(input('Input number of rows for the signature matrix: '))
	# create signature matrix stored as numpy 2D array
	sig_mat, time_taken1 = signature_matrix(chr_mat, N, D)
	print('Created signature matrix (%.4f seconds)' % float(time_taken1))
	# compute estimated Jaccard similarity between requested documents
	print('Estimated Jaccard similarity for documents %s and %s: %.4f' %
		(doc1, doc2, estimated_jaccard(sig_mat[:,doc1-1],
							sig_mat[:,doc2-1])))
	print('------------------------------------------------------------------------------------------------')
	# get user input for number of nearest neighbors K
	K = int(input('Input number of nearest neighbors: '))
	# compute average Jaccard similarity in data set using brute force
	avg_acc1, time_taken2 = knn_brute_force(chr_mat, K=K, D=D)
	print('Average Jaccard similarity across all documents using brute force: %.4f (%.4f seconds)' \
		% (avg_acc1, float(time_taken2)))
	# get user input for number of rows per band R
	R = int(input('Input number of rows in each band: '))
	# compute average estimated Jaccard similarity using LSH
	avg_acc2, time_taken3, num_random = knn_lsh(sig_mat, K=K, D=D, R=R)
	print('Average estimated Jaccard similarity across all documents using banding and LSH: %.4f (%.4f seconds)' \
		% (avg_acc2, float(time_taken3)))
	print('Number of documents whose neighbors were chosen at random:', num_random)

if __name__ == '__main__':
	main()


import seaborn as sns
import matplotlib.pyplot as plt
import numpy as np

def buildJointPlot(embedded_array, n_trial_data, classes, strains, unique_classes, usingFullData=True, fname='full'):
    '''
    Create bivariate plot for TSNE visualization

    embedded_array
        2-dimensional array output from TSNE

    n_trial_data
        Dictionary with key = strain, value = number of flies recorded for this strain

    classes
        List of colors applied to each fly

    strains
        array containg the corresponding strain to each fly

    unique_classes
        List of colors applied to each strain

    usingFullData
        If true, displays the legend for the scatter plot (as there would be multiple strains)
        If only one strain is to be displayed, use False (the name of the used strain will appear in the title)

    fname
        Filename to save the plot
        "full" is left by default for the plot of the whole data
        For singular strain plots, the suggested filename is the name of the strain (e.g. 'MDN.png' for MDN)
    '''

    g = sns.JointGrid(embedded_array[:,0], embedded_array[:,1])
    g.plot_joint(sns.kdeplot)

    #Clear the axes containing the scatter plot
    g.ax_joint.cla()
    g.ax_marg_x.set_axis_off()
    g.ax_marg_y.set_axis_off()

    # set the current axis to be the joint plot's axis
    plt.sca(g.ax_joint)

    sc = plt.scatter(embedded_array[:,0], embedded_array[:,1], c = classes, label = strains)

    if usingFullData:
        lp = lambda i: plt.plot([], color = unique_classes[i], mec="none",
                                label=list(n_trial_data.keys())[i], ls="", marker="o")[0]

        handles = [lp(i) for i in np.arange(len(np.unique(strains)))]
        plt.legend(bbox_to_anchor=(1.01, 1), loc='upper left', handles=handles, ncol=2, prop={'size': 8})
    else:
        plt.title(fname)

    plt.show()
    plt.savefig(fname + '.png')


from flask import Blueprint
from flask import render_template
from flask import request, session, redirect, url_for
from routes import menu

loginController = Blueprint("loginController", __name__)

@loginController.route('/login', methods=['POST', 'GET'])
def login():
    """
    Обрабатывает функциональность входа в систему.

    Параметры:
    - Нет

    Возвращает:
    - Если пользователь уже авторизован, перенаправляет на страницу профиля
        пользователя.
    - Если метод запроса POST, устанавливает переменную сеанса 'userLogged'
        и перенаправляет на страницу профиля пользователя.
    - В противном случае, отображает страницу входа с указанным
        заголовком и меню.

    """
    if 'userLogged' in session:
        return redirect(url_for(
            'profileController.profile',
            username=session['userLogged']
        ))
    elif request.method == 'POST':
        session['userLogged'] = request.form['username']
        return redirect(
            url_for(
                'profileController.profile',
                username=session['userLogged']
            )
        )

    return render_template('login.html', title='Авторизация', menu=menu)


import random
import re
import signal
from collections import defaultdict
from contextlib import contextmanager
from typing import Any, Dict, List, Optional, Set, Tuple

from llama_index.program.predefined.evaporate.prompts import (
    DEFAULT_EXPECTED_OUTPUT_PREFIX_TMPL,
    DEFAULT_FIELD_EXTRACT_QUERY_TMPL,
    FN_GENERATION_PROMPT,
    SCHEMA_ID_PROMPT,
    FnGeneratePrompt,
    SchemaIDPrompt,
)
from llama_index.schema import BaseNode, MetadataMode, NodeWithScore, QueryBundle
from llama_index.service_context import ServiceContext

class TimeoutException(Exception):
    pass

@contextmanager
def time_limit(seconds: int) -> Any:
    """Time limit context manager.

    NOTE: copied from https://github.com/HazyResearch/evaporate.

    """

    def signal_handler(signum: Any, frame: Any) -> Any:
        raise TimeoutException("Timed out!")

    signal.signal(signal.SIGALRM, signal_handler)
    signal.alarm(seconds)
    try:
        yield
    finally:
        signal.alarm(0)

def get_function_field_from_attribute(attribute: str) -> str:
    """Get function field from attribute.

    NOTE: copied from https://github.com/HazyResearch/evaporate.

    """
    return re.sub(r"[^A-Za-z0-9]", "_", attribute)

def extract_field_dicts(result: str, text_chunk: str) -> Set:
    """Extract field dictionaries."""
    existing_fields = set()
    result = result.split("---")[0].strip("\n")
    results = result.split("\n")
    results = [r.strip("-").strip() for r in results]
    results = [r[2:].strip() if len(r) > 2 and r[1] == "." else r for r in results]
    for result in results:
        try:
            field = result.split(": ")[0].strip(":")
            value = ": ".join(result.split(": ")[1:])
        except Exception:
            print(f"Skipped: {result}")
            continue
        field_versions = [
            field,
            field.replace(" ", ""),
            field.replace("-", ""),
            field.replace("_", ""),
        ]
        if not any(f.lower() in text_chunk.lower() for f in field_versions):
            continue
        if not value:
            continue
        field = field.lower().strip("-").strip("_").strip(" ").strip(":")
        if field in existing_fields:
            continue
        existing_fields.add(field)

    return existing_fields

# since we define globals below
class EvaporateExtractor:
    """Wrapper around Evaporate.

    Evaporate is an open-source project from Stanford's AI Lab:
    https://github.com/HazyResearch/evaporate.
    Offering techniques for structured datapoint extraction.

    In the current version, we use the function generator
    from a set of documents.

    Args:
        service_context (Optional[ServiceContext]): Service Context to use.
    """

    def __init__(
        self,
        service_context: Optional[ServiceContext] = None,
        schema_id_prompt: Optional[SchemaIDPrompt] = None,
        fn_generate_prompt: Optional[FnGeneratePrompt] = None,
        field_extract_query_tmpl: str = DEFAULT_FIELD_EXTRACT_QUERY_TMPL,
        expected_output_prefix_tmpl: str = DEFAULT_EXPECTED_OUTPUT_PREFIX_TMPL,
        verbose: bool = False,
    ) -> None:
        """Initialize params."""
        # TODO: take in an entire index instead of forming a response builder
        self._service_context = service_context or ServiceContext.from_defaults()
        self._schema_id_prompt = schema_id_prompt or SCHEMA_ID_PROMPT
        self._fn_generate_prompt = fn_generate_prompt or FN_GENERATION_PROMPT
        self._field_extract_query_tmpl = field_extract_query_tmpl
        self._expected_output_prefix_tmpl = expected_output_prefix_tmpl
        self._verbose = verbose

    def identify_fields(
        self, nodes: List[BaseNode], topic: str, fields_top_k: int = 5
    ) -> List:
        """Identify fields from nodes.

        Will extract fields independently per node, and then
        return the top k fields.

        Args:
            nodes (List[BaseNode]): List of nodes to extract fields from.
            topic (str): Topic to use for extraction.
            fields_top_k (int): Number of fields to return.

        """
        field2count: dict = defaultdict(int)
        for node in nodes:
            llm = self._service_context.llm
            result = llm.predict(
                self._schema_id_prompt,
                topic=topic,
                chunk=node.get_content(metadata_mode=MetadataMode.LLM),
            )

            existing_fields = extract_field_dicts(
                result, node.get_content(metadata_mode=MetadataMode.LLM)
            )

            for field in existing_fields:
                field2count[field] += 1

        sorted_tups: List[Tuple[str, int]] = sorted(
            field2count.items(), key=lambda x: x[1], reverse=True
        )
        sorted_fields = [f[0] for f in sorted_tups]
        return sorted_fields[:fields_top_k]

    def extract_fn_from_nodes(
        self, nodes: List[BaseNode], field: str, expected_output: Optional[Any] = None
    ) -> str:
        """Extract function from nodes."""
        # avoid circular import
        from llama_index.response_synthesizers import (
            ResponseMode,
            get_response_synthesizer,
        )

        function_field = get_function_field_from_attribute(field)
        # TODO: replace with new response synthesis module

        if expected_output is not None:
            expected_output_str = (
                f"{self._expected_output_prefix_tmpl}{expected_output!s}\n"
            )
        else:
            expected_output_str = ""

        qa_prompt = self._fn_generate_prompt.partial_format(
            attribute=field,
            function_field=function_field,
            expected_output_str=expected_output_str,
        )

        response_synthesizer = get_response_synthesizer(
            service_context=self._service_context,
            text_qa_template=qa_prompt,
            response_mode=ResponseMode.TREE_SUMMARIZE,
        )

        # ignore refine prompt for now
        query_str = self._field_extract_query_tmpl.format(field=function_field)
        query_bundle = QueryBundle(query_str=query_str)
        response = response_synthesizer.synthesize(
            query_bundle,
            [NodeWithScore(node=n, score=1.0) for n in nodes],
        )
        fn_str = f"""def get_{function_field}_field(text: str):
    \"""
    Function to extract {field}.
    \"""
    {response!s}
"""

        # format fn_str
        return_idx_list = [i for i, s in enumerate(fn_str.split("\n")) if "return" in s]
        if not return_idx_list:
            return ""

        return_idx = return_idx_list[0]
        fn_str = "\n".join(fn_str.split("\n")[: return_idx + 1])
        fn_str = "\n".join([s for s in fn_str.split("\n") if "print(" not in s])
        return "\n".join(
            [s for s in fn_str.split("\n") if s.startswith((" ", "\t", "def"))]
        )

    def run_fn_on_nodes(
        self, nodes: List[BaseNode], fn_str: str, field_name: str, num_timeouts: int = 1
    ) -> List:
        """Run function on nodes.

        Calls python exec().

        There are definitely security holes with this approach, use with caution.

        """
        function_field = get_function_field_from_attribute(field_name)
        results = []
        for node in nodes:
            global result
            global node_text
            node_text = node.get_content()  # type: ignore[name-defined]
            # this is temporary
            result = []  # type: ignore[name-defined]
            try:
                with time_limit(1):
                    exec(fn_str, globals())
                    exec(f"result = get_{function_field}_field(node_text)", globals())
            except TimeoutException:
                raise
            results.append(result)  # type: ignore[name-defined]
        return results

    def extract_datapoints_with_fn(
        self,
        nodes: List[BaseNode],
        topic: str,
        sample_k: int = 5,
        fields_top_k: int = 5,
    ) -> List[Dict]:
        """Extract datapoints from a list of nodes, given a topic."""
        idxs = list(range(len(nodes)))
        sample_k = min(sample_k, len(nodes))
        subset_idxs = random.sample(idxs, sample_k)
        subset_nodes = [nodes[si] for si in subset_idxs]

        # get existing fields
        existing_fields = self.identify_fields(
            subset_nodes, topic, fields_top_k=fields_top_k
        )

        # then, for each existing field, generate function
        function_dict = {}
        for field in existing_fields:
            fn = self.extract_fn_from_nodes(subset_nodes, field)
            function_dict[field] = fn

        # then, run function for all nodes
        result_dict = {}
        for field in existing_fields:
            result_list = self.run_fn_on_nodes(nodes, function_dict[field], field)
            result_dict[field] = result_list

        # convert into list of dictionaries
        result_list = []
        for i in range(len(nodes)):
            result_dict_i = {}
            for field in existing_fields:
                result_dict_i[field] = result_dict[field][i]
            result_list.append(result_dict_i)
        return result_list