# -*- coding: utf-8 -*-
"""
Created on Fri Oct 29 10:51:03 2021

@author: Dina
"""

# imports

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from pyrolite.plot import pyroplot
import plotly.express as px
import ternary
from sklearn.metrics import mean_squared_error, r2_score
from sklearn.preprocessing import PolynomialFeatures
from sklearn.feature_extraction import DictVectorizer
# some values are below detection limit; these will be replaced with half of the detection limit
# there are occasional Xs instead of a number; these will be replaced by zeros

def item_cleanup(s):
    if type(s) != str:
        c = pd.to_numeric(s)
    elif s == "X":
        c = 0
    elif "<" in s:
        c = pd.to_numeric(s[1:]) / 2
    else:
        c = pd.to_numeric(s)
    return c

def data_cleanup(s):
    c = s.copy()
    for i in range(s.size):
        c.values[i] = item_cleanup(s.values[i])
    return c

# read the data
df = pd.read_csv('greenstone_geochemistry_rock_selected_groups.csv',encoding='cp1252')
# 1. EDA
# We will look at the Cu, Au, Ni (key ore elements), As and SO3 (maybe pathfinder elements, or elements that are
# contained in the ore minerals, but are not useful by themselves):
els = ['SO3_pct',
'Au_ppb',
'Ni_ppm',
'Cu_ppm',
'As_ppm']

ore_els = df[els].copy()
ore_els = ore_els.dropna()

ore_els = ore_els.apply(data_cleanup) # convert to numeric values

sns.histplot(ore_els['SO3_pct'], bins = 50) # bins is how many bars we have
plt.show()
sns.histplot(ore_els['Au_ppb'], bins = 50) # bins is how many bars we have
plt.show()
sns.histplot(ore_els['Ni_ppm'], bins = 50) # bins is how many bars we have
plt.show()
sns.histplot(ore_els['Cu_ppm'], bins = 50) # bins is how many bars we have
plt.show()
sns.histplot(ore_els['As_ppm'], bins = 50) # bins is how many bars we have
plt.show()

# all long tails except Au (very few significant values) and Ni (sort of bimodal)
# will need to apply log-transform

# 1.2 Basic plots for looking at rock composition (from Scott Halley's tutorial).

# plot the density plots of Si vs Mg, Fe, Cr, Al, Sc, Ti (page 4 of the tutorial)

silica_els = ['SiO2_pct',
                'TiO2_pct',
                'Al2O3_pct',
                'Fe2O3T_pct',
                'MgO_pct',
                'Sc_ppm',
                'Cr_ppm',
                'MnO_pct',
                'CaO_pct',
                'Na2O_pct',
                'K2O_pct',
                'P2O5_pct',
                'SO3_pct']

silica_et_al = df[silica_els].copy()
silica_et_al = silica_et_al.dropna()
silica_et_al = silica_et_al.apply(data_cleanup)

# multiple detection limits! leave for the moment, but for the future this has to be half of the lowest detection limit?

# Convert oxides to els by dividing by the molar weight of oxide and multiplyting by molar weight of element:
# Mg = MgO/40.3044*24.305
# Fe = Fe2O3/159.69*55.845
# Al = Al2O3/101.96*26.9815
# Ti = TiO2/79.866*47.867
silica_et_al['Mg'] = silica_et_al['MgO_pct']/40.3044*24.305
silica_et_al['Fe'] = silica_et_al['Fe2O3T_pct']/159.69*55.845
silica_et_al['Al'] = silica_et_al['Al2O3_pct']/101.96*26.9815
silica_et_al['Ti'] = silica_et_al['TiO2_pct']/79.866*47.867

# plot with pyrolite library
fig, ax = plt.subplots(2, 3, figsize=(14, 8))       # create a composite plot with 6 figures, 3 in 2 rows
ax = ax.flat                                       # this iterates through the figures...

Mg_vs_Si = ['SiO2_pct', 'Mg']
Fe_vs_Si = ['SiO2_pct', 'Fe']
Cr_vs_Si = ['SiO2_pct', 'Cr_ppm']
Al_vs_Si = ['SiO2_pct', 'Al']
Sc_vs_Si = ['SiO2_pct', 'Sc_ppm']
Ti_vs_Si = ['SiO2_pct', 'Ti']

silica_et_al.loc[:, Mg_vs_Si].pyroplot.density(ax=ax[0])
silica_et_al.loc[:, Fe_vs_Si].pyroplot.density(ax=ax[1])
silica_et_al.loc[:, Cr_vs_Si].pyroplot.density(ax=ax[2])
silica_et_al.loc[:, Al_vs_Si].pyroplot.density(ax=ax[3])
silica_et_al.loc[:, Sc_vs_Si].pyroplot.density(ax=ax[4])
silica_et_al.loc[:, Ti_vs_Si].pyroplot.density(ax=ax[5])

titles = ["SiO2,% vs Mg, %", "SiO2,% vs Fe, %", "SiO2,% vs Cr, ppm", "SiO2,% vs Al, %", "SiO2,% vs Sc, ppm", "SiO2,% vs Ti, %"]
for t, a in zip(titles + [i + " (log-log)" for i in titles], ax):    # put headers on the diagrams - leaving this pyroplot cycle for future log-log plots
    a.set_title(t)
plt.tight_layout()
plt.show()

fig, ax = plt.subplots(2, 3, figsize=(14, 8))       # create a composite plot with 6 figures, 3 in 2 rows
ax = ax.flat                                       # this iterates through the figures...
contours = [0.95, 0.7, 0.5, 0.3]  # set the density contours: cumulative 95% ... 30% of points

silica_et_al.loc[:, Mg_vs_Si].pyroplot.density(ax=ax[0], contours=contours)
silica_et_al.loc[:, Fe_vs_Si].pyroplot.density(ax=ax[1], contours=contours)
silica_et_al.loc[:, Cr_vs_Si].pyroplot.density(ax=ax[2], contours=contours)
silica_et_al.loc[:, Al_vs_Si].pyroplot.density(ax=ax[3], contours=contours)
silica_et_al.loc[:, Sc_vs_Si].pyroplot.density(ax=ax[4], contours=contours)
silica_et_al.loc[:, Ti_vs_Si].pyroplot.density(ax=ax[5], contours=contours)

for t, a in zip(titles + [i + " (log-log)" for i in titles], ax):    # put headers on the diagrams - leaving this pyroplot cycle for future log-log plots
    a.set_title(t)
plt.tight_layout()
plt.show()

# 1.2 Jensen plot
# calculate cation percentage for Fe+Ti, Al and Mg, normalize to sum of those three to plot on a ternary diagram
silica_et_al['cat_Si'] = silica_et_al['SiO2_pct']/60.09*1
silica_et_al['cat_Ti'] = silica_et_al['TiO2_pct']/79.9*1
silica_et_al['cat_Al'] = silica_et_al['Al2O3_pct']/101.96*2
silica_et_al['cat_Fe'] = silica_et_al['Fe2O3T_pct']/159.69*2
silica_et_al['cat_Mg'] = silica_et_al['MgO_pct']/40.3*1
silica_et_al['cat_Mn'] = silica_et_al['MnO_pct']/70.94*1
silica_et_al['cat_Ca'] = silica_et_al['CaO_pct']/56.08*1
silica_et_al['cat_Na'] = silica_et_al['Na2O_pct']/61.98*2
silica_et_al['cat_K'] = silica_et_al['K2O_pct']/94.2*2
silica_et_al['cat_P'] = silica_et_al['P2O5_pct']/141.95*2
silica_et_al['cat_S'] = silica_et_al['SO3_pct']/80.06*1

silica_et_al['cat_total'] = silica_et_al['cat_Si'] + silica_et_al['cat_Ti'] + silica_et_al['cat_Al'] + silica_et_al['cat_Fe'] + \
    silica_et_al['cat_Mg'] + silica_et_al['cat_Mn'] + silica_et_al['cat_Ca'] + silica_et_al['cat_Na'] + silica_et_al['cat_K'] + \
    silica_et_al['cat_P'] + silica_et_al['cat_S']

silica_et_al['cat_total_ternary'] = silica_et_al['cat_Al'] + silica_et_al['cat_Mg'] + silica_et_al['cat_Fe'] + silica_et_al['cat_Ti']

silica_et_al['cat_Fe_Ti_ternary'] = (silica_et_al['cat_Ti'] + silica_et_al['cat_Fe'])/silica_et_al['cat_total_ternary']
silica_et_al['cat_Mg_ternary'] = silica_et_al['cat_Mg']/silica_et_al['cat_total_ternary']
silica_et_al['cat_Al_ternary'] = silica_et_al['cat_Al']/silica_et_al['cat_total_ternary']

# make a ternary plot based on: https://colab.research.google.com/github/agile-geoscience/xlines/blob/master/notebooks/12_Ternary_diagrams.ipynb#scrollTo=NuDIFksmNObR

fig, tax = ternary.figure(scale=1)
fig.set_size_inches(5, 4.5)
# tax.horizontal_line(16)
# draw lines (Rollinson, page 62)
# p1 = (22, 8, 10)
# p2 = (2, 22, 16)
# tax.line(p1, p2, linewidth=3., marker='s', color='green', linestyle=":")

tax.scatter(silica_et_al[['cat_Mg_ternary', 'cat_Fe_Ti_ternary', 'cat_Al_ternary']].values, marker='o')
tax.gridlines(multiple=10)
tax.get_axes().axis('off')
# axis labels
fontsize = 12
offset = 0.14
tax.set_title("Jensen plot attempt\n", fontsize=fontsize)
tax.right_corner_label("Mg cation %", fontsize=fontsize)
tax.top_corner_label("Ti + Fe (total) cation %", fontsize=fontsize)
tax.left_corner_label("Al cation %", fontsize=fontsize)

# make a density plot, or a heatmap (color by Cu content)
# draw lines (Rollinson, page 62) - work in progress, doesn't want to plot
# p1 = (22, 8, 10)
# p2 = (2, 22, 16)
# tax.line(p1, p2, linewidth=3., marker='s', color='green', linestyle=":")
# p1, p2 = (90, 10,), (53.5, 28.5)
# tax.line(p1, p2, linewidth=3, color='k', alpha=0.35, linestyle="-")
# p1, p2 = (75, 25, 0), (75, 0, 25)
# tax.line(p1, p2, linewidth=3, color='k', alpha=0.35, linestyle="-")
# 90, 10, 0; 53.5, 28.5, 18; 52.5, 29,
# 18.5; 51.5, 29, 19.5; 50.5, 27.5, 22;
# 50.3, 25, 24.7; 50.8, 20, 29.2; 51.5,
# 12.5, 36

plt.savefig('ternary.jpg', dpi = 300)

# select the data that is likely to be of interest: all major elements, plus some indicator elements
# we test whether we can predict Cu and Ni from other selected elements

# define list of elements for each dataset separately

Cu_and_els = ['SiO2_pct',
         'TiO2_pct',
         'Al2O3_pct',
         'Fe2O3T_pct',
         'MgO_pct',
         'Sc_ppm',
         'Cr_ppm',
         'MnO_pct',
         'CaO_pct',
         'Na2O_pct',
         'K2O_pct',
         'P2O5_pct',
         'SO3_pct',
         'Sr_ppm',
         'Zr_ppm',
         'Y_ppm',
         'V_ppm',
         'Hf_ppm',
         'Ba_ppm',
         'Nb_ppm',
         'Cu_ppm',
         'Preferred chemical classification']

Ni_and_els = ['SiO2_pct',
         'TiO2_pct',
         'Al2O3_pct',
         'Fe2O3T_pct',
         'MgO_pct',
         'Sc_ppm',
         'Cr_ppm',
         'MnO_pct',
         'CaO_pct',
         'Na2O_pct',
         'K2O_pct',
         'P2O5_pct',
         'SO3_pct',
         'Sr_ppm',
         'Zr_ppm',
         'Y_ppm',
         'V_ppm',
         'Hf_ppm',
         'Ba_ppm',
         'Nb_ppm',
         'Ni_ppm']

# clean up data and convert to numerical
df_Cu = df[Cu_and_els].copy().dropna()
for i in Cu_and_els[:-1]:
    df_Cu[i] = df_Cu[i].apply(item_cleanup)

# df_Ni = df[Ni_and_els].copy()
# df_Ni = df_Ni.dropna()
# df_Ni = df_Ni.apply(data_cleanup)

# Split the dataset into training, validation and test sets. We should not really do it like that in
# geology, because the datasets are often not balanced and we need another kind of split that takes into
# account rock types

from sklearn.model_selection import train_test_split
df_full_train, df_test = train_test_split(df_Cu, test_size = 0.2, random_state = 1)

# convert objects to numerical
# we still have "objects" instead of numeric values in the dataset, I have no clue why
# so I convert them here explicitly

df_full_train['Cu_ppm'] = df_full_train['Cu_ppm'].astype(float)
df_full_train['Sc_ppm'] = df_full_train['Sc_ppm'].astype(float)
df_full_train['Cr_ppm'] = df_full_train['Cr_ppm'].astype(float)
df_full_train['MnO_pct'] = df_full_train['MnO_pct'].astype(float)
df_full_train['CaO_pct'] = df_full_train['CaO_pct'].astype(float)
df_full_train['Na2O_pct'] = df_full_train['Na2O_pct'].astype(float)
df_full_train['K2O_pct'] = df_full_train['K2O_pct'].astype(float)
df_full_train['SO3_pct'] = df_full_train['SO3_pct'].astype(float)
df_full_train['Sr_ppm'] = df_full_train['Sr_ppm'].astype(float)
df_full_train['Zr_ppm'] = df_full_train['Zr_ppm'].astype(float)
df_full_train['V_ppm'] = df_full_train['V_ppm'].astype(float)
df_full_train['Hf_ppm'] = df_full_train['Hf_ppm'].astype(float)
df_full_train['Ba_ppm'] = df_full_train['Ba_ppm'].astype(float)
df_full_train['Nb_ppm'] = df_full_train['Nb_ppm'].astype(float)

df_train, df_val = train_test_split(df_full_train, test_size = 0.25)

# reset indices
df_train = df_train.reset_index(drop = True)
df_val = df_train.reset_index(drop = True)
df_test = df_train.reset_index(drop = True)

y_train = df_train.Cu_ppm.values
y_val = df_val.Cu_ppm.values
y_test = df_test.Cu_ppm.values

# just in case our Cu long tail is real, we do a log transform and see if it changes the RMSE

y_train_log = np.log1p(df_train.Cu_ppm).values
y_val_log = np.log1p(df_val.Cu_ppm).values
y_test_log = np.log1p(df_test.Cu_ppm).values

# delete the target variable - could cause overfitting if occasionally used for training

del df_train['Cu_ppm']
del df_val['Cu_ppm']
del df_test['Cu_ppm']
# 1.2 Feature importance

# check correlation
X_columns = ['SiO2_pct',
         'TiO2_pct',
         'Al2O3_pct',
         'Fe2O3T_pct',
         'MgO_pct',
         'Sc_ppm',
         'Cr_ppm',
         'MnO_pct',
         'CaO_pct',
         'Na2O_pct',
         'K2O_pct',
         'P2O5_pct',
         'SO3_pct',
         'Sr_ppm',
         'Zr_ppm',
         'Y_ppm',
         'V_ppm',
         'Hf_ppm',
         'Ba_ppm',
         'Nb_ppm']

# check the correlation between each input element and the output element, Cu
corr_check = df_full_train[X_columns].corrwith(df_full_train.Cu_ppm)

# None of the elements are highly correlated with Cu

# Logistic regression with scikit learn
from sklearn.linear_model import LinearRegression
model_LR = LinearRegression()
X_train = df_train[X_columns]
X_val = df_val[X_columns]
X_test = df_test[X_columns]
model_LR.fit(X_train, y_train)
y_pred = model_LR.predict(X_val)
rmse_LR = np.sqrt(mean_squared_error(y_val, y_pred))

# rmse of 65 for predicting ppms of Cu means that whenever we are predicting Cu values from other elements with
# linear regression, we will be on average 65 ppm off from the true value for the validation dataset
# plot the predictions
plt.figure()
sns.histplot (y_pred, color = 'red', alpha = 0.5, bins = 50)
sns.histplot(y_val, color = 'blue', alpha = 0.5, bins = 50)
plt.xlim(0, 500)
plt.show()
# from the figure we see that the predictions are kind of off, and Cu had a long tail, so we will test on the log-transformed variable

model_LR_log = LinearRegression()
X_train = df_train[X_columns]
X_val = df_val[X_columns]
X_test = df_test[X_columns]
model_LR_log.fit(X_train, y_train_log)
y_pred_log = model_LR_log.predict(X_val)
rmse_LR_log = np.sqrt(mean_squared_error(y_val_log, y_pred_log))
rmse_LR_log_exp = np.exp(rmse_LR_log)

# plot the predictions that resulted from linear regression on the log-transformed values;
# to be comparable with the prediction on the non-log-transformed values, take the exponent
y_pred_LR_log = np.exp(y_pred_log)

plt.figure()
sns.histplot (y_pred_LR_log, color = 'red', alpha = 0.5, bins = 50)
sns.histplot(y_val, color = 'blue', alpha = 0.5, bins = 50)
plt.xlim(0, 500)
plt.show()

# logtransform really means improvement in rmse and in the character of data distribution;
# however, it looks like it was safe to predict low values because of the structure of the dataset,
# so the linear regression got pessimistic

# let's try with polynomial regression on log-transformed data
# we train the model with several order of the polynomials and compare the results
order_polynomial = [2, 3, 4]

for i in order_polynomial:
    polynomial_features = PolynomialFeatures(degree=i)
    X_train_poly = polynomial_features.fit_transform(X_train)
    X_val_poly = polynomial_features.fit_transform(X_val)

    model_polynomial = LinearRegression()
    model_polynomial.fit(X_train_poly, y_train)
    y_poly_pred = model_polynomial.predict(X_val_poly)
    plt.figure()
    sns.histplot (y_poly_pred, color = 'red', alpha = 0.5, bins = 50)
    sns.histplot(y_val, color = 'blue', alpha = 0.5, bins = 50)
    # plt.xlim(0, 500)
    plt.show()
    rmse = np.sqrt(mean_squared_error(y_val, y_poly_pred))
    r2 = r2_score(y_val, y_poly_pred)
    print('polynomial of order', i, 'rmse', rmse)

# I choose the 4th order, although it seems it's overfitting, it reproduces the shape of the data distribution best
# besides, both 2nd and 3rd order polynomials ended up predicting some of the Cu values as negative
# does it make sense to work with polynomials of the log-transformed values?

# Now let's try to use dictvectorizer to code the output variable and predict rock types from chemistry
# first additional manual data clean-up: removing the spaces, checking the rock types for typos
# the dataset is not balanced, but we will see how it gets split;
# an idea is to use only rock types with more than 50 samples in the group

# Logistic regression

from sklearn.linear_model import LogisticRegression
numerical = ['SiO2_pct',
         'TiO2_pct',
         'Al2O3_pct',
         'Fe2O3T_pct',
         'MgO_pct',
         'Sc_ppm',
         'Cr_ppm',
         'MnO_pct',
         'CaO_pct',
         'Na2O_pct',
         'K2O_pct',
         'P2O5_pct',
         'SO3_pct',
         'Sr_ppm',
         'Zr_ppm',
         'Y_ppm',
         'V_ppm',
         'Hf_ppm',
         'Ba_ppm',
         'Nb_ppm']

# choose rock types as y variable

categorical = ['Preferred chemical classification']

# make a new split of the training, validation and test sets trying to keep it balanced with stratify
df_full_train_rocks, df_test_rocks = train_test_split(df_Cu, test_size = 0.2, random_state = 1, stratify=df_Cu[categorical])

# convert objects to numerical
# we still have "objects" instead of numeric values in the dataset, I have no clue why
# so I convert them here explicitly

df_full_train['Cu_ppm'] = df_full_train['Cu_ppm'].astype(float)
df_full_train['Sc_ppm'] = df_full_train['Sc_ppm'].astype(float)
df_full_train['Cr_ppm'] = df_full_train['Cr_ppm'].astype(float)
df_full_train['MnO_pct'] = df_full_train['MnO_pct'].astype(float)
df_full_train['CaO_pct'] = df_full_train['CaO_pct'].astype(float)
df_full_train['Na2O_pct'] = df_full_train['Na2O_pct'].astype(float)
df_full_train['K2O_pct'] = df_full_train['K2O_pct'].astype(float)
df_full_train['SO3_pct'] = df_full_train['SO3_pct'].astype(float)
df_full_train['Sr_ppm'] = df_full_train['Sr_ppm'].astype(float)
df_full_train['Zr_ppm'] = df_full_train['Zr_ppm'].astype(float)
df_full_train['V_ppm'] = df_full_train['V_ppm'].astype(float)
df_full_train['Hf_ppm'] = df_full_train['Hf_ppm'].astype(float)
df_full_train['Ba_ppm'] = df_full_train['Ba_ppm'].astype(float)
df_full_train['Nb_ppm'] = df_full_train['Nb_ppm'].astype(float)

df_train_rocks, df_val_rocks = train_test_split(df_full_train_rocks, test_size = 0.25, stratify=df_full_train_rocks[categorical])

# reset indices
df_train_rocks = df_train_rocks.reset_index(drop = True)
df_val_rocks = df_train_rocks.reset_index(drop = True)
df_test_rocks = df_train_rocks.reset_index(drop = True)

X_train_rocks = df_train_rocks[numerical].values
X_val_rocks = df_val_rocks[numerical].values
X_test_rocks = df_test_rocks[numerical].values

y_train_rocks = df_train_rocks[categorical].values
y_val_rocks = df_val_rocks[categorical].values
y_test_rocks = df_test_rocks[categorical].values

lr = LogisticRegression()

lr.fit(X_train_rocks, y_train_rocks)

y_pred_rocks = lr.predict(X_val_rocks)
accuracy_LR = (y_pred_rocks == y_val_rocks).mean()

# This was done on the dataset with labels "as is", and whoops, we can only guess the rock type in 10% of the cases. A random number generator could do a better job.
# Let's take a smaller subset which is balanced

# increase of accuracy to 12% - where did it all go wrong?

from sklearn.tree import DecisionTreeClassifier

dt = DecisionTreeClassifier(max_depth = 4)
dt.fit(X_train_rocks, y_train_rocks)
dt_predict = dt.predict(X_val_rocks)

# confusion matrix
from sklearn.metrics import confusion_matrix
confusion_matrix(y_val_rocks, dt_predict)
accuracy_dt = (dt_predict == y_val_rocks).mean()

rocks_in_train = np.unique(df_train_rocks[categorical].values)
rocks_in_val = np.unique(df_val_rocks[categorical].values)
rocks_in_test = np.unique(df_test_rocks[categorical].values)

# save the confusion matrix
conf = confusion_matrix(y_val_rocks, dt_predict)
df = pd.DataFrame(conf)

plt.figure(figsize = (15,15))
cm = conf
ax = plt.subplot()
sns.heatmap(cm, annot=True, ax = ax, cmap='Greens'); #annot=True to annotate cells

# labels, title and ticks
ax.set_xlabel('Predicted labels');ax.set_ylabel('True labels');
ax.set_title('Confusion Matrix, decision trees');
ax.xaxis.set_ticklabels(['Basalt High-Ti', 'Dolerite LTB', 'Dolerite/gabbro LTB', 'EMI',
       'Felsic dyke (dacitic)', 'Felsic dyke (rhyolitic)',
       'Felsic vol/volclas (andesitic)', 'Felsic vol/volclas (dacitic)',
       'Felsic vol/volclas (rhyolitic)', 'Granite', 'HTSB', 'ITB',
       'Komatiite', 'Komatiite High-Al', 'LTB', 'LTB High-Nb',
       'Problematic data'], rotation = 90);
ax.yaxis.set_ticklabels(['Basalt High-Ti', 'Dolerite LTB', 'Dolerite/gabbro LTB', 'EMI',
       'Felsic dyke (dacitic)', 'Felsic dyke (rhyolitic)',
       'Felsic vol/volclas (andesitic)', 'Felsic vol/volclas (dacitic)',
       'Felsic vol/volclas (rhyolitic)', 'Granite', 'HTSB', 'ITB',
       'Komatiite', 'Komatiite High-Al', 'LTB', 'LTB High-Nb',
       'Problematic data'], rotation = 45);

plt.savefig('confusion matrix dtree.jpg', dpi = 300)

# I have 17 groups! Why are there 20?
res = pd.DataFrame(dt_predict)
res.to_csv("prediction_results.csv")

# when in doubt, use Gradient Boost

from sklearn.ensemble import GradientBoostingClassifier
gb = GradientBoostingClassifier(n_estimators=100, learning_rate=1.0, max_depth=1, random_state=0).fit(X_train_rocks, y_train_rocks)
gb.fit(X_train_rocks, y_train_rocks)
gb_predict = gb.predict(X_val_rocks)
gb_score = gb.score(X_val_rocks, y_val_rocks)

confusion_matrix(y_val_rocks, gb_predict)
accuracy_gb = (gb_predict == y_val_rocks).mean()

# save the confusion matrix
conf_gb = confusion_matrix(y_val_rocks, gb_predict)
df = pd.DataFrame(conf_gb)

plt.figure(figsize = (15,15))
cm = conf_gb
ax = plt.subplot()
sns.heatmap(cm, annot=True, ax = ax, cmap='Greens'); #annot=True to annotate cells

# labels, title and ticks
ax.set_xlabel('Predicted labels');ax.set_ylabel('True labels');
ax.set_title('Confusion Matrix, Gradient boost');
ax.xaxis.set_ticklabels(['Basalt High-Ti', 'Dolerite LTB', 'Dolerite/gabbro LTB', 'EMI',
       'Felsic dyke (dacitic)', 'Felsic dyke (rhyolitic)',
       'Felsic vol/volclas (andesitic)', 'Felsic vol/volclas (dacitic)',
       'Felsic vol/volclas (rhyolitic)', 'Granite', 'HTSB', 'ITB',
       'Komatiite', 'Komatiite High-Al', 'LTB', 'LTB High-Nb',
       'Problematic data'], rotation = 90);
ax.yaxis.set_ticklabels(['Basalt High-Ti', 'Dolerite LTB', 'Dolerite/gabbro LTB', 'EMI',
       'Felsic dyke (dacitic)', 'Felsic dyke (rhyolitic)',
       'Felsic vol/volclas (andesitic)', 'Felsic vol/volclas (dacitic)',
       'Felsic vol/volclas (rhyolitic)', 'Granite', 'HTSB', 'ITB',
       'Komatiite', 'Komatiite High-Al', 'LTB', 'LTB High-Nb',
       'Problematic data'], rotation = 45);

plt.savefig('confusion matrix gboost.jpg', dpi = 300)

# whatever. Random forest.
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV
n_estimators = [50,100,200,500,1000]
max_depth = [3,5,10,15,20,None]
##
parameters = {'n_estimators':n_estimators, 'max_depth':max_depth}
classifier = RandomForestClassifier(random_state=27)
modelGS = GridSearchCV(estimator=classifier,param_grid=parameters,cv=10,n_jobs=-1)
modelGS.fit(X_train_rocks, y_train_rocks)

modelGS_predict = modelGS.predict(X_val_rocks)
IGS_score = modelGS.score(X_val_rocks, y_val_rocks)

confusion_matrix(y_val_rocks, modelGS_predict)
accuracy_rf = (modelGS_predict == y_val_rocks).mean()

# save the confusion matrix
conf_rf = confusion_matrix(y_val_rocks, modelGS_predict)
df = pd.DataFrame(conf_rf)

plt.figure(figsize = (15,15))
cm = conf_rf
ax = plt.subplot()
sns.heatmap(cm, annot=True, ax = ax, cmap='Greens'); #annot=True to annotate cells

# labels, title and ticks
ax.set_xlabel('Predicted labels');ax.set_ylabel('True labels');
ax.set_title('Confusion Matrix, random forest');
ax.xaxis.set_ticklabels(['Basalt High-Ti', 'Dolerite LTB', 'Dolerite/gabbro LTB', 'EMI',
       'Felsic dyke (dacitic)', 'Felsic dyke (rhyolitic)',
       'Felsic vol/volclas (andesitic)', 'Felsic vol/volclas (dacitic)',
       'Felsic vol/volclas (rhyolitic)', 'Granite', 'HTSB', 'ITB',
       'Komatiite', 'Komatiite High-Al', 'LTB', 'LTB High-Nb',
       'Problematic data'], rotation = 90);
ax.yaxis.set_ticklabels(['Basalt High-Ti', 'Dolerite LTB', 'Dolerite/gabbro LTB', 'EMI',
       'Felsic dyke (dacitic)', 'Felsic dyke (rhyolitic)',
       'Felsic vol/volclas (andesitic)', 'Felsic vol/volclas (dacitic)',
       'Felsic vol/volclas (rhyolitic)', 'Granite', 'HTSB', 'ITB',
       'Komatiite', 'Komatiite High-Al', 'LTB', 'LTB High-Nb',
       'Problematic data'], rotation = 45);

plt.savefig('confusion random forest.jpg', dpi = 300)

# let's see how it performs on the test set

dt_test_predict = dt.predict(X_test_rocks)
# save the confusion matrix
conf_dt = confusion_matrix(y_test_rocks, dt_test_predict)
df = pd.DataFrame(conf_dt)

plt.figure(figsize = (15,15))
cm = conf_dt
ax = plt.subplot()
sns.heatmap(cm, annot=True, ax = ax, cmap='Greens'); #annot=True to annotate cells

# labels, title and ticks
ax.set_xlabel('Predicted labels');ax.set_ylabel('True labels');
ax.set_title('Confusion Matrix, decision trees, test set');
ax.xaxis.set_ticklabels(['Basalt High-Ti', 'Dolerite LTB', 'Dolerite/gabbro LTB', 'EMI',
       'Felsic dyke (dacitic)', 'Felsic dyke (rhyolitic)',
       'Felsic vol/volclas (andesitic)', 'Felsic vol/volclas (dacitic)',
       'Felsic vol/volclas (rhyolitic)', 'Granite', 'HTSB', 'ITB',
       'Komatiite', 'Komatiite High-Al', 'LTB', 'LTB High-Nb',
       'Problematic data'], rotation = 90);
ax.yaxis.set_ticklabels(['Basalt High-Ti', 'Dolerite LTB', 'Dolerite/gabbro LTB', 'EMI',
       'Felsic dyke (dacitic)', 'Felsic dyke (rhyolitic)',
       'Felsic vol/volclas (andesitic)', 'Felsic vol/volclas (dacitic)',
       'Felsic vol/volclas (rhyolitic)', 'Granite', 'HTSB', 'ITB',
       'Komatiite', 'Komatiite High-Al', 'LTB', 'LTB High-Nb',
       'Problematic data'], rotation = 45);

plt.savefig('confusion matrix dtree test.jpg', dpi = 300)

gb_predict_test = gb.predict(X_test_rocks)
gb_score = gb.score(X_test_rocks, y_test_rocks)

confusion_matrix(y_test_rocks, gb_predict)
accuracy_gb = (gb_predict_test == y_test_rocks).mean()

# save the confusion matrix
conf_gb_test = confusion_matrix(y_test_rocks, gb_predict_test)
df = pd.DataFrame(conf_gb_test)

plt.figure(figsize = (15,15))
cm = conf_gb_test
ax = plt.subplot()
sns.heatmap(cm, annot=True, ax = ax, cmap='Greens'); #annot=True to annotate cells

# labels, title and ticks
ax.set_xlabel('Predicted labels');ax.set_ylabel('True labels');
ax.set_title('Confusion Matrix, Gradient boost, test set');
ax.xaxis.set_ticklabels(['Basalt High-Ti', 'Dolerite LTB', 'Dolerite/gabbro LTB', 'EMI',
       'Felsic dyke (dacitic)', 'Felsic dyke (rhyolitic)',
       'Felsic vol/volclas (andesitic)', 'Felsic vol/volclas (dacitic)',
       'Felsic vol/volclas (rhyolitic)', 'Granite', 'HTSB', 'ITB',
       'Komatiite', 'Komatiite High-Al', 'LTB', 'LTB High-Nb',
       'Problematic data'], rotation = 90);
ax.yaxis.set_ticklabels(['Basalt High-Ti', 'Dolerite LTB', 'Dolerite/gabbro LTB', 'EMI',
       'Felsic dyke (dacitic)', 'Felsic dyke (rhyolitic)',
       'Felsic vol/volclas (andesitic)', 'Felsic vol/volclas (dacitic)',
       'Felsic vol/volclas (rhyolitic)', 'Granite', 'HTSB', 'ITB',
       'Komatiite', 'Komatiite High-Al', 'LTB', 'LTB High-Nb',
       'Problematic data'], rotation = 45);

plt.savefig('confusion matrix gboost test.jpg', dpi = 300)

modelGS_predict_test = modelGS.predict(X_test_rocks)
IGS_score_test = modelGS.score(X_test_rocks, y_test_rocks)

confusion_matrix(y_test_rocks, modelGS_predict)
accuracy_rf = (modelGS_predict == y_test_rocks).mean()

# save the confusion matrix
conf_rf_test = confusion_matrix(y_test_rocks, modelGS_predict_test)
df = pd.DataFrame(conf_rf_test)

plt.figure(figsize = (15,15))
cm = conf_rf_test
ax = plt.subplot()
sns.heatmap(cm, annot=True, ax = ax, cmap='Greens'); #annot=True to annotate cells

# labels, title and ticks
ax.set_xlabel('Predicted labels');ax.set_ylabel('True labels');
ax.set_title('Confusion Matrix, random forest, test set');
ax.xaxis.set_ticklabels(['Basalt High-Ti', 'Dolerite LTB', 'Dolerite/gabbro LTB', 'EMI',
       'Felsic dyke (dacitic)', 'Felsic dyke (rhyolitic)',
       'Felsic vol/volclas (andesitic)', 'Felsic vol/volclas (dacitic)',
       'Felsic vol/volclas (rhyolitic)', 'Granite', 'HTSB', 'ITB',
       'Komatiite', 'Komatiite High-Al', 'LTB', 'LTB High-Nb',
       'Problematic data'], rotation = 90);
ax.yaxis.set_ticklabels(['Basalt High-Ti', 'Dolerite LTB', 'Dolerite/gabbro LTB', 'EMI',
       'Felsic dyke (dacitic)', 'Felsic dyke (rhyolitic)',
       'Felsic vol/volclas (andesitic)', 'Felsic vol/volclas (dacitic)',
       'Felsic vol/volclas (rhyolitic)', 'Granite', 'HTSB', 'ITB',
       'Komatiite', 'Komatiite High-Al', 'LTB', 'LTB High-Nb',
       'Problematic data'], rotation = 45);

plt.savefig('confusion random forest test set.jpg', dpi = 300)


from luggfinder.db_models import Process, db
from flask import render_template, Blueprint, request, redirect, url_for

user_bp = Blueprint('user', __name__)

@user_bp.route('/user')
def home():
    query = Process.query.all()
    suppliers = {"", "Gv Express", "Três Company", "Voucher", "Retirou"}

    return render_template("user_pages/user.html",
                            title="User",
                            query=query,
                            suppliers=suppliers)

@user_bp.route('/update_process', methods=['POST'])
def update_process():
    json_data = request.get_json()

    for i in json_data:
        update_process = Process.query.filter_by(process=json_data[i]["process"]).first();

        if update_process:
            update_process.date = json_data[i]["date"]
            update_process.name = json_data[i]["name"]
            update_process.pnr = json_data[i]["pnr"]
            update_process.process_type = json_data[i]["process_type"]
            update_process.fault_station = json_data[i]["fault_station"]
            update_process.process_reason = json_data[i]["process_reason"]
            update_process.cost = json_data[i]["cost"]
            update_process.supplier = json_data[i]["supplier"]

            db.session.commit()

    return "received"

# @user_bp.route('/user/new_process')
# def new_process():
#     add_process = Process(
#             process = json_data[i]["process"],
#             date = json_data[i]["date"],
#             name = json_data[i]["name"],
#             pnr = json_data[i]["pnr"],
#             process_type = json_data[i]["process_type"],
#             fault_station = json_data[i]["fault_station"],
#             process_reason = json_data[i]["process_reason"],
#             cost = json_data[i]["cost"],
#             supplier = json_data[i]["supplier"]
#             )

#     db.session.add(add_process)

@user_bp.route('/delete/<process>')
def delete_process(process):
    to_delete = Process.query.filter_by(process=process.upper()).first();
    print(to_delete)
    db.session.delete(to_delete)
    db.session.commit()

    return redirect(url_for('user.home'))


import functools
from sklearn.utils.class_weight import compute_class_weight
import numpy as np
import tensorflow.compat.v1 as tf
tf.disable_v2_behavior()
from tensorflow.compat.v1  import set_random_seed
from tensorflow.python.framework import ops
from tensorflow.python.client import device_lib
print('device_lib.list_local_devices()',device_lib.list_local_devices())

BATCH_SIZE = 128
EPOCHS = 12
KEEP_PROB = 0.8
REGULARIZATION = 0.001
NUM_HIDDEN = [512, 512]
RANDOM = 0
NUM_CLASSES = 4

def lazy_property(function):
    attribute = '_' + function.__name__

    @property
    @functools.wraps(function)
    def wrapper(self):
        if not hasattr(self, attribute):
            setattr(self, attribute, function(self))
        return getattr(self, attribute)
    return wrapper

class VariableSequenceLabelling:

    def __init__(self, data, target, dropout_prob, reg, num_hidden, class_weights):
        self.data = data
        self.target = target
        self.dropout_prob = dropout_prob
        self.reg = reg
        self._num_hidden = num_hidden
        self._num_layers = len(num_hidden)
        self.num_classes = len(class_weights)
        self.attn_length = 0
        self.class_weights = class_weights
        self.prediction
        self.error
        self.optimize

    @lazy_property
    def make_rnn_cell(self,
                      attn_length=0,
                      base_cell=tf.compat.v1.nn.rnn_cell.BasicRNNCell):

        attn_length = self.attn_length
        input_dropout = self.dropout_prob
        output_dropout = self.dropout_prob

        cells = []
        for num_units in self._num_hidden:
            cell = base_cell(num_units)
            cell = tf.nn.rnn_cell.DropoutWrapper(cell, input_keep_prob=input_dropout, output_keep_prob=output_dropout,
                                                seed=RANDOM)
            cells.append(cell)

        cell = tf.nn.rnn_cell.MultiRNNCell(cells)

        return cell

    # predictor for slices
    @lazy_property
    def prediction(self):

        cell = self.make_rnn_cell

        # Recurrent network.
        output, final_state = tf.nn.dynamic_rnn(cell,
            self.data,
            dtype=tf.float32
        )

        with tf.variable_scope("model") as scope:
            tf.get_variable_scope().reuse_variables()

            # final weights
            num_classes = self.num_classes
            weight, bias = self._weight_and_bias(self._num_hidden[-1], num_classes)

            # flatten + sigmoid
            if self.attn_length > 0:
                logits = tf.matmul(final_state[0][-1][-1], weight) + bias
            else:
                logits = tf.matmul(final_state[-1][-1], weight) + bias

            prediction = tf.nn.softmax(logits)

            return logits, prediction

    @lazy_property
    def cross_ent(self):
        predictions = self.prediction[0]
        real = tf.cast(tf.squeeze(self.target), tf.int32)
        weights = tf.gather(self.class_weights, real)

        xent = tf.losses.sparse_softmax_cross_entropy(labels=real, logits=predictions, weights=weights)
        loss = tf.reduce_mean(xent) #shape 1
        ce = loss
        l2 = self.reg * sum(tf.nn.l2_loss(tf_var) for tf_var in tf.trainable_variables())
        ce += l2
        return ce

    @lazy_property
    def optimize(self):
        learning_rate = 0.0003
        optimizer = tf.train.AdamOptimizer(learning_rate)
        return optimizer.minimize(self.cross_ent)

    @lazy_property
    def error(self):
        prediction = tf.argmax(self.prediction[1], 1)
        real = tf.cast(self.target, tf.int32)
        prediction = tf.cast(prediction, tf.int32)
        mistakes = tf.not_equal(real, prediction)
        mistakes = tf.cast(mistakes, tf.float32)
        mistakes = tf.reduce_sum(mistakes, reduction_indices=0)
        total = 128
        mistakes = tf.divide(mistakes, tf.to_float(total))
        return mistakes

    @staticmethod
    def _weight_and_bias(in_size, out_size):
        weight = tf.truncated_normal([in_size, out_size], stddev=0.01)
        bias = tf.constant(0.1, shape=[out_size])
        return tf.Variable(weight), tf.Variable(bias)

    @lazy_property
    def summaries(self):
        tf.summary.scalar('loss', tf.reduce_mean(self.cross_ent))
        tf.summary.scalar('error', self.error)
        merged = tf.summary.merge_all()
        return merged

def LSTM_Model(x_train, y_train, x_test, x_val):
    class_weight = compute_class_weight('balanced', classes=np.unique(y_train), y=y_train)
    print('class_weight', class_weight)
    classes = np.unique(y_train)
    print('classes', classes)
    print('LSTM starting')
    ops.reset_default_graph()
    set_random_seed(RANDOM)
    config = tf.compat.v1.ConfigProto(allow_soft_placement = True)

    sess = tf.compat.v1.Session(config = config)
    with  sess, tf.device('/cpu:0'):
        _, length, num_features = x_train.shape
        num_data_cols = num_features

        data = tf.placeholder(tf.float32, [None, length, num_data_cols])
        target =  tf.placeholder(tf.float32, [None])
        dropout_prob =  tf.placeholder(tf.float32)
        reg =  tf.placeholder(tf.float32)

        # initialization
        model = VariableSequenceLabelling(data, target, dropout_prob, reg, num_hidden=NUM_HIDDEN, class_weights=class_weight)
        sess.run(tf.global_variables_initializer())

        batch_size = BATCH_SIZE
        dp = KEEP_PROB
        rp = REGULARIZATION
        train_samples = x_train.shape[0]
        indices = list(range(train_samples))
        num_classes = NUM_CLASSES

        # for storing results
        test_data = x_test
        val_data = x_val

        val_aucs = []
        test_aucs = []
        val_aucs_macro = []
        test_aucs_macro = []
        test_accuracys = []
        test_f1s = []
        test_auprcs = []

    epoch = -1


import os
import mmcv
import urllib.request

dir_path = os.path.dirname(os.getcwd())

labelbox_file = dir_path + '/cleaning/export-2019-07-11T20_41_53.065Z.json'
out_dir = dir_path + '/cleaning/711_with_rotate/'

def main():
    mmcv.mkdir_or_exist(out_dir)

    items = mmcv.load(labelbox_file)
    print('downloading {} images. '.format(len(items)))

    count = 1
    for item in items:
        name = item['DataRow ID']
        image_link = item['Labeled Data']

        # img_data = requests.get(image_link).content
        # with open(out_dir+'{}.jpg'.format(name), 'wb') as handler:
        #     handler.write(img_data)
        urllib.request.urlretrieve(image_link, out_dir+'{}'.format(name))

        print(count)
        count = count + 1

if __name__ == '__main__':
    main()


import os

from datetime import datetime

pathw1="/var/www/html/webcam1/"
pathw2="/var/www/html/webcam2/"
path="/home/pi/motion/"
path_arch = "archive/"
dest="/MalinaBencol/"

year = datetime.now ().strftime ('%Y')
month = datetime.now ().strftime ('%m')
day = datetime.now ().strftime ('%d')
def upload_files():

    if not os.path.exists(pathw1):

        return

    dir_list1 = os.listdir(pathw1)

    for file_name1 in dir_list1:

        if file_name1.endswith(".avi"):

            cmd = "sudo /home/pi/dropbox_uploader.sh upload " + pathw1 + file_name1 + " "+ datetime.now().strftime('%Y-%m-%d') + "/webcam1/" + file_name1

            os.system(cmd)

            if not os.path.exists(path + path_arch):
                os.system ('mkdir ' + path + path_arch)

            if not os.path.exists (path + path_arch + "/" + year + "/"):
                os.system ("mkdir " + path + path_arch + "/" + year)

            os.system("sudo mv /var/www/html/webcam1/" + file_name1 + " /home/pi/motion/archive/" + file_name1)

    if not os.path.exists(pathw2):
        return

    dir_list2 = os.listdir(pathw2)
    for file_name2 in dir_list2:
        if file_name2.endswith(".avi"):
            cmd = "sudo /home/pi/dropbox_uploader.sh upload " + pathw2 + file_name2 + " "+ datetime.now().strftime('%Y-%m-%d') + "/webcam2/" + file_name2
            os.system(cmd)
            if not os.path.exists(path + path_arch):
                os.system ('mkdir ' + path + path_arch)
            if not os.path.exists (path + path_arch + "/" + year + "/"):
                os.system ("mkdir " + path + path_arch + "/" + year)
            os.system("sudo mv /var/www/html/webcam2/" + file_name2 + " /home/pi/motion/archive/" + file_name2)

#if _name_ == "_main_":

upload_files()


import datetime
import os
import yaml
from flask import request
from flask_restful import Api, Resource
from sqlalchemy import exc
import sqlalchemy.exc as sqlalchemy_exc
from config import db, basedir
from models.searched_scripts import SearchedScripts
from schemas.searchedscripts_schema import *
import yfinance as yf

class GetAddSearchedScripts(Resource):
    def __init__(self):
        pass
    # SearchedScripts get call

    def get(self):
        try:
            obj = db.session.query(SearchedScripts).order_by(SearchedScripts.id).all()
            if obj:
                schema = GetSearchedScriptsSchema(many=True)
                data = schema.dump(obj).data
                response = {"success": True, "message":"data fetched successfully" ,"data": data}
                return (response)
            else:
                response = {"success": False,
                            "message": "no data found in SearchedScripts"}
                return(response)
        except Exception as e:
            return({"success": False, "message": "server is down try after sometime", "error": str(e.args)})

    # SearchedScripts post call
    def post(self):
        try:
            request_json_data = request.get_json()
            msft = yf.Ticker(request_json_data['stockscript'])
            # print(msft)
            output = msft.history(period="max")
            output.index = output.index.map(str)

            # print(data,type(data))
            output = output.to_dict('index')

            if len(output)==0:
                output = {"success":False}
            else:
                output["success"]=True
            request_json_data["stockscriptoutput"]=output
            print(request_json_data)
            schema = AddSearchedScriptsSchema()
            obj = schema.load(
                request_json_data, session=db.session).data
            db.session.add(obj)
            db.session.commit()
            data = schema.dump(obj).data
            role_response = {"success": True, "message": "data added successfully","data": data}
            return (role_response)
        except KeyError as e :
            return({"success": False, "message": "Incorrect key name", "error":str(e.args)})
        except exc.IntegrityError as e:
            errorInfo = e.orig.args
            message=errorInfo[1]
            return({"success": False, "message": message})
        except sqlalchemy_exc.SQLAlchemyError as e:
            return({"success": False, "message":str(e.args)})
        except Exception as e:
            return({"success": False, "message": str(e.args)})

class GetUserSearchedScripts(Resource):
    def __init__(self):
        pass
    # SearchedScripts get call

    def get(self,id):
        try:
            obj = SearchedScripts.query.filter(SearchedScripts.userId == id).order_by(SearchedScripts.id.desc()).all()
            # obj = db.session.query(SearchedScripts).order_by(SearchedScripts.id).all()
            if obj:
                schema = GetUserSearchedScriptsSchema(many=True)
                data = schema.dump(obj).data
                response = {"success": True, "message":"data fetched successfully" ,"data": data}
                return (response)
            else:
                response = {"success": False,
                            "message": "no data found in SearchedScripts"}
                return(response)
        except Exception as e:
            return({"success": False, "message": "server is down try after sometime", "error": str(e.args)})


import logging
import time
import numpy as np
from numpy import array, ones, sqrt
from numpy.random import rand
from .pyutils import is_number
from .core import ttm, nvecs, norm

__all__ = [
    'hooi',
    'hosvd',
]

_log = logging.getLogger('TUCKER')
__DEF_MAXITER = 500
__DEF_INIT = 'nvecs'
__DEF_CONV = 1e-7

def hooi(X, rank, **kwargs):
    """
    Compute Tucker decomposition of a tensor using Higher-Order Orthogonal
    Iterations.

    Parameters
    ----------
    X : tensor_mixin
        The tensor to be decomposed
    rank : array_like
        The rank of the decomposition for each mode of the tensor.
        The length of ``rank`` must match the number of modes of ``X``.
    init : {'random', 'nvecs'}, optional
        The initialization method to use.
            - random : Factor matrices are initialized randomly.
            - nvecs : Factor matrices are initialzed via HOSVD.
        default : 'nvecs'

    Examples
    --------
    Create dense tensor

    >>> T = np.zeros((3, 4, 2))
    >>> T[:, :, 0] = [[ 1,  4,  7, 10], [ 2,  5,  8, 11], [3,  6,  9, 12]]
    >>> T[:, :, 1] = [[13, 16, 19, 22], [14, 17, 20, 23], [15, 18, 21, 24]]
    >>> T = dtensor(T)

    Compute Tucker decomposition of ``T`` with n-rank [2, 3, 1] via higher-order
    orthogonal iterations

    >>> Y = hooi(T, [2, 3, 1], init='nvecs')

    Shape of the core tensor matches n-rank of the decomposition.

    >>> Y['core'].shape
    (2, 3, 1)
    >>> Y['U'][1].shape
    (3, 2)

    References
    ----------
    .. [1] L. De Lathauwer, B. De Moor, J. Vandewalle: On the best rank-1 and
           rank-(R_1, R_2, \ldots, R_N) approximation of higher order tensors;
           IEEE Trans. Signal Process. 49 (2001), pp. 2262-2271
    """
    # init options
    ainit = kwargs.pop('init', __DEF_INIT)
    maxIter = kwargs.pop('maxIter', __DEF_MAXITER)
    conv = kwargs.pop('conv', __DEF_CONV)
    dtype = kwargs.pop('dtype', X.dtype)
    if not len(kwargs) == 0:
        raise ValueError('Unknown keywords (%s)' % (kwargs.keys()))

    ndims = X.ndim
    if is_number(rank):
        rank = rank * ones(ndims)

    normX = norm(X)

    U = __init(ainit, X, ndims, rank, dtype)
    fit = 0
    exectimes = []
    for itr in range(maxIter):
        tic = time.clock()
        fitold = fit

        for n in range(ndims):
            Utilde = ttm(X, U, n, transp=True, without=True)
            U[n] = nvecs(Utilde, n, rank[n])

        # compute core tensor to get fit
        core = ttm(Utilde, U, n, transp=True)

        # since factors are orthonormal, compute fit on core tensor
        normresidual = sqrt(normX ** 2 - norm(core) ** 2)

        # fraction explained by model
        fit = 1 - (normresidual / normX)
        fitchange = abs(fitold - fit)
        exectimes.append(time.clock() - tic)

        _log.debug(
            '[%3d] fit: %.5f | delta: %7.1e | secs: %.5f'
            % (itr, fit, fitchange, exectimes[-1])
        )
        if itr > 1 and fitchange < conv:
            break
    return core, U

def hosvd(X, rank, dims=None, dtype=None, compute_core=True):
    U = [None for _ in range(X.ndim)]
    if dims is None:
        dims = range(X.ndim)
    if dtype is None:
        dtype = X.dtype
    for d in dims:
        U[d] = array(nvecs(X, d, rank[d]), dtype=dtype)
    if compute_core:
        core = X.ttm(U, transp=True)
        return U, core
    else:
        return U

def __init(init, X, N, rank, dtype):
    # Don't compute initial factor for first index, gets computed in
    # first iteration
    Uinit = [None]
    if isinstance(init, list):
        Uinit = init
    elif init == 'random':
        for n in range(1, N):
            Uinit.append(array(rand(X.shape[n], rank[n]), dtype=dtype))
    elif init == 'nvecs':
        Uinit = hosvd(X, rank, range(1, N), dtype=dtype, compute_core=False)
    return Uinit


import sys
from collections import Counter

n4_offsets = [(0, -1), (0, 1), (-1, 0), (1, 0)]
n8_offsets = [(1, 0), (1, -1), (0, -1), (-1, -1), (-1, 0), (-1, 1), (0, 1), (1, 1)]
dircheck_offsets = [
        [o for o in n8_offsets if o[1] == -1],
        [o for o in n8_offsets if o[1] == 1],
        [o for o in n8_offsets if o[0] == -1],
        [o for o in n8_offsets if o[0] == 1]
    ]

def propose(pos, startdir):
    x, y = pos
    if any((x + o[0], y + o[1]) in elves for o in n8_offsets):
        for dir_offset in range(4):
            dir = (startdir + dir_offset) % 4
            if all((x + o[0], y + o[1]) not in elves for o in dircheck_offsets[dir]):
                return (x + n4_offsets[dir][0], y + n4_offsets[dir][1]), dir
    return pos, None

elves = set()
for y, line in enumerate(sys.stdin.read().splitlines()):
    for x, c in enumerate(line):
        if c == '#':
            elves.add((x, y))

startdir = 0
for _ in range(10):
    counters = Counter()
    targets = {}
    for pos in elves:
        target, dir = propose(pos, startdir)
        counters[target] += 1
        targets.update({target : pos})
    for target, pos in targets.items():
        if counters[target] == 1:
            elves.remove(pos)
            elves.add(target)
    startdir = (startdir + 1) % 4

xmin = min(p[0] for p in elves)
xmax = max(p[0] for p in elves)
ymin = min(p[1] for p in elves)
ymax = max(p[1] for p in elves)
print((xmax-xmin+1) * (ymax-ymin+1) - len(elves))


# -*- coding: utf-8 -*-
"""
Created on Mon Feb  1 17:38:38 2021

@author: KeliDu
"""
# =================================
# Import statements
# =================================

import os
import pandas as pd
import numpy as np
from sklearn import preprocessing as prp
import random
from measures import welsh_t
from measures import wilcoxon_ranksum
#from measures import KL_Divergence
from measures import chi_square
from measures import LLR
from measures import DP_Gries
# =================================
# Functions: calculate
# =================================

def make_idlists(metadatafile, separator, contrast):
    """
    This function creates lists of document identifiers based on the metadata.
    Depending on the contrast defined, the two lists contain various identifiers.
    """
    with open(metadatafile, "r", encoding='utf-8') as infile:
        metadata = pd.read_csv(infile, sep=separator)
        print("\nmetadata\n", metadata.head())
        #metadata = metadata.drop("Unnamed: 0", axis=1)
        metadata.set_index("idno", inplace=True)
        print("\nmetadata\n", metadata.head())
        if contrast[0] != "random":
            list1 = list(metadata[metadata[contrast[0]].isin([contrast[1]])].index)
            list2 = list(metadata[metadata[contrast[0]].isin([contrast[2]])].index)
            #print("list1", list1)
            #print("list2", list2)
        elif contrast[0] == "random":
            allidnos = list(metadata.loc[:, "idno"])
            allidnos = random.sample(allidnos, len(allidnos))
            list1 = allidnos[:int(len(allidnos)/2)]
            list2 = allidnos[int(len(allidnos)/2):]
            #print(list1[0:5])
            #print(list2[0:5])
        idlists = [list1, list2]
        #print(idlists)
        return idlists

def filter_dtm(dtmfolder, parameterstring, idlists, absolutefreqs, relativefreqs, binaryfreqs, tf_frame):
    """
    This function splits the DTM in two parts.
    Each part consists of the segments corresponding to one partition.
    Each segment is chosen based on the file id it corresponds to.
    """
    ids1 = "|".join([str(idno)+".*" for idno in idlists[0]])
    print(ids1)
    ids2 = "|".join([str(id)+".*" for id in idlists[1]])
    binary = binaryfreqs
    relative = relativefreqs
    absolute = absolutefreqs
    binary1 = binary.T.filter(regex=ids1, axis=1)
    binary2 = binary.T.filter(regex=ids2, axis=1)
    relative1 = relative.T.filter(regex=ids1, axis=1)
    relative2 = relative.T.filter(regex=ids2, axis=1)
    absolute1 = absolute.T.filter(regex=ids1, axis=1)
    absolute2 = absolute.T.filter(regex=ids2, axis=1)
    tf_frame1 = tf_frame.T.filter(regex=ids1, axis=1)
    tf_frame2 = tf_frame.T.filter(regex=ids2, axis=1)
    print("\nbinary1\n", binary1.head())
    return binary1, binary2, relative1, relative2, absolute1, absolute2, tf_frame1, tf_frame2

def get_indicators(binary1, binary2, relative1, relative2, tf_frame1, tf_frame2):
    """
    Indicators are the mean relative frequency or the document proportions,
    depending on the method chosen.
    """
    docprops1 = np.mean(binary1, axis=1)
    docprops1 = pd.Series(docprops1, name="docprops1")
    docprops2 = np.mean(binary2, axis=1)
    docprops2 = pd.Series(docprops2, name="docprops2")
    relfreqs1 = np.mean(relative1, axis=1)*1000
    relfreqs1 = pd.Series(relfreqs1, name="relfreqs1")
    relfreqs2 = np.mean(relative2, axis=1)*1000
    relfreqs2 = pd.Series(relfreqs2, name="relfreqs2")
    tf_framefreqs1 = np.mean(tf_frame1, axis=1)
    tf_framefreqs2 = np.mean(tf_frame2, axis=1)
    print("\ndocprops1\n", docprops1.head(20))
    print("\ndocprops2\n", docprops2.head(20))
    print("\nrelfreqs1\n", relfreqs1.head())
    print("\nrelfreqs2\n", relfreqs2.head())
    print("\tf_framefreqs2\n", tf_framefreqs2.head())
    print("\tf_framefreqs1\n", tf_framefreqs1.head())
    return docprops1, docprops2, relfreqs1, relfreqs2, tf_framefreqs1, tf_framefreqs2

def scaling_results (scaler, Series):
    Series_index = Series.index
    Series = scaler.fit_transform(Series.values.reshape(-1, 1))
    Series = [value[0] for value in Series]
    scaled_Series = pd.Series(data=Series, index=Series_index)
    return scaled_Series

def calculate_scores(docprops1, docprops2, absolute1, absolute2, relfreqs1, relfreqs2, logaddition, segmentlength, idlists, tf_framefreqs1, tf_framefreqs2, scaling):
    """
    This function implements several distinctive measures.
    """
    # Define logaddition and division-by-zero avoidance addition
    logaddition = logaddition+1
    divaddition = 0.00000000001
    print("---calculating scores: 1/9, 'original Zeta'---")
    try:
        # sd0 - Subtraction, docprops, untransformed a.k.a. "original Zeta"
        zeta_sd0 = docprops1 - docprops2
        zeta_sd0 = pd.Series(zeta_sd0, name="zeta_sd0")
        # Prepare scaler to rescale variants to range of sd0 (original Zeta)
        scaler = prp.MinMaxScaler(feature_range=(min(zeta_sd0),max(zeta_sd0)))
    except:
        print("Something went wrong while calculating 'original Zeta'")
        zeta_sd0 = pd.Series()
        scaler = prp.MinMaxScaler(feature_range=(-1,1))

    print("---calculating scores: 2/9, 'Zeta_log2-transformed'---")
    try:
        # sd2 - Subtraction, docprops, log2-transformed
        zeta_sd2 = np.log2(docprops1 + logaddition) - np.log2(docprops2 + logaddition)
        zeta_sd2 = pd.Series(zeta_sd2, name="zeta_sd2")
    except:
        print("Something went wrong while calculating 'Zeta_log2-transformed'")
        zeta_sd2 = pd.Series()

    print("---calculating scores: 3/9, 'ratio of relative frequencies'---")
    try:
        rrf_dr0 = (relfreqs1 + divaddition) / (relfreqs2 + divaddition)
        rrf_dr0 = pd.Series(rrf_dr0, name="rrf_dr0")
    except:
        print("Something went wrong while calculating 'ratio of relative frequencies'")
        rrf_dr0 = pd.Series()

    print("---calculating scores: 4/9, 'Eta, deviation of proportions based distinctiveness'---")
    # == Calculate subtraction variants ==
    # sg0 - Subtraction, devprops, untransformed a.k.a. "dpd", ("g" for Gries)
    try:
        devprops1 = DP_Gries.main(absolute1, segmentlength)
        devprops2 = DP_Gries.main(absolute2, segmentlength)
        eta_sg0 = (devprops1 - devprops2) * -1
        eta_sg0 = pd.Series(eta_sg0, name="eta_sg0")
    except:
        print("Something went wrong while calculating 'Eta, deviation of proportions based distinctiveness'")
        eta_sg0 = pd.Series()

    #Calculate Welshs-t-test
    print("---calculating scores: 5/9, 'Welshs-t-test'---")
    try:
        welsh_t_value = welsh_t.main(absolute1, absolute2)
    except:
        print("Something went wrong while calculating 'Welshs-t-test'")
        welsh_t_value = pd.Series()

    #Calculate Wilcoxon rank-sum test
    print("---calculating scores: 6/9, 'Wilcoxon rank-sum test'---")
    try:
        ranksumtest_value = wilcoxon_ranksum.main(absolute1, absolute2)
    except:
        print("Something went wrong while calculating 'Wilcoxon rank-sum test'")
        ranksumtest_value = pd.Series()
    '''
    #Calculate Kullback-Leibler Divergence
    print("---calculating scores: 7/10, 'Kullback-Leibler Divergence'---")
    try:
        KLD_value = KL_Divergence.main(absolute1, absolute2, 2, logaddition)
    except:
        print("Something went wrong while calculating 'Kullback-Leibler Divergence'")
        KLD_value = pd.Series()
    '''
    #Calculate Chi-squared test
    print("---calculating scores: 7/9, 'Chi-squared test'---")
    try:
        chi_square_value = chi_square.main(absolute1, absolute2)
    except:
        print("Something went wrong while calculating 'Chi-squared test'")
        chi_square_value = pd.Series()

    #Calculate Log-likelihood-Ratio test
    print("---calculating scores: 8/9, 'Log-likelihood-Ratio test'---")
    try:
        LLR_value = LLR.main(absolute1, absolute2)
    except:
        print("Something went wrong while calculating 'Log-likelihood-Ratio test'")
        LLR_value = pd.Series()

    print("---calculating scores; 9/9, 'Tf-idf weighted absolutefreqs absbased distinctiveness'---")
    try:
        tf_idf = tf_framefreqs1 - tf_framefreqs2
    except:
        print("Something went wrong while calculating 'Tf-idf weighted absolutefreqs absbased distinctiveness'")
        tf_idf = pd.Series()

    if scaling == True:
        zeta_sd2 = scaling_results (scaler, zeta_sd2)
        rrf_dr0 = scaling_results (scaler, rrf_dr0)
        eta_sg0 = scaling_results (scaler, eta_sg0)
        welsh_t_value = scaling_results (scaler, welsh_t_value)
        ranksumtest_value = scaling_results (scaler, ranksumtest_value)
        chi_square_value = scaling_results (scaler, chi_square_value)
        LLR_value = scaling_results (scaler, LLR_value)
        tf_idf = scaling_results (scaler, tf_idf)
        return zeta_sd0, zeta_sd2, rrf_dr0, eta_sg0, welsh_t_value, ranksumtest_value, chi_square_value, LLR_value, tf_idf #KLD_value

    if scaling == False:
        return zeta_sd0, zeta_sd2, rrf_dr0, eta_sg0, welsh_t_value, ranksumtest_value, chi_square_value, LLR_value, tf_idf #KLD_value

def get_meanrelfreqs(dtmfolder, parameterstring, relativefreqs):
    meanrelfreqs = relativefreqs.T
    print("\nrelfreqs_df\n", meanrelfreqs.head())
    meanrelfreqs_index = meanrelfreqs.index
    meanrelfreqs = np.mean(meanrelfreqs, axis=1)*1000
    meanrelfreqs = pd.Series(data=meanrelfreqs, index=meanrelfreqs_index)
    print("\nmeanrelfreqs_series\n", meanrelfreqs.head(10))
    return meanrelfreqs

def combine_results(docprops1, docprops2, relfreqs1, relfreqs2, meanrelfreqs, zeta_sd0, zeta_sd2, rrf_dr0, eta_sg0, welsh, ranksum, chi_square, LLR, tf_idf):
    results = pd.DataFrame({
    "docprops1" : docprops1,
    "docprops2" : docprops2,
    "relfreqs1" : relfreqs1,
    "relfreqs2" : relfreqs2,
    "meanrelfreqs" :meanrelfreqs,
    "zeta_sd0" : zeta_sd0,
    "zeta_sd2" : zeta_sd2,
    "rrf_dr0" : rrf_dr0,
    "eta_sg0" : eta_sg0,
    "welsh" : welsh,
    "ranksum" : ranksum,
    #"KL_Divergence": KL_Divergence,
    "chi_square": chi_square,
    "LLR": LLR,
    "tf_idf": tf_idf})
    results = results[[
    "docprops1",
    "docprops2",
    "relfreqs1",
    "relfreqs2",
    "meanrelfreqs",
    "zeta_sd0",
    "zeta_sd2",
    "rrf_dr0",
    "eta_sg0",
    "welsh",
    "ranksum",
    #"KL_Divergence",
    "chi_square",
    "LLR",
    "tf_idf"]]
    results.sort_values(by="zeta_sd0", ascending=False, inplace=True)
    print("\nresults-head\n", results.head(10), "\nresults-tail\n", results.tail(10))
    return results

def save_results(results, resultsfile):
    with open(resultsfile, "w", encoding='utf-8') as outfile:
        results.to_csv(outfile, sep="\t")

# =================================
# Function: main
# =================================

def main(datafolder, dtmfolder, metadatafile, separator, contrast, logaddition, resultsfolder, segmentlength, featuretype, absolutefreqs, relativefreqs, binaryfreqs, tf_frame, scaling=True):
    print("--calculate")
    if not os.path.exists(resultsfolder):
        os.makedirs(resultsfolder)
    parameterstring = str(segmentlength) +"-"+ str(featuretype[0]) +"-"+ str(featuretype[1])
    #print(parameterstring)
    contraststring = str(contrast[0]) +"_"+ str(contrast[2]) +"-"+ str(contrast[1])
    #print(contraststring)
    resultsfile = resultsfolder + "results_" + parameterstring +"_"+ contraststring +".csv"
    idlists = make_idlists(metadatafile, separator, contrast)
    #print(idlists)
    binary1, binary2, relative1, relative2, absolute1, absolute2, tf_frame1, tf_frame2 = filter_dtm(dtmfolder, parameterstring, idlists, absolutefreqs, relativefreqs, binaryfreqs, tf_frame)
    #print(binary1)
    docprops1, docprops2, relfreqs1, relfreqs2, tf_framefreqs1, tf_framefreqs2 = get_indicators(binary1, binary2, relative1, relative2, tf_frame1, tf_frame2)
    zeta_sd0, zeta_sd2, rrf_dr0, eta_sg0, welsh, ranksum, chi_square, LLR, tf_idf = calculate_scores(docprops1, docprops2, absolute1, absolute2, relfreqs1, relfreqs2, logaddition, segmentlength, idlists, tf_framefreqs1, tf_framefreqs2, scaling)
    meanrelfreqs = get_meanrelfreqs(dtmfolder, parameterstring, relativefreqs)
    results = combine_results(docprops1, docprops2, relfreqs1, relfreqs2, meanrelfreqs, zeta_sd0, zeta_sd2, rrf_dr0, eta_sg0, welsh, ranksum, chi_square, LLR, tf_idf)
    save_results(results, resultsfile)


import cv2
import numpy as np

colorPath = "Images/week1/boy.jpg"

img = cv2.imread(colorPath, cv2.IMREAD_COLOR)
new_img = img.copy()

#creating an empty matrix of desired dimensions
empty_mat1 = np.zeros((100,200,3), dtype = 'uint8')
empty_mat2 = 255 * np.ones((100,200,3), dtype = 'uint8')

#creating an empty matrix similar to a existing image
mat1 = np.zeros_like(img)
mat2 = 255 * np.ones_like(img)

print(mat1.shape)
cv2.imshow("hello", mat2)
cv2.waitKey(0)
cv2.destroyAllWindows()


from copy import deepcopy
import csv
from datetime import datetime
import glob
import operator
from data import getStocks, getData
from objects import Indicator, ReqParam, Stock, DayTrades
from talib_processes import getIndicatorValues, getPeriodStr, getRequirementData
import pandas as pd
import numpy as np
import util
import time
from util import removeNaN

indicatorMap = {}
resMap = {}
adxMap = {}
chaikinMap = {}
histMap = {}
volMap = {}
natrMap = {}

def getRes(indicator: Indicator, max_buy_val, min_sell_val):
    stocks = getStocks()
    for count, ticker in enumerate(stocks):
        df = pd.DataFrame()
        df[indicator.name] = indicatorMap.get(ticker + getPeriodStr(indicator))
        df['Res'] = np.where(df[indicator.name] <= max_buy_val, 1, np.where(df[indicator.name] >= min_sell_val, -1, 0))
        resMap[ticker + getPeriodStr(indicator) + '_mbv_' + str(max_buy_val) + '_msv_' + str(min_sell_val)] = util.removeNaN(df['Res'].values)

def checkRequirement(req: ReqParam, position: int=None, vals: list=None) -> bool:
    if req.condition == 0:
        return True
    elif req.condition != 0 and len(vals) == 0:
        return False
    elif req.condition == 1:
        if position is None or vals is None or req.adx_min is None:
            raise ValueError('Need values for position, adx and adx_min')
        return vals[position] > req.adx_min
    elif req.condition == 2:
        if position is None or vals is None:
            raise ValueError('Need values for position and chaikin')
        return vals[position] > 0
    elif req.condition == 3:
        if position is None or vals is None or req.isBuy is None:
            raise ValueError('Need values for position, isBuy and hist')
        return vals[position] > 0 if req.isBuy else vals[position] < 0
    elif req.condition == 4:
        if position is None or vals is None:
            raise ValueError('Need values for position and vol')
        return vals[position] > 0
    elif req.condition == 5:
        if position is None or vals is None or req.isBuy is None:
            raise ValueError('Need values for position, isBuy and vals')
        return vals[position - 1] <= vals[position] if req.isBuy else vals[position - 1] >= vals[position]
    elif req.condition == 6:
        if position is None or vals is None or req.natr_min is None:
            raise ValueError('Need values for position, adx and adx_min')
        return vals[position] > req.natr_min
    else:
        raise ValueError('Need condition needs to be between 0 & 6')

def getRequirementValues(req: ReqParam, ticker, indicator: Indicator):
    if req.condition == 0:
        return []
    elif req.condition == 1:
        return adxMap[ticker + '_' + str(req.indicatorPeriod)]
    elif req.condition == 2:
        return chaikinMap[ticker + '_' + str(req.fastPeriod) + '_' + str(req.slowPeriod)]
    elif req.condition == 3:
        return histMap[ticker + '_' + str(req.indicatorPeriod) + '_' + str(req.fastPeriod) + '_' + str(req.slowPeriod)]
    elif req.condition == 4:
        return volMap[ticker + '_' + str(req.rollingPeriod)]
    elif req.condition == 5:
        return indicatorMap[ticker + getPeriodStr(indicator)].values
    elif req.condition == 6:
        return natrMap[ticker + '_' + str(req.indicatorPeriod)]

def addTrade(trades, ticker, date, val, volume=0, indicatorVal=0, isBuy=True):
    s = Stock()
    s.symbol = ticker
    s.wap = val

    if isBuy:
        s.indicatorVal = indicatorVal
        s.max_quantity = volume // 10

    if trades.get(date) is not None:
        ndt = trades.get(date)
        if isBuy:
            ndt.buys.append(s)
        else:
            ndt.sells.append(s)
        trades[date] = ndt

    else:
        dt = DayTrades()
        dt.date = date
        if isBuy:
            dt.buys.append(s)
        else:
            dt.sells.append(s)
        trades[date] = dt

def doCalculation(file, max_buy_val, min_sell_val, indicator, req1=ReqParam(), req2=ReqParam(), req3=ReqParam(), req4=ReqParam()):
    trades = {}
    total_res = np.zeros(15)
    stocks = getStocks(file)
    for count, ticker in enumerate(stocks):
        df = deepcopy(data[ticker])

        dates = df.index.tolist()
        vals = df['VWAP'].values
        volume = df['Volume'].values
        res = resMap[ticker + getPeriodStr(indicator) + '_mbv_' + str(max_buy_val) + '_msv_' + str(min_sell_val)]
        indicatorVals = removeNaN(indicatorMap[ticker + getPeriodStr(indicator)].values)

        r1 = getRequirementValues(req1, ticker, indicator)
        r2 = getRequirementValues(req2, ticker, indicator)
        r3 = getRequirementValues(req3, ticker, indicator)
        r4 = getRequirementValues(req4, ticker, indicator)

        max_len = util.getMaxDataPoints(res, indicatorVals, r1, r2, r3, r4)
        dates = util.getUsableDataPoints(dates, max_len)
        vals = util.getUsableDataPoints(vals, max_len)
        volume = util.getUsableDataPoints(volume, max_len)
        res = util.getUsableDataPoints(res, max_len)
        indicatorVals = util.getUsableDataPoints(indicatorVals, max_len)
        r1 = util.getUsableDataPoints(r1, max_len)
        r2 = util.getUsableDataPoints(r2, max_len)
        r3 = util.getUsableDataPoints(r3, max_len)
        r4 = util.getUsableDataPoints(r4, max_len)

        amt = np.zeros(len(res))
        gain = np.zeros(len(res))
        day_diff = np.zeros(len(res)).astype(int)
        sell_days = np.zeros(len(res)).astype(int)
        buy_signal = 0
        sell_signal = 0

        i = 1
        while i < len(res) - 1:
            if res[i] > buy_signal and checkRequirement(req1, i, r1) and checkRequirement(req2, i, r2):
                addTrade(trades, ticker, dates[i + 1][1], vals[i + 1], indicatorVal=indicatorVals[i],
                         volume=volume[i + 1])
                for j in range(i + 1, len(res) - 1):
                    if res[j] < sell_signal and checkRequirement(req3, j, r3) and checkRequirement(req4, j, r4):
                        addTrade(trades, ticker, dates[j + 1][1], vals[j + 1], isBuy=False)
                        amt[j] = vals[j] - vals[i]
                        gain[j] = (vals[j] - vals[i]) * 100 / vals[i]
                        day_diff[j] = j - i
                        i = j
                        break

                    if j == len(res) - 2:
                        i = j

            i += 1

        # to match the df with the values of the arrays
        df.drop(df.head(len(df) - max_len).index, inplace=True)

        df['Amt'] = amt[:]
        df['Gain'] = gain[:]
        df['Day_Diff'] = day_diff[:]
        df['Sell_Days'] = sell_days[:]

        total_res[0] += len(df['Gain'])
        total_res[1] += len(df['Gain'][df['Gain'] > 0])
        total_res[2] += len(df['Gain'][df['Gain'] < 0])
        total_res[3] += df['Gain'][(df['Gain'] > 0) | (df['Gain'] < 0)].sum()
        total_res[4] += df['Gain'][df['Gain'] > 0].sum()
        total_res[5] += df['Gain'][df['Gain'] < 0].sum()
        total_res[6] += df['Day_Diff'][(df['Gain'] > 0) | (df['Gain'] < 0)].sum()
        total_res[7] += df['Day_Diff'][df['Gain'] > 0].sum()
        total_res[8] += df['Day_Diff'][df['Gain'] < 0].sum()
        total_res[9] += len(df['Gain'][(df['Gain'] > 0) | (df['Gain'] < 0)])
        total_res[10] += len(df['Sell_Days'][df['Sell_Days'] > 0])

    result = np.zeros(13)
    result[0] = total_res[0]
    result[1] = total_res[1]
    result[2] = total_res[2]
    result[3] = total_res[1]/total_res[9] if total_res[1] > 100 else 0
    result[4] = total_res[3] / total_res[9]
    result[5] = total_res[4] / total_res[1]
    result[6] = total_res[5] / total_res[2]
    result[7] = total_res[6] / total_res[9]
    result[8] = total_res[7] / total_res[1]
    result[9] = total_res[8] / total_res[2]
    result[10] = total_res[9]

    group = file[32: len(file) - 8]

    result[11] = backTestStrategy(trades, group == '500')
    result[12] = backTestStrategy(trades, group == '500', date='2017-01-01')

    row = [indicator.name, group, max_buy_val, min_sell_val, indicator.period, indicator.fastperiod, indicator.slowperiod, indicator.timeperiod1, indicator.timeperiod2, indicator.timeperiod3, req1.__repr__(), req2.__repr__(), req3.__repr__(),
           req4.__repr__()]

    # check for diff append and extend https://stackoverflow.com/a/252711/5512020
    row.extend([element for element in result])
    # without newline = '' , a row is skipped in csv
    with open('./analyses/analyses.csv', 'a', newline='') as f:
        writer = csv.writer(f)
        writer.writerow(row)
    f.close()

def backTestStrategy(trades, isNifty500, date=None):
    buyDict = {}
    iniVal = 250000
    money = deepcopy(iniVal)
    nav = deepcopy(iniVal)

    if date is None:
        tradeSubset = trades
    else:
        # to analyse trades only for 2017, use filter
        timeStamp = datetime.strptime(date, "%Y-%m-%d").timestamp()
        tradeSubset = dict(filter(lambda x: datetime.strptime(x[0], "%Y-%m-%d").timestamp() > timeStamp, trades.items()))

    for key, dayTrade in sorted(tradeSubset.items(), key=lambda t: datetime.strptime(t[0], "%Y-%m-%d").timestamp()):
        for stock in dayTrade.sells:
            # cases where date is not None, buyDict will return none for early dates
            if buyDict.get(stock.symbol) is not None:
                # .4% loss due to brockerage and taxes
                money += buyDict.get(stock.symbol)[0] * stock.wap * 0.997
                nav += buyDict.get(stock.symbol)[0] * (stock.wap * 0.997 - buyDict.get(stock.symbol)[1])
                buyDict[stock.symbol] = (0, 0)

        # // gives int division so no decimal
        max_stocks = 15 if isNifty500 else 4
        max_per_stock = nav // max_stocks
        sortedBuys = sorted(dayTrade.buys, key=operator.attrgetter('indicatorVal'))
        for s in sortedBuys:
            amount = min(max_per_stock, money)
            quantity = min(amount // (s.wap * 1.003), s.max_quantity)
            buyDict[s.symbol] = (quantity, s.wap)
            money -= amount - amount % s.wap

    return nav - iniVal

def analyseIndicator(count):
    periods = np.array([6, 8, 10, 12, 14, 16, 18])
    maxBuyVals = np.array([-85, -80, -75, -70])
    minSellVals = np.array([-25, -20, -15])
    adx_periods = np.array([6, 8, 10, 12, 14])
    adx_mins = np.array([25, 30, 35])

    for i in range(len(periods)):
        indicator = Indicator()
        indicator.period = periods[i]
        indicator.name = 'WILLR'

        req2 = ReqParam()
        req2.condition = 5
        req2.indicatorPeriod = periods[i]
        req2.isBuy = True

        req3 = ReqParam()
        req3.condition = 5
        req3.indicatorPeriod = periods[i]
        req3.isBuy = False

        getIndicatorValues(indicator, indicatorMap)

        for j in range(len(maxBuyVals)):
            for k in range(len(minSellVals)):
                getRes(indicator, maxBuyVals[j], minSellVals[k])

                for l in range(len(adx_periods)):
                    for m in range(len(adx_mins)):
                        req1 = ReqParam()
                        req1.condition = 1
                        req1.indicatorPeriod = adx_periods[l]
                        req1.adx_min = adx_mins[m]
                        getRequirementData(req1, data, adxMap)

                        for f in allFiles:
                            count += 1
                            print("Analyses: ", f[32: len(f) - 8], count, periods[i], maxBuyVals[j],
                                  minSellVals[k], adx_periods[l], adx_mins[m])

                            doCalculation(f, max_buy_val=maxBuyVals[j], min_sell_val=minSellVals[k], indicator=indicator, req1=req1, req2=req2, req3=req3)

pd.options.mode.chained_assignment = None

st = int(round(time.time() * 1000))
count = 0

path = r'F:\data\nse500\indices'
allFiles = glob.glob(path + "\*.csv")

data = getData()
analyseIndicator(count)
totalTime = int(round(time.time() * 1000)) - st
print('Total Time:', totalTime)


import setuptools

# from src.aio_dtls import __version__

with open("README.md", "r", encoding="utf-8") as fh:
    long_description = fh.read()

setuptools.setup(
    name='aio_dtls',
    version='0.0.2',  # __version__,
    test_requires=[],
    url='https://github.com/businka/aio_dtls',
    license='MIT',
    author='Razgovorov Mikhail',
    author_email='1338833@gmail.com',
    description='asyncio implementation of the dtls protocol in python',
    long_description=long_description,
    long_description_content_type="text/markdown",
    classifiers=[
        'Development Status :: 2 - Pre-Alpha',
        'Natural Language :: Russian',
        'License :: OSI Approved :: MIT License',
        'Programming Language :: Python :: 3',
        'Topic :: Software Development',
        "Operating System :: OS Independent"
    ],
    keywords='python tls dtls mbed asyncio',
    python_requires='>=3.9',
    zip_safe=False,
    install_requires=[
        "construct",
        "cryptography",
    ],
    package_dir={"": "src"},
    packages=setuptools.find_packages(where="src")
)


import sys

from mainwindow import Ui_MainWindow
from dataloader import DataLoader
from xbee import XBeeCommunicator

from functools import partial

from PyQt5 import QtCore, QtGui, QtWidgets
import matplotlib.pyplot as plt
from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas
from matplotlib.backends.backend_qt5agg import NavigationToolbar2QT as NavigationToolbar
from matplotlib.figure import Figure

from mpl_toolkits.mplot3d import Axes3D

from random import randint
from datetime import datetime

import warnings

HEADERS = ["TeamID", "Time", "Packet", "Altitude", "Pressure", "Airspeed", "Temperature", "Voltage", "Latitude", "Longitude",
"GPSAlt", "Satellites", "GPSSpeed", "Heading", "ImageCount", "State"]

# Values below graph
v1 = "Time"
v2 = "Altitude"
v3 = "GPSSpeed"
v4 = "Airspeed"
v5 = "Temperature"

# Values below graph - units
v1_u = "s"
v2_u = "m"
v3_u = "mph"
v4_u = "mph"
v5_u = "°F"

class Wrapper():

    def __init__(self, ui, xbee_communicator, dataloader=None):
        self.ui = ui
        self.ax = None
        self.xbee_communicator = xbee_communicator

        # reference to 3d plot
        self.plot3d = None

        # hold the smallest min and largest max limits for the y axis
        # this lets us know the highest or lowest we can scale the graph
        self.yLimits = [0, 0]
        self.xLimits = [0, 0]

        # hold the current min and max limits for the y axis
        # this keeps track of the values the sliders are at
        self.minY = 0
        self.maxY = 0
        self.minX = 0
        self.maxX = 0

        # populate plot select menu
        self.ui.comboBox.clear()
        # self.ui.comboBox.addItem("All")
        # should create separate array of items we want to plot
        self.ui.comboBox.addItems(HEADERS[2:])

        self.ui.horizontalSlider_2.setValue(100)
        self.ui.horizontalSlider_4.setValue(100)

        # store the current option select in plot select
        self.currentPlot = self.ui.comboBox.currentText()

        if dataloader:
            # Create default dataloader? Dataloader should be initialized with directory/file name
            self.dataloader = dataloader
            pass

    def codeUpdatesToUI(self):
        self.ui.figure = Figure()
        self.ui.canvas = FigureCanvas(self.ui.figure)
        self.ui.canvas.setObjectName("gridCanvas")
        # add the plot to the main layout
        self.ui.gridLayout.addWidget(self.ui.canvas, 0, 0, 1, 1)

        # add a plot to the sub layout
        self.ui.figure2 = Figure()
        self.ui.canvas2 = FigureCanvas(self.ui.figure2)
        self.ui.canvas2.setObjectName("gridCanvas2")
        self.ui.gridLayout_2.addWidget(self.ui.canvas2, 0, 0, 1, 1)

    def initNewUI(self):

        # Change session name
        if self.dataloader is not None:

            # Update session name
            self.update_session_name(self.dataloader.file_name)

            if self.ax is not None:
                # Clear old plot to plot new data
                self.plotClear()

            # Plot initial data
            data = self.dataloader.fetch(["Time", self.currentPlot])

            # Compute new limits
            self.compute_plot_limits(data)

            # Update limits
            self.update_plot_limits()

            # Update plot controls
            self.update_plot_controls(True)

            # Plot
            self.plot(data, "Time", self.currentPlot)

            # plot 3d
            self.plotOrientation(0, 0, 0, 0.2, -0.4, 1.8)

            # Update text vals
            self.update_text_vals()

        else:
            # Update session name
            self.update_session_name("No File Loaded")

    def compute_plot_limits(self, data):

        data = data.astype(float)

        if not data.empty:
            self.maxX = int(data["Time"].max())
            self.maxY = int(data[self.currentPlot].max()) + 1
            self.minX = int(data["Time"].min())
            self.minY = int(data[self.currentPlot].min()) + 1
        else:
            self.maxX = 10
            self.maxY = 10
            self.minX = 0
            self.minY = 0

    def update_plot_limits(self):
        axes = self.ui.figure.gca()
        axes.set_xlim([self.minX, self.maxX])
        axes.set_ylim([self.minY, self.maxY])
        self.ui.canvas.draw()

    def update_plot_controls(self, set_slider_max):

        minVals = [self.minX, self.minY]
        maxVals = [self.maxX, self.maxY]
        mintxtedits = [self.ui.textEdit, self.ui.textEdit_3]
        maxtxtedits = [self.ui.textEdit_2, self.ui.textEdit_4]
        minsliders = [self.ui.horizontalSlider_3, self.ui.horizontalSlider]
        maxsliders = [self.ui.horizontalSlider_4, self.ui.horizontalSlider_2]

        for minval, minte, minslider, maxval, maxte, maxslider in zip(minVals, mintxtedits, minsliders, maxVals, maxtxtedits, maxsliders):
            minte.setText(str(minval))
            maxte.setText(str(maxval))

            if set_slider_max:
                minslider.setMinimum(minval)
                minslider.setMaximum(maxval)
                maxslider.setMinimum(minval)
                maxslider.setMaximum(maxval)

            minslider.setValue(minval)
            maxslider.setValue(maxval)

    # plot the data
    """
    need to use data.astype(float) so it can process the data
    need ax = self.ui.figure.add_subplot(111) to show the graph
    set x axis to Time and y axis to altitude
    """
    def plot(self, data, x, y):
        self.ax = self.ui.figure.add_subplot(111)
        self.ax.plot(data[x].astype(float), data[y].astype(float),
            color = "xkcd:teal")
        self.ax.set_title("{} vs {}".format(x, y))
        self.ax.set_xlabel(x)
        self.ax.set_ylabel(y)
        self.ax.grid()
        self.ui.canvas.draw()

    # plots new points
    def plot_points(self, data, x, y):
        self.ax.plot(data[x].tail(2).astype(float),
            data[y].tail(2).astype(float), color = "xkcd:teal")
        self.ui.canvas.draw()

    # Clear the plot
    def plotClear(self):
        self.ax.clear()
        self.ui.canvas.draw()

    # plots 3d
    # use u v w for orientation/direction of arrow
    # x y z is for initial position of tail
    def plotOrientation(self, x, y, z, u, v, w):
        ax = self.ui.figure2.add_subplot(111, projection = "3d")
        ax.set_xlabel("x")
        ax.set_ylabel("y")
        ax.set_zlabel("z")
        ax.quiver(x, y, z, u, v, w)
        ax.set_xlim([-1, 1])
        ax.set_ylim([-1, 1.5])
        ax.set_zlim([-1, 2])
        self.ui.canvas2.draw()

    # Connects buttons to given functions
    def setUpHandlers(self):
        self.ui.actionConnect.triggered.connect(self.select_port)
        self.ui.actionStart.triggered.connect(self.xbee_start)
        self.ui.actionPause.triggered.connect(self.xbee_pause)
        self.ui.actionStop.triggered.connect(self.xbee_stop)

        # detect when a new plot has been selected
        self.ui.comboBox.activated.connect(self.setComboBox)

        # detect when max sliders have changed
        self.ui.horizontalSlider_2.valueChanged.connect(partial(self.maxSliderChange, self.ui.horizontalSlider_2)) # Y Max
        self.ui.horizontalSlider_4.valueChanged.connect(partial(self.maxSliderChange, self.ui.horizontalSlider_4)) # X Max

        # detect when min sliders have changed
        self.ui.horizontalSlider.valueChanged.connect(partial(self.minSliderChange, self.ui.horizontalSlider)) # Y Min
        self.ui.horizontalSlider_3.valueChanged.connect(partial(self.minSliderChange, self.ui.horizontalSlider_3)) # X Min

        self.ui.actionNew_Session.triggered.connect(self.new_session)
        self.ui.actionLoad_Session.triggered.connect(self.load_session)

        # Snapshot detection
        self.ui.pushButton.clicked.connect(self.snapshot)

    # Starts a new session (new data file)
    def new_session(self):
        if self.yesno_prompt("New Session", "Are you sure you want to start a new session?"):

            # Prompt user to choose filename
            fn = self.saveFileNameDialog()
            if fn == '':
                return

            # Save current data
            if self.dataloader is not None:
                self.dataloader.save_as_csv()

            # Use current time as filename
            self.dataloader = DataLoader(fn, HEADERS)
            self.dataloader.save_as_csv()

            self.initNewUI()

    # Loads a session from an old csv file
    def load_session(self):
        if self.yesno_prompt("Load Session", "Are you sure you want to load a session?"):

            # Prompt user to choose filename
            fn = self.openFileNameDialog()
            if fn == '':
                return

            # Save current data loader
            if self.dataloader is not None:
                self.dataloader.save_as_csv()

            # Load data from file
            self.dataloader = DataLoader(fn, HEADERS)
            self.dataloader.read_file()

            self.initNewUI()

    def xbee_update(self, data_row):
        if self.dataloader is None:
            print("Error: Session has not been started")
            return

        self.dataloader.update(HEADERS, data_row)

        x = "Time"
        y = self.currentPlot

        # Fetch data
        data = self.dataloader.fetch([x, y])

        # Compute new limits
        self.compute_plot_limits(data)

        # Update limits
        self.update_plot_limits()

        # Update plot controls
        self.update_plot_controls(True)

        print("here")

        # Update plot
        self.plot_points(data, x, y)

        # Update text below graph
        self.update_text_vals()

    def select_port(self):
        valid = False
        while not valid:
            port, choice = self.inputdialog("Port", "Input Port (/dev/tty or COM)")
            if not choice:
                print("None selected")
                return
            else:
                print("Chosen port: {}".format(port))
                valid = xbee_communicator.connect(port)
                if not valid:
                    self.warningdialog("Not a valid port, try again.")

    def xbee_start(self):
        valid = xbee_communicator.start(self.xbee_update)
        if not valid:
            self.warningdialog("Not connected; cannot start.")

    def xbee_pause(self):
        valid = xbee_communicator.pause()
        if not valid:
            self.warningdialog("Not connected; cannot pause.")

    def xbee_stop(self):
        valid = xbee_communicator.stop()
        if not valid:
            self.warningdialog("Not connected; cannot stop.")

    def inputdialog(self, title, message):
        return QtWidgets.QInputDialog.getText(self.ui.mainwindow, title, message)

    # update the current plot and replace the previous plot
    def setComboBox(self, index):
        self.currentPlot = self.ui.comboBox.itemText(index)

        if (self.currentPlot == "All"):
            pass
        else:
            data = self.dataloader.fetch(["Time", self.currentPlot])
            warnings.filterwarnings("ignore",module="matplotlib")
            self.plotClear()
            self.plot(data, "Time", self.currentPlot)

            # Compute new limits
            self.compute_plot_limits(data)

            # Update limits
            self.update_plot_limits()

            # Update plot controls
            self.update_plot_controls(True)

    # change the max range of the y axis using slider
    def maxSliderChange(self, slider):
        newMaxValue = slider.value()
        axes = self.ui.figure.gca()

        # slider for Y Max
        if (slider == self.ui.horizontalSlider_2):
            if self.minY < newMaxValue:
                self.maxY = newMaxValue
                self.update_plot_limits()
        # slider for X Max
        else:
            if self.minX < newMaxValue:
                self.maxX = newMaxValue
                self.update_plot_limits()
        warnings.filterwarnings("ignore",module="matplotlib")

        self.update_plot_controls(False)

    # change the min range of the y axis using slider
    def minSliderChange(self, slider):
        newMinValue = slider.value()
        axes = self.ui.figure.gca()

        # Slider for Y Min
        if (slider == self.ui.horizontalSlider):
            if newMinValue < self.maxY:
                self.minY = newMinValue
                self.update_plot_limits()
        # Slider for X Min
        else:
            if newMinValue < self.maxX:
                self.minX = newMinValue
                self.update_plot_limits()

        warnings.filterwarnings("ignore",module="matplotlib")

        self.update_plot_controls(False)

    def getCurrentPlot(self):
        return self.currentPlot

    def snapshot(self):
        if not self.xbee_communicator.snapshot():
            self.warningdialog("Not connected to XBee")

    def warningdialog(self, message):
        msg = QtWidgets.QMessageBox()
        msg.setIcon(QtWidgets.QMessageBox.Warning)
        msg.setText(message)
        msg.setStandardButtons(QtWidgets.QMessageBox.Ok)
        msg.setWindowTitle("WARNING!")
        ret = msg.exec_()

    def yesno_prompt(self, title, msg):
        reply = QtWidgets.QMessageBox.question(self.ui.mainwindow, title,
                 msg, QtWidgets.QMessageBox.Yes, QtWidgets.QMessageBox.No)
        return True if reply == QtWidgets.QMessageBox.Yes else False

    def openFileNameDialog(self):
        options = QtWidgets.QFileDialog.Options()
        options |= QtWidgets.QFileDialog.DontUseNativeDialog
        fileName, _ = QtWidgets.QFileDialog.getOpenFileName(self.ui.mainwindow,"QFileDialog.getOpenFileName()",
            "","All Files (*);;Python Files (*.py)", options=options)
        return fileName

    def saveFileNameDialog(self):
        options = QtWidgets.QFileDialog.Options()
        options |= QtWidgets.QFileDialog.DontUseNativeDialog
        fileName, _ = QtWidgets.QFileDialog.getSaveFileName(self.ui.mainwindow,"QFileDialog.getSaveFileName()",
            "","All Files (*);;Python Files (*.py)", options=options)
        return fileName

    def update_session_name(self, name):
        self.ui.session_label.setText(name)

    def update_text_vals(self):
        label_order = [self.ui.label_11, self.ui.label_9, self.ui.label_6, self.ui.label_5, self.ui.label_4]
        for field, unit, label in zip([v1, v2, v3, v4, v5], [v1_u, v2_u, v3_u, v4_u, v5_u], label_order):
            txt_data = self.dataloader.fetch([field])
            text = str(txt_data[field].iloc[-1]) + unit if len(txt_data[field]) > 1 else "N/A"
            label.setText(text)

# Instantiate UI
if __name__ == "__main__":
    app = QtWidgets.QApplication(sys.argv)
    MainWindow = QtWidgets.QMainWindow()
    ui = Ui_MainWindow()
    ui.setupUi(MainWindow)
    ui.mainwindow = MainWindow

    # Setup necessary components
    # Optional
    dataloader = DataLoader("./data/Data.txt", HEADERS)
    dataloader.read_file()

    xbee_communicator = XBeeCommunicator()

    # Create wrapper around UI Window
    window = Wrapper(ui, xbee_communicator, dataloader)
    window.codeUpdatesToUI()
    window.setUpHandlers()
    window.initNewUI()

    # Show window
    MainWindow.show()

    sys.exit(app.exec_())