import json
import math
import time
import csv
import graphviz
import numpy as np
import pandas as pd
from io import BytesIO
from sklearn import svm
from sklearn import tree
from sklearn import metrics
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import cross_val_score
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split

# Count the number of values occurrence in dictionary
# Input type: dic. Example: price_data.values()
# Output type: dic
def count_values(value):
	result = {}
	for i in value:
		result[i] = result.get(i, 0) + 1
	return result

# Take the hour out from created column
# Input type: dic. Example: data.get("created", "")
# Output type: list
def extrac_hour(data):
	time = []
	hour = []
	values = data.values()
	for i in values:
		time.append(i.split(' ')[1])
	for i in time:
		hour.append(i.split(':')[0])
	return hour

# Count the same address and use the index in the list for the category
# Input type: data -> train data or test data, column -> column name, df -> train data frame or test data frame
# Output type: list
def calculate_address_category(data, column, df):
	address = data.get(column, '')
	count = count_values(address.values())

	# After count, put all the address into list
	# get a list of distinct addresses
	# Ex: ['Suffolk Street', 'Thompson Street' ...]
	address_collection = []
	for key, value in count.items():
		address_collection.append(key)

	# Get the address from data frame and
	# compare to address_collection to get the
	# index and the index is the category
	# Ex: [0, 1, 2, 1, 0, 500 ...]
	address_category = []
	for i in df[column]:
		address_category.append(address_collection.index(i))

	return address_category

def listToString(s):
	# initialize an empty string
	str1 = ""
	# traverse in the string
	for ele in s:
		str1 += " "
		str1 += ele
	# return string
	return str1

def count_feature_number(feature_names):
	def f2(s):
		result_list = []
		for ele in feature_names:
			if ele in s:
				result_list.append(1)
			else:
				result_list.append(0)
		return result_list
	result = f2
	return result

def count_feature(data):
	corpus = data['features'].values
	with open("decision_tree_text_features.txt", "r") as f:
		feature_names = f.read().splitlines()

	for i in range(len(feature_names)):
		feature_names[i] = feature_names[i].capitalize()

	count_f = count_feature_number(feature_names)
	feature_result = list(map(count_f, corpus))

	count_feature = []
	for i in feature_result:
		count_feature.append(i[0])
	return count_feature

def x_ysquare(data,value):
	square = []
	for i in data:
		square.append((i-value)**2)
	return square
def distance(data1,data2):
	f_distance = []
	for x in range(len(data1)):
		f_distance.append(math.sqrt(data1[x]+data2[x]))
	return f_distance

def decision_tree():
	print ("Running Decision Tree...\n")
	with open('test.json') as json_file:
		test_data = json.load(json_file)

	with open('train.json') as json_file:
		data = json.load(json_file)

		data_frame = pd.DataFrame(data)
		test_data_frame = pd.DataFrame(test_data)

		# Count the number of photos in the training set
		count_photo = []
		for i in data_frame['photos']:
			count_photo.append(len(i))
		# Count the number of photos in the test set
		count_test_photo = []
		for i in test_data_frame['photos']:
			count_test_photo.append(len(i))
		# Append the data as columns into data_frame
		data_frame['count_photos'] = count_photo
		test_data_frame['count_photos'] = count_test_photo

		# Append the address as columns into data_frame
		data_frame['address_category'] = calculate_address_category(data, 'display_address', data_frame)
		test_data_frame['address_category'] = calculate_address_category(test_data, 'display_address', test_data_frame)

		# Extracting hour and append them into data frame
		hour = extrac_hour(data.get("created", ""))
		test_hour = extrac_hour(test_data.get("created", ""))
		data_frame['hour'] = hour
		test_data_frame['hour'] = test_hour

		# Append the features word counting as columns into data_frame
		data_frame['count_feature'] = count_feature(data_frame)
		test_data_frame['count_feature'] = count_feature(test_data_frame)

		# Get the target and features
		target_values = data_frame['interest_level'].values.reshape(-1, 1)
		features = ['price', 'address_category', 'hour', 'latitude', 'longitude', 'bathrooms', 'count_photos', 'count_feature']

		# Set up the classifier
		clf = DecisionTreeClassifier(min_samples_split=5, max_depth=6)
		# Using cross_val_score to calculate the score of the training set
		score = cross_val_score(clf, data_frame[features], target_values, cv=10)

		print ("The score of the training set is: \n", score)
		print ("Mean of score: ", score.mean())
		print ("Accuracy: %0.2f (+/- %0.2f)" % (score.mean(), score.std() * 2))

		# Train the model
		clf.fit(data_frame[features], target_values)

		# Get the predict probability of test set
		predict = clf.predict_proba(test_data_frame[features])

		# Put the prediction into data frame with listing id
		# Export it to csv file for submitting to Kaggle
		result = pd.DataFrame({"listing_id": test_data_frame["listing_id"].values, "high": predict[:, 0], "medium": predict[:, 2], "low": predict[:, 1]})
		result.to_csv('decision_tree_result.csv', index=False)
		print ("\nFile: decision_tree_result.csv generated.")
		# Draw the decision tree into pdf
		dot_data = tree.export_graphviz(clf, out_file=None,
										filled=True, rounded=True,
										special_characters=True)
		graph = graphviz.Source(dot_data)
		graph.render("Decision Tree")
		print ("\nFile: Decision Tree.pdf generated.")

def SVM():
	print ('\nRunning SVM...\n')
	#open tain.jason
	with open('train.json') as json_file:
		data = json.load(json_file)
		data_frame = pd.DataFrame(data)

		print ("Extracting features from text...\n")

		corpus = data_frame['features'].values
		with open("text_features.txt", "r") as f:
			feature_names = f.read().splitlines()

		for i in range(len(feature_names)):
			feature_names[i] = feature_names[i].capitalize()

		# print(corpus)
		# print(feature_names)
		count_f = count_feature_number(feature_names)
		result = list(map(count_f, corpus))
		# print(result)

		myfeatures = result
		mytarget = data_frame['interest_level'].values
		# X_train, X_test, y_train, y_test = train_test_split(myfeatures, mytarget, test_size = 0.2)
		clf = svm.SVC(kernel = 'linear')
		clf.fit(myfeatures, mytarget)
		y_pred = clf.predict(myfeatures)
		print(y_pred)

		print("Accuracy:",metrics.accuracy_score(mytarget, y_pred))

		#start cross validation by k-fold
		from sklearn.model_selection import KFold # import KFold
		X = np.array(result) # create an array
		y = np.array(mytarget) # Create another array
		kf = KFold(n_splits=5) # Define the split - into 2 folds
		kf.get_n_splits(myfeatures) # returns the number of splitting iterations in the cross-validator
		clf2 = svm.SVC(kernel='linear',probability=True) # Linear Kernel
		scores = []
		proba = []
		for train_index, test_index in kf.split(X):
			X_train, X_test = X[train_index], X[test_index]
			y_train, y_test = y[train_index], y[test_index]
			#Create a svm Classifier

			#Train the model using the training sets
			clf2.fit(X_train, y_train)

			#Predict the response for test dataset
			y_pred = clf2.predict(X_test)
			scores.append(metrics.accuracy_score(y_test, y_pred))

			y_prob = clf2.predict_proba(myfeatures)
			proba.append(y_prob)
		print("done")

	#test the model by test data
	with open('test.json') as json_file:
		data_test = json.load(json_file)
		data_frame_test = pd.DataFrame(data_test)

		corpus_test = data_frame_test['features'].values
		with open("text_features.txt", "r") as f:
			feature_names_test = f.read().splitlines()

		for i in range(len(feature_names_test)):
			feature_names_test[i] = feature_names_test[i].capitalize()

		# print(corpus)
		# print(feature_names)
		count_f_test = count_feature_number(feature_names_test)
		result_test = list(map(count_f_test, corpus_test))

		myfeatures_test = result_test
		y_pred_test = clf2.predict(myfeatures_test)

		y_prob_test = clf2.predict_proba(myfeatures_test)

		y_prob_test2 = y_prob_test
		#y_prob_test
		y_prob_test = []
		for i in range(len(y_prob_test2)):
			temp = []
			temp.append(y_prob_test2[i][0])
			temp.append(y_prob_test2[i][2])
			temp.append(y_prob_test2[i][1])
			y_prob_test.append(temp)

		listing_ids_test = list(data_frame_test['listing_id'].values)
		i_test = 0
		output_test = []
		for i in range(len(y_prob_test)):
			temp_test = []
			temp_test.append(listing_ids_test[i])

			temp_test.extend(y_prob_test[i])
			output_test.append(temp_test)
			i_test = i_test + 1

		with open('SVM_result.csv', 'w', newline='') as mycsvfile:
			thedatawriter = csv.writer(mycsvfile, delimiter=',', lineterminator = '\n')
			thedatawriter.writerows(output_test)
		with open('SVM_result.csv',newline='') as f:
			r = csv.reader(f)
			data = [line for line in r]
		with open('SVM_result.csv','w',newline='') as f:
			w = csv.writer(f)
			w.writerow(['listing_id', 'high', 'medium', 'low'])
			w.writerows(data)

def logistic_regression():
	print ("\nRunning Logistic Regression...\n")
	with open('test.json') as json_file:
		test_data = json.load(json_file)

	with open('train.json') as json_file:
		data = json.load(json_file)

		data_frame = pd.DataFrame(data)
		test_data_frame = pd.DataFrame(test_data)

		city_latitude = 40.730610
		city_longitude = -73.935242

		x_square = x_ysquare(data_frame['latitude'],city_latitude)
		y_square = x_ysquare(data_frame['longitude'],city_longitude)

		x_square_test = x_ysquare(test_data_frame['latitude'],city_latitude)
		y_square_test = x_ysquare(test_data_frame['longitude'],city_longitude)

		# print (max(calculate_address_category(data, 'display_address', data_frame)))

		data_frame['address_category'] = calculate_address_category(data, 'display_address', data_frame)
		test_data_frame['address_category'] = calculate_address_category(test_data, 'display_address', test_data_frame)

		data_frame['center_distance'] = distance(x_square,y_square)
		test_data_frame['center_distance'] = distance(x_square_test,y_square_test)

		hour = extrac_hour(data.get("created", ""))
		test_hour = extrac_hour(test_data.get("created", ""))
		data_frame['hour'] = hour
		test_data_frame['hour'] = test_hour

		target_values = data_frame["interest_level"].values.ravel()
		features = ['price', 'address_category', 'hour', 'latitude', 'longitude', 'bathrooms','center_distance']
		# kf = KFold(n_splits=10)

		lr = LogisticRegression(multi_class ='multinomial', max_iter=5000)
		lr.fit(data_frame[features], target_values)

		score = cross_val_score(lr, data_frame[features], target_values, cv=10)

		print (score)
		print (score.mean())

		print("=================")

		accuracy = cross_val_score(lr, data_frame[features], target_values, cv=10,scoring='balanced_accuracy')
		print(accuracy)
		print (accuracy.mean())

		predict = lr.predict_proba(test_data_frame[features])

		result = pd.DataFrame({"listing_id": test_data_frame["listing_id"].values, "high": predict[:, 0], "medium": predict[:, 2], "low": predict[:, 1]})
		result.to_csv('logistic_regression_result.csv', index=False)

if __name__ == "__main__":
	decision_tree()
	logistic_regression()
	SVM()


from typing import Optional

import py3Dmol
from rdkit.Chem import AllChem, Mol, MolToMolBlock
from rdkit.Chem.Draw.IPythonConsole import ShowMols

def draw3d(m: Mol, dimensions: tuple[int, int] = (500, 300), p: Optional[py3Dmol.view] = None):
    AllChem.EmbedMultipleConfs(m, clearConfs=True, numConfs=50)
    opt = AllChem.MMFFOptimizeMoleculeConfs(m)
    conf = min(range(len(opt)), key=lambda x: opt[x][1] if opt[x][0] == 0 else float("inf"))
    mb = MolToMolBlock(m, confId=conf)

    if p is None:
        p = py3Dmol.view(width=dimensions[0], height=dimensions[1])

    p.removeAllModels()
    p.addModel(mb, 'sdf')
    p.setStyle({'stick': {}})
    p.setBackgroundColor('0xeeeeee')
    p.zoomTo()
    return p.show()


import collections
import keras
import keras.utils.np_utils
import pandas as pd
import glob
import os
from os.path import exists
import fileinput as fi
import numpy as np
import sklearn.preprocessing
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import f_classif
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.neighbors import KNeighborsClassifier

import tensorflow as tf
import matplotlib.pyplot as plt

# Changing max size for printing pandas df.

def fetch_metric(fp, set_target):
    """
    A function that takes a list of filepaths, creates a pd.Dataframe containing the info from each file. If the
    file contain more than one row of samples all the rows below the first data row is removed before appending it to
    the final df that gets returned.

    :param fp: A list of the filepaths we want to read in.
    :param set_target: The target we want for the files in the folder. 0 for acl, 1 for control, 2 for athletes.
    :return: A dataframe containing the data from the files we just read in.
    """
    # Iterating over all filepaths given.
    dict_with_values = collections.defaultdict(list)
    prev_entries = 0
    current_max = 0
    for filepath in fp:
        print(f'IMPORTING FILE: {filepath}')
        # Loading data as df and extracting the rows needed.

        temp_df = pd.read_csv(filepath, delimiter="\t")
        #print(temp_df)
        temp_df.drop(temp_df.columns[0], axis=1, inplace=True)
        patient_name = [i for i in temp_df.columns]

        # Combining the sensors location with the axis the datapoint is regarding.
        # skipping inclution of node since they still have unique names. No Metric data with the same name.
        categories_of_data = temp_df.iloc[0] + temp_df.iloc[3]

        # Getting a list of all the measured values.
        actual_datapoints = [i for i in temp_df.iloc[4]]
        list_of_columns_to_create = [str(i) for i in categories_of_data]
        # Checking for missing data.
        d = dict()
        for value in list_of_columns_to_create:
            if value not in d:
                d[value] = 1
            else:
                d[value] += 1
        number_reps = max(d.values())
        unique_cols = list(d.keys())
        # Making a new list of all correct columns. Meaning the missing columns have been added.
        correct_cols = unique_cols * number_reps
        index_to_add_nan = []
        # Finding which categories actually are missing.
        missing_categories = [k for k, v in d.items() if float(v) < number_reps]
        fixed = 0
        for i in range(len(list_of_columns_to_create)):
            if correct_cols[i + fixed] != list_of_columns_to_create[i]:
                index_to_add_nan.append(i)
                fixed += 1

        for index in index_to_add_nan:
            actual_datapoints.insert(index, np.nan)

        # Making sure each datapoint have a corresponding description of location for the sensor.
        if len(actual_datapoints) != len(correct_cols):
            print('Something wrong with the import. Number of datapoints do not match categories')
            break

        # Setting up list containing the pairs of sensor location and the actual value.
        setup_for_dict = []
        for i in range(len(correct_cols)):
            setup_for_dict.append([correct_cols[i], actual_datapoints[i]])

        # Setting up dict to create dataframe of based on the setup list above.
        for sublist in setup_for_dict:
            if sublist[0] in dict_with_values.keys():
                dict_with_values[sublist[0]].append(sublist[1])
            else:
                dict_with_values[sublist[0]] = [sublist[1]]

        for k, v in dict_with_values.items():
            current_max = max(current_max, len(list(filter(None, v))))
        number_of_repetitions_added = current_max - prev_entries
        prev_entries = current_max
        for i in range(number_of_repetitions_added):
            if 'Patient_name' in dict_with_values.keys():
                dict_with_values['Target'].append(set_target)
                dict_with_values['Patient_name'].append(patient_name[1] + str(i+1))
            else:
                dict_with_values['Target'] = [set_target]
                dict_with_values['Patient_name'] = [patient_name[1] + str(i+1)]

        # Creating df for the patient.
    folder_df = pd.DataFrame(dict([(k, pd.Series(v)) for k, v in dict_with_values.items()]))

    #print(folder_df.head(500))
    #print(folder_df[folder_df.isnull().any(axis=1)])
    folder_df.dropna(inplace=True)
    return folder_df

def fetch_p2d(fp, set_target):
    """
        A function that takes a list of filepaths, creates a pd.Dataframe containing the info from each file. If the
        file contain more than one row of samples all the rows below the first data row is removed before appending it to
        the final df that gets returned.

        :param fp: A list of the filepaths we want to read in.
        :param set_target: The target we want for the files in the folder. 0 for acl, 1 for controll, 2 for athletes.
        :return: A dataframe containing the data from the files we just read in.
        """
    # Iterating over all filepaths given.
    dict_with_values = collections.defaultdict(list)
    dict_list_of_list = collections.defaultdict(list)
    prev_entries = 0
    breaking = 0
    current_max = 0
    for filepath in fp:
        print(f'IMPORTING FILE: {filepath}')
        # Loading data as df and extracting the rows needed.
        temp_df = pd.read_csv(filepath, delimiter="\t")
        temp_df.drop(temp_df.columns[0], axis=1, inplace=True)
        patient_name = [i for i in temp_df.columns]

        # Combining the sensors location with the axis the datapoint is regarding.
        # skipping inclution of node since they still have unique names. No Metric data with the same name.
        categories_of_data = temp_df.iloc[0] + temp_df.iloc[3]

        # Getting a list of all the measured values.
        actual_datapoints = [temp_df[i].iloc[4:].tolist() for i in temp_df.columns]
        list_of_columns_to_create = [i for i in categories_of_data]

        # Making sure each datapoint have a corresponding description of location for the sensor.
        if len(actual_datapoints) != len(list_of_columns_to_create):
            print('Something wrong with the import. Number of datapoints do not match categories')
            breaking += 1
            break

        # Setting up list containing the pairs of sensor location and the actual value.
        setup_for_dict = []
        for i in range(len(list_of_columns_to_create)):
            setup_for_dict.append([list_of_columns_to_create[i], actual_datapoints[i]])

        # Setting up dict to create dataframe of based on the setup list above.
        for sublist in setup_for_dict:
            if sublist[0] in dict_with_values.keys():
                dict_with_values[sublist[0]].append(sublist[1])
            else:
                dict_with_values[sublist[0]] = [sublist[1]]

        number_of_cols_in_temp_df = len(list_of_columns_to_create)

        # Getting the amount of unique cols on first iteration before target and name is added.
        if prev_entries == 0:
            number_of_unique_cols = len(dict_with_values.keys())
        number_of_repetitions = number_of_cols_in_temp_df / number_of_unique_cols
        temp_df_cols = [i for i in temp_df.columns]
        # Slicing df width so that that repetitions don't get added as columns.
        for i in range(1, int(number_of_repetitions) + 1):
            # Creating df with the slice of the columns containing one repetitions data.
            new_df = temp_df.loc[:, (temp_df_cols[(i - 1) * int(number_of_unique_cols):(i * int(number_of_unique_cols))])]
            columns_of_data = new_df.iloc[0] + new_df.iloc[3]

            # Dropping all values but the actual datapoints.
            new_df.drop([0, 1, 2, 3], inplace=True)
            # Checking for nan values and only adding if the set is intact.
            if not new_df.isna().values.any():
                # Making the datapoints into a list of lists.
                actual_data = new_df.values.tolist()

                # Filling dictionary to save each repetitions data.
                if 'patient_name' in dict_list_of_list:
                    dict_list_of_list['data'].append(actual_data)
                    dict_list_of_list['patient_name'].append(patient_name[1] + str(i))
                    dict_list_of_list['Target'].append(set_target)
                else:
                    dict_list_of_list['data'] = [actual_data]
                    dict_list_of_list['patient_name'] = [patient_name[1] + str(i)]
                    dict_list_of_list['Target'] = [set_target]

        # Keeping this segment because we use prev_entries to build dict above.
        for k, v in dict_with_values.items():
            current_max = max(current_max, len(list(filter(None, v))))
        number_of_repetitions_added = current_max - prev_entries
        prev_entries = current_max
        for i in range(number_of_repetitions_added):
            if 'Patient_name' in dict_with_values.keys():
                dict_with_values['Target'].append(set_target)
                dict_with_values['Patient_name'].append(patient_name[1])
            else:
                dict_with_values['Target'] = [set_target]
                dict_with_values['Patient_name'] = [patient_name[1]]

        # Creating df for the patient.
        # saknas värden för vissa sensorer så df blir olika lång -.- fix it.
    #print('BUILDING DF')
    folder_df = pd.DataFrame(dict([(k, pd.Series(v)) for k, v in dict_list_of_list.items()]))
    #fo_df = pd.DataFrame(dict_list_of_list)
    #print(fo_df.head(20))
    #print(dict_list_of_list.keys())
    folder_df.dropna(inplace=True)
    #print(folder_df)
    # Changing df structure from each col containing values to one col containing a list of lists
    #print(breaking)
    #print(folder_df.head(200))
    return folder_df

def build_sets(folder_path):
    """
    A function that gets all patient files on the affected or non-dominant leg from each folderpath specified below.
    In order for function to work you will have to change the paths to be the location where you store the data locally.
    The dataframes created contain the data points with set targets (0 for acl, 1 for control and 2 for athlete) and
    the patient name in the form of one sample filename.

    :param folder_path: Is the path to the folder containing all the datasets.
    :return: Two dataframes containing metric and p2d data from all different tests except olvj.

    Comments: Making individual models for each test ( OLHD, SD etc.) and then combine outputs from each test
    into an input for a new model that predicts category. Might need more data and patient name must be taken into
    consideration in order for the model to get accurate inputs. Meaning every patient must have corresponding data for
    each test.
    """
    # Setting up variables containing the paths to the 3 groups.
    ########## ACL GROUP #############

    # OLHD data paths
    olhd_acl_p2d_fp = glob.glob(f"{folder_path}/OLHD AI export/ACL Injured/anonymized/*_A_p2d.txt")
    olhd_acl_metric_fp = glob.glob(f"{folder_path}/OLHD AI export/ACL Injured/anonymized/*_A_metric.txt")

    # OLVJ data paths
    olvj_acl_p2d_fp = glob.glob(f"{folder_path}/OLVJ AI export/ACL Injured/anonymized/*_A_p2d.txt")
    # No point using metric data, only one sample avalible. Will only be an outliner decreasing performance.
    # olvj_acl_metric_fp = glob.glob("/home/wubba/ai_umu/raw_data/OLVJ AI export/ACL Injured/anonymized/*_A_metric.txt")

    # SD data paths
    sd_acl_p2d_fp = glob.glob(f"{folder_path}/SD AI export/ACL Injured/anonymized/*_A_p2d.txt")
    sd_acl_metric_fp = glob.glob(f"{folder_path}/SD AI export/ACL Injured/anonymized/*_A_metric.txt")

    # SDPS data paths
    sdps_acl_p2d_fp = glob.glob(f"{folder_path}/SDPS AI export/ACL Injured/anonymized/*_A2_p2d.txt")
    sdps_acl_metric_fp = glob.glob(f"{folder_path}/SDPS AI export/ACL Injured/anonymized/*_A2_metric.txt")

    # SH data paths
    sh_acl_p2d_fp = glob.glob(f"{folder_path}/SH AI export/ACL Injured/anonymized/*_A_p2d.txt")
    sh_acl_metric_fp = glob.glob(f"{folder_path}/SH AI export/ACL Injured/anonymized/*_A_metric.txt")

    ######### ATHLETE GROUP ##############

    # OLHD data paths
    olhd_athletes_p2d_fp = glob.glob(f"{folder_path}/OLHD AI export/Athletes/anonymized/*_ND_p2d.txt")
    olhd_athletes_metric_fp = glob.glob(f"{folder_path}/OLHD AI export/Athletes/anonymized/*_ND_metric.txt")

    # OLVJ data paths
    olvj_athletes_p2d_fp = glob.glob(f"{folder_path}/OLVJ AI export/Athletes/anonymized/*_ND_p2d.txt")
    # No point using metric data, only one sample avalible. Will only be an outliner decreasing performance.
    # olvj_acl_metric_fp = glob.glob("/home/wubba/ai_umu/raw_data/OLVJ AI export/ACL Injured/anonymized/*_A_metric.txt")

    # SD data paths
    sd_athletes_p2d_fp = glob.glob(f"{folder_path}/SD AI export/Athletes/anonymized/*_ND_p2d.txt")
    sd_athletes_metric_fp = glob.glob(f"{folder_path}/SD AI export/Athletes/anonymized/*_ND_metric.txt")

    # SDPS data paths
    sdps_athletes_p2d_fp = glob.glob(f"{folder_path}/SDPS AI export/Athletes/anonymized/*_ND2_p2d.txt")
    sdps_athletes_metric_fp = glob.glob(f"{folder_path}/SDPS AI export/Athletes/anonymized/*_ND2_metric.txt")

    # SH data paths
    sh_athletes_p2d_fp = glob.glob(f"{folder_path}/SH AI export/Athletes/anonymized/*_ND_p2d.txt")
    sh_athletes_metric_fp = glob.glob(f"{folder_path}/SH AI export/Athletes/anonymized/*_ND_metric.txt")

    ######### Controls group #############

    # OLHD data paths
    olhd_controls_p2d_fp = glob.glob(f"{folder_path}/OLHD AI export/Controls/anonymized/*_ND_p2d.txt")
    olhd_controls_metric_fp = glob.glob(f"{folder_path}/OLHD AI export/Controls/anonymized/*_ND_metric.txt")

    # OLVJ data paths
    olvj_controls_p2d_fp = glob.glob(f"{folder_path}/OLVJ AI export/Controls/anonymized/*_ND_p2d.txt")
    # No point using metric data, only one sample avalible. Will only be an outliner decreasing performance.

    # SD data paths
    sd_controls_p2d_fp = glob.glob(f"{folder_path}/SD AI export/Controls/anonymized/*_ND_p2d.txt")
    sd_controls_metric_fp = glob.glob(f"{folder_path}/SD AI export/Controls/anonymized/*_ND_metric.txt")

    # SDPS data paths
    sdps_controls_p2d_fp = glob.glob(f"{folder_path}/SDPS AI export/Controls/anonymized/*_ND2_p2d.txt")
    sdps_controls_metric_fp = glob.glob(f"{folder_path}/SDPS AI export/Controls/anonymized/*_ND2_metric.txt")

    # SH data paths
    sh_controls_p2d_fp = glob.glob(f"{folder_path}/SH AI export/Controls/anonymized/*_ND_p2d.txt")
    sh_controls_metric_fp = glob.glob(f"{folder_path}/SH AI export/Controls/anonymized/*_ND_metric.txt")

    ########## IMPORTING ############

    # OLHD
    olhd_acl_metric_df = fetch_metric(olhd_acl_metric_fp, 0)
    olhd_controls_metric_df = fetch_metric(olhd_controls_metric_fp, 1)
    olhd_athletes_metric_df = fetch_metric(olhd_athletes_metric_fp, 2)
    olhd_metric_df = pd.concat([olhd_acl_metric_df, olhd_controls_metric_df, olhd_athletes_metric_df], ignore_index=True)

    olhd_acl_p2d_df = fetch_p2d(olhd_acl_p2d_fp, 0)
    olhd_controls_p2d_df = fetch_p2d(olhd_controls_p2d_fp, 1)
    olhd_athletes_p2d_df = fetch_p2d(olhd_athletes_p2d_fp, 2)
    olhd_p2d_df = pd.concat([olhd_acl_p2d_df, olhd_controls_p2d_df, olhd_athletes_p2d_df], ignore_index=True)

    # OLVJ
    olvj_acl_p2d_df = fetch_p2d(olvj_acl_p2d_fp, 0)
    olvj_controls_p2d_df = fetch_p2d(olvj_controls_p2d_fp, 1)
    olvj_athletes_p2d_df = fetch_p2d(olvj_athletes_p2d_fp, 2)
    olvj_p2d_df = pd.concat([olvj_acl_p2d_df, olvj_controls_p2d_df, olvj_athletes_p2d_df], ignore_index=True)

    # SD
    sd_acl_metric_df = fetch_metric(sd_acl_metric_fp, 0)
    sd_controls_metric_df = fetch_metric(sd_controls_metric_fp, 1)
    sd_athletes_metric_df = fetch_metric(sd_athletes_metric_fp, 2)
    sd_metric_df = pd.concat([sd_acl_metric_df, sd_controls_metric_df, sd_athletes_metric_df], ignore_index=True)

    sd_acl_p2d_df = fetch_p2d(sd_acl_p2d_fp, 0)
    sd_controls_p2d_df = fetch_p2d(sd_controls_p2d_fp, 1)
    sd_athletes_p2d_df = fetch_p2d(sd_athletes_p2d_fp, 2)
    sd_p2d_df = pd.concat([sd_acl_p2d_df, sd_controls_p2d_df, sd_athletes_p2d_df], ignore_index=True)

    # SDPS
    sdps_acl_metric_df = fetch_metric(sdps_acl_metric_fp, 0)
    sdps_controls_metric_df = fetch_metric(sdps_controls_metric_fp, 1)
    sdps_athletes_metric_df = fetch_metric(sdps_athletes_metric_fp, 2)
    sdps_metric_df = pd.concat([sdps_acl_metric_df, sdps_controls_metric_df, sdps_athletes_metric_df], ignore_index=True)

    sdps_acl_p2d_df = fetch_p2d(sdps_acl_p2d_fp, 0)
    sdps_controls_p2d_df = fetch_p2d(sdps_controls_p2d_fp, 1)
    sdps_athletes_p2d_df = fetch_p2d(sdps_athletes_p2d_fp, 2)
    sdps_p2d_df = pd.concat([sdps_acl_p2d_df, sdps_controls_p2d_df, sdps_athletes_p2d_df], ignore_index=True)

    # SH
    sh_acl_metric_df = fetch_metric(sh_acl_metric_fp, 0)
    sh_controls_metric_df = fetch_metric(sh_controls_metric_fp, 1)
    sh_athletes_metric_df = fetch_metric(sh_athletes_metric_fp, 2)
    sh_metric_df = pd.concat([sh_acl_metric_df, sh_controls_metric_df, sh_athletes_metric_df], ignore_index=True)

    sh_acl_p2d_df = fetch_p2d(sh_acl_p2d_fp, 0)
    sh_controls_p2d_df = fetch_p2d(sh_controls_p2d_fp, 1)
    sh_athletes_p2d_df = fetch_p2d(sh_athletes_p2d_fp, 2)
    sh_p2d_df = pd.concat([sh_acl_p2d_df, sh_controls_p2d_df, sh_athletes_p2d_df], ignore_index=True)

    # Cant assemble complete datasets for all tests since the data are from different sensors making different columns.
    #metric_df = pd.concat([olhd_metric_df, sd_metric_df, sdps_metric_df, sh_metric_df], ignore_index=True)
    #p2d_df = pd.concat([olhd_p2d_df, olvj_p2d_df, sd_p2d_df, sdps_p2d_df, sh_p2d_df], ignore_index=True)
    #print(sh_metric_df.head(1000))
    #print(olhd_metric_df.head(1000))
    #print(sd_metric_df.head(100))
    #print(sh_metric_df.head(100))
    #print(p2d_df.head(1000))
    return olhd_metric_df, olhd_p2d_df, olvj_p2d_df, sd_metric_df, sd_p2d_df, sdps_metric_df, sdps_p2d_df, sh_metric_df, sh_p2d_df

def check_metric_features(df, n_features):
    """
    This is a function which uses 2 different types of feature selection to reduce the number of columns our df
    consists of. It combines the top n_features from each test and only keeps the calculated columns. This means
    that the df will most likely consist of more than n_features columns when it's done.

    :param df: The dataframe of which we want to reduce the number of columns in.
    :param n_features: The number of top features we want from each feature search.
    :return: Returns the df with the selected features.
    """

    targets = df['Target_x'].values
    train = df.iloc[:, 0:-2]
    df_columns = df.columns
    scaler = sklearn.preprocessing.MinMaxScaler(feature_range=(-1, 1))
    train_scaled = scaler.fit_transform(train)
    # Score of columns.
    bestfeatures = SelectKBest(score_func=f_classif, k=n_features)
    fit = bestfeatures.fit(train_scaled, targets)
    score_df = pd.DataFrame(fit.scores_)
    columns_df = pd.DataFrame(train.columns)
    feature_scores = pd.concat([columns_df, score_df], axis=1)
    #print(targets)
    feature_scores.columns = ['category', 'Score']
    top_20 = feature_scores.nlargest(n_features, 'Score')
    highscore_features = top_20['category'].tolist()
    print(highscore_features)

    # Plot of feature importance.
    cate_targets = keras.utils.np_utils.to_categorical(targets)
    model = ExtraTreesClassifier()
    model.fit(train_scaled, cate_targets)
    feature_importance = pd.Series(model.feature_importances_, index=train.columns)
    feature_importance.nlargest(n_features).plot(kind='barh')
    important_features = feature_importance.nlargest(n_features)
    top_features = important_features.index.tolist()
    features_to_use = np.unique(highscore_features + top_features).tolist()
    features_to_use.append('Target_x')
    features_to_use.append('Patient_name')
    features_to_remove = [x for x in df_columns if x not in features_to_use]
    df.drop(features_to_remove, axis=1, inplace=True)

    return df, features_to_use

def baseline_dense_model(xtrain, target, epochs, patience):
    """
    NOT BEING USED.
    :param xtrain:
    :param target:
    :param epochs:
    :param patience:
    :return:
    """
    input_shape = xtrain.shape
    train_stop_index = int(0.7 * input_shape[0])

    train_data = xtrain[:train_stop_index, :]
    train_target = target[:train_stop_index, :]

    validation_data = xtrain[train_stop_index:, :]
    validation_target = target[train_stop_index:, :]
    print(validation_target.shape)
    print(validation_data.shape)

    lstm_test_model = tf.keras.models.Sequential([
        tf.keras.layers.Dense(units=2056, activation='relu'),
        tf.keras.layers.Dropout(0.8),
        tf.keras.layers.Dense(units=2056, activation='relu'),
        tf.keras.layers.Dropout(0.8),
        tf.keras.layers.Dense(units=3, activation='softmax')
    ])
    callbacks = tf.keras.callbacks.EarlyStopping(
        monitor="val_accuracy",
        min_delta=0.001,
        patience=patience,

        mode="auto",
        restore_best_weights=True)

    lstm_test_model.compile(loss='binary_crossentropy',
                            optimizer=tf.optimizers.Adam(learning_rate=0.001),
                            metrics=['accuracy'])
    history = lstm_test_model.fit(train_data, train_target,
                                  epochs=epochs,
                                  validation_data=(validation_data, validation_target),
                                  callbacks=[callbacks],
                                  shuffle=True)

    best_accuracy = max(history.history.get('val_accuracy'))
    return best_accuracy

def baseline_model(xtrain, ytrain, xval, yval):
    """
    NOT USED ATM.
    Was a function to evaluate the models.
    :param xtrain:
    :param ytrain:
    :param xval:
    :param yval:
    :return:
    """
    lstm_test_model = tf.keras.models.Sequential([
        tf.keras.layers.LSTM(64, return_sequences=True),
        tf.keras.layers.LSTM(128, return_sequences=False),
        # tf.keras.layers.LSTM(256, return_sequences=False),
        # tf.keras.layers.Dropout(0.2),
        # tf.keras.layers.Dense(units=512, activation='relu'),
        # maxpooling
        tf.keras.layers.Dense(units=256, activation='relu'),
        tf.keras.layers.Dense(units=3, activation='sigmoid')
    ])
    callbacks = tf.keras.callbacks.EarlyStopping(
        monitor="val_accuracy",
        min_delta=0.001,
        patience=40,
        verbose=0,
        mode="auto",
        restore_best_weights=True)

    lstm_test_model.compile(loss='binary_crossentropy',
                            optimizer=tf.optimizers.Adam(learning_rate=0.01),
                            metrics=['accuracy'])
    history = lstm_test_model.fit(xtrain, ytrain,
                                  epochs=600,
                                  verbose=0,
                                  validation_data=(xval, yval),
                                  callbacks=[callbacks],
                                  shuffle=True)

    best_accuracy = max(history.history.get('val_accuracy'))
    return best_accuracy

def check_p2d_features(df, n_features, patience_of_search):
    """
    NOT USED WHEN TRAINING THE MODELS ATM.

    Kolla på scaling av np array direkt istället för att gå tillbaka i df och skala därifrån.
    Se även över om vi kanske borde skala metric data innan vi gör feature selection.
    Se även över hur vi kan lägga till noise för endast en kolumn itaget i numpy format.

    Kanske sluta testa den som ger störst negativ effekt på acc varje itteration också.

    IMPROVEMENTS:
    Set fixed batch size to smooth out the training time and maybe lock/remove more
    than one feature per iteration.
    Change value assigned to the locked indices since they get assigned the prev avg
    value which makes the new avg higher since it was the lowest in the
    prev iteration. Meaning the algorithm might not stop at the peak but instead
    keep removing features even though no improvement will be made.

    This function takes a dataframe and test the data on a simple LSTM network while iteratively trying to remove one
    feature. The networks with the highest accuracy means that that feature affects the models negatively and can be
    removed for a possible improvement. Besides removing one feature we also lock the feature that performed the worst
    when removed from being tested or removed in future iterations to speed up the selection. The locked indices is
    shown in as a output list "Locked indices".
    The feature selection stops when the resulting avg accuracy when a feature is removed is lower than the previous
    iterations avg.
    :param df: A dataframe that we got from fetch p2d.
    :param n_features: Minimum number of features we want the model to have.
    :param patience_of_search: The number of times each model will be tested with each feature removed. Big numbers will
    make this process expensive so keep it relatively small.
    :return:
    """
    dict_with_acurracy = collections.defaultdict(list)
    # shuffle dataframe
    df = df.sample(frac=1).reset_index()
    # Get targets and convert them from single value 0-2 to 3d vector.
    targets = df['Target_y'].values
    cate_targets = keras.utils.np_utils.to_categorical(targets)

    # Reshaping the train data to be of shape (number_of_patients, samples_from_each, number_of_sensors)
    train = df['data'].values.tolist()
    test_df = pd.DataFrame(train)
    train_array = np.array(train).astype(float)
    input_shape = train_array.shape
    train_array_min = train_array.min(axis=(0, 1), keepdims=True)
    train_array_max = train_array.max(axis=(0, 1), keepdims=True)
    scaled_train_array = (2 * (train_array - train_array_min) / (train_array_max - train_array_min)) - 1

    # Creating a loop removing one feature each iteration untill only n_features are left.
    # input_shape[2] - n_features
    n_of_iterations = 0
    improvement = True
    index_to_lock = []
    base_acc_list = []
    while improvement:
        index_to_test = 0
        array_size = scaled_train_array.shape
        if index_to_lock != None:
            if len(index_to_lock) == array_size[2]:
                improvement = False
        # Clearing the dict
        dict_with_acurracy.clear()

        # i loop range input_shape[2]
        for feature in range(array_size[2]):
            print(f'Testing index {index_to_test}')
            if index_to_lock != None:
                print(f'Locked indices: {index_to_lock}')
            if index_to_test in index_to_lock:
                index_to_test += 1
                dict_with_acurracy[index_to_test].append(avg_overall)
                continue
            train_stop_index = int(0.7 * input_shape[0])
            # Dropping one feature from the data set.

            reduced_array = np.delete(scaled_train_array, index_to_test, 2)
            # Slicing array into train and validation sets.
            reduced_train_data = reduced_array[:train_stop_index, :, :]
            print(f'Current array shape: {reduced_train_data.shape}')
            reduced_train_target = cate_targets[:train_stop_index, :]

            reduced_validation_data = reduced_array[train_stop_index:, :, :]
            reduced_validation_target = cate_targets[train_stop_index:, :]
            # Checking baseline acc. Meaning no reduction of number of columns.
            if n_of_iterations == 0 and index_to_test == 0:
                train_data = scaled_train_array[:train_stop_index, :, :]
                print(f'Current array shape: {scaled_train_array.shape}')
                train_target = cate_targets[:train_stop_index, :]

                validation_data = scaled_train_array[train_stop_index:, :, :]
                validation_target = cate_targets[train_stop_index:, :]
                print(f'RUNNING {patience_of_search} TRIAL RUNS WITH NO MODIFICATION OF ARRAY SIZE, THIS MIGHT TAKE A WHILE')
                for i in range(patience_of_search):
                    base_acc = baseline_model(train_data, train_target, validation_data, validation_target)

                    base_acc_list.append(base_acc)
                base_accuracy = sum(base_acc_list) / len(base_acc_list)
                print(f'BASE MODEL ACC: {base_accuracy}')
            # Buildning test-model to compare features.
            for i in range(patience_of_search):
                best_accuracy = baseline_model(reduced_train_data, reduced_train_target, reduced_validation_data, reduced_validation_target)
                print(f'Best accuracy: {best_accuracy}')
                if index_to_test in dict_with_acurracy:
                    dict_with_acurracy[index_to_test].append(best_accuracy)
                else:
                    dict_with_acurracy[index_to_test] = [best_accuracy]
            index_to_test += 1

        accuracy_avg = []
        for index, values in dict_with_acurracy.items():
            accuracy_avg.append(sum(values) / float(len(values)))
        print(accuracy_avg)
        # Evaluating the improvement and breaking if no improvement was made.
        if n_of_iterations != 0:

            prev_avg = avg_overall
            avg_overall = sum(accuracy_avg) / float(len(accuracy_avg))
            print(f'prev: {prev_avg} achieved this iteration: {avg_overall}')
            if prev_avg < avg_overall:
                index_to_lock.append(accuracy_avg.index(min(accuracy_avg)))
                index_to_remove = accuracy_avg.index(max(accuracy_avg))
                print(f'Removing index: {index_to_remove}')
                scaled_train_array = np.delete(scaled_train_array, index_to_remove, 2)
                # Changing indices in the list of locked indices to stay updated after the removal of index_to_remove.
                for i in range(len(index_to_lock)):
                    if index_to_lock[i] > index_to_remove:
                        index_to_lock[i] -= 1
                n_of_iterations += 1
            elif array_size == n_features:
                improvement = False
            else:
                improvement = False
        else:
            avg_overall = sum(accuracy_avg) / float(len(accuracy_avg))
            print(f'Overall acc: {avg_overall}')
            index_to_lock.append(accuracy_avg.index(min(accuracy_avg)))
            index_to_remove = accuracy_avg.index(max(accuracy_avg))
            max_acc = max(accuracy_avg)

            if index_to_remove in index_to_lock:
                print('NOT REMOVING A LOCKED INDEX!')
                break
            if max_acc < base_accuracy:
                print('NO IMPROVEMENT BY REDUCING ARRAY SIZE. KEEP ALL DATA')
                break
            else:
                print(f'Removing index: {index_to_remove}')
                scaled_train_array = np.delete(scaled_train_array, index_to_remove, 2)
                if index_to_remove < index_to_lock[0]:
                    index_to_lock[0] -= 1
                n_of_iterations += 1

    return scaled_train_array, cate_targets

def same_samples(metric_df, p2d_df):
    """
        Compares the patients from the metric and p2d data between ONE exercise so that the same patient and number of
        repetitions exists in both.
        :param metric_df: Metric df.
        :param p2d_df: p2d df.
        :return: Returns new dfs containing the same patients.
    """
    # Checking metric df.
    s = metric_df.merge(p2d_df, left_on='Patient_name', right_on='patient_name', how='left', indicator=True)
    s.dropna(inplace=True)

    new_metric_df = s.iloc[:, :-4]
    new_p2d_df = s.iloc[:, -4:-1]

    return new_metric_df, new_p2d_df

def same_patients(met_df, p_df):
    """
    Compares the patients from the metric and p2d data between the SD and SDPS data.
    :param met_df: Metric df.
    :param p_df: p2d df.
    :return: Returns new dfs containing the same patients.
    """
    met_df['patient_num'] = met_df.Patient_name.str[:3] + met_df.Patient_name.str[-2:]
    #print(sd_df.head(100))
    p_df['Patient_num'] = p_df.patient_name.str[:3] + p_df.patient_name.str[-2:]
    df = met_df.merge(p_df, left_on='patient_num', right_on='Patient_num', how='left', indicator=True)
    df.dropna(inplace=True)
    fixed_met_df = df.iloc[:, :-6]
    fixed_p_df = df.iloc[:, -5:-2]
    return fixed_met_df, fixed_p_df

"""
CHANGE folder_path TO THE ROOT OF THE DATA. The folder rå data below contains the SD AI EXPORT folder etc. See function
build_sets. To see the expected structure of the folder containing data see paths in the function build_sets.
"""
folder_path = '/home/wubba/umeå_projekt/rå data'
# Importing datasets
olhd_metric_df, olhd_p2d_df, olvj_p2d_df, sd_metric_df, sd_p2d_df, sdps_metric_df, sdps_p2d_df, sh_metric_df, sh_p2d_df = build_sets(folder_path)

# Making sure the same samples exists in both metric and p2d dataframe.
same_sd_metric_df, same_sd_p2d_df = same_samples(sd_metric_df, sd_p2d_df)
same_sdps_metric_df, same_sdps_p2d_df = same_samples(sdps_metric_df, sdps_p2d_df)

# Making same patients and number of reps in sd and sdps data.
same_patient_sd_metric, same_patient_sdps_p2d = same_patients(same_sd_metric_df, same_sdps_p2d_df)
same_patient_sdps_metric, same_patient_sd_p2d = same_patients(same_sdps_metric_df, same_sd_p2d_df)

if not exists('sd_metric.csv'):
    same_patient_sd_metric, saved_sd_metrics = check_metric_features(same_patient_sd_metric, 30)
    same_patient_sd_metric.to_csv('sd_metric.csv', index=False)
    textfile = open('saved_sd_metrics', 'w')
    for metric in saved_sd_metrics:
        textfile.write(metric + '\n')
    textfile.close()

if not exists('sdps_metric.csv'):
    same_patient_sdps_metric, saved_sdps_metrics = check_metric_features(same_patient_sdps_metric, 30)
    same_patient_sdps_metric.to_csv('sdps_metric.csv', index=False)
    textfile = open('saved_sdps_metrics', 'w')
    for metric in saved_sdps_metrics:
        textfile.write(metric + '\n')
    textfile.close()

#sh_metric_df = check_metric_features(sh_metric_df, 20)

# Checking if we can remove any columns of the input data and saving a dataset on disk for easy access.
# This process is quite compute intensive. Improvement of check_p2d_features are possible by setting a bigger batchsize
# and perhaps lock more than one feature per iteration.
#olhd_p2d_train, olhd_p2d_cate_targets = check_p2d_features(olhd_p2d_df, 10, patience_of_search=10)
#np.save('olhd_p2d_training', olhd_p2d_train, allow_pickle=False)
#np.save('olhd_p2d_targets', olhd_p2d_cate_targets, allow_pickle=False)

#olvj_p2d_train, olvj_p2d_cate_targets = check_p2d_features(olvj_p2d_df, 10, patience_of_search=10)
#np.save('olvj_p2d_training', olvj_p2d_train, allow_pickle=False)
#np.save('olvj_p2d_targets', olvj_p2d_cate_targets, allow_pickle=False)

#sd_p2d_train, sd_p2d_cate_targets = check_p2d_features(sd_p2d_df, 10, patience_of_search=10)
#np.save('sd_p2d_training', sd_p2d_train, allow_pickle=False)
#np.save('sd_p2d_targets', sd_p2d_cate_targets, allow_pickle=False)

#sdps_p2d_train, sdps_p2d_cate_targets = check_p2d_features(sdps_p2d_df, 10, patience_of_search=10)
#np.save('sdps_p2d_training', sdps_p2d_train, allow_pickle=False)
#np.save('sdps_p2d_targets', sdps_p2d_cate_targets, allow_pickle=False)

#sh_p2d_train, sh_p2d_cate_targets = check_p2d_features(sh_p2d_df, 10, patience_of_search=10)
#np.save('sh_p2d_training', sh_p2d_train, allow_pickle=False)
#np.save('sh_p2d_targets', sh_p2d_cate_targets, allow_pickle=False)

"""

    Bygg modell där noise introduceras i för en column i taget, jämnför resultat för att se vilka som ger störst
    negativ påverkan när de har noise. Detta betyder att denna feature har påverkan på resultat. Om ingen eller låg
    påverkan sker så kan vi ta bort den kolumnen med data.

"""


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from keras.layers import *
from keras.models import Model
from keras.callbacks import EarlyStopping, TensorBoard
from keras import backend as K

import numpy as np
import matplotlib.pyplot as plt
import argparse
import os

# reparameterization trick
# instead of sampling from Q(z|X), sample eps = N(0,I)
# then z = z_mean + sqrt(var)*eps
from showcase.models import AutoencoderModel
from showcase.image_sequence import create_rgbdseq_from_files

def plot_results(models,
                 data,
                 batch_size=128,
                 model_name="vae_mnist"):
    """Plots labels and MNIST digits as function of 2-dim latent vector

    # Arguments:
        models (tuple): encoder and decoder models
        data (tuple): test data and label
        batch_size (int): prediction batch size
        model_name (string): which model is using this function
    """

    encoder, decoder = models
    x_test, y_test = data
    os.makedirs(model_name, exist_ok=True)

    filename = os.path.join(model_name, "vae_mean.png")
    # display a 2D plot of the digit classes in the latent space
    z_mean, _, _ = encoder.predict(x_test,
                                   batch_size=batch_size)
    plt.figure(figsize=(12, 10))
    plt.scatter(z_mean[:, 0], z_mean[:, 1], c=y_test)
    plt.colorbar()
    plt.xlabel("z[0]")
    plt.ylabel("z[1]")
    plt.savefig(filename)
    plt.show()

    return
    filename = os.path.join(model_name, "digits_over_latent.png")
    # display a 30x30 2D manifold of digits
    n = 30
    digit_size = 28
    figure = np.zeros((digit_size * n, digit_size * n))
    # linearly spaced coordinates corresponding to the 2D plot
    # of digit classes in the latent space
    grid_x = np.linspace(-4, 4, n)
    grid_y = np.linspace(-4, 4, n)[::-1]

    for i, yi in enumerate(grid_y):
        for j, xi in enumerate(grid_x):
            z_sample = np.array([[xi, yi]])
            x_decoded = decoder.predict(z_sample)
            digit = x_decoded[0].reshape(digit_size, digit_size)
            figure[i * digit_size: (i + 1) * digit_size,
                   j * digit_size: (j + 1) * digit_size] = digit

    plt.figure(figsize=(10, 10))
    start_range = digit_size // 2
    end_range = n * digit_size + start_range + 1
    pixel_range = np.arange(start_range, end_range, digit_size)
    sample_range_x = np.round(grid_x, 1)
    sample_range_y = np.round(grid_y, 1)
    plt.xticks(pixel_range, sample_range_x)
    plt.yticks(pixel_range, sample_range_y)
    plt.xlabel("z[0]")
    plt.ylabel("z[1]")
    plt.imshow(figure, cmap='Greys_r')
    plt.savefig(filename)
    plt.show()

def plot_history(history):
    plt.plot(history.history['loss'])
    plt.plot(history.history['val_loss'])
    plt.title('model loss')
    plt.ylabel('loss')
    plt.xlabel('epoch')
    plt.legend(['train', 'test'], loc='upper left')
    plt.show()

# data set
image_height = 96
image_width = 128
n_chans = 4
batch_size = 128
input_shape = (image_height, image_width, n_chans)

train_gen = create_rgbdseq_from_files(glob_pattern="data/*.npz", batch_size=batch_size, is_validation=False)
val_gen = create_rgbdseq_from_files(glob_pattern="data/*.npz", batch_size=batch_size, is_validation=True)

x_train = np.vstack(filter(lambda x: isinstance(x, bool)==False, [train_gen[i] for i in range(600)]))
x_test = np.vstack(filter(lambda x: isinstance(x, bool)==False, [val_gen[i] for i in range(80)]))

model = AutoencoderModel(input_shape)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("-w", "--weights", help="Load h5 model trained weights")
    parser.add_argument("-e", "--epochs", default=25)

    args = parser.parse_args()

    if args.weights:
        model.load_model(args.weights)
    else:
        # train the autoencoder
        hist = model.fit(x_train, x_test)
        model.save_model('autoenc_cnn_rgbd_180526.h5')
        plot_history(hist)

    #plot_results(models, data, batch_size=batch_size, model_name="vae_cnn")


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
import django.utils.timezone

class Migration(migrations.Migration):

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='Year',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('start_year', models.IntegerField(unique=True, db_index=True)),
                ('added_time', models.DateTimeField(default=django.utils.timezone.now, auto_now_add=True)),
                ('update_time', models.DateTimeField(default=django.utils.timezone.now, auto_now=True)),
            ],
            options={
            },
            bases=(models.Model,),
        ),
    ]


import json
import requests
import sys

sys.path.append("..")

from common.aidevs_authorize import get_task, get_token, send_answer, BASE_URL
from common.openai_requests import send_moderations, send_chat_completion

task_name = "liar"
token = get_token(task_name)
task = get_task(token)

question = "what is capital of Poland? Answer ultra-briefly, possibly in one word."
resp = requests.post(f"{BASE_URL}/task/{token}", data={"question": question})
liar_answer = json.loads(resp.text)["answer"]

resp_gpt = send_chat_completion(
    model_version="gpt-4",
    system_content="Provide brief Yes/No answer",
    user_content=f"""You will be provided with a question and answer. Tell if the answer is correct or not.\n
    ###Question: {question}\n
    ###Answer: {liar_answer}""",
)

guard_rail = resp_gpt["choices"][0]["message"]["content"]

resp_after_guard = send_answer(token, guard_rail)
print(json.loads(resp_after_guard))


import numpy as np
from scipy.optimize import minimize

import utils

len_x = 2
len_y = 4
len_z = 3

r = [None] * len_x

for i in range(len_x):
    r[i] = [None] * len_y
    for j in range(len_y):
        r[i][j] = [None] * len_z
        for k in range(len_z):
            r[i][j][k] = i * j * k

# print(r)

vec = utils.toVector(r)
multiDim = utils.to3D(vec, len_x, len_y, len_z)

print(r)
print(len(vec))
print(multiDim)

# arr_1d = np.arange(12)
# arr_3d = arr_1d.reshape(2, 3, 2)

# print(arr_3d)


"""Perform fitting with a range of regularizers"""
from pyhdx.batch_processing import yaml_to_hdxmset, StateParser
from pathlib import Path
from pyhdx.fitting import fit_gibbs_global_batch
import yaml
import time
from datetime import datetime
from pyhdx.local_cluster import default_client
from pyhdx.fileIO import csv_to_protein
from pyhdx.models import HDXMeasurementSet
from pyhdx.support import pprint_df_to_file
from pyhdx import VERSION_STRING

current_dir = Path(__file__).parent
test_data_dir = current_dir.parent / "tests" / "test_data"
input_dir = test_data_dir / "input"
yaml_stream = Path(current_dir / "yaml_files" / "SecB.yaml").read_text()
data_dict = yaml.safe_load(yaml_stream)

output_dir = current_dir / "fit"
output_dir.mkdir(exist_ok=True)

parser = StateParser(data_dict, data_src=input_dir)
hdx_set = parser.load_hdxmset()

rates_list = [
    csv_to_protein(current_dir / "guesses" / f"{name}_rates_guess.csv")["rate"]
    for name in data_dict.keys()
]
gibbs_guess = hdx_set.guess_deltaG(rates_list)

log_file = output_dir / f"fitting_log.txt"
now = datetime.now()
date = f'# {now.strftime("%Y/%m/%d %H:%M:%S")} ({int(now.timestamp())})'

lines = [VERSION_STRING, date]

r2 = 0.5
for r1 in [0, 0.01, 0.25, 0.5, 1]:
    t0 = time.time()
    result = fit_gibbs_global_batch(hdx_set, gibbs_guess, epochs=1000, r1=r1, r2=r2)
    t1 = time.time()

    block = "--------------------------"
    regularizers = f"Regualizer 1: {r1}  Regualizer 2: {r2}"
    loss = (
        f"Total_loss {result.total_loss:.2f}, mse_loss {result.mse_loss:.2f}, reg_loss {result.reg_loss:.2f}"
        f"({result.regularization_percentage:.2f}%)"
    )
    time_elapsed = f"Time elapsed: {(t1 - t0):.2f} s"
    epochs = f"Number of epochs: {result.metadata['epochs_run']}"

    result.output.to_csv(output_dir / f"fit_output_r1_{r1}_r2_{r2}.csv")  # , na_rep='NaN')
    result.output.to_file(
        output_dir / f"fit_output_r1_{r1}_r2_{r2}.txt", fmt="pprint", na_rep="NaN"
    )

    lines += ["", block, regularizers, loss, epochs, time_elapsed, block]

log_file.write_text("\n".join(lines))


import io
from PIL import Image as PilImage
from typing import Tuple, Type, Union

from .attachments import Thumbnail
from .descriptors import StreamDescriptor
from .helpers import validate_width_height_ratio
from .typing_ import FileLike

def generate_thumbnail(
    original_image: Union[FileLike, StreamDescriptor],
    width: int = None,
    height: int = None,
    ratio: float = None,
    ratio_precision: int = 5,
    thumbnail_type: Type[Thumbnail] = Thumbnail,
) -> Tuple[int, int, float, Thumbnail]:
    width, height, ratio = validate_width_height_ratio(
        width, height, ratio
    )
    thumbnail_buffer = io.BytesIO()
    format_ = 'jpeg'
    extension = '.jpg'

    # generating thumbnail and storing in buffer
    img = PilImage.open(original_image)

    # JPEG does not handle Alpha channel, switch to PNG
    if img.mode == 'RGBA':
        format_ = 'png'
        extension = '.png'

    with img:
        original_size = img.size

        if callable(width):
            width = width(original_size)
        if callable(height):
            height = height(original_size)

        width = int(width)
        height = int(height)

        thumbnail_image = img.resize((width, height))
        thumbnail_image.save(thumbnail_buffer, format_)

    thumbnail_buffer.seek(0)

    ratio = round(width / original_size[0], ratio_precision)
    thumbnail = thumbnail_type.create_from(
        thumbnail_buffer,
        content_type=f'image/{format_}',
        extension=extension,
        dimension=(width, height)
    )

    return width, height, ratio, thumbnail


import serial
import time
import win32com.client
import numpy as np
from matplotlib import pyplot as plt

class connect:
    # COM-poort zoeken waar Arduino op aangesloten is
    connected = False
    wmi = win32com.client.GetObject("winmgmts:")
    for port in wmi.InstancesOf("Win32_SerialPort"):
        if "Arduino" in port.Name:
            comPort = port.DeviceID
            ser = serial.Serial(comPort, 19200, timeout=0)
            connected = True
            if ser != 0:
                print("Arduino connected to", comPort)
        else:
            print("Arduino is not connected")
    intvalue = 1
    plt.ion()  # set plot to animated

    ydata = [0] * 50
    ax1 = plt.axes()

    # make plot
    line, = plt.plot(ydata)
    plt.ylim([0, 400])  # set the y-range to 0 to 400

    # Deze functie zet een waarde om naar een int
    def to_int(s):
        try:
            return int(s)
        except ValueError:
            return float(s)

    def light(self):
        while True:
            data = connect.ser.readline().rstrip()
            data = int.from_bytes(data, byteorder = 'big')  # read data from serial
            # port and strip line endings
            ymin = float(min(connect.ydata)) - 10
            ymax = float(max(connect.ydata)) + 10
            plt.ylim([ymin, ymax])
            connect.ydata.append(data)
            del connect.ydata[0]
            connect.line.set_xdata(np.arange(len(connect.ydata)))
            connect.line.set_ydata(connect.ydata)  # update the data
            plt.draw()  # update the plot

        """ while True:
            for x in connect.ser:
                # data = connect.ser.readline().rstrip().decode('utf-8')
                data = (int.from_bytes(x, byteorder = 'big'))
                connect.ydata.append(data)
                del connect.ydata[0]
                connect.line.set_xdata(np.arange(len(connect.ydata)))
                connect.line.set_ydata(connect.ydata)  # update the data
                plt.draw()  # update the plot
                time.sleep(1) """

connect.light(connect)


"""
URL configuration for social_app project.

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/4.2/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.conf import settings
from django.conf.urls.static import static
from django.contrib import admin
from django.urls import path, include

from drf_yasg import openapi
from drf_yasg.views import get_schema_view

from rest_framework.permissions import AllowAny

# Swagger-UI.
schema_view = get_schema_view(
    openapi.Info(
        title="MLSA SOCIAL IMPACT - GREEN AIR API ",
        default_version='v1',
        description="Django Backend API",
        terms_of_service="",
        contact=openapi.Contact(email="rooseveltabandy@gmail.com"),
        license=openapi.License(name="MIT License"),
    ),
    url="https://backendgreenair.azurewebsites.net/",
    public=True,
    permission_classes=[AllowAny, ],
)
admin.site.site_header = "Green Air"
urlpatterns = [
    path('', schema_view.with_ui('swagger', cache_timeout=0), name='schema-swagger-ui'),
    path('admin/', admin.site.urls),
    path('auth/', include('core.urls')),
    path('air/', include('air_quality_app.urls')),
    path('tinymce/', include('tinymce.urls')),
    path("docs/", schema_view.with_ui("redoc", cache_timeout=0), name="schema-redoc"),
]+ static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)


import tensorflow as tf
from tensorflow import keras
from keras.models import load_model
import os
import matplotlib.pyplot as plt
import cv2 as cv
import numpy as np
from p1_utils import resize_image

os.environ["KMP_DUPLICATE_LIB_OK"]="TRUE"
new_model = load_model('C:/Niranjan/Ashoka/Research/DCV/dcv_src/P1_Edges/Results/my_model.h5')

test_data_path = "C:/Niranjan/Ashoka/Research/DCV/Datasets/BIPED/edges/imgs/test/rgbr/"
results_path = "C:/Niranjan/Ashoka/Research/DCV/dcv_src/P1_Edges/Results/Test_Output/"

test_imgs = os.listdir(test_data_path)

fig, axs = plt.subplots(nrows=1, ncols=2, figsize=(10, 5))
for img in test_imgs:
    img_arr = cv.imread(test_data_path+img)
    img_x = resize_image(img_arr, 256, 256)

    pred = new_model.predict(np.expand_dims(img_x, axis=0),batch_size=1)

    axs[0].imshow(img_arr)
    axs[0].axis('off')
    axs[0].set_title('Input Image')
    axs[1].imshow(pred[0], cmap='Greys')
    axs[1].axis('off')
    axs[1].set_title('Predicted Edge Map')

    plt.savefig(results_path+img)
    print("Image saved: ", img)

print("Inference Complete! Images saved in: ", results_path)

# img_x = cv.imread("C:/Niranjan/Ashoka/Research/DCV/Datasets/BIPED/edges/imgs/test/rgbr/RGB_017.jpg")

# pred = new_model.predict(np.expand_dims(img_x, axis=0),batch_size=1)

# fig = plt.imshow(pred[0], cmap='Greys')

# plt.savefig("C:/Niranjan/Ashoka/Research/DCV/dcv_src/P1_Edges/Results/pred.png")
# plt.close(fig)


#This file includes the core function for text detection in a given image

import sys
import os
import argparse
import numpy as np
import re
import cv2
import time
import logging
from ctypes import *
from PIL import Image, ImageEnhance
from itertools import takewhile
import xml.etree.cElementTree as ET

#Define logging level
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# create XML file from text by xml.etree.cElementTree
def xml_create_unit(text_block, duration, stime, keywords, filename):
    hits = ET.Element("HITLIST")

    for i, text in enumerate(text_block):

        tb = (' ').join(ks for ks in text).strip()

        wordsli = re.split('[ )(":.\[\];/,\n]', tb) # split sentence into words
        if '' in wordsli:
            wordsli.remove('')
        begrec_all=[]
        wordrec_all=[]
        temprec_all=[]
        lenrec_all=[]
        # match with keywords from file
        for wordnum in xrange(len(wordsli),0,-1):
            for indword in xrange(len(wordsli)-wordnum+1): # check multiple words unit in a line
                word =u''
                for s in xrange(wordnum):
                    word += (u' ' + wordsli[indword+s])
                word_without_acc = unicodedata.normalize('NFKD', word.strip()).encode('ASCII','ignore') # convert characters with French accent to normal English chracters
                word=word.strip().encode('utf8')
                flag2=0
                # French accent matching
                if (word.lower() in keywords)>0:#np.sum([find_substring(word.lower(), s) for s in keywords]) > 0:# French words matching
                    pla = np.where(keywords==word.lower())[0]
                    temprec_all = temprec_all + pla.tolist()
                    begrec_all = begrec_all+ [indword]*len(pla)
                    wordrec_all = wordrec_all+ [word]*len(pla)
                    lenrec_all = lenrec_all + [wordnum]*len(pla)
                    flag2=1
                # matching without accent
                if flag2==0 and (word_without_acc.lower() in keywords)>0:#np.sum([find_substring(word.lower(), s) for s in keywords]) > 0:# converted English words matching
                    pla = np.where(keywords==word_without_acc.lower())[0]
                    temprec_all = temprec_all + pla.tolist()
                    begrec_all = begrec_all+ [indword]*len(pla)
                    wordrec_all = wordrec_all+ [word]*len(pla)
                    lenrec_all = lenrec_all + [wordnum]*len(pla)

        # save to xml structure
        if len(temprec_all)==0:#if no matched keywords with Keywords list
            hit = ET.SubElement(hits,"HIT")# {0}".format(count))
            ET.SubElement(hit, "frame", stime="{0}".format(stime[i]), dur="{0}".format(duration[i])).text = "{0} second".format(stime[i])
            for indw, word in enumerate(wordsli):
                if word=='':
                        continue
                word=word.strip()
                ET.SubElement(hit, "Word", stime="{0}".format(stime[i]), dur="{0}".format(duration[i])).text = "{0}".format(word)
        else: # if there is matched words to keywords
            count=1
            while len(temprec_all) > 0:# multiple matched words or word segments
                flag = 1
                wordrec = wordrec_all.pop()
                temprec = temprec_all.pop()
                begrec = begrec_all.pop()
                lenrec = lenrec_all.pop()

                hit = ET.SubElement(hits,"HIT", DUP="{0}".format(count))
                ET.SubElement(hit, "frame", stime="{0}".format(stime[i]), dur="{0}".format(duration[i])).text = "{0} second".format(stime[i])
                for indw, word in enumerate(wordsli):# if there is multiple mathced keywords in a frame, output for multiple times
                    if word=='':
                        continue
                    word=word.strip()

                    if indw==begrec and flag == 1:#np.sum([find_substring(word.lower(), s) for s in keywords]) > 0:#
                        ET.SubElement(hit, "Word", keyword="{0}".format(wordrec),  classement="{0}".format(keywords[temprec,1]), nature="{0}".format(keywords[temprec,0]), stime="{0}".format(stime[i]), dur="{0}".format(duration[i])).text = "{0}".format(wordrec)
                        flag=2
                    elif flag==2 and begrec+lenrec>indw>begrec :
                        pass
                    else:
                        ET.SubElement(hit, "Word", stime="{0}".format(stime[i]), dur="{0}".format(duration[i])).text = "{0}".format(word)

                count +=1
    tree = ET.ElementTree(hits)
    tree.write(filename, xml_declaration=True, encoding='UTF-8')

def text_recognition(img, Debug, debugPath,i):
    #Preprocessing to enhance text structure in the image
    #Image enhancement using PIL
    image = Image.fromarray(img)
    contrast = ImageEnhance.Contrast(image)
    contrasted = contrast.enhance(1)

    #contrasted.show()
    img = np.array(contrasted)
    (rows, cols) = (img.shape[0], img.shape[1])

    #sharpen the image
    kernel = np.zeros((9, 9), np.float32)
    kernel[4, 4] = 2.0  # Identity, times two!
    boxFilter = np.ones((9, 9), np.float32) / 81.0
    kernel = kernel - boxFilter
    img = cv2.filter2D(img, -1, kernel)

    # Load the dll
    mydll = cdll.LoadLibrary("TextMSER/libText.so")
    #mydll.text_recognition.restype = c_char_p
    mydll.text_recognition(img.ctypes.data_as(POINTER(c_ubyte)), rows, cols,i, Debug, debugPath, 0)

def text_detect_image(img, Debug):

    pathname = os.path.dirname(os.path.abspath(sys.argv[0]))
    # for visualization
    vis = img.copy()
    # Extract channels to be processed individually
    channels = cv2.text.computeNMChannels(img)
    # Append negative channels to detect ER- (bright regions over dark background)
    cn = len(channels) - 1
    for c in range(0, cn):
        channels.append((255 - channels[c]))

    # Apply the default cascade classifier to each independent channel (could be done in parallel)
    #print("Extracting Class Specific Extremal Regions from " + str(len(channels)) + " channels ...")
    #print("    (...) this may take a while (...)")
    for channel in channels:
        erc1 = cv2.text.loadClassifierNM1(pathname + '/trained_classifierNM1.xml')
        er1 = cv2.text.createERFilterNM1(erc1, 8, 0.00015, 0.13, 0.2, True, 0.1)#16

        erc2 = cv2.text.loadClassifierNM2(pathname + '/trained_classifierNM2.xml')
        er2 = cv2.text.createERFilterNM2(erc2, 0.5)

        regions = cv2.text.detectRegions(channel, er1, er2)
        #rects = cv2.text.erGrouping(img, channel, [r.tolist() for r in regions])
        rects, regions_groupes = cv2.text.erGrouping(img,channel,[x.tolist() for x in regions], cv2.text.ERGROUPING_ORIENTATION_ANY,pathname + '/trained_classifier_erGrouping.xml',0.5)
        #Visualization
        for r in range(0,np.shape(rects)[0]):
            rect = rects[r]
            cv2.rectangle(vis, (rect[0],rect[1]), (rect[0]+rect[2],rect[1]+rect[3]), (0, 0, 0), 2)
            cv2.rectangle(vis, (rect[0],rect[1]), (rect[0]+rect[2],rect[1]+rect[3]), (255, 0, 255), 1)

    #Visualization
    if Debug:
        cv2.imshow("Text detection result", vis)
        cv2.waitKey(0)

    #Text Recognition OCR
    out_img=img.copy()
    out_img_detection = img.copy()
    out_img_segmentation = np.zeros(img.shape[0], img.shape[1], np.uint8)
    scale_img = 600/float(img.shape[0])
    scale_font = (float)(2-scale_img)/1.4

    for i in xrange(0,np.shape(rects)[0]):
        rect = rects[i]
        cv2.rectangle(out_img_detection, (rect[0],rect[1]), (rect[0]+rect[2],rect[1]+rect[3]), (255, 0, 255), 3 )
        #group_img = np.zeros(img.shape[0], img.shape[1], np.uint8)

    return rects

#the main function if used standalone
def main():
    #Read input arguments
    parser = argparse.ArgumentParser(description='Detect and recognize the text present in the input image')
    parser.add_argument("-i", "--input", help="input image file")
    parser.add_argument("-o", "--outputpath", help="the path to save the output files")
    parser.add_argument("-n", "--outputname", help="the name to save the output files")
    parser.add_argument("-d", "--debug", help="Debug mode")

    args = parser.parse_args()

    #Check the input arguments
    if not os.path.isfile(args.input):
        logging.error('---input image file does not exist---')
        sys.exit(1)

    outputpath = args.outputpath
    if outputpath != '' and not os.path.exists(outputpath):
        os.makedirs(outputpath)

    img = cv2.imread(args.input)
    (rows, cols)= (img.shape[0], img.shape[1])
    Debug = int(args.debug)

    starttime = time.time()
    text_recognition(img, Debug,0)
    #text_detect_image(img,Debug)
    endtime = time.time()

    if Debug:
        logging.debug('Text zone detection and groupping time: ' + str(endtime - starttime) +'seconds')
    sys.exit(0)

if __name__ == "__main__":

    main()


import torch
import torch.nn.functional as F

train_num = 200000

class Net(torch.nn.Module):

    def __init__(self, n_feature, n_hidden, n_output):
        super(Net, self).__init__()
        self.hidden = torch.nn.Linear(n_feature, n_hidden)
        self.predict = torch.nn.Linear(n_hidden, n_output)

    def forward(self, x):
        inter = F.relu(self.hidden(x))
        y = self.predict(inter)
        return y

net = Net(n_feature=4, n_hidden=10, n_output=1)

optimizer = torch.optim.SGD(net.parameters(), lr=0.5)
loss_func = torch.nn.MSELoss()

file_train = open('dataset/train_data.csv','r')
line = file_train.readline()

x = torch.FloatTensor(4).zero_()
y = torch.FloatTensor(1).zero_()

for t in range(train_num):
    line = file_train.readline()
    if line == '':
        break
    line = line.split(',')

    x[0] = float(line[6])
    x[1] = float(line[7])/10000
    x[2] = float(line[8])
    x[3] = float(line[9])/10000
    y[0] = float(line[3])

    prediction = net(x)
    loss = loss_func(prediction, y)
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

file_train.close()
torch.save(net.state_dict(),'net_data_single.pkl')
print('train complete: ',train_num)


#!/usr/bin/env python

import os, sys, psutil, time

# THIS SCRIPT SHOULD BE RUN AS ROOT

def is_flask(proc):
  try:
    return "LZAPI_MODE" in proc.environ() and len(proc.children()) > 0 and proc.name() == "gunicorn"
  except:
    return False

def get_process(pid):
  try:
    return psutil.Process(pid=pid)
  except:
    return None

class FlaskServerInfo(object):
  def __init__(self,mode,pid,ppid,cwd,cons):
    self.mode = mode
    self.pid = pid
    self.ppid = ppid
    self.cwd = cwd
    self.cons = cons

  def __str__(self):
    constrings = []
    for c in self.cons:
      if c.status == "LISTEN":
        constrings.append("Listening on {} to port {}".format(*(c.laddr)))

    template = """
      Server: {0.mode}
      Working directory: {0.cwd}
      PID: {0.pid}
      {1}
    """

    template = "\n".join([x.lstrip() for x in template.strip().split("\n")])
    return template.format(self,"\n".join(constrings))

def find_flask_servers():
  servers = []
  for p in psutil.process_iter():
    if is_flask(p):
      s = FlaskServerInfo(
        p.environ()["LZAPI_MODE"],
        p.pid,
        p.ppid(),
        p.cwd(),
        p.connections()
      )

      servers.append(s)

  return servers

if __name__ == "__main__":
  servers = find_flask_servers()
  for s in servers:
    print(s)
    print()


import logging
from datetime import datetime, timedelta

from heapq import heappush, heappop, heapify
from flask import request, jsonify;
from codeitsuisse import app;

logger = logging.getLogger(__name__)

# """
# def flight(sorted_flights, lengthRun):
#     for counter, flight in enumerate(sorted_flights):
#         if(counter >= lengthRun):
#             before = datetime.strptime(sorted_flights[counter-lengthRun]["Time"], "%H%M")
#             after = datetime.strptime(sorted_flights[counter]["Time"], "%H%M")
#             if after < before + timedelta(seconds=int(reservedTime)):
#                  sorted_flights[counter]["Time"] = (before + timedelta(seconds=int(reservedTime))).strftime("%H%M")
#         flight["Runway"] = runways[counter%lengthRun]
#     return sorted_flights
# """
# @app.route('/airtrafficcontroller', methods=['POST'])
# def airtraffic():
#     data = request.get_json()
#     print("data sent for evaluation {}".format(data))
#
#     flights = data.get("Flights")
#     numOfFlights = len(flights)
#     sorted_flights = sorted(flights, key=lambda kv: (kv["Time"], kv["PlaneId"]))
#     reservedTime = data.get("Static").get("ReserveTime")
#
#     if "Runways" in data.get("Static").keys():
#         runways = data.get("Static").get("Runways")
#         sorted_runways = sorted(runways)
#         lengthRun = len(sorted_runways)
#
#         for flight in sorted_flights:
#             if(flight.get("Distressed", False)):
#                 flight.pop("Distressed")
#                 reserved_flight = flight
#                 sorted_flights.remove(flight)
#                 sorted_flights.insert(0, reserved_flight)
#                 for counter, flight in enumerate(sorted_flights):
#                     if(counter >= lengthRun):
#                         before = datetime.strptime(sorted_flights[counter-lengthRun]["Time"], "%H%M")
#                         after = datetime.strptime(sorted_flights[counter]["Time"], "%H%M")
#                         if after < before + timedelta(seconds=int(reservedTime)):
#                              sorted_flights[counter]["Time"] = (before + timedelta(seconds=int(reservedTime))).strftime("%H%M")
#                     flight["Runway"] = runways[counter%lengthRun]
#                 results = {"Flights" : sorted_flights}
#                 return jsonify(results)
#
#         for counter, flight in enumerate(sorted_flights):
#             if(counter >= lengthRun):
#                 for i in range(lengthRun):
#                     before = datetime.strptime(sorted_flights[counter-i-lengthRun]["Time"], "%H%M")
#                     after = datetime.strptime(sorted_flights[counter]["Time"], "%H%M")
#                     if after > before + timedelta(seconds=int(reservedTime)):
#                          sorted_flights[counter]["Time"] = (before + timedelta(seconds=int(reservedTime))).strftime("%H%M")
#                     flight["Runway"] = runways[counter%lengthRun]
#
#                 if after < before + timedelta(seconds=int(reservedTime)):
#                      sorted_flights[counter]["Time"] = (before + timedelta(seconds=int(reservedTime))).strftime("%H%M")
#             else:
#                 flight["Runway"] = sorted_runways[counter%lengthRun]
#
#     else:
#         for i in range(1, numOfFlights):
#             before = datetime.strptime(sorted_flights[i-1]["Time"], "%H%M")
#             after = datetime.strptime(sorted_flights[i]["Time"], "%H%M")
#             if after < before + timedelta(seconds=int(reservedTime)):
#                  sorted_flights[i]["Time"] = (before + timedelta(seconds=int(reservedTime))).strftime("%H%M")
#
#     print("My result :{}".format(sorted_flights))
#     results = {"Flights" : sorted_flights}
#     return jsonify(results)

def cmp_to_key(mycmp):
    'Convert a cmp= function into a key= function'
    class K:
        def __init__(self, obj, *args):
            self.obj = obj
        def __lt__(self, other):
            return mycmp(self.obj, other.obj) < 0
        def __gt__(self, other):
            return mycmp(self.obj, other.obj) > 0
        def __eq__(self, other):
            return mycmp(self.obj, other.obj) == 0
        def __le__(self, other):
            return mycmp(self.obj, other.obj) <= 0
        def __ge__(self, other):
            return mycmp(self.obj, other.obj) >= 0
        def __ne__(self, other):
            return mycmp(self.obj, other.obj) != 0
    return K

def cmp(x, y):
    print('fuck', x, y)
    if x['Distressed'] != y['Distressed']:
        return x['Distressed']
    if x['Time'] != y['Time']:
        print(x['Time'], y['Time'], x['Time'] < y['Time'])
        return x['Time'] < y['Time']
    return x['PlaneId'] < y['PlaneId']

@app.route('/airtrafficcontroller', methods=['POST'])
def air():
    data = request.get_json()
    print("data sent for evaluation {}".format(data))

    flights = data['Flights']
    for i in range(len(flights)):
        flights[i]['Time'] = datetime(1, 1, 1, int(flights[i]['Time']) // 100, int(flights[i]['Time']) % 100, 0)
        if 'Distressed' not in flights[i]:
            flights[i]['Distressed'] = 1
        elif flights[i]['Distressed'] == 'true':
            flights[i]['Distressed'] = 0
        else:
            flights[i]['Distressed'] = 1
    reserve = int(data['Static']['ReserveTime'])

    flights = sorted(flights, key=lambda k: (k['Distressed'], k['Time'], k['PlaneId']))
    print(flights)
    if 'Runways' in data['Static']:
        runways = data['Static']['Runways']
        times = {}

        for i in range(len(runways)):
            times[runways[i]] = datetime(1,1,1,0,0,0)

        response = []
        for flight in flights:
            #time, runway = heappop (runways)
            rw = ''

            flightTime = flight['Time']
            for i in times:
                print(times[i], flightTime)
                if times[i] <= flightTime:
                    if rw == '' or rw > i:
                        rw = i
            if rw == '':
                for i in times:
                    if rw == '' or rw > i:
                        rw = i
            time = times[rw]
            time = max(time, flightTime)
            response.append({'PlaneId': flight['PlaneId'], 'Time': time.strftime("%H%M"), 'Runway': rw})
            time = time + timedelta(0, reserve)
            times[rw] = time

            print('Response: {}'.format(response))
            #heappush(runways, (time, runway))
        return jsonify(Flights=response)
    else:
        response = []
        time = datetime(1,1,1,0,0,0)
        for flight in flights:
            flightTime = flight['Time']
            time = max(time, flightTime)
            response.append({'PlaneId': flight['PlaneId'], 'Time': time.strftime("%H%M")})
            time = time + timedelta(0, reserve)
        return jsonify(Flights=response)


import os
import requests
from logging import StreamHandler, Formatter, INFO, getLogger
from requests.exceptions import Timeout, ConnectionError, ConnectTimeout
import json
"""requestsモジュールサンプル

    aiohttp,asyncioとの比較

"""

def init_logger():
    handler = StreamHandler()
    handler.setLevel(INFO)
    handler.setFormatter(
        Formatter("[%(asctime)s] [%(threadName)s] [%(levelname)s] %(message)s"))
    logger = getLogger()
    logger.addHandler(handler)
    logger.setLevel(INFO)

def get():
    # proxyを使う場合はget(url, proxies=proxies)の形で使う
    # proxies = {
    #     "http": "http://xxxxx",
    #     "https": "http://xxxxx"
    # }
    url = 'https://api.github.com/events'
    with requests.get(url) as r:
        getLogger().info(f'url: {r.url}')
        getLogger().info(f'encoding: {r.encoding}')
        # getLogger().info(f'text:{r.text}') # データ量多いので注意

def post():
    # proxyを使う場合はpost(url, data=payload, proxies=proxies)の形で使う
    # proxies = {
    #     "http": "http://xxxxx",
    #     "https": "http://xxxxx"
    # }
    # クエリを使う場合はpost(url, params=payload)の形で使う
    url = 'http://hoge.com/post/abcde'
    payload = {'key': 'value'}
    headers = {'content-type': 'application/json'}
    with requests.post(url, data=json.dumps(payload), headers=headers) as r:
        getLogger().info(f'url: {r.url} text:{r.text}')

def file_download(url: str, path: str = './'):
    """URLからファイルをダウンロード

    Parameters
    ----------
    url : str
        ダウンロードURL
    path : str, optional
        格納先パス, by default './'

    Returns
    -------
    str
        ダウンロードファイル名
    """
    filename = url.split('/')[-1]
    filepath = os.path.join(path, filename)
    # connect timeout, read timeoutは必要
    connect_timeout = 1.0001
    read_timeout = 1.0001
    try:
        with requests.get(url, timeout=(connect_timeout, read_timeout), stream=True) as r:
            with open(filepath, 'wb') as f:
                for chunk in r.iter_content(chunk_size=1024):
                    if chunk:
                        f.write(chunk)
                        f.flush()
        getLogger().info(f'filename: {filename}')
    except ConnectionError as e:
        # [connect timeout]
        # HTTPSConnectionPool(host='hira-tla2-test.s3.ap-northeast-1.amazonaws.com', port=443): Max retries exceeded with url: /helloworld.zip (Caused by ProxyError('Cannot connect to proxy.', timeout('timed out')))
        print('ConnectionError')
        getLogger().error(e)
    except ConnectTimeout as e:
        # 先にConnectionErrorが発生してしまう為、発生方法不明
        print('ConnectTimeout')
        getLogger().error(e)
    except Timeout as e:
        # [read timeout]
        # HTTPSConnectionPool(host='hira-tla2-test.s3.ap-northeast-1.amazonaws.com', port=443): Read timed out. (read timeout=0.001)
        print('Timeout')
        getLogger().error(e)
    except Exception as e:
        print('Exception')
        getLogger().error(e)

    return filename

if __name__ == "__main__":
    init_logger()

    getLogger().info('start')
    # get()
    post()
    file_url = 'https://aaa/helloworld.zip'
    # file_download(file_url)

    getLogger().info('end')


import xml.etree.ElementTree as e_tree
import re

from main.aho_korasick.pattern import Pattern
from main.model.enzyme_entity import EnzymeEntity
from main.utils.transformation_helper import TransformationHelper

class EnzymesReader:
    # SIB_ENZYMES_XML_PATH = '../../resources/static/xml/syb_enzymes.xml'
    # NEB_ENZYMES_XML_PATH = '../../resources/static/xml/neb_enzymes.xml'
    SIB_ENZYMES_XML_PATH = 'resources/static/xml/syb_enzymes.xml'
    NEB_ENZYMES_XML_PATH = 'resources/static/xml/neb_enzymes.xml'

    all_sib_enzymes_data = None
    all_neb_enzymes_data = None

    def __init__(self):
        self.all_sib_enzymes_data = self.get_enzymes_data(self.SIB_ENZYMES_XML_PATH)
        self.all_neb_enzymes_data = self.get_enzymes_data(self.NEB_ENZYMES_XML_PATH)

    def get_sib_enzymes_data(self):
        return self.all_sib_enzymes_data

    def get_neb_enzymes_data(self):
        return self.all_neb_enzymes_data

    def get_enzymes_data(self, enzyme_file_path):
        enzymes_data = []
        tree = e_tree.parse(enzyme_file_path)
        root_element = tree.getroot()
        enzyme_objs = root_element.findall('enzyme')
        for enzyme_obj in enzyme_objs:
            name = str(enzyme_obj.find('name').text)
            top_site = str(enzyme_obj.find('recognition_site_1').text)
            bottom_site = str(enzyme_obj.find('recognition_site_2').text)
            # TODO: fill link property of enzyme object
            link = ''
            enzyme = EnzymeEntity(name, top_site, bottom_site, link)
            enzymes_data.append(enzyme)
        # print(enzymes_data)
        return enzymes_data

    def get_sib_simple_patterns(self):
        return self.get_simple_patterns(self.all_sib_enzymes_data)

    def get_neb_simple_pattens(self):
        return self.get_simple_patterns(self.all_neb_enzymes_data)

    def get_simple_patterns(self, enzymes_list):
        enzymes_seq_names = {}
        for enzyme in enzymes_list:
            if enzyme.get_top_site().find('N') == -1:
                sequence = enzyme.get_top_site().replace('↑', '')
                if sequence not in enzymes_seq_names:
                    enzymes_seq_names[sequence] = list()
                enzymes_seq_names[sequence].append(enzyme.get_e_name())
        simple_patterns = self.process_multi_nucleotides(enzymes_seq_names)
        return simple_patterns

    def get_sib_wildcard_patterns(self):
        return self.get_wildcard_patterns(self.all_sib_enzymes_data)

    def get_neb_wildcard_patterns(self):
        return self.get_wildcard_patterns(self.all_neb_enzymes_data)

    def get_wildcard_patterns(self, enzymes_list):
        enzymes_seq_names = {}
        for enzyme in enzymes_list:
            sequence = enzyme.get_top_site()
            if sequence.find('N') != -1:
                sequence = sequence.replace('(N)', '')
                start_digit_pos = 0
                digits = []
                while start_digit_pos < len(sequence):
                    if sequence[start_digit_pos].isdigit():
                        end_digit_pos = start_digit_pos + 1
                        while sequence[end_digit_pos].isdigit():
                            end_digit_pos += 1
                        digits.append(sequence[start_digit_pos:end_digit_pos])
                        start_digit_pos = end_digit_pos
                    else:
                        start_digit_pos += 1
                sequence = self.replace_all_digits_by_n(sequence, digits)
                sequence = sequence.replace('↑', '')
                if sequence not in enzymes_seq_names:
                    enzymes_seq_names[sequence] = list()
                enzymes_seq_names[sequence].append(enzyme.get_e_name())
        wildcard_patterns = self.process_multi_nucleotides(enzymes_seq_names)
        return wildcard_patterns

    def get_all_sib_patterns(self):
        return self.get_all_patterns(self.all_sib_enzymes_data)

    def get_all_neb_patterns(self):
        return self.get_all_patterns(self.all_neb_enzymes_data)

    def get_all_patterns(self, enzymes_list):
        patterns = {}
        for enzyme in enzymes_list:
            sequence = enzyme.get_top_site()
            e_name = enzyme.get_e_name()
            if sequence.find('N') != -1:
                sequence = sequence.replace('(N)', '')
                start_digit_pos = 0
                digits = []
                while start_digit_pos < len(sequence):
                    if sequence[start_digit_pos].isdigit():
                        end_digit_pos = start_digit_pos + 1
                        while sequence[end_digit_pos].isdigit():
                            end_digit_pos += 1
                        digits.append(sequence[start_digit_pos:end_digit_pos])
                        start_digit_pos = end_digit_pos
                    else:
                        start_digit_pos += 1
                sequence = self.replace_all_digits_by_n(sequence, digits)
            sequence = sequence.replace('↑', '')
            if sequence in patterns:
                patterns[sequence].add_name(e_name)
            else:
                p = Pattern(sequence, [e_name])
                patterns[sequence] = p
        return list(patterns.values())

    def replace_all_digits_by_n(self, sequence, digits):
        for digit in digits:
            n_seq = 'N' * int(digit)
            sequence = sequence.replace(digit, n_seq)
        return sequence

    def process_multi_nucleotides(self, enzymes_seq_names):
        transformer = TransformationHelper()
        patterns = []
        for seq, names in enzymes_seq_names.items():
            if transformer.check_multi_nucleotides(seq):
                formatted_seqs = transformer.get_formatted_patterns(seq)
                for formatted_seq in formatted_seqs:
                    pattern = Pattern(seq, names, formatted_seq)
                    patterns.append(pattern)
            else:
                pattern = Pattern(seq, names)
                patterns.append(pattern)
        return patterns

if __name__ == "__main__":
    pass
    # patts = EnzymesReader.get_sib_enzymes_seq_names_data()
    # enzymes_seq_names_test = {'aRggY': ['name1', 'name2'], 'cgggWWttt': ['name3']}
    # patts = EnzymesReader.process_multi_nucleotides(enzymes_seq_names_test)
    # for p in patts:
    #    print(str(p))
    er = EnzymesReader()
    patts = er.get_all_sib_patterns()
    for p in patts:
        print(str(p))


# -*- coding: utf-8 -*-
import scrapy
from Tencent.items import TencentItem

#scrapy.CrawlSpider

class TencentSpider(scrapy.Spider):
    name = 'tencent'
    allowed_domains = ['hr.tencent.com']

    base_url = "http://hr.tencent.com/position.php?&start="
    offset = 0
    start_urls = [base_url + str(offset)]

    # start_urls 里的url地址不做去重（不记录请求指纹）
    #start_urls = [base_url + str(num) for num in range(0, 2200 + 1, 10)]

    def parse(self, response):
        node_list = response.xpath("//tr[@class='even'] | //tr[@class='odd']")

        for node in node_list:
            item = TencentItem()

            item['position_name'] = node.xpath("./td[1]/a/text()").extract()[0]
            item['position_link'] = "http://hr.tencent.com/" + node.xpath("./td[1]/a/@href").extract()[0]

            try:
                item['position_type'] = node.xpath("./td[2]/text()").extract()[0]
            except:
                item['position_type'] = "None"

            #position_type_list = node.xpath("./td[2]/text()").extract()
            #item['position_type'] = position_type_list[0] if position_type_list else "None"

            item['people_number'] = node.xpath("./td[3]/text()").extract()[0]
            item['work_location'] = node.xpath("./td[4]/text()").extract()[0]
            item['publish_times'] = node.xpath("./td[5]/text()").extract()[0]

            yield item

        #if self.offset < 2200:
        #    self.offset += 10
        #    # scrapy.Request() 构建一个请求，第一个参数是url，callback 指定回调函数，表示该请求返回的response由哪个方法来解析
        #    yield scrapy.Request(self.base_url + str(self.offset), callback = self.parse)

        # 如果是不同标签里判断属性, 用 | 表示或
        #"//tr[@class='even'] | //tr[@class='odd']"

        # 如果同一个标签内判断属性，用 and  表示与
        # 如果到了最后一页，则列表为空，判断条件为False，不再发送请求；not 取反表示继续发送请求
        if not response.xpath("//a[@class='noactive' and @id='next']"):
            next_link = "http://hr.tencent.com/" + response.xpath("//a[@id='next']/@href").extract()[0]
            yield scrapy.Request(next_link, callback = self.parse)


from fastapi import APIRouter, HTTPException, Depends
from sqlalchemy.orm import Session
from .database import session
from .authorizator import verify_token, verify_admin
from . import crud, schemas, models

router = APIRouter()

@router.get('/list')
async def get_airdrops(
    page: int = 1,
    sort: int = 1,
    count: int = 20,
    db: Session = Depends(session)
):
    airdrops, total = crud.list_airdrop(db, page, sort, count)
    airdrops = [
        {
            "id": p.id,
            "name": p.name,
            "description": p.description,
            "amount": p.amount,
            "interval": p.interval
        }
        for p in airdrops
    ]
    pages, extra = divmod(total, count)
    if extra:
        pages += 1
    return {
        "text": "ok",
        "airdrops": airdrops,
        "pages": pages,
        "current": page,
        "total": total
    }

@router.get('/list_with_status')
async def get_airdrop_status(
    page: int = 1,
    sort: int = 1,
    count: int = 20,
    user: dict = Depends(verify_token),
    db: Session = Depends(session)
):
    airdrops, total = crud.list_airdrop_with_stat(
        db,
        page,
        sort,
        count,
        user["id"]
    )
    pages, extra = divmod(total, count)
    if extra:
        pages += 1
    return {
        "text": "ok",
        "airdrops": airdrops,
        "pages": pages,
        "current": page,
        "total": total
    }

@router.post('/create')
async def create_airdrop_as_admin(
    airdrop: schemas.AirdropCreateRequest,
    admin: dict = Depends(verify_admin),
    db: Session = Depends(session)
):
    crud.create_airdrop(
        db,
        airdrop.name,
        airdrop.description,
        airdrop.amount,
        airdrop.interval,
        airdrop.mode
    )
    return {"text": "ok"}

@router.get('/{airdrop_id}')
async def get_airdrop(
    airdrop_id: int,
    db: Session = Depends(session)
):
    airdrop = crud.get_airdrop(db, airdrop_id)
    return {
        "text": "ok",
        "id": airdrop.id,
        "name": airdrop.name,
        "description": airdrop.description,
        "amount": airdrop.amount,
        "interval": airdrop.interval,
        "mode": airdrop.mode
    }

@router.get('/{airdrop_id}/status')
async def get_airdrop_with_stat(
    airdrop_id: int,
    user: dict = Depends(verify_token),
    db: Session = Depends(session)
):
    airdrop_data = crud.get_airdrop_with_stat(db, airdrop_id, user["id"])
    return airdrop_data

@router.put('/{airdrop_id}')
async def set_airdrop_info_as_admin(
    airdrop: schemas.AirdropEditRequest,
    admin: dict = Depends(verify_admin),
    db: Session = Depends(session)
):
    crud.put_airdrop(
        db,
        airdrop.id,
        airdrop.name,
        airdrop.description,
        airdrop.amount,
        airdrop.interval,
        airdrop.mode
    )
    return {"text": "ok"}

@router.delete('/{airdrop_id}')
async def delete_airdrop_as_admin(
    airdrop_id: int,
    admin: dict = Depends(verify_admin),
    db: Session = Depends(session)
):
    crud.delete_airdrop(db, airdrop_id)
    return {"text": "ok"}

@router.post('/{airdrop_id}/claim')
async def claim_airdrop(
    airdrop_id: int,
    user: dict = Depends(verify_token),
    db: Session = Depends(session)
):
    _, before, after = crud.claim_airdrop(db, airdrop_id, user['id'])
    return {"text": "ok", "before": before, "after": after}