# -*- coding: utf-8 -*-
"""
Created on Sat Mar 20 13:30:40 2021

@author: Who
"""
import kerastuner as kt
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
import pandas as pd
import numpy as np
import cv2 as cv
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler

def get_data_palm(PATH, NUM, X, label):
    for x in range(1, NUM+1, 1):
        # tidak dinormalisasi
        temp_dem = cv.imread(PATH+'/dem/'+str(x)+'.tif', cv.IMREAD_UNCHANGED)
        temp_dsm = cv.imread(PATH+'/dsm/'+str(x)+'.tif', cv.IMREAD_UNCHANGED)
        # temp_intensity = cv.imread(PATH+'/intensity/'+str(x)+'.tif', cv.IMREAD_UNCHANGED)
        # temp_r = cv.imread(PATH+'/r/'+str(x)+'.tif', cv.IMREAD_UNCHANGED)
        # temp_g = cv.imread(PATH+'/g/'+str(x)+'.tif', cv.IMREAD_UNCHANGED)
        # temp_b = cv.imread(PATH+'/b/'+str(x)+'.tif', cv.IMREAD_UNCHANGED)

        scaler = MinMaxScaler(feature_range=(np.min(temp_dem), np.max(temp_dem)))

        #dengan dinormalisasi
        # temp_dem = scaler.fit_transform(cv.imread(PATH+'/dem/'+str(x)+'.tif', cv.IMREAD_UNCHANGED))
        # temp_dsm = scaler.fit_transform(cv.imread(PATH+'/dsm/'+str(x)+'.tif', cv.IMREAD_UNCHANGED))
        temp_intensity = scaler.fit_transform(cv.imread(PATH+'/intensity/'+str(x)+'.tif', cv.IMREAD_UNCHANGED))
        temp_r = scaler.fit_transform(cv.imread(PATH+'/r/'+str(x)+'.tif', cv.IMREAD_UNCHANGED))
        temp_g = scaler.fit_transform(cv.imread(PATH+'/g/'+str(x)+'.tif', cv.IMREAD_UNCHANGED))
        temp_b = scaler.fit_transform(cv.imread(PATH+'/b/'+str(x)+'.tif', cv.IMREAD_UNCHANGED))

        temp_concat = list(np.dstack((temp_dem, temp_dsm, temp_intensity)))
        X.append(temp_concat)
        label.append(1)

def get_data_non_palm(PATH, NUM, X, label):
    for x in range(1, NUM+1, 1):
        # tidak dinormalisasi
        temp_dem = cv.imread(PATH+'/dem/'+str(x)+'.tif', cv.IMREAD_UNCHANGED)
        temp_dsm = cv.imread(PATH+'/dsm/'+str(x)+'.tif', cv.IMREAD_UNCHANGED)
        # temp_intensity = cv.imread(PATH+'/intensity/'+str(x)+'.tif', cv.IMREAD_UNCHANGED)
        # temp_r = cv.imread(PATH+'/r/'+str(x)+'.tif', cv.IMREAD_UNCHANGED)
        # temp_g = cv.imread(PATH+'/g/'+str(x)+'.tif', cv.IMREAD_UNCHANGED)
        # temp_b = cv.imread(PATH+'/b/'+str(x)+'.tif', cv.IMREAD_UNCHANGED)

        scaler = MinMaxScaler(feature_range=(np.min(temp_dem), np.max(temp_dem)))

        #dengan dinormalisasi
        # temp_dem = scaler.fit_transform(cv.imread(PATH+'/dem/'+str(x)+'.tif', cv.IMREAD_UNCHANGED))
        # temp_dsm = scaler.fit_transform(cv.imread(PATH+'/dsm/'+str(x)+'.tif', cv.IMREAD_UNCHANGED))
        temp_intensity = scaler.fit_transform(cv.imread(PATH+'/intensity/'+str(x)+'.tif', cv.IMREAD_UNCHANGED))
        temp_r = scaler.fit_transform(cv.imread(PATH+'/r/'+str(x)+'.tif', cv.IMREAD_UNCHANGED))
        temp_g = scaler.fit_transform(cv.imread(PATH+'/g/'+str(x)+'.tif', cv.IMREAD_UNCHANGED))
        temp_b = scaler.fit_transform(cv.imread(PATH+'/b/'+str(x)+'.tif', cv.IMREAD_UNCHANGED))

        temp_concat = list(np.dstack((temp_dem, temp_dsm, temp_intensity)))
        X.append(temp_concat)
        label.append(0)

num_oil_palm1 = 586
num_non_oil_palm1 = 1210
num_oil_palm2 = 709
num_non_oil_palm2 = 381

X = []
label = []

PATH_PALM_OIL1 = 'dataset/img_rgb3000_1500/kelas_sawit'
PATH_PALM_OIL2 = 'dataset/img_rgb0_4500/kelas_sawit'
PATH_NON_PALM_OIL1 = 'dataset/img_rgb3000_1500/bukan_sawit'
PATH_NON_PALM_OIL2 = 'dataset/img_rgb3000_1500/bukan_sawit'

get_data_palm(PATH_PALM_OIL1, num_oil_palm1, X, label)
get_data_palm(PATH_PALM_OIL2, num_oil_palm2, X, label)
get_data_non_palm(PATH_NON_PALM_OIL1, num_non_oil_palm1, X, label)
get_data_non_palm(PATH_NON_PALM_OIL2, num_non_oil_palm2, X, label)

SAMPLE = len(label)
X = tf.cast(tf.constant(X), dtype="float32")
label = tf.cast(tf.constant(label), dtype="float32")
train_X, test_X, val_X = tf.split(X, [int(SAMPLE*0.7), int(SAMPLE*0.1)+1, int(SAMPLE*0.2)], 0)
train_label, test_label, val_label = tf.split(label, [int(SAMPLE*0.7), int(SAMPLE*0.1)+1, int(SAMPLE*0.2)], 0)

print(len(train_X), 'train examples')
print(len(val_X), 'validation examples')
print(len(test_X), 'test examples')

def model_builder(hp):
    model = keras.Sequential()

    model.add(layers.Conv2D(hp.Int("units_1", min_value=16, max_value=512, step=16), (3, 3), activation=hp.Choice("act_1", ['relu', 'tanh']), padding=hp.Choice("pad_1", ['same', 'valid']), input_shape=(60, 60, 3)))
    model.add(layers.MaxPooling2D(pool_size=(2, 2)))
    model.add(layers.Conv2D(hp.Int("units_2", min_value=16, max_value=512, step=16), (3, 3), activation=hp.Choice("act_2", ['relu', 'tanh'])))
    model.add(layers.MaxPooling2D(pool_size=(2, 2)))
    model.add(layers.Flatten())
    for i in range(hp.Int('num_hidden_layers', 1, 5)):
        model.add(layers.Dense(units=hp.Int('units_' + str(i+2), min_value=16, max_value=512, step=16), activation=hp.Choice("act_" + str(i+2), ['relu', 'tanh'])))
    model.add(layers.Dropout(hp.Float('dropout_1', min_value=0.1, max_value=0.9, step=0.1)))
    model.add(layers.Dense(1))

    hp_learning_rate = hp.Choice('learning_rate', values=[1e-2, 1e-3, 1e-4])
    model.compile(optimizer=keras.optimizers.Adam(learning_rate=hp_learning_rate),
                loss=keras.losses.BinaryCrossentropy(from_logits=True),
                metrics=['accuracy'])

    return model

tuner = kt.Hyperband(model_builder,
                     objective='val_accuracy',
                     max_epochs=10,
                     factor=3,
                     directory='my_dir',
                     project_name='kelapa_sawit_clf_kt')

stop_early = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=5)

tuner.search(train_X, train_label, epochs=50, validation_data=(val_X, val_label), callbacks=[stop_early])

# Get the optimal hyperparameters
best_hps=tuner.get_best_hyperparameters(num_trials=1)[0]

print(f"""
The optimal number of units in the first Conv2D layer is {best_hps.get('units_1')} , activation {best_hps.get('act_1')} and padding {best_hps.get('pad_1')}.
Second Conv2D layer best unit is {best_hps.get('units_2')}, activation {best_hps.get('act_2')}. The optimal learning rate for the optimizer
is {best_hps.get('learning_rate')}. And the best Dropout is {best_hps.get('dropout_1')}
""")

best_num_hidden_layers = best_hps.get('num_hidden_layers')

if best_num_hidden_layers == 1:
    print(f"""
    The optimal number of units in the first dense layer is {best_hps.get('units_3')} , activation {best_hps.get('act_3')}.
    """)
elif best_num_hidden_layers == 2:
    print(f"""
    The optimal number of units in the first dense layer is {best_hps.get('units_3')} , activation {best_hps.get('act_3')}.
    Second dense layer units is {best_hps.get('units_4')} , activation {best_hps.get('act_4')}
    """)
elif best_num_hidden_layers == 3:
    print(f"""
    The optimal number of units in the first dense layer is {best_hps.get('units_3')} , activation {best_hps.get('act_3')}.
    Second dense layer units is {best_hps.get('units_4')} , activation {best_hps.get('act_4')}
    Third dense layer units is {best_hps.get('units_5')} , activation {best_hps.get('act_5')}
    """)
elif best_num_hidden_layers == 4:
    print(f"""
    The optimal number of units in the first dense layer is {best_hps.get('units_3')} , activation {best_hps.get('act_3')}.
    Second dense layer units is {best_hps.get('units_4')} , activation {best_hps.get('act_4')}
    Third dense layer units is {best_hps.get('units_5')} , activation {best_hps.get('act_5')}
    Third dense layer units is {best_hps.get('units_6')} , activation {best_hps.get('act_6')}
    """)

# Build the model with the optimal hyperparameters and train it on the data for 50 epochs
model = tuner.hypermodel.build(best_hps)
history = model.fit(train_X, train_label, epochs=100, validation_data=(val_X, val_label))

val_acc_per_epoch = history.history['val_accuracy']
best_epoch = val_acc_per_epoch.index(max(val_acc_per_epoch)) + 1
print('Best epoch: %d' % (best_epoch,))

hypermodel = tuner.hypermodel.build(best_hps)

# Retrain the model
hypermodel.fit(train_X, train_label, epochs=best_epoch, validation_data=(val_X, val_label))

eval_result = hypermodel.evaluate(test_X, test_label)
print("[test loss, test accuracy]:", eval_result)

# Save model
hypermodel.save("model/model_cnn_kerastuner.h5")


from ml_utils.logging.tb_log import Logger
from typing import Union, List
from datetime import datetime
import shutil
import os

def create_directories(name: Union[List[str], str]) -> None:
    """
    Creates one or multiple directories. If a string value is provided, then the single directory is created. If a list
    is provided, then all directory names within the list are created.

    Parameters
    ----------
    name: A list, containing the names of directories, which should be created.
    """

    if type(name) == 'str':
        name = [name]

    for directory in name:
        if not os.path.exists(directory):
            os.makedirs(directory)

def copy_files(directory: str, files: Union[str, List[str]]) -> None:
    """
    Copies provided files into a provided directory.

    Parameters
    ----------
    directory: Path to the directory, in which the files should be stored.
    files: Files which should be copied into the provided directory.
    """

    if not os.path.exists(directory):
        os.makedirs(directory)

    if type(files) == 'str':
        files = [files]

    for source_file in files:
        shutil.copy(source_file, directory)

def create_date_time_string() -> str:
    """
    Creates a string consisting of YYYY_MM_DD__hh_mm_ss, where Y represents numbers of the current year, M represents
    numbers of the current month, D represents numbers of the current day, h represents numbers of the current hour, m
    represents numbers of the current minute and s represents numbers of the current seconds.

    Returns
    -------
    now: Generated string from the current date and time.
    """

    now = str(datetime.now()).replace('-', '_').replace(':', '_').replace(' ', '__')[0:20]
    return now

def init_experiment_dir(dir_root, tb_log: bool = False, tb_log_names: Union[None, list] = None) -> tuple:
    """
    Creates a new directory, in which results and logs of a new experiment can be stored.

    Parameters
    ----------
    dir_root: Path to a directory, in which the root directory for a new experiment is created.
    tb_log: Variable, which defines, whether to use Tensorboard or not. If the value of this variable is None, then the
            experiment is initialized without tensorboard. If the value of this variable is an integer, than this number
            of tb_logger instances is created.
    tb_log_names: Names of the directories of the Tensorboard logger (if the names are not None). Each directory is
                  created in the as sub-directory of the tb directory in the experience root path.

    Returns
    -------
    dir_name: Path to the experiment directory
    tb_logger_tuple: Tuple containing the tensorboard logger instances.
    """

    # get the name of the root directory
    dir_name = dir_root + create_date_time_string() + '/'
    source_file_dir = dir_name + 'src/'
    dir_list = [dir_name, source_file_dir]
    output = tuple(dir_name)

    # if we want to include tensorboard logging
    if tb_log:

        # get the name of the tensorboard directory
        tb_root_dir = dir_name + 'tb/'
        dir_list.append(tb_root_dir)

        # if multiple logging instances should be initialized, get the name of their directories
        if tb_log_names is not None:
            tb_logger_list = []
            for name in tb_log_names:

                # compute the directory of the logger instance
                logger_dir = tb_root_dir + name + '/'

                # create a logger instance and store the instance as well as the corresponding directory
                tb_logger_list.append(Logger(logger_dir))
                dir_list.append(logger_dir)
                output = (dir_name, tb_logger_list)

    # create the directories and return the root directory of the new created experiment
    create_directories(dir_list)
    return output


"""
https://www.youtube.com/watch?v=PjhlUzp_cU0&t=960s

나도코딩
파이썬 코딩 무료 강의 (활용편5) - 데이터 분석 및 시각화, 이 영상 하나로 끝내세요

2022-07-29 03:05:23 ~ 03:11:24
"""

import matplotlib.pyplot as plt
import matplotlib
matplotlib.rcParams['font.family'] = 'AppleGothic'
matplotlib.rcParams['font.size'] = 15
matplotlib.rcParams['axes.unicode_minus'] = False

x = [1, 2, 3]
y = [2, 4, 8]

'''
2. 축
'''

# title

plt.plot(x, y)
plt.title('꺾은선 그래프', fontdict={'family': 'Maplestory OTF', 'size': 20})
# 기본 폰트는 애플고딕이지만, 개별적으로 설정도 가능

# label

plt.plot(x, y)
plt.xlabel('X축', color='red', loc='right')
# x축 loc : left, center, right
plt.ylabel('Y축', color='#00aa00', loc='top')
# y축 loc : top, center, bottom

# tick

plt.plot(x, y)
plt.xticks([1, 2, 3])
plt.yticks([3, 6, 9, 12])
plt.show()


import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
import time

hidden_sz = 32
d_grad_steps = 1
batch = 128
n_mix = 2

def discriminator(x, keep, reuse):
	with tf.variable_scope("discriminator", reuse=reuse):
		d_fc1 = tf.layers.dense(x,  hidden_sz, activation=tf.nn.relu)
		d_dropout = tf.nn.dropout(d_fc1, keep)
		d_fc2 = tf.layers.dense(d_dropout, hidden_sz, activation=tf.nn.relu)
		d_out = tf.sigmoid(tf.layers.dense(d_fc2, 1))
		return d_out, keep
def main():

	input_shape = [2]
	latent_shape = [2]

	with tf.variable_scope("generator"):
		g_in = tf.placeholder(dtype=tf.float32, shape=[None]+latent_shape,
			name="generator_latent")
		g_fc1 = tf.layers.dense(g_in,  hidden_sz, activation=tf.nn.relu)
		g_fc2 = tf.layers.dense(g_fc1, hidden_sz, activation=tf.nn.relu)
		g_out = tf.layers.dense(g_fc2, input_shape[0])

	d_keep = tf.placeholder(dtype=tf.float32, shape=())
	d_in_real = tf.placeholder(dtype=tf.float32, shape=[None]+input_shape,
		name="discriminator_real_input")
	d_real, d_keep = discriminator(d_in_real, d_keep, reuse=False)

	d_fake, _ = discriminator(g_out, d_keep, reuse=True)

	with tf.variable_scope("loss"):
		d_loss = -tf.reduce_mean(tf.log(d_real) + tf.log(1.0 - d_fake))
		g_loss = tf.reduce_mean(tf.log(1.0 - d_fake))

	vars = tf.trainable_variables()

	d_adam = tf.train.AdamOptimizer()
	d_step = tf.Variable(0, name="d_step", trainable=False)
	d_optimize = d_adam.minimize(d_loss, global_step=d_step,
		var_list=[v for v in vars if v.name.startswith("disc")])

	g_adam = tf.train.AdamOptimizer()
	g_step = tf.Variable(0, name="g_step", trainable=False)
	g_optimize = g_adam.minimize(g_loss, global_step=g_step,
		var_list=[v for v in vars if v.name.startswith("gen")])

	def random_gaussian(dim):
		randn = np.random.normal
		mean = 3*randn(size=dim)
		#cov_chol = randn(size=(dim,dim))
		#cov = np.matmul(cov_chol.T, cov_chol)
		cov = 0.3 * np.diag(1 + 2 * np.random.uniform(size=2))
		return mean, cov

	gaussians = [random_gaussian(input_shape[0]) for _ in range(n_mix)]

	#tf_gaussians = [tf.distributions.Normal(loc=m, scale=np.diag(c).flat)
		#for m, c in gaussians]
	#density_in = tf.placeholder(dtype=tf.float32, shape=[None]+input_shape)
	#true_density = tf.reduce_sum([g.prob(density_in) for g in tf_gaussians]) / float(n_mix)

	def sample(N):
		n = N // len(gaussians)
		assert n * len(gaussians) == N
		return np.vstack(
			np.random.multivariate_normal(mu, cov, size=n)
				for mu, cov in gaussians)

	sess = tf.Session()
	sess.run(tf.global_variables_initializer())

	def distmap():
		plt.clf()
		plt.hold(True)

		res = 64
		lim = 10
		t = np.linspace(-lim, lim, 64)
		x, y = np.meshgrid(t,t)
		mesh = np.float32(np.c_[x.flat, y.flat])

		#dens_true = sess.run(true_density, feed_dict={density_in: mesh})
		#dsurface = sess.run(d_real, feed_dict={d_in_real: mesh, d_keep: 1.0})
		#dsurface = dsurface.reshape(x.shape)

		r_sample = sample(n_mix * 10000)
		r_hist, _, _ = np.histogram2d(r_sample[:,0], r_sample[:,1],
			bins=res, range=[[-lim, lim], [-lim, lim]])

		g_latent = np.random.normal(size=[10000] + latent_shape)
		g_sample = sess.run(g_out, feed_dict={g_in: g_latent})
		g_hist, _, _ = np.histogram2d(g_sample[:,0], g_sample[:,1],
			bins=res, range=[[-lim, lim], [-lim, lim]])

		def normalize(x):
			m = np.max(x.flat)
			if m > 0:
				return x / m
			return x

		img = np.zeros((res,res,3))
		img[:,:,0] = normalize(r_hist)
		img[:,:,1] = normalize(g_hist)

		plt.imshow(img, extent=(-lim, lim, -lim, lim))

	n_batches = 40000
	for b in range(n_batches):
		# train the discriminator
		nstep = 100 if b == 0 else d_grad_steps
		for _ in range(nstep):
			# generate real data
			r_sample = sample(batch)

			# sample from generator
			g_latent = np.random.normal(size=[batch] + latent_shape)

			# train the discriminator
			_, d_loss_value = sess.run([d_optimize, d_loss],
				feed_dict={d_in_real: r_sample, g_in: g_latent, d_keep: 0.5})

		# train the generator
		g_latent = np.random.normal(size=[batch] + latent_shape)
		_, g_sample, g_loss_value = sess.run([g_optimize, g_out, g_loss],
			feed_dict={g_in: g_latent, d_keep: 1.0})

		if False and b % 100 == 0:
			plt.clf()
			plt.hold(True)
			lim = 10
			t = np.linspace(-lim, lim, 64)
			x, y = np.meshgrid(t,t)
			mesh = np.c_[x.flat, y.flat]
			dsurface = sess.run(d_real, feed_dict={d_in_real: mesh, d_keep: 1.0})
			dsurface = dsurface.reshape(x.shape)
			plt.imshow(dsurface, extent=(-lim, lim, -lim, lim), alpha=0.4)
			plt.scatter(r_sample[:,0], r_sample[:,1], color=(0.7, 0.1, 0.1))
			plt.scatter(g_sample[:,0], g_sample[:,1], color=(0.1, 0.6, 0.3))
			plt.show(block=False)
			plt.xlim([-lim, lim])
			plt.ylim([-lim, lim])
			plt.pause(0.001)

		if b % 100 == 0:
			distmap()
			plt.show(block=False)
			plt.pause(0.001)

	plt.show(block=True)

main()


#!/usr/bin/env python
# coding: utf-8

# In[2]:

import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler
from sklearn.linear_model import LinearRegression
from sklearn.metrics import r2_score
from sklearn.neighbors import KNeighborsRegressor
import numpy as np
from sklearn import metrics
from sklearn import neighbors
from sklearn.metrics import mean_squared_error
from math import sqrt
import statsmodels.api as sm
from scipy.stats import zscore
from sklearn.linear_model import BayesianRidge
from sklearn.linear_model import Lasso

# # Importing Data

# In[47]:

data = pd.read_csv(r"C:\Users\serka\Desktop\YTÜ\YTÜ 3-2\proje\notCodeSmell_merged.csv")
data

# # Removing Missing Values and Some Features

# In[48]:

data = data.dropna()
data = data.drop(["JIRA_KEY","HATA_KAYDI_OZET","Unnamed: 0","securityRating","reliabilityRating","Unnamed: 0.1", "KLOC_EFOR_y","ESTIMATED_EFOR_y","faultFixingCommitHash","effort_y","confirmedIssues","nloc","complexity","effort_x","linesAdded","linesRemoved"],axis=1)
data = data.drop_duplicates()
data = data.reset_index(drop=True)

# In[49]:

data

# # Dividing Real Effort and Estimated Effort and Plotting Our Formula

# In[36]:

div_lines = data["KLOC_EFOR_x"]/data["ESTIMATED_EFOR_x"]
data["bölüm"]=div_lines

# In[37]:

data["bölüm"].plot()

# In[39]:

print(data['bölüm'].skew())
data['bölüm'].describe()

# In[40]:

data_plot = pd.DataFrame()
data_plot["KLOC_EFOR"] = data["KLOC_EFOR_x"]
data_plot["ESTIMATED_EFOR"] = data["ESTIMATED_EFOR_x"]

# In[41]:

data_plot

# In[23]:

print(data_plot['ESTIMATED_EFOR'].skew())
data_plot['ESTIMATED_EFOR'].describe()

# In[43]:

data_plot = data_plot[data_plot["ESTIMATED_EFOR"]<4028]
data_plot = data_plot[data_plot["ESTIMATED_EFOR"]>269]

# In[44]:

lines = data_plot.plot.line()

# In[46]:

r2_score(data["KLOC_EFOR_x"], data["ESTIMATED_EFOR_x"])

# # Train Test Split

# In[50]:

x = data.drop(["KLOC_EFOR_x"],axis=1)
y = data["KLOC_EFOR_x"]

# In[51]:

X_train, X_test, y_train, y_test = train_test_split(x, y,test_size=0.3)

# In[52]:

y_train

# # Corelation Matrix

# In[53]:

plt.figure(figsize=(20,20))
cor = X_train.corr()
sns.heatmap(cor, annot=True, cmap=plt.cm.CMRmap_r)
plt.show()

# # Creating Datasets of Different Correlation Levels

# In[152]:

def correlation(dataset, threshold):
    col_corr = set()  # Set of all the names of correlated columns
    corr_matrix = dataset.corr()
    for i in range(len(corr_matrix.columns)):
        for j in range(i):
            if abs(corr_matrix.iloc[i, j]) > threshold: # we are interested in absolute coeff value
                colname = corr_matrix.columns[i]  # getting the name of column
                col_corr.add(colname)
    return col_corr

# In[153]:

corr_features = correlation(X_train, 0.3)
KLOC_data_clean_0_3_train = X_train.drop(corr_features,axis=1)
KLOC_data_clean_0_3_test = X_test.drop(corr_features,axis=1)
KLOC_data_clean_0_3_train

# In[154]:

corr_features = correlation(X_train, 0.4)
KLOC_data_clean_0_4_train = X_train.drop(corr_features,axis=1)
KLOC_data_clean_0_4_test = X_test.drop(corr_features,axis=1)
KLOC_data_clean_0_4_train

# In[156]:

corr_features = correlation(X_train, 0.55)
KLOC_data_clean_0_5_train = X_train.drop(corr_features,axis=1)
KLOC_data_clean_0_5_test = X_test.drop(corr_features,axis=1)
KLOC_data_clean_0_5_train

# In[159]:

corr_features = correlation(X_train, 0.7)
KLOC_data_clean_0_6_train = X_train.drop(corr_features,axis=1)
KLOC_data_clean_0_6_test = X_test.drop(corr_features,axis=1)
KLOC_data_clean_0_6_train

# In[160]:

corr_features = correlation(X_train, 0.8)
KLOC_data_clean_0_7_train = X_train.drop(corr_features,axis=1)
KLOC_data_clean_0_7_test = X_test.drop(corr_features,axis=1)
KLOC_data_clean_0_7_train

# # Normalization

# In[18]:

scaler = MinMaxScaler(feature_range=(0, 1))

x_train_scaled = scaler.fit_transform(KLOC_data_clean_0_3_train)
KLOC_data_clean_0_3_train_scaled = pd.DataFrame(x_train_scaled)
x_test_scaled = scaler.fit_transform(KLOC_data_clean_0_3_test)
KLOC_data_clean_0_3_test_scaled = pd.DataFrame(x_test_scaled)

x_train_scaled = scaler.fit_transform(KLOC_data_clean_0_4_train)
KLOC_data_clean_0_4_train_scaled = pd.DataFrame(x_train_scaled)
x_test_scaled = scaler.fit_transform(KLOC_data_clean_0_4_test)
KLOC_data_clean_0_4_test_scaled = pd.DataFrame(x_test_scaled)

x_train_scaled = scaler.fit_transform(KLOC_data_clean_0_5_train)
KLOC_data_clean_0_5_train_scaled = pd.DataFrame(x_train_scaled)
x_test_scaled = scaler.fit_transform(KLOC_data_clean_0_5_test)
KLOC_data_clean_0_5_test_scaled = pd.DataFrame(x_test_scaled)

x_train_scaled = scaler.fit_transform(KLOC_data_clean_0_6_train)
KLOC_data_clean_0_6_train_scaled = pd.DataFrame(x_train_scaled)
x_test_scaled = scaler.fit_transform(KLOC_data_clean_0_6_test)
KLOC_data_clean_0_6_test_scaled = pd.DataFrame(x_test_scaled)

x_train_scaled = scaler.fit_transform(KLOC_data_clean_0_7_train)
KLOC_data_clean_0_7_train_scaled = pd.DataFrame(x_train_scaled)
x_test_scaled = scaler.fit_transform(KLOC_data_clean_0_7_test)
KLOC_data_clean_0_7_test_scaled = pd.DataFrame(x_test_scaled)
KLOC_data_clean_0_3_train_scaled

# # LinearRegression

# In[161]:

reg=LinearRegression()

# In[162]:

reg.fit(KLOC_data_clean_0_3_train,y_train)
y_pred=reg.predict(KLOC_data_clean_0_3_test)
plt.figure(figsize=(15,10))
plt.scatter(y_test,y_pred)
plt.xlabel('Actual')
plt.ylabel('Predicted')
plt.title('corr < 0.3')
r2_score(y_test,y_pred)

# In[163]:

reg.fit(KLOC_data_clean_0_4_train,y_train)
y_pred=reg.predict(KLOC_data_clean_0_4_test)
plt.figure(figsize=(15,10))
plt.scatter(y_test,y_pred)
plt.xlabel('Actual')
plt.ylabel('Predicted')
plt.title('corr < 0.5')
r2_score(y_test,y_pred)

# In[164]:

reg.fit(KLOC_data_clean_0_5_train,y_train)
y_pred=reg.predict(KLOC_data_clean_0_5_test)
plt.figure(figsize=(15,10))
plt.scatter(y_test,y_pred)
plt.xlabel('Actual')
plt.ylabel('Predicted')
plt.title('corr < 0.7')
r2_score(y_test,y_pred)

# In[165]:

reg.fit(KLOC_data_clean_0_6_train,y_train)
y_pred=reg.predict(KLOC_data_clean_0_6_test)
plt.figure(figsize=(15,10))
plt.scatter(y_test,y_pred)
plt.xlabel('actual')
plt.ylabel('predicted')
plt.title('corr < 0.9')
r2_score(y_test,y_pred)

# In[166]:

reg.fit(KLOC_data_clean_0_7_train,y_train)
y_pred=reg.predict(KLOC_data_clean_0_7_test)
plt.figure(figsize=(15,10))
plt.scatter(y_test,y_pred)
plt.xlabel('Actual')
plt.ylabel('Predicted')
plt.title('corr < 0.95')
r2_score(y_test,y_pred)

# In[172]:

reg.coef_
pd.DataFrame(reg.coef_,
             KLOC_data_clean_0_7_train.columns,
             columns=['coef'])\
            .sort_values(by='coef', ascending=False)


from flask import Flask, request, send_file, render_template
from markupsafe import escape
import os, sys, time, subprocess, glob, uuid, config, zipfile, sqlite3, shutil, signal, threading, multiprocessing, psutil
from datetime import datetime
import util, json, re
import xml.etree.ElementTree as ET

from ruamel.yaml import YAML

import girder_client

app = Flask(__name__, template_folder="templates")

@app.route('/api/docs')
def get_docs():
    return render_template('swaggerui.html')

@app.route('/get-workflow-templates', methods=['POST'])
def get_workflow_templates():

    folders_cwl = [f.split('/')[-2] for f in glob.glob(config.CWL  + "**/")]

    folders_toil = [f.split('/')[-2] for f in glob.glob(config.TOIL  + "**/")]

    query = 'SELECT name, description FROM templates'
    conn = sqlite3.connect(config.DATABASE)
    c = conn.cursor()
    c.execute(query)
    rows = c.fetchall()
    conn.close()

    cwl = []
    toil = []

    for wf in folders_cwl:
        descr = ""
        for row in rows:
            if (row[0] == wf):
                descr = row[1]
        cwl.append({
            "workflow-template": wf,
            "description": descr
        })

    for wf in folders_toil:
        descr = ""
        for row in rows:
            if (row[0] == wf):
                descr = row[1]
        toil.append({
            "workflow-template": wf,
            "description": descr
        })

    return {
        "status": 'OK',
        "templates": {
            "cwl": cwl,
            "toil": toil
        }
    }

@app.route('/template-description', methods=["GET", "POST", "PUT", "DELETE"])
def template_descriptions():
    if request.method == 'GET':
        template = request.args.get("workflow-template")
        if (template is None):
            return {
                "success": False,
                "message": "No workflow template specified."
            }

        query = 'SELECT description FROM templates WHERE name="'+template+'"'
        conn = sqlite3.connect(config.DATABASE)
        c = conn.cursor()
        c.execute(query)
        row = c.fetchone()
        if (row is None):
            return {
                "success": False,
                "message": "Workflow template doesn't have a description."
            }
        description = row[0]
        conn.close()

        return {
            "success": True,
            "description": description
        }
    elif (request.method == "POST"):
        data = dict(request.form)
        template = data.get("workflow-template")
        description = data.get("description")
        type = data.get("type")
        if (template is None):
            return {
                "success": False,
                "message": "No workflow template specified."
            }

        if (description is None):
            return {
                "success": False,
                "message": "No description."
            }

        if (type is None):
            return {
                "success": False,
                "message": "No type specified."
            }
        TypeID = 0
        if (type == "cwl"):
            TypeID = 1
        elif (type == "toil"):
            TypeID = 2
        else:
            return{
                "success": False,
                "message": "Uknown type"
            }

        query = 'SELECT description FROM templates WHERE name="'+template+'"'
        conn = sqlite3.connect(config.DATABASE)
        c = conn.cursor()
        c.execute(query)
        row = c.fetchone()

        if (row is None):
            query = 'INSERT INTO templates (name, description, TypeID) VALUES("'+template+'","'+description+'",'+str(TypeID)+')'
            try:
                c.execute(query)
                conn.commit()
                conn.close()
            except Exception as e :
                print(e)
                return {
                    "success": False,
                    "message": "Server error."
                }

            return {
                "success": True
            }

        else:
            return {
                "success": False,
                "message": "Workflow template already has a description."
            }
    elif (request.method == "PUT"):
        data = dict(request.form)
        template = data.get("workflow-template")
        description = data.get("description")

        if (template is None):
            return {
                "success": False,
                "message": "No workflow template specified."
            }

        if (description is None):
            return {
                "success": False,
                "message": "No description."
            }

        query = 'SELECT description FROM templates WHERE name="'+template+'"'
        conn = sqlite3.connect(config.DATABASE)
        c = conn.cursor()
        c.execute(query)
        row = c.fetchone()

        if (row is not None):
            query = 'UPDATE templates SET description = "'+description+'" WHERE name="'+template+'"'
            try:
                c.execute(query)
                conn.commit()
                conn.close()
            except Exception as e :
                print(e)
                return {
                    "success": False,
                    "message": "Server error."
                }

            return {
                "success": True
            }

        else:
            return {
                "success": False,
                "message": "Workflow template doesn't exists."
            }
    else:
        data = dict(request.form)
        template = data.get("workflow-template")
        query = 'DELETE FROM templates WHERE name="'+template+'"'
        try:
            conn = sqlite3.connect(config.DATABASE)
            c = conn.cursor()
            c.execute(query)
            conn.commit()
            conn.close()
        except Exception as e :
            print(e)
            return {
                "success": False,
                "message": "Server error."
            }

        return {
            "sucess": True
        }

@app.route('/get-workflows', methods=["GET"])
def get_workflows():
    try:
        userID = request.args.get("userID")
        wf_type = request.args.get("type")
        template = request.args.get("workflow-template")
        status = request.args.get("status")

        query = 'SELECT w.*, t.Type_Name FROM workflows w join Types t on w.TypeId=t.ID'
        if (wf_type is not None and template is not None):
            query += ' WHERE Name="'+template+'" and Type_Name="'+wf_type+'"'
        elif (wf_type is not None):
            query += ' WHERE t.Type_Name="'+wf_type+'"'
        elif (template is not None):
            query += ' WHERE w.Name="'+template+'"'

        conn = sqlite3.connect(config.DATABASE)
        c = conn.cursor()
        c.execute(query)
        rows = c.fetchall()
        wfs = list()
        for row in rows:
            wf = dict()
            wf['workflow-template'] = row[2]
            wf['type']= row[6]
            wf['GUID'] = row[0]
            wf['status'] = util.get_wf_status(row[3], row[0])
            wf['metadata'] = row[4]
            wf['creationDate'] = row[5]

            if (status is None and userID is None):
                wfs.append(wf)
            elif (status is None and userID is not None):
                if(wf['metadata'] is not None):
                    try:
                        tree = ET.fromstring(wf['metadata'])
                    except Exception as e:
                        print(e)
                        continue
                    if tree.find('userID').text == userID:
                        wfs.append(wf)
            elif (status is not None and userID is None):
                if (wf['status']==status):
                    wfs.append(wf)
            else:
                if (wf['status']==status and wf['metadata'] is not None):
                    try:
                        tree = ET.fromstring(wf['metadata'])
                    except Exception as e:
                        print(e)
                        continue
                    if tree.find('userID').text == userID:
                        wfs.append(wf)

        return {
            "success":True,
            "workflows":wfs
        }
    except Exception as e:
        print(e)
        return {
            "success":False,
            "message":"Server error."
        }

@app.route('/get-workflow-info', methods=["GET"])
def get_workflow_info():

    GUID = request.args['GUID']

    if GUID is None:
        return{
            "success": False,
            "message": "GUID field not set."
        }

    try:
        conn = sqlite3.connect(config.DATABASE)
        c = conn.cursor()
        c.execute("SELECT * FROM workflows where GUID='"+GUID+"'")
        row = c.fetchone()

        if (row == None):
            conn.close()
            return{
                "success": False,
                "message": "Workflow does't exist"
            }

        typeID = row[1]
        wf = dict()
        wf['workflow-template'] = row[2]
        wf['GUID'] = row[0]
        wf['status'] = util.get_wf_status(row[3], GUID)
        wf['metadata'] = row[4]
        wf['creationDate'] = row[5]

        c.execute('SELECT * FROM Types WHERE ID='+str(typeID))
        row = c.fetchone()
        if (row is not None):
            wf['type']=row[1]
        conn.close()
        return {
            "success":True,
            "workflow":wf
        }
    except Exception as e:
        print(e)
        return {
            "success":False,
            "message":"Server error."
        }

@app.route('/create-workflow', methods=['POST'])
def create_wf():

    try:
        req_data = dict(request.form)
    except Exception as e:
        print(e)
        return{
            "success": False,
            "message": "Bad request."
        }

    yaml_field = 'yaml'
    zip_field = 'input_zip'
    try:
        conn = sqlite3.connect(config.DATABASE)
        if yaml_field not in request.files or request.files[yaml_field].filename=='':
            return{
                'success': False,
                'message': 'Yaml file not selected.'
            }

        if ('workflow-template' not in req_data.keys()):
            return{
                "success":False,
                "message": "Workflow template not selected."
            }

        if ('type' not in req_data.keys()):
            return{
                "success":False,
                "message": "type field not set"
            }

        if not (req_data['type'] == 'toil' or req_data['type'] == 'cwl'):
            return{
                "success": False,
                "message": "type field is not 'toil' or 'cwl'"
            }

        cwl_templates = [f.split('/')[-2] for f in glob.glob(config.CWL  + "**/")]

        toil_templates = [f.split('/')[-2] for f in glob.glob(config.TOIL  + "**/")]

        if (req_data['type'] == 'toil'):
            templates = [f.split('/')[-2] for f in glob.glob(config.TOIL  + "**/")]
        else:
            templates = [f.split('/')[-2] for f in glob.glob(config.CWL  + "**/")]

        if not (req_data['workflow-template'] in templates):
            return {
                "success": False,
                "message": "Workflow template doesn't exist."
            }

        yaml = request.files[yaml_field]
        if zip_field in request.files:
            inputs = request.files[zip_field]

        GUID = uuid.uuid4()
        in_dir = os.path.join(os.path.abspath(config.INPUTS), str(GUID))
        if (os.path.isdir(in_dir) == True):
            return{
                "success": False,
                "message": "GUID already exists. Try sending the same request again."
            }
        os.mkdir(in_dir)

        if zip_field in request.files:
            if (not zipfile.is_zipfile(inputs)):
                return {
                    "success":False,
                    "message":"input_zip is not a .zip"
                }

        yaml.save(os.path.join(in_dir, 'inputs.yaml'))

        try:
            if zip_field in request.files:
                with zipfile.ZipFile(inputs, 'r') as zip_ref:
                    zip_ref.extractall(in_dir)
        except Exception as e:
            print(e)
            shutil.rmtree(in_dir)
            return{
                "success": False,
                "message": "Bad .zip file."
            }

        if (len(util.getGirderIds(in_dir)) > 0):
            if (request.headers.get("girder-api-key") is not None):
                try:
                    for girderId in util.getGirderIds(in_dir):
                        util.downloadGirderFile(girderId,in_dir, request.headers.get("girder-api-key"))
                except Exception as e:
                    print(e)
                    shutil.rmtree(in_dir)
                    return {
                        "success": False,
                        "message": "Girder error.",
                    }

        missing = util.validate_yaml(in_dir)
        if len(missing) > 0:
            shutil.rmtree(in_dir)
            return {
                "success": False,
                "message": "Input files are missing",
                "missing": missing
            }

        if (req_data['type'] == 'cwl'):
            typeId = 1
        else:
            typeId = 2

        nowDate = datetime.now()
        currentDate = nowDate.isoformat("T")

        if ('metadata' in req_data.keys() and req_data['metadata'] is not None):
            try:
                tree = ET.fromstring(req_data['metadata'])
                sanitizedName = re.sub(r'[^0-9a-zA-Z]','_',tree.find('comments').text).strip('_')
                sanitizedName = re.sub("\_+","_",sanitizedName)
                outputName = nowDate.isoformat(sep='_', timespec='seconds')+"_" + sanitizedName

                outputElement = ET.SubElement(tree, 'output')
                outputElement.text = outputName
                metadata = ET.tostring(tree).decode('utf-8')
            except Exception as e:
                print(e)
                pass

        else:
            metadata = None

        c = conn.cursor()

        if (metadata is not None):
            c.execute("INSERT INTO workflows VALUES ('"+str(GUID)+"',"+str(typeId)+",'"+req_data['workflow-template']+"',NULL, '"+metadata+"', '"+currentDate+"')")
        else:
            c.execute("INSERT INTO workflows VALUES ('"+str(GUID)+"',"+str(typeId)+",'"+req_data['workflow-template']+"',NULL, NULL, '"+currentDate+"')")
        conn.commit()

        return {
            'success': True,
            'GUID':GUID
        }
    except Exception as e:
        print(e)
        shutil.rmtree(in_dir)
        return{
            "success": False,
            "message": "Server error."
        }

@app.route('/run-workflow', methods=['POST'])
def run_workflow():

    girderApiKey = request.headers.get("girder-api-key")

    if (girderApiKey is None):
        girderApiKey = "--"

    req_data = request.get_json()

    if ('GUID' not in req_data.keys()):
        return{
            "success": False,
            "message": "GUID field not set."
        }

    GUID = req_data['GUID']
    try:
        conn = sqlite3.connect(config.DATABASE)
        c = conn.cursor()
        c.execute('SELECT * FROM workflows WHERE GUID="'+GUID+'"')
        row = c.fetchone()
        req_data['workflow'] = row[2]
        if (row == None):
            conn.close()
            return {
                'success':False,
                "message": "Workflow doesn't exist."
            }

        if (not row[3] == None):
            conn.close()
            return {
                "success": False,
                "message": "Workflow already run."
            }

        c.execute('SELECT Type_Name FROM Types where ID='+str(row[1]))
        row = c.fetchone()
        req_data['type'] = row[0]

        input_path = os.path.join(config.INPUTS,GUID)
        job_store_path = os.path.join(config.RUNNING_WORKFLOWS,str(GUID))
        log_file_path = os.path.join(config.LOGS, str(GUID))

        out_dir = os.path.join(config.RESULTS, str(GUID))
        if (os.path.isdir(out_dir)):
            return{
                "success": False,
                "message": "Workflow has already been run."
            }

        rootFolder = os.path.abspath(os.getcwd())
        if (req_data['type'] == 'cwl'):

            os.mkdir(out_dir)
            os.mkdir(log_file_path)
            log_file_path = os.path.join(log_file_path,"log.txt")
            cwl_path = os.path.abspath(os.path.join(config.CWL,req_data['workflow'], 'workflow.cwl'))
            yaml_path = os.path.abspath(os.path.join(input_path, 'inputs.yaml'))

            if (util.ifReadOnlyWf(req_data['workflow'])):
                flag = "y"
            else:
                flag = "n"
            if (config.CWL_RUNNER == 'toil-cwl-runner'):
                script = os.path.abspath('run_cwltoil.sh')

                processArgs = ['bash', script, flag, os.path.abspath(job_store_path), os.path.abspath(log_file_path), cwl_path, yaml_path, GUID, girderApiKey, rootFolder]
                process = subprocess.Popen(processArgs, cwd=os.path.abspath(out_dir))
            elif (config.CWL_RUNNER == 'mesos'):
                script = os.path.abspath('run_mesos.sh')

                processArgs = ['bash', script, flag, os.path.abspath(job_store_path), os.path.abspath(log_file_path), cwl_path, yaml_path, GUID, girderApiKey,config.MESOS_NODE_TYPE, rootFolder]
                process = subprocess.Popen(processArgs, cwd=os.path.abspath(out_dir))

            else:

                script = os.path.abspath('run_cwltool.sh')
                processArgs = ['bash', script, flag, cwl_path, yaml_path, GUID, girderApiKey, rootFolder]
                process = subprocess.Popen(processArgs, cwd=os.path.abspath(out_dir), stdout = open(log_file_path,'w'), stderr = open(log_file_path,'w'))

            pid = process.pid

            c.execute('UPDATE workflows SET PID='+str(pid)+', creationDate="'+datetime.now().isoformat("T")+'" WHERE GUID="'+GUID+'"')

            conn.commit()
        elif (req_data['type'] == 'toil'):

            os.mkdir(out_dir)
            os.mkdir(log_file_path)
            log_file_path = os.path.join(log_file_path,"log.txt")
            toil_path = os.path.join(config.TOIL, req_data["workflow"] , 'main.py')
            processArgs = ["bash", "run_toil.sh", toil_path, job_store_path, input_path, out_dir, os.path.abspath(log_file_path), GUID, girderApiKey, rootFolder]

            process = subprocess.Popen(processArgs)
            pid = process.pid

            c.execute('UPDATE workflows SET PID='+str(pid)+', creationDate="'+datetime.now().isoformat("T")+'" WHERE GUID="'+GUID+'"')
            conn.commit()

        conn.close()
        return {
            "success": True
        }
    except Exception as e:
        print(e)
        return{
            "success":True,
            "message": "Server error."
        }

@app.route('/get-log', methods=['GET'])
def get_log():
    GUID = request.args['GUID']
    if (GUID is None):
        return {
            "success":False,
            "message": "GUID field not set."
        }
    try:
        log_file_path = os.path.join(config.LOGS, str(GUID))
        log_file_path = os.path.join(log_file_path,"log.txt")
        if (os.path.isfile(log_file_path)):
            return send_file(log_file_path)
        else:
            return {
                "success": False,
                "message": "Log does not exist."
            }
    except Exception as e:
        print(e)
        return {
            "success":False,
            "message": "Server error."
        }

@app.route('/get-status', methods=['GET'])
def get_status():
    GUID = request.args['GUID']
    if GUID is None:
        return{
            "success": False,
            "message": "GUID field not set."
        }
    try:
        conn = sqlite3.connect(config.DATABASE)
        c = conn.cursor()
        c.execute('SELECT * FROM workflows WHERE GUID="'+GUID+'"')
        row = c.fetchone()

        conn.close()
    except Exception as e:
        print(e)
        return {
            "success": False,
            "message": "Server error.",
            "status": "Error"

        }
    if (row == None):
        return{
                "success": False,
                "message": "Workflow doesn't exist",
                "status": "Error"
            }

    status = util.get_wf_status(row[3], GUID)

    if (row[3] == None):
        return {
            'success':True,
            "message": "Workflow has not been started yet.",
            "status": status
        }
    if (row[3] == -1):
        return {
            'success': True,
            "message": "Workflow has been terminated by user.",
            "status": status
        }
    if (row[3] == -2):
        return {
            'success': True,
            "message": "Workflow has been terminated due to timeout.",
            "status": status
        }

    job_store = os.path.abspath(os.path.join(config.RUNNING_WORKFLOWS, GUID))

    if (config.CWL_RUNNER=="toil-cwl-runner"):
        message = subprocess.check_output(['toil', 'status', job_store])
        return {
            "success": True,
            "message": str(message, encoding="UTF-8"),
            "status": status
        }
    else:
        return {
            "success": True,
            "status": status
        }

@app.route('/get-results', methods=['GET'])
def get_results():

    GUID = request.args['GUID']
    if request.args['GUID'] == None:
        return{
            "success": False,
            "message": "GUID field not set."
        }
    out_dir = os.path.join(config.RESULTS, GUID)
    if (os.path.isdir(out_dir)):
        if (util.get_wf_status(util.get_wf_pid(GUID), GUID) == 'FINISHED_OK'):
            try:
                dir_name = os.path.join(config.RESULTS, GUID)
                if (not os.path.isfile(dir_name+".zip")):

                    zip_file = zipfile.ZipFile(dir_name+".zip", 'w')

                    if os.path.isdir(os.path.join(dir_name, "tmp")):
                        shutil.rmtree(os.path.join(dir_name, "tmp"))

                    for root, directories, files in os.walk(dir_name):
                        directories[:] = [d for d in directories if d not in ['tmp']]
                        for filename in files:
                            filePath = os.path.join(root, filename)
                            zip_file.write(filePath, filename)

                    zip_file.close()
                return send_file(dir_name+".zip")
            except Exception as e:
                print(e)
                return {
                    "success": False,
                    "message": "Server error."
                }

        elif (util.get_wf_status(util.get_wf_pid(GUID), GUID) == 'FINISHED_ERROR'):
            return {
                "success": False,
                "message": "Job finished with error"
            }

        else:
            return {
                "success": False,
                "message": "Job not finished"
            }
    else:
        return{
            "success": False,
            "message": "Workflow either doesn't exist or it hasn't been run."
        }

@app.route('/upload-results', methods=['POST'])
def upload_results():
    if (request.headers.get('girder-api-key') is None ):
        return {
            "success": False,
            "message": "API KEY not provided."
        }

    req_data = request.get_json()
    if 'GUID' not in req_data.keys():
        return{
            "success": False,
            "message": "GUID field not set."
        }

    if 'girder-parent-id' not in req_data.keys():
        return{
            "success": False,
            "message": "Girder parent folder field not set."
        }

    GUID = req_data['GUID']

    out_dir = os.path.join(config.RESULTS, GUID)
    if (os.path.isdir(out_dir)):
        if (util.get_wf_status(util.get_wf_pid(GUID), GUID) == 'FINISHED_OK'):
            try:
                if os.path.isdir(os.path.join(out_dir, "tmp")):
                    shutil.rmtree(os.path.join(out_dir, "tmp"))

                try:
                    folder = util.uploadToGirder(out_dir,request.headers.get('girder-api-key'), req_data['gider-parent-id'])

                    return {
                        "success": True,
                        "uploadedFolder": folder
                    }
                except Exception as e:
                    print(e)
                    return {
                    "success": False,
                    "message": "Error uploading to girder."
                }

            except Exception as e:
                print(e)
                return {
                    "success": False,
                    "message": "Server error."
                }

        elif (util.get_wf_status(util.get_wf_pid(GUID), GUID) == 'FINISHED_ERROR'):
            return {
                "success": False,
                "message": "Job finished with error"
            }

        else:
            return {
                "success": False,
                "message": "Job not finished"
            }
    else:
        return{
            "success": False,
            "message": "Workflow either doesn't exist or it hasn't been run."
        }

@app.route('/stop-workflow', methods=['POST'])
def stop_workflow():

    req_data = request.get_json()
    if 'GUID' not in req_data.keys():
        return{
            "success": False,
            "message": "GUID field not set."
        }
    GUID = req_data['GUID']
    try:

        conn = sqlite3.connect(config.DATABASE)
        c = conn.cursor()
        c.execute('SELECT * FROM workflows WHERE GUID="'+GUID+'"')
        row = c.fetchone()

        conn.close()
    except Exception as e:
        print(e)
        return{
            "success": False,
            "message": "Server error."
        }

    if (row == None):
        return {
            "success": False,
            "message": "Workflow doesn't exist."
        }
    if (row[3] == None):
        return {
            "success": False,
            "message": "Workflow is not running"
        }

    if (row[3] == -1):
        return {
            'success': False,
            "message": "Workflow has been terminated by user."
        }
    if (row[3] == -2):
        return{
            'success': False,
            "message": "Workflow has been terminated due to timeout."
        }

    if (not util.check_pid(row[3])):
        return {
            "success": False,
            "message": "Workflow is finished."
        }
    else:
        util.terminate(GUID, row[3], -1)
        return {
            "success":True,
            "message": "Workflow terminated."
        }

@app.route('/delete-workflow', methods=['POST'])
def delete_wf():
    girderApiKey = request.headers.get("girder-api-key")
    req_data = request.get_json()
    if 'GUID' not in req_data.keys():
        return{
            "success": False,
            "message": "GUID field not set."
        }
    try:
        GUID = req_data['GUID']
        conn = sqlite3.connect(config.DATABASE)
        c = conn.cursor()
        c.execute('SELECT * FROM workflows WHERE GUID="'+GUID+'"')
        row = c.fetchone()

        if (row == None):
            conn.close()
            return {
                "success": False,
                "message": "Workflow doesn't exist."
            }

        if (row[3] is not None and row[3] > 0 and util.check_pid(row[3])):
            conn.close()
            return {
                "success":False,
                "message": "Workflow is running"
            }

        jobStore = os.path.abspath(os.path.join(config.RUNNING_WORKFLOWS,GUID))
        input_dir = os.path.abspath(os.path.join(config.INPUTS, GUID))
        output_dir = os.path.abspath(os.path.join(config.RESULTS, GUID))
        log_dir = os.path.abspath(os.path.join(config.LOGS,GUID))

        if (os.path.isdir(jobStore)):
            shutil.rmtree(jobStore)
        if (os.path.isdir(input_dir)):
            shutil.rmtree(input_dir)
        if (os.path.isdir(output_dir)):
            shutil.rmtree(output_dir)
        if (os.path.isdir(log_dir)):
            shutil.rmtree(log_dir)

        if (girderApiKey is not None):
            util.deleteFolderFromGirder(row, girderApiKey)

        c.execute('DELETE FROM workflows WHERE GUID="'+GUID+'"')
        conn.commit()
        conn.close()

        return {
            "success":True
        }
    except Exception as e:
        print(e)
        return{
            "success": False,
            "message": "Server error."
        }

@app.route('/download-workflow', methods=['GET'])
def download_wf():
    try:
        GUID = request.args['GUID']
        if request.args['GUID'] == None:
            return{
                "success": False,
                "message": "GUID field not set."
            }

        wf = util.get_wf(GUID)
        if (wf is None):
            return{
                "success":False,
                "message": "Workflow doesn't exist."
            }

        out_dir = os.path.join(config.RESULTS, GUID)

        wf_dir = ""
        if (wf[1] == 1):
            wf_dir = os.path.join(config.CWL,wf[2])
        elif (wf[1] == 2):
            wf_dir = os.path.join(config.TOIL,wf[2])
        else:
            return{
                "success":False,
                "message": "Server error."
            }

        in_dir = os.path.join(config.INPUTS, GUID)
        zip_wf = zipfile.ZipFile(wf[2]+".zip", 'w')

        execute_path = ""
        for root, directories, files in os.walk(wf_dir):
            execute_path = root
            for filename in files:
                filePath = os.path.join(root,filename)

                zipFilePath = "./wf/"+filePath.split(root)[1]
                zip_wf.write(filePath, zipFilePath)

        input_path = ""
        for root,directories,files in os.walk(in_dir):
            input_path = root
            for filename in files:
                filePath = os.path.join(root, filename)
                zipFilePath = "./inputs/"+filePath.split(root)[1]
                zip_wf.write(filePath, zipFilePath)

        f = open('start.sh','w')
        f.write("mkdir outputs\n")
        f.write("cd outputs\n")
        command = ""
        if (wf[1] == 1):
            execute_file = os.path.join('..','wf', 'workflow.cwl')
            input_file = os.path.join('..','inputs', 'inputs.yaml')
            command = command+"cwltoil "
            command = command+execute_file+" "+input_file
        elif (wf[1] == 2):
            execute_file = os.path.join('wf', 'main.py')
            input_file = os.path.join('inputs', 'inputs.yaml')
            command = command+"python "
            command = command+execute_file+" . "+input_file+ " ."

        f.write(command)
        f.close()

        zip_wf.write('start.sh')

        if (os.path.isdir(out_dir)):
            if (util.get_wf_status(util.get_wf_pid(GUID), GUID) == 'FINISHED_OK'):

                zip_file = zipfile.ZipFile(wf[2]+"-out.zip", 'w')
                for root, directories, files in os.walk(out_dir):
                    directories[:] = [d for d in directories if d not in ['tmp']]
                    for filename in files:
                        filePath = os.path.join(root, filename)
                        zip_file.write(filePath, filename)

                zip_file.close()
                zip_wf.write(zip_file.filename)
                os.remove(zip_file.filename)

        log_dir = os.path.join(config.LOGS,GUID)

        if (os.path.isdir(log_dir)):
            if (os.path.isfile(os.path.join(log_dir,'log.txt'))):
                zip_wf.write(os.path.join(log_dir,'log.txt'), 'log.txt')

        zip_wf.close()

        return send_file(zip_wf.filename)
    except Exception as e:
        print(e)
        return {
            "success": False,
            "message": "Server error."
        }

@app.route('/get-output-file', methods=['GET'])
def download_output_file():
    try:
        GUID = request.args['GUID']
        if request.args['GUID'] == None:
            return{
                "success": False,
                "message": "GUID field not set."
            }

        if request.args['filepath'] == None:
            return{
                "success": False,
                "message": "Filepath not sent."
            }

        pid = util.get_wf_pid(GUID)
        status = util.get_wf_status(pid,GUID)
        if (not status == 'FINISHED_OK'):
            return {
                "success": False,
                "message": "Workflow does not have a FINISHED_OK status."
            }

        filepath = os.path.join(util.getWfOutputDir(GUID),request.args['filepath'])
        return send_file(os.path.abspath(filepath))
    except Exception as e:
        print(e)
        return {
            "success": False,
            "message": "Server error."
        }

@app.route('/get-output-metadata', methods=['GET'])
def get_output_struct():
    try:
        GUID = request.args['GUID']
        if request.args['GUID'] == None:
            return{
                "success": False,
                "message": "GUID field not set."
            }

        conn = sqlite3.connect(config.DATABASE)
        c = conn.cursor()
        c.execute("SELECT * FROM workflows where GUID='"+GUID+"'")
        row = c.fetchone()

        if (row == None):
            conn.close()
            return{
                "success": False,
                "message": "Workflow does't exist"
            }

        typeID = row[1]
        wf = dict()
        wf['workflow-template'] = row[2]
        wf['GUID'] = row[0]
        wf['status'] = util.get_wf_status(row[3], GUID)
        wf['metadata'] = row[4]
        wf['creationDate'] = row[5]

        c.execute('SELECT * FROM Types WHERE ID='+str(typeID))
        row = c.fetchone()
        if (row is not None):
            wf['type']=row[1]
        conn.close()

        if (not wf['status'] == 'FINISHED_OK'):
            return {
                "success": False,
                "message": "Workflow does not have a FINISHED_OK status."
            }

        extensions = json.loads(open("extesions.json").read())

        outputFiles = dict()
        for key in extensions:
            outputFiles[key] = list()
        outputFiles['other'] = list()

        for root, dirs, files in os.walk(util.getWfOutputDir(GUID)):
            files.sort()
            for f in files:
                link = util.getDownloadOutputLink(GUID,os.path.relpath(os.path.join(root,f),util.getWfOutputDir(GUID)))

                foundFlag = False
                for ext in extensions:
                    if f.endswith(tuple(extensions[ext])):
                        foundFlag = True
                        outputFiles[ext].append({
                            "filename": f,
                            "link": request.host_url + link
                        })
                        continue
                if (not foundFlag):
                    outputFiles['other'].append({
                            "filename": f,
                            "link": request.host_url + link
                    })

        empty = list()
        for ext in outputFiles:
             if not outputFiles[ext]:
                empty.append(ext)

        for ext in empty:
            outputFiles.pop(ext)

        wf['outputs'] = outputFiles
        wf["downloadOutputURL"] = request.host_url + util.getZipLink(GUID)
        return {
            "success":True,
            "workflow":wf
        }
    except Exception as e:
        print(e)
        return {
            "success": False,
            "message": "Server error."
        }

@app.route('/get-input-file', methods=['GET'])
def download_input_file():
    try:
        GUID = request.args['GUID']
        if request.args['GUID'] == None:
            return{
                "success": False,
                "message": "GUID field not set."
            }

        if request.args['filepath'] == None:
            return{
                "success": False,
                "message": "Filepath not sent."
            }

        filepath = os.path.join(util.getWfInputDir(GUID),request.args['filepath'])
        return send_file(os.path.abspath(filepath))
    except Exception as e:
        print(e)
        return {
            "success": False,
            "message": "Server error."
        }

@app.route('/get-input-metadata', methods=['GET'])
def get_input_struct():
    try:
        GUID = request.args['GUID']
        if request.args['GUID'] == None:
            return{
                "success": False,
                "message": "GUID field not set."
            }

        conn = sqlite3.connect(config.DATABASE)
        c = conn.cursor()
        c.execute("SELECT * FROM workflows where GUID='"+GUID+"'")
        row = c.fetchone()

        if (row == None):
            conn.close()
            return{
                "success": False,
                "message": "Workflow does't exist"
            }

        typeID = row[1]
        wf = dict()
        wf['workflow-template'] = row[2]
        wf['GUID'] = row[0]
        wf['status'] = util.get_wf_status(row[3], GUID)
        wf['metadata'] = row[4]
        wf['creationDate'] = row[5]
        wf['inputs'] = list()

        c.execute('SELECT * FROM Types WHERE ID='+str(typeID))
        row = c.fetchone()
        if (row is not None):
            wf['type']=row[1]
        conn.close()
        # inputFiles = dict()
        # inputFiles['inputs']= list()

        for root, dirs, files in os.walk(util.getWfInputDir(GUID)):
            files.sort()
            for f in files:
                link = util.getDownloadInputLink(GUID,os.path.relpath(os.path.join(root,f),util.getWfInputDir(GUID)))
                print(link)
                print(wf['inputs'])
                wf['inputs'].append({
                            "filename": f,
                            "link": request.host_url + link
                    })

        # empty = list()
        # for ext in outputFiles:
        #      if not outputFiles[ext]:
        #         empty.append(ext)

        # for ext in empty:
        #     outputFiles.pop(ext)

        # wf['inputs'] = inputFiles
        wf["downloadOutputURL"] = request.host_url + util.getZipLink(GUID)
        return {
            "success":True,
            "workflow":wf
        }
    except Exception as e:
        print(e)
        return {
            "success": False,
            "message": "Server error."
        }


from __future__ import annotations

from typing import IO, TYPE_CHECKING, Dict

from flask import current_app, g

from enums import Dep, LeafState, MangoType
from exts import storage
from exts.id_tools import generate_id
from exts.sqlalchemy_ import (BaseModel, Column, ForeignKey, UniqueConstraint,
                              relationship)
from exts.sqlalchemy_.types import (ARRAY, JSON, Boolean, DateTime, Enum,
                                    LargeBinary, String)

from .audio_tools import get_audio_info
from .errors import NotQualifiedToPick, StateLocked, UnallowedType, UnknowType
from .file_tools import EXTS, is_allowed_type, special_save

if TYPE_CHECKING:
    from proj.models import Proj
    from pink.models import Pink

class Leaf(BaseModel):
    __tablename__ = 'leaf'

    id = Column(String, primary_key=True)
    proj_id = Column(String, ForeignKey('proj.id', ondelete='CASCADE'))
    dep = Column(Enum(Dep))
    role = Column(String)
    pink_id = Column(String, ForeignKey('pink.id'))
    state = Column(Enum(LeafState), default=LeafState.waiting)
    track = Column(ARRAY(String), default=list())
    timestamp = Column(DateTime)

    pink = relationship('Pink', back_populates='leafs')
    proj = relationship('Proj', back_populates='leafs')
    mangos = relationship('Mango',
                          back_populates='leaf',
                          cascade='all, delete-orphan',
                          passive_deletes=True)
    schedule = relationship('Schedule',
                            back_populates='leaf',
                            uselist=False,
                            cascade='all, delete-orphan',
                            passive_deletes=True)
    __table_args__ = (UniqueConstraint('proj_id',
                                       'dep',
                                       'role',
                                       'pink_id',
                                       name='_leaf_uc'), )

    def __init__(self, freerole, pink):
        if freerole.dep not in pink.deps:
            raise NotQualifiedToPick()
        super().__init__(proj_id=freerole.progress_id,
                         dep=freerole.dep,
                         role=freerole.role,
                         pink=pink,
                         timestamp=g.now)
        self.id = generate_id(15)
        self.proj.call_elf(dep=None, mango=None, pos=None)
        freerole.take(leaf_id=self.id)

    def drop(self, force: bool = False) -> None:
        if self.state != LeafState.normal and not force:
            raise StateLocked(current=self.state)
        self.state = LeafState.droped_m if not force else LeafState.droped_f
        self.track.append(f'drop|{g.now}' if not force else f'f_drop|{g.now}')
        self.proj.progress.put_role(leaf_id=self.id)

    def f_drop(self) -> None:
        self.pink.cc += 1
        self.drop(force=True)

    def submit(self, f: IO) -> None:
        if self.state != LeafState.normal:
            raise StateLocked(current=self.state)
        mango = Mango(leaf=self, f=f)
        self.state = LeafState.paused_fin
        self.proj.run_elfs(dep=self.dep, mango=mango, pos=True)
        self.track.append(f'fin|{g.now}')
        return mango

    def download(self):
        if self.state != LeafState.paused_fin:
            raise StateLocked(current=self.state)
        mango = self.mangos[-1]
        mango.download()
        return mango.id, mango.name

    def redo(self) -> None:
        if self.state != LeafState.paused_fin:
            raise StateLocked(current=self.state)
        mango: Mango = self.mangos[-1]
        mango.delete()
        self.state = LeafState.normal
        self.proj.run_elfs(dep=self.dep, mango=mango, pos=False)
        self.track.append(f'redo|{g.now}')

    def to_dict(self, lv: int) -> Dict[str, str]:
        if lv == 0:
            return {
                'id': self.id,
                'proj_id': self.proj_id,
                'dep_': self.dep.name,
                'role': self.role,
                'pink_id': self.pink.id,
                'state': self.state.name
            }

class Mango(BaseModel):
    __tablename__ = 'mango'

    id = Column(String, primary_key=True)
    leaf_id = Column(String, ForeignKey('leaf.id', ondelete='CASCADE'))
    mtype = Column(Enum(MangoType))
    fp = Column(LargeBinary(32))
    deleted = Column(Boolean, default=False)
    timestamp = Column(DateTime)
    metainfo = Column(JSON)

    leaf = relationship('Leaf', back_populates='mangos')

    @property
    def name(self):
        leaf = self.leaf
        return f'{leaf.dep}-{leaf.role} [{leaf.pink.name}-{self.id[:6]}].{EXTS[self.mtype]}'

    def __init__(self, leaf: Leaf, f: IO):
        super().__init__(leaf=leaf)
        self.id = generate_id(18)
        self.timestamp = g.now
        path = f'{current_app.root_path}/{self.id}'
        self.fp, self.mtype = special_save(f=f, path=path)
        if self.mtype is MangoType.unknown:
            raise UnknowType()
        if not is_allowed_type(self.dep, self.mtype):
            raise UnallowedType(mtype=self.mtype)
        # fetch info
        if self.mtype // 10 == 5:
            self.metainfo = get_audio_info(path=path)
        else:
            self.metainfo = None

        storage.upload(mango=self, path=path)

    def download(self) -> None:
        storage.download(mango=self)

    def delete(self) -> None:
        self.deleted = True

class Schedule(BaseModel):
    __tablename__ = 'schedule'

    leaf_id = Column(String,
                     ForeignKey('leaf.id', ondelete='CASCADE'),
                     primary_key=True)
    begin = Column(DateTime)
    due = Column(DateTime)
    paused_at = Column(DateTime)

    leaf = relationship('Leaf', back_populates='schedule')


import bpy
from math import degrees

def exportAnimationToCSV():
    sce = bpy.context.scene
    ob = bpy.context.active_object
    if ob.animation_data is not None:
        if ob.animation_data.action is not None:
            actionName = ob.animation_data.action.name
            fileName = f"{actionName}.csv" # Will name the file as the action name
            currentFrame = sce.frame_current
            file=open(fileName,'w')
            for frame in range(sce.frame_start,sce.frame_end+1):
                sce.frame_set(frame)
                base=ob.pose.bones["Base"].rotation_euler.y
                shoulder=ob.pose.bones["Shoulder"].rotation_euler.z
                elbow=ob.pose.bones["Elbow"].rotation_euler.z
                wrist=ob.pose.bones["Wrist vertical"].rotation_euler.z
                toArmRot = lambda angle:int((degrees(angle)+90)%360)
                file.write(f"{toArmRot(base)};{toArmRot(shoulder)};{toArmRot(elbow)};{toArmRot(wrist)}\n")
            file.close()
            sce.frame_set(currentFrame)

if __name__ == "__main__":
    exportAnimationToCSV()


from datetime import date

nome = 'Pedro Lisboa'
idade = 28
altura = 1.90
peso = 130
ano_atual = date.today().year
ano_nascimento = ano_atual - idade
imc = (peso / altura ** 2)

print(f'{nome} nasceu em {ano_nascimento} portanto tem {idade} anos')
print(f'Seu peso e de {peso}kg e sua altura {altura} m seu imc = {imc:.2f}')
print('Teste : é è ç ^ ã ê ')


import os
import json
import pandas as pd
import altair as alt
import pathlib
import random
import argparse

HERE = pathlib.Path(__file__).parent

PLOTS = HERE / "plots"
PLOTS.mkdir(exist_ok=True)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()

    # Add an argument for the path to the results file
    parser.add_argument("results_file", type=str)

    # Add an optional argument for the chart title.
    parser.add_argument("--title", type=str, default=None)

    # Parse the command line arguments
    args = parser.parse_args()
    bench_group, git_hash = args.results_file[:-5].split("_")

    path = pathlib.Path(args.results_file)
    data = json.loads(path.read_text())

    chart_title = args.title or f"Results from {path.name}"

    rows = []

    for result in data["results"]:
        command = result["command"]
        for time in result["times"]:
            rows.append(
                {
                    "Run Group": command,
                    "Time (s)": time,
                }
            )

    df = pd.DataFrame(rows)

    chart = (
        alt.Chart(df)
        .mark_point(size=18)
        .encode(
            y="Run Group",
            x=alt.X(
                "Time (s):Q",
            ).scale(domain=(df["Time (s)"].min() - 2, df["Time (s)"].max() + 2)),
            yOffset="jitter:Q",
            color="Run Group",
        )
        .properties(
            width=200,
            height=200,
            title=chart_title,
        )
        .configure_axisY(title=None)
        .transform_calculate(
            # Generate Gaussian jitter with a Box-Muller transform
            jitter="sqrt(-1*log(random()))*cos(1*PI*random())"
        )
        .resolve_scale(yOffset="independent")
    )

    chart.save(PLOTS / f"{path.name}.svg")


#!/usr/bin/python3
from pygame import mixer
import RPi.GPIO as GPIO
import random
import glob
import time

class LightShow(object):

    def __init__(self, volume=1.0, gpio_speed=0.166):
        self.base_audio_path = ("./audio/")
        self.base_sequence_path = ("./sequences/")
        self.songs = self.gather_songs()
        self.sequences = self.gather_sequences()
        self.volume = volume
        GPIO.setwarnings(False)   # Disable GPIO in use warnings
        GPIO.setmode(GPIO.BOARD)  # Set the GPIO mode to BOARD using pin integer
        self.channels = self.configure_channels()
        self.speed=gpio_speed

    def gather_songs(self):
        songs = glob.glob("".join([self.base_audio_path, "*"]))
        return sorted(songs)

    def gather_sequences(self):
        sequences = glob.glob("".join([self.base_sequence_path, "*"]))
        return sorted(sequences)

    # Channels are Solid State Relay Channels
    # Actual numbers are the RPi 4 BOARD pin integers
    def configure_channels(self):
        channel_0 = 11
        channel_1 = 12
        channel_2 = 13
        channel_3 = 15
        channel_4 = 16
        channel_5 = 18
        channels = [ channel_0, channel_1, channel_2, channel_3, channel_4, channel_5 ]
        return channels

    # Pair channel: 0,3
    def pair_one(self, custom_speed=False):
        for channel in [self.channels[0], self.channels[3]]:
            GPIO.setup(channel, GPIO.OUT)
            GPIO.output(channel, True)
        if custom_speed:
            time.sleep(float(int(custom_speed) / 1000))
        else:
            time.sleep(self.speed)
        for channel in [self.channels[0], self.channels[3]]:
            GPIO.output(channel, False)

    # Pair channel: 1,4
    def pair_two(self, custom_speed=False):
        for channel in [self.channels[1], self.channels[4]]:
            GPIO.setup(channel, GPIO.OUT)
            GPIO.output(channel, True)
        if custom_speed:
            time.sleep(float(int(custom_speed) / 1000))
        else:
            time.sleep(self.speed)
        for channel in [self.channels[1], self.channels[4]]:
            GPIO.output(channel, False)

    # Pair channel: 2,5
    def pair_three(self, custom_speed=False):
        for channel in [self.channels[2], self.channels[5]]:
            GPIO.setup(channel, GPIO.OUT)
            GPIO.output(channel, True)
        if custom_speed:
            time.sleep(float(int(custom_speed) / 1000))
        else:
            time.sleep(self.speed)
        for channel in [self.channels[2], self.channels[5]]:
            GPIO.output(channel, False)

    # Pair channel: 0,5
    def diagonal_one(self, custom_speed=False):
        for channel in [self.channels[0], self.channels[5]]:
            GPIO.setup(channel, GPIO.OUT)
            GPIO.output(channel, True)
        if custom_speed:
            time.sleep(float(int(custom_speed) / 1000))
        else:
            time.sleep(self.speed)
        for channel in [self.channels[0], self.channels[5]]:
            GPIO.output(channel, False)

    # Pair channel: 2,3
    def diagonal_two(self, custom_speed=False):
        for channel in [self.channels[2], self.channels[3]]:
            GPIO.setup(channel, GPIO.OUT)
            GPIO.output(channel, True)
        if custom_speed:
            time.sleep(float(int(custom_speed) / 1000))
        else:
            time.sleep(self.speed)
        for channel in [self.channels[2], self.channels[3]]:
            GPIO.output(channel, False)

    # Pair channel: 0,4
    def down_right_one(self, custom_speed=False):
        for channel in [self.channels[0], self.channels[4]]:
            GPIO.setup(channel, GPIO.OUT)
            GPIO.output(channel, True)
        if custom_speed:
            time.sleep(float(int(custom_speed) / 1000))
        else:
            time.sleep(self.speed)
        for channel in [self.channels[0], self.channels[4]]:
            GPIO.output(channel, False)

    # Pair channel: 1,5
    def down_right_two(self, custom_speed=False):
        for channel in [self.channels[1], self.channels[5]]:
            GPIO.setup(channel, GPIO.OUT)
            GPIO.output(channel, True)
        if custom_speed:
            time.sleep(float(int(custom_speed) / 1000))
        else:
            time.sleep(self.speed)
        for channel in [self.channels[1], self.channels[5]]:
            GPIO.output(channel, False)

    # Pair channel: 1,3
    def down_left_one(self, custom_speed=False):
        for channel in [self.channels[1], self.channels[3]]:
            GPIO.setup(channel, GPIO.OUT)
            GPIO.output(channel, True)
        if custom_speed:
            time.sleep(float(int(custom_speed) / 1000))
        else:
            time.sleep(self.speed)
        for channel in [self.channels[1], self.channels[3]]:
            GPIO.output(channel, False)

    # Pair channel: 2,4
    def down_left_two(self, custom_speed=False):
        for channel in [self.channels[2], self.channels[4]]:
            GPIO.setup(channel, GPIO.OUT)
            GPIO.output(channel, True)
        if custom_speed:
            time.sleep(float(int(custom_speed) / 1000))
        else:
            time.sleep(self.speed)
        for channel in [self.channels[2], self.channels[4]]:
            GPIO.output(channel, False)

    def all_on(self, custom_speed=False):
        for channel in self.channels:
            GPIO.setup(channel, GPIO.OUT)
            GPIO.output(channel, True)
        if custom_speed:
            time.sleep(float(int(custom_speed) / 1000))
        else:
            time.sleep(self.speed)
        for channel in self.channels:
            GPIO.output(channel, False)

    # Turn off all GPIOs
    def all_off(self):
        try:
            for channel in self.channels:
                GPIO.output(channel, False)
        except RuntimeError:
            # Maybe all (or one) of the GPIOs are already off
            pass

    # Cycle GPIOs forward
    def forward(self, custom_speed=False):
        channels = [
            self.channels[0], self.channels[1], self.channels[2],
            self.channels[5], self.channels[4], self.channels[3]
        ]
        for channel in channels:
            GPIO.setup(channel, GPIO.OUT)
            GPIO.output(channel, True)
            if custom_speed:
                time.sleep(float(int(custom_speed) / 1000))
            else:
                time.sleep(self.speed)
            GPIO.output(channel, False)

    # Cycle GPIOs reverse
    def reverse(self, custom_speed=False):
        channels = [
            self.channels[3], self.channels[4], self.channels[5],
            self.channels[2], self.channels[1], self.channels[0]
        ]
        for channel in channels:
            GPIO.setup(channel, GPIO.OUT)
            GPIO.output(channel, True)
            if custom_speed:
                time.sleep(float(int(custom_speed) / 1000))
            else:
                time.sleep(self.speed)
            GPIO.output(channel, False)

    # Cycle GPIOs randomly (using a LCG)
    def random_selection(self, custom_speed=False):
        for channel in [random.choice(self.channels) for _ in range(0, 12)]:
            GPIO.setup(channel, GPIO.OUT)
            GPIO.output(channel, True)
            if custom_speed:
                time.sleep(float(int(custom_speed) / 1000))
            else:
                time.sleep(self.speed)
            GPIO.output(channel, False)

    # Cycle GPIOs from the outside channels to the inside channels
    def outside_in(self, custom_speed=False):
        for channel in [self.channels[0], self.channels[2], self.channels[3], self.channels[5]]:
            GPIO.setup(channel, GPIO.OUT)
            GPIO.output(channel, True)
        if custom_speed:
            time.sleep(float(int(custom_speed) / 1000))
        else:
            time.sleep(self.speed)
        for channel in [self.channels[0], self.channels[2], self.channels[3], self.channels[5]]:
            GPIO.output(channel, False)

        for channel in [self.channels[1], self.channels[4]]:
            GPIO.setup(channel, GPIO.OUT)
            GPIO.output(channel, True)
        if custom_speed:
            time.sleep(float(int(custom_speed) / 1000))
        else:
            time.sleep(self.speed)
        for channel in [self.channels[1], self.channels[4]]:
            GPIO.output(channel, False)

    # Cycle GPIOs from the inside channels to the outside channels
    def inside_out(self, custom_speed=False):
        for channel in [self.channels[1], self.channels[4]]:
            GPIO.setup(channel, GPIO.OUT)
            GPIO.output(channel, True)
        if custom_speed:
            time.sleep(float(int(custom_speed) / 1000))
        else:
            time.sleep(self.speed)
        for channel in [self.channels[1], self.channels[4]]:
            GPIO.output(channel, False)

        for channel in [self.channels[0], self.channels[2], self.channels[3], self.channels[5]]:
            GPIO.setup(channel, GPIO.OUT)
            GPIO.output(channel, True)
        if custom_speed:
            time.sleep(float(int(custom_speed) / 1000))
        else:
            time.sleep(self.speed)
        for channel in [self.channels[0], self.channels[2], self.channels[3], self.channels[5]]:
            GPIO.output(channel, False)

    def split_channels_one(self, custom_speed=False):
        split_channels = [ self.channels[0], self.channels[4], self.channels[2] ]
        for channel in split_channels:
            GPIO.setup(channel, GPIO.OUT)
            GPIO.output(channel, True)
        if custom_speed:
            time.sleep(float(int(custom_speed) / 1000))
        else:
            time.sleep(self.speed)
        for channel in split_channels:
            GPIO.output(channel, False)

    def split_channels_two(self, custom_speed=False):
        split_channels = [ self.channels[1], self.channels[3], self.channels[5] ]
        for channel in split_channels:
            GPIO.setup(channel, GPIO.OUT)
            GPIO.output(channel, True)
        if custom_speed:
            time.sleep(float(int(custom_speed) / 1000))
        else:
            time.sleep(self.speed)
        for channel in split_channels:
            GPIO.output(channel, False)

    def run(self):
        for song, sequence in zip(self.songs, self.sequences):
            print(song, sequence)

            with open(sequence, "r") as f:
                sequence_data = f.readlines()
                for i in range(len(sequence_data)):
                    sequence_data[i] = sequence_data[i].rstrip()

            # Play the song #
            mixer.init()
            mixer.music.set_volume(self.volume)
            mixer.music.load(song)
            mixer.music.play()

            # Set the start time and current time
            start_time = int(round(time.time() * 1000))
            current_time = (int(round(time.time() * 1000)) - start_time)

            # Instead of blindly blocking on a while True, step through each timestep.
            # When the timestep in the sequence (approximately) matches, trigger the GPIO actions in the sequence file
            # In practice, this approximate timing has always been close enough to be accurate.
            for step in sequence_data[1:]:
                while (int(step.split(",")[0]) >= current_time):
                    current_time = (int(round(time.time() * 1000)) - start_time)
                    time.sleep(0.1)  # Can decrease for higher accuracy - causes more CPU load

                # Once the timestep is reached, trigger the action in the sequence file
                timestep = step.split(",")[0]
                action = step.split(",")[1]
                duration = step.split(",")[2]
                print(f"{timestep} :: {action} {duration}")
                if action == "ALL_ON":
                    if duration != "":
                        self.all_on(custom_speed=duration)
                    else:
                        self.all_on()
                elif action == "ALL_OFF":
                    if duration != "":
                        self.all_off(custom_speed=duration)
                    else:
                        self.all_off()
                elif action == "PAIR_ONE":
                    if duration != "":
                        self.pair_one(custom_speed=duration)
                    else:
                        self.pair_one()
                elif action == "PAIR_TWO":
                    if duration != "":
                        self.pair_two(custom_speed=duration)
                    else:
                        self.pair_two()
                elif action == "PAIR_THREE":
                    if duration != "":
                        self.pair_three(custom_speed=duration)
                    else:
                        self.pair_three()
                elif action == "DIAGONAL_ONE":
                    if duration != "":
                        self.diagonal_one(custom_speed=duration)
                    else:
                        self.diagonal_one()
                elif action == "DIAGONAL_TWO":
                    if duration != "":
                        self.diagonal_two(custom_speed=duration)
                    else:
                        self.diagonal_two()
                elif action == "DOWN_RIGHT_ONE":
                    if duration != "":
                        self.down_right_one(custom_speed=duration)
                    else:
                        self.down_right_one()
                elif action == "DOWN_RIGHT_TWO":
                    if duration != "":
                        self.down_right_two(custom_speed=duration)
                    else:
                        self.down_right_two()
                elif action == "DOWN_LEFT_ONE":
                    if duration != "":
                        self.down_left_one(custom_speed=duration)
                    else:
                        self.down_left_one()
                elif action == "DOWN_LEFT_TWO":
                    if duration != "":
                        self.down_left_two(custom_speed=duration)
                    else:
                        self.down_left_two()
                elif action == "FORWARD":
                    if duration != "":
                        self.forward(custom_speed=duration)
                    else:
                        self.forward()
                elif action == "REVERSE":
                    if duration != "":
                        self.reverse(custom_speed=duration)
                    else:
                        self.reverse()
                elif action == "RANDOM":
                    if duration != "":
                        self.random_selection(custom_speed=duration)
                    else:
                        self.random_selection()
                elif action == "OUTSIDE_IN":
                    if duration != "":
                        self.outside_in(custom_speed=duration)
                    else:
                        self.outside_in()
                elif action == "INSIDE_OUT":
                    if duration != "":
                        self.inside_out(custom_speed=duration)
                    else:
                        self.inside_out()
                elif action == "SPLIT_CHANNELS_ONE":
                    if duration != "":
                        self.split_channels_one(custom_speed=duration)
                    else:
                        self.split_channels_one()
                elif action == "SPLIT_CHANNELS_TWO":
                    if duration != "":
                        self.split_channels_two(custom_speed=duration)
                    else:
                        self.split_channels_two()
                elif action == "END":
                    # Move on to the next song-sequence pair or exit 0
                    break

def main():
    x = LightShow()
    x.run()

if ((__name__) == ("__main__")):
    main()