'''
Importing modules
'''
import os
import pandas as pd
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import pdb
import pickle
import datetime
import random

import torch
import torchvision
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as opt
import numpy as np

from tqdm import tqdm
from skimage.transform import resize
from torch.utils.data import DataLoader, TensorDataset
from torch import Tensor
from torch.autograd import Variable

'''
Loading the input and output data
'''
root = os.getcwd()
ipath = os.path.join(root, 'projects/def-kjerbi/sirish01/Data/Processed/TrainData/')
opath = os.path.join(root, 'projects/def-kjerbi/sirish01/Data/Processed/TrainTruth/')

ifiles = sorted(os.listdir(ipath))
ofiles = sorted(os.listdir(opath))
input_data = []
output_data = []

for x in ifiles:
	input_data.append(np.load(ipath + x))
for x in ofiles:
	output_data.append(np.load(opath + x))

input_data = np.array(input_data)
output_data = np.array(output_data)

'''
LFADS module
'''
# print("Load Success")

#-------------------------
# COST FUNCTION COMPONENTS
#-------------------------
def KLCostGaussian(post_mu, post_lv, prior_mu, prior_lv):
	'''
	KLCostGaussian(post_mu, post_lv, prior_mu, prior_lv)

	KL-Divergence between a prior and posterior diagonal Gaussian distribution.

	Arguments:
		- post_mu (torch.Tensor): mean for the posterior
		- post_lv (torch.Tensor): logvariance for the posterior
		- prior_mu (torch.Tensor): mean for the prior
		- prior_lv (torch.Tensor): logvariance for the prior
	'''
	klc = 0.5 * (prior_lv - post_lv + torch.exp(post_lv - prior_lv) \
		 + ((post_mu - prior_mu)/torch.exp(0.5 * prior_lv)).pow(2) - 1.0).sum()
	return klc
#--------
# NETWORK
#--------
class LFADS(nn.Module):

	#------------------------------------------------------------------------------
	#------------------------------------------------------------------------------
	def __init__(self, count, inputs_dim, output_dim, T, value1, value2, model_hyperparams = None,
				 device = 'cpu', save_variables = False,
				 seed = None):

		# -----------------------
		# BASIC INIT STUFF
		# -----------------------

		# call the nn.Modules constructor
		super(LFADS, self).__init__()

		# Default hyperparameters
		default_hyperparams  = {### DATA PARAMETERS ###
								'dataset_name'             : 'best',
								'run_name'                 : 'CV',

								### MODEL PARAMETERS ###
								'g_dim'                    : value1,
								'u_dim'                    : 2,
								'factors_dim'              : value2,
								'g0_encoder_dim'           : value1,
								'c_encoder_dim'            : value1,
								'controller_dim'           : value1,
								'g0_prior_kappa'           : 0.1,
								'u_prior_kappa'            : 0.1,
								'keep_prob'                : 0.97,
								'clip_val'                 : 5.0,
								'max_norm'                 : 200,

								### OPTIMIZER PARAMETERS
								'learning_rate'            : 0.001,
								'learning_rate_min'        : 1e-5,
								'learning_rate_decay'      : 0.95,
								'scheduler_on'             : True,
								'scheduler_patience'       : 4,
								'scheduler_cooldown'       : 4,
								'epsilon'                  : 0.1,
								'betas'                    : (0.9, 0.99),
								'l2_gen_scale'             : 0,
								'l2_con_scale'             : 0,
								'kl_weight_schedule_start' : 0,
								'kl_weight_schedule_dur'   : 2000,
								'l2_weight_schedule_start' : 0,
								'l2_weight_schedule_dur'   : 2000,
								'ew_weight_schedule_start' : 0,
								'ew_weight_schedule_dur'   : 2000}

		# Store the hyperparameters
		self._update_params(count,default_hyperparams, model_hyperparams)

		self.inputs_dim                = inputs_dim
		self.output_dim                = output_dim
		self.T                         = T

		self.device                    = device
		self.save_variables            = save_variables
		self.seed                      = seed

		if self.seed is None:
			self.seed = random.randint(1, 10000)
			print('Random seed: {}'.format(self.seed))
		else:
			print('Preset seed: {}'.format(self.seed))

		random.seed(self.seed)
		np.random.seed(self.seed)
		torch.manual_seed(self.seed)
		if self.device == 'cuda':
			torch.cuda.manual_seed_all(self.seed)

		# Store loss
		self.full_loss_store = {'train_loss' : {}, 'train_recon_loss' : {}, 'train_kl_loss' : {},
								'valid_loss' : {}, 'valid_recon_loss' : {}, 'valid_kl_loss' : {},
								'l2_loss' : {}}

		self.train_loss_store = []
		self.valid_loss_store = []
		self.best = np.inf

		# Training variable
		self.epochs = 0
		self.current_step = 0
		self.last_decay_epoch = 0

		self.cost_weights = {'kl' : {'weight': 0, 'schedule_start': self.kl_weight_schedule_start,
									 'schedule_dur': self.kl_weight_schedule_dur},
							 'l2' : {'weight': 0, 'schedule_start': self.l2_weight_schedule_start,
									 'schedule_dur': self.l2_weight_schedule_dur}}

		# -----------------------
		# NETWORK LAYERS INIT
		#
		# Notation:
		#
		#   layertype_outputvariable(_direction)
		#
		#   Examples: fc_factors = "fully connected layer, variable = factors"
		#             gru_Egen_forward = "gated recurrent unit layer, encoder for generator, forward direction"
		# -----------------------

		# ----
		# RNN layers
		# ----

		# Generator Forward Encoder
		self.gru_Egen_forward  = nn.GRUCell(input_size= self.inputs_dim, hidden_size= self.g0_encoder_dim)

		# Generator Backward Encoder
		self.gru_Egen_backward = nn.GRUCell(input_size= self.inputs_dim, hidden_size= self.g0_encoder_dim)

		# Controller Forward Encoder
		self.gru_Econ_forward  = nn.GRUCell(input_size= self.inputs_dim, hidden_size= self.c_encoder_dim)

		# Controller Backward Encoder
		self.gru_Econ_backward = nn.GRUCell(input_size= self.inputs_dim, hidden_size= self.c_encoder_dim)

		# Controller
		self.gru_controller    = nn.GRUCell(input_size= self.c_encoder_dim * 2 + self.factors_dim, hidden_size= self.controller_dim)

		# Generator
		self.gru_generator     = nn.GRUCell(input_size= self.u_dim, hidden_size= self.g_dim)
		# -----------
		# Fully connected layers
		# -----------

		# mean and logvar of the posterior distribution for the generator initial conditions (g0 from E_gen)
		# takes as inputs:
		#  - the forward encoder for g0 at time T (g0_enc_f_T)
		#  - the backward encoder for g0 at time 1 (g0_enc_b_0]

		self.fc_g0mean   = nn.Linear(in_features= 2 * self.g0_encoder_dim, out_features= self.g_dim)
		self.fc_g0logvar = nn.Linear(in_features= 2 * self.g0_encoder_dim, out_features= self.g_dim)

		# mean and logvar of the posterior distribution for the inferred inputs (u provided to g)
		# takes as inputs:
		#  - the controller at time t (c_t)

		self.fc_umean   = nn.Linear(in_features= self.controller_dim, out_features= self.u_dim)
		self.fc_ulogvar = nn.Linear(in_features= self.controller_dim, out_features= self.u_dim)

		# factors from generator output
		self.fc_factors = nn.Linear(in_features= self.g_dim, out_features= self.factors_dim)

		self.f1 = nn.Linear(in_features = self.factors_dim, out_features = 80)
		self.fc_clean   = nn.Linear(in_features= 80, out_features = self.output_dim)

		# -----------
		# Dropout layer
		# -----------
		self.dropout = nn.Dropout(1.0 - self.keep_prob)

		# -----------------------
		# WEIGHT INIT
		#
		# The weight initialization is modified from the standard PyTorch, which is uniform. Instead,
		# the weights are drawn from a normal distribution with mean 0 and std = 1/sqrt(K) where K
		# is the size of the input dimension. This helps prevent vanishing/exploding gradients by
		# keeping the eigenvalues of the Jacobian close to 1.
		# -----------------------

		# Step through all layers and adjust the weight initiazition method accordingly
		for m in self.modules():

			# GRU layer, update using input weight and recurrent weight dimensionality
			if isinstance(m, nn.GRUCell):
				k_ih = m.weight_ih.shape[1] # dimensionality of the inputs to the GRU
				k_hh = m.weight_hh.shape[1] # dimensionality of the GRU outputs
				m.weight_ih.data.normal_(std = k_ih ** -0.5) # inplace resetting of W ~ N(0,1/sqrt(N))
				m.weight_hh.data.normal_(std = k_hh ** -0.5) # inplace resetting of W ~ N(0,1/sqrt(N))

			# FC layer, update using input dimensionality
			elif isinstance(m, nn.Linear):
				k = m.in_features # dimensionality of the inputs
				m.weight.data.normal_(std = k ** -0.5) # inplace resetting of W ~ N(0,1/sqrt(N))

		torch.nn.init.normal_(self.fc_factors.weight, std = 1.0)
		torch.nn.init.normal_(self.f1.weight, std = 1.0)
		torch.nn.init.normal_(self.fc_clean.weight, std = 1.0)

		# Row-normalise fc_factors (See bullet-point 11 of section 1.9 of online methods)
		self.fc_factors.weight.data = F.normalize(self.fc_factors.weight.data, dim = 1)

		# --------------------------
		# LEARNABLE PRIOR PARAMETERS INIT
		# --------------------------

		self.g0_prior_mu = nn.parameter.Parameter(torch.tensor(0.0))
		self.u_prior_mu  = nn.parameter.Parameter(torch.tensor(0.0))

		from math import log
		self.g0_prior_logkappa = nn.parameter.Parameter(torch.tensor(log(self.g0_prior_kappa)))
		self.u_prior_logkappa  = nn.parameter.Parameter(torch.tensor(log(self.u_prior_kappa)))
		# --------------------------
		# OPTIMIZER INIT
		# --------------------------
		self.optimizer = opt.Adam(self.parameters(), lr=self.learning_rate, eps=self.epsilon, betas=self.betas)
		# --------------------------
		# LOSS FUNCTION
		# --------------------------
		self.loss = nn.MSELoss()

	#------------------------------------------------------------------------------
	#------------------------------------------------------------------------------
	def initialize(self, batch_size=None):
		'''
		initialize()

		Initialize dynamic model variables. These need to be reinitialized with each forward pass to
		ensure we don't need to retain graph between each .backward() call.

		See https://discuss.pytorch.org/t/what-exactly-does-retain-variables-true-in-loss-backward-do/3508/2
		for discussion and explanation

		Note: The T + 1 terms  accommodate learnable biases for all variables, except for the generator,
		which is provided with a g0 estimate from the network

		optional arguments:
		  batch_size (int) : batch dimension. If None, use self.batch_size.

		'''

		batch_size = batch_size if batch_size is not None else self.batch_size

		self.g0_prior_mean = torch.ones(batch_size, self.g_dim).to(self.device)*self.g0_prior_mu            # g0 prior mean
		self.u_prior_mean  = torch.ones(batch_size, self.u_dim).to(self.device)*self.u_prior_mu             # u prior mean

		self.g0_prior_logvar = torch.ones(batch_size, self.g_dim).to(self.device)*self.g0_prior_logkappa    # g0 prior logvar
		self.u_prior_logvar  = torch.ones(batch_size, self.u_dim).to(self.device)*self.u_prior_logkappa     # u prior logvar

		self.c = Variable(torch.zeros((batch_size, self.controller_dim)).to(self.device))  # Controller hidden state

		self.efgen = Variable(torch.zeros((batch_size, self.g0_encoder_dim)).to(self.device))  # Forward generator encoder
		self.ebgen = Variable(torch.zeros((batch_size, self.g0_encoder_dim)).to(self.device))  # Backward generator encoder

		self.efcon = torch.zeros((batch_size, self.T+1, self.c_encoder_dim)).to(self.device)   # Forward controller encoder
		self.ebcon = torch.zeros((batch_size, self.T+1, self.c_encoder_dim)).to(self.device)   # Backward controller encoder

		self.raw_data		= torch.zeros(batch_size, self.T, self.inputs_dim)
		self.clean_data 	= torch.zeros(batch_size, self.T, self.output_dim)
		self.output        = torch.zeros(batch_size, self.T, self.output_dim)
		self.factors       = torch.zeros(batch_size, self.T, self.factors_dim)
		self.inferred_inputs        = torch.zeros(batch_size, self.T, self.u_dim)
		self.inferred_inputs_mean   = torch.zeros(batch_size, self.T, self.u_dim)
		self.inferred_inputs_logvar = torch.zeros(batch_size, self.T, self.u_dim)

	def encode(self, x):
		'''
		encode(x)

		Function to encode the data with the forward and backward encoders.

		Arguments:
		  - x (torch.Tensor): Variable tensor of size batch size x time-steps x input dimension
		'''

		# Dropout some data
		if self.keep_prob < 1.0:
			x = self.dropout(x)

		# Encode data into forward and backward generator encoders to produce E_gen
		# for generator initial conditions.
		for t in range(1, self.T+1):

			# generator encoders
			self.efgen = torch.clamp(self.gru_Egen_forward(x[:, t-1], self.efgen),max=self.clip_val)
			self.ebgen = torch.clamp(self.gru_Egen_backward(x[:, -t], self.ebgen),max=self.clip_val)

			# controller encoders
			self.efcon[:, t]      = torch.clamp(self.gru_Econ_forward(x[:, t-1], self.efcon[:, t-1].clone()),max=self.clip_val)
			self.ebcon[:, -(t+1)] = torch.clamp(self.gru_Econ_backward(x[:, -t], self.ebcon[:, -t].clone()),max=self.clip_val)

		# Concatenate efgen_T and ebgen_1 for generator initial condition sampling
		egen = torch.cat((self.efgen, self.ebgen), dim=1)
		# Dropout the generator encoder output
		if self.keep_prob < 1.0:
			egen = self.dropout(egen)

		# Sample initial conditions for generator from g0 posterior distribution
		self.g0_mean   = self.fc_g0mean(egen)
		self.g0_logvar = torch.clamp(self.fc_g0logvar(egen), min=np.log(0.0001))
		self.g         = Variable(torch.randn(self.batch_size, self.g_dim).to(self.device))*torch.exp(0.5*self.g0_logvar)\
						 + self.g0_mean

		# KL cost for g(0)
#         pdb.set_trace()
		self.kl_loss   = KLCostGaussian(self.g0_mean, self.g0_logvar,
										self.g0_prior_mean, self.g0_prior_logvar)/x.shape[0]
		# print(x.shape[0])
		# Initialise factors
		self.f         = self.fc_factors(self.g)

	#------------------------------------------------------------------------------
	#------------------------------------------------------------------------------
	def generate(self, x, y):
		'''
		generate()

		Generates the rates using the controller encoder outputs and the sampled initial conditions for
		generator.
		'''

		self.recon_loss = 0

		for t in range(self.T):

			# Concatenate ebcon and efcon outputs at time t with factors at time t+1 as input to controller
			# Note: we take efcon at t+1, because the learnable biases are at first index for efcon
			econ_and_fac = torch.cat((self.efcon[:, t+1].clone(), self.ebcon[:,t].clone(), self.f), dim = 1)

			# Dropout the controller encoder outputs and factors
			if self.keep_prob < 1.0:
				econ_and_fac = self.dropout(econ_and_fac)

			# Update controller with controller encoder outputs
			self.c = torch.clamp(self.gru_controller(econ_and_fac, self.c),max=self.clip_val)

			# Calculate posterior distribution parameters for inferred inputs from controller state
			self.u_mean   = self.fc_umean(self.c)
			self.u_logvar = self.fc_ulogvar(self.c)
			# Sample inputs for generator from u(t) posterior distribution
			self.u = Variable(torch.randn(self.batch_size, self.u_dim).to(self.device))*torch.exp(0.5*self.u_logvar) \
						+ self.u_mean

			# KL cost for u(t)
			self.kl_loss = self.kl_loss + KLCostGaussian(self.u_mean, self.u_logvar,
										self.u_prior_mean, self.u_prior_logvar)/x.shape[0]

			# Update generator
			self.g = torch.clamp(self.gru_generator(self.u,self.g), min=0.0, max=self.clip_val)

			# Dropout on generator output
			if self.keep_prob < 1.0:
				self.g = self.dropout(self.g)

			# Generate factors from generator state
			self.f = self.fc_factors(self.g)
			# Generate rates from factor state
			out1 = F.relu(self.f1(self.f))
			self.res = self.fc_clean(out1)
			# if t and t % 10 == 0:
			# 	print("Epoch number is: %s"%(self.epochs))
			# 	print("Target Data")
			# 	print(y[:, t])
			# 	print(y[:, t].min(), y[:, t].max())
			# 	print("LFADS output")
			# 	print(self.res)
			# 	print(self.res.min(), self.res.max())

			L = self.loss(y[:, t], self.res)
			self.recon_loss = self.recon_loss + L

			self.raw_data[:,t] = x[:, t].detach().cpu()
			self.clean_data[:, t] = y[:, t].detach().cpu()
			self.output[:, t]   = self.res.detach().cpu()
			self.factors[:, t] = self.f.detach().cpu()
			self.inferred_inputs[:, t] = self.u.detach().cpu()
			self.inferred_inputs_mean[:, t] = self.u_mean.detach().cpu()
			self.inferred_inputs_logvar[:, t] = self.u_logvar.detach().cpu()
		self.recon_loss = self.recon_loss / self.T

	#------------------------------------------------------------------------------
	#------------------------------------------------------------------------------
	def forward(self, x, y):
		'''
		forward(x)

		Runs a forward pass through the network.

		Arguments:
		  - x (torch.Tensor): Single-trial spike data. Tensor of size batch size x time-steps x input dimension
		'''
		batch_size, steps_dim, inputs_dim = x.shape

		assert steps_dim  == self.T
		assert inputs_dim == self.inputs_dim

		self.batch_size = x.shape[0]
		self.initialize(batch_size=x.shape[0])
		self.encode(x)
		self.generate(x, y)

	def weight_schedule_fn(self, step):
		'''
		weight_schedule_fn(step)

		Calculate the KL and L2 regularization weights from the current training step number. Imposes
		linearly increasing schedule on regularization weights to prevent early pathological minimization
		of KL divergence and L2 norm before sufficient data reconstruction improvement. See bullet-point
		4 of section 1.9 in online methods

		required arguments:
		step (int) : training step number
		'''
		for cost_key in self.cost_weights.keys():
			# Get step number of scheduler
			weight_step = max(step - self.cost_weights[cost_key]['schedule_start'], 0)

			# Calculate schedule weight
			self.cost_weights[cost_key]['weight'] = min(weight_step/ self.cost_weights[cost_key]['schedule_dur'], 1.0)

	#------------------------------------------------------------------------------
	#------------------------------------------------------------------------------

	def apply_decay(self, current_loss):
		'''
		apply_decay(current_loss)

		Decrease the learning rate by a defined factor (self.learning_rate_decay) if loss is greater
		than the loss in the last six training steps and if the loss has not decreased in the last
		six training steps. See bullet point 8 of section 1.9 in online methods
		'''
		if len(self.train_loss_store) >= self.scheduler_patience:
			if all((current_loss > past_loss for past_loss in self.train_loss_store[-self.scheduler_patience:])):
				if self.epochs >= self.last_decay_epoch + self.scheduler_cooldown:
					self.learning_rate  = self.learning_rate * self.learning_rate_decay
					self.last_decay_epoch = self.epochs
					for g in self.optimizer.param_groups:
						g['lr'] = self.learning_rate
					print('Learning rate decreased to %.8f'%self.learning_rate)

	def test(self, valid_dl, batch_size):
		self.eval()
		self.batch_size = 64
		test_loss = 0
		test_recon_loss = 0
		test_kl_loss = 0
		for i, (xtest,ytest) in enumerate(valid_dl):
			with torch.no_grad():
				xtest = Variable(xtest)
				ytest = Variable(ytest)
				# print("Testing")
				self(xtest, ytest)
				loss = self.recon_loss
				# print("KL Loss per batch in test set: %.4f"%(self.kl_loss.data))
				test_loss += loss.data
				test_recon_loss += self.recon_loss.data
				test_kl_loss += self.kl_loss.data

		test_loss /= (i+1)
		test_recon_loss /= (i+1)
		test_kl_loss /= (i+1)
		return test_loss, test_recon_loss, test_kl_loss

	def randomloss(self, train_dl, test_dl, cv):
		self.batch_size = 64
		with torch.no_grad():
			train_loss = 0
			self.train()
			for i, (xtrain, ytrain) in enumerate(train_dl):
				xtrain = Variable(xtrain)
				ytrain = Variable(ytrain)
				self(xtrain, ytrain)
				loss = self.recon_loss
				train_loss += loss.data
			train_loss /= (i + 1)
			test_loss = 0
			self.eval()
			for i, (xtest, ytest) in enumerate(test_dl):
				xtest = Variable(xtest)
				ytest = Variable(ytest)
				self(xtest,ytest)
				loss = self.recon_loss
				test_loss += loss.data
			test_loss /= (i + 1)
			print("Random Train Reconstruction Loss: %.4f, Random Test Reconstruction Loss: %.4f"%(train_loss, test_loss))
			random = {}
			random['Train'] = train_loss
			random['Test'] = test_loss
			save_loc = '%s/models/%s/%s/'%('./projects/def-kjerbi/sirish01', self.dataset_name, self.run_name)
			if not os.path.isdir(save_loc):
				os.makedirs(save_loc)
			with open(save_loc + 'randomloss_' + str(cv) + '.p','wb') as f:
					pickle.dump(random, f, protocol=pickle.HIGHEST_PROTOCOL)

	#------------------------------------------------------------------------------
	#------------------------------------------------------------------------------

	def fit(self, input_data, clean_data, train, test, cv, batch_size=64, max_epochs = 100, use_tensorboard=False, health_check=False, output='.'):

		self.batch_size = 64
		# create the dataloader

		# Initialize directory to save checkpoints
		save_loc = '%s/models/%s/%s/checkpoints/'%(output, self.dataset_name, self.run_name)
		# Create model_checkpoint directory if it doesn't exist
		if not os.path.isdir(save_loc):
			os.makedirs(save_loc)
		# print a message
		# for each epoch...

		train_data = torch.tensor(np.concatenate(input_data[train])).type('torch.FloatTensor').to(device)
		train_truth = torch.tensor(np.concatenate(clean_data[train])).type('torch.FloatTensor').to(device)
		train_dataset = torch.utils.data.TensorDataset(train_data,train_truth)
		train_dl = torch.utils.data.DataLoader(train_dataset,batch_size = 64,shuffle = True)

		valid_data = torch.tensor(np.concatenate(input_data[test])).type('torch.FloatTensor').to(device)
		valid_truth = torch.tensor(np.concatenate(clean_data[test])).type('torch.FloatTensor').to(device)
		valid_dataset = torch.utils.data.TensorDataset(valid_data,valid_truth)
		valid_dl = torch.utils.data.DataLoader(valid_dataset,batch_size = 64,shuffle = True)
		self.randomloss(train_dl, valid_dl, cv)

		print(train_data.size(),train_truth.size())
		print(valid_data.size(),valid_truth.size())
		print(len(train_dataset),len(valid_dataset))
		for epoch in range(max_epochs):
			self.train()
			# If minimum learning rate reached, break training loop
			if self.learning_rate <= self.learning_rate_min:
				break
			# cumulative training loss for this epoch
			train_loss = 0
			train_recon_loss = 0
			train_kl_loss = 0

			for i, (xtrain,ytrain) in enumerate(train_dl, 0):
				self.current_step += 1
				# apply Variable wrapper to batch
				xtrain = Variable(xtrain)
				ytrain = Variable(ytrain)

				# zero the parameter gradients
				self.optimizer.zero_grad()

				# Calculate regularizer weights
				self.weight_schedule_fn(self.current_step)

				# Forward
				# print("Training")
				self(xtrain, ytrain)
				# Calculate l2 regularisation penalty
				l2_loss = self.l2_gen_scale * self.gru_generator.weight_hh.norm(2)/self.gru_generator.weight_hh.numel() + \
						self.l2_con_scale * self.gru_controller.weight_hh.norm(2)/self.gru_controller.weight_hh.numel()

				# Collect separate weighted losses
				kl_weight = self.cost_weights['kl']['weight']
				l2_weight = self.cost_weights['l2']['weight']
				loss = self.recon_loss + kl_weight * self.kl_loss + l2_weight * l2_loss
				#print("KL Loss per batch in training set: %.4f"%(self.kl_loss.data))
				# return
				# Backward
				loss.backward()

				# clip gradient norm
				torch.nn.utils.clip_grad_norm_(self.parameters(), max_norm=self.max_norm)

				# update the weights
				self.optimizer.step()

				# Row-normalise fc_factors (See bullet-point 11 of section 1.9 of online methods)
				self.fc_factors.weight.data = F.normalize(self.fc_factors.weight.data, dim=1)

				if use_tensorboard:
					self.health_check(writer)

				train_loss += loss.data
				train_recon_loss += self.recon_loss.data
				train_kl_loss += self.kl_loss.data

			train_loss /= (i+1)
			train_recon_loss /= (i+1)
			train_kl_loss /= (i+1)

			valid_loss, valid_recon_loss, valid_kl_loss = self.test(valid_dl,batch_size = 64)

			print('Epoch: %4d, tloss: %.3f, tr loss: %.3f, tkl loss: %.3f, vr loss: %.3f, v_loss: %.3f, vkl loss: %.3f' %(self.epochs+1, train_loss, train_recon_loss, train_kl_loss, valid_recon_loss,valid_loss, valid_kl_loss))
			# Apply learning rate decay function
			if self.scheduler_on:
				self.apply_decay(train_loss)

			# Store loss
			self.train_loss_store.append(float(train_loss))
			self.valid_loss_store.append(float(valid_loss))

			self.full_loss_store['train_loss'][self.epochs]       = float(train_loss)
			self.full_loss_store['train_recon_loss'][self.epochs] = float(train_recon_loss)
			self.full_loss_store['train_kl_loss'][self.epochs]    = float(train_kl_loss)
			self.full_loss_store['valid_loss'][self.epochs]       = float(valid_loss)
			self.full_loss_store['valid_recon_loss'][self.epochs] = float(valid_recon_loss)
			self.full_loss_store['valid_kl_loss'][self.epochs]    = float(valid_kl_loss)
			self.full_loss_store['l2_loss'][self.epochs]          = float(l2_loss.data)
			self.epochs += 1
			if self.current_step >= max(self.cost_weights['kl']['schedule_start'] + self.cost_weights['kl']['schedule_dur'],
										self.cost_weights['l2']['schedule_start'] + self.cost_weights['l2']['schedule_dur']):
				if self.valid_loss_store[-1] < self.best:
					self.last_saved = epoch
					self.best = self.valid_loss_store[-1]
					self.save_checkpoint(cv,output=output)

		import pandas as pd
		df = pd.DataFrame(self.full_loss_store)
		df.to_csv('%s/models/%s/%s/Total_%s.csv'%(output, self.dataset_name, self.run_name,cv), index_label='epoch')
		# Save a final checkpoint
		self.save_checkpoint(cv,force=True, output=output)

		print('...training complete.')

	def save_checkpoint(self,cv,force=False, purge_limit=50, output='.'):
		'''
		Save checkpoint of network parameters and optimizer state

		Arguments:
			force (bool) : force checkpoint to be saved (default = False)
			purge_limit (int) : delete previous checkpoint if there have been fewer
								epochs than this limit before saving again
		'''

		# output_filename of format [timestamp]_epoch_[epoch]_loss_[training].pth:
		#  - timestamp   (YYMMDDhhmm)
		#  - epoch       (int)
		#  - loss        (float with decimal point replaced by -)

		save_loc = '%s/models/%s/%s/checkpoints/'%(output, self.dataset_name, self.run_name)

		if force:
			pass
		else:
			if purge_limit:
				# Get checkpoint filenames
				try:
					_,_,filenames = list(os.walk(save_loc))[0]
					split_filenames = [os.path.splitext(fn)[0].split('_') for fn in filenames]
					avail_files = []
					for fn in split_filenames:
						if int(fn[6]) == cv:
							avail_files.append(fn)
					split_filenames = avail_files
					print(avail_files)
					epochs = [att[2] for att in split_filenames]
					epochs.sort()
					last_saved_epoch = epochs[-1]
					if self.epochs - 50 <= int(last_saved_epoch):
						rm_filename = [filename for filename in filenames if last_saved_epoch in filename][0]
						os.remove(save_loc+rm_filename)

				except IndexError:
					pass

		# Get current time in YYMMDDhhmm format
		timestamp = datetime.datetime.now().strftime('%y%m%d%H%M')

		# Get epoch_num as string
		epoch = str('%i'%self.epochs)

		# Get training_error as string
		loss = str(self.valid_loss_store[-1]).replace('.','-')

		output_filename = '%s_epoch_%s_loss_%s_CV_%s.pth'%(timestamp, epoch, loss, cv)
		assert os.path.splitext(output_filename)[1] == '.pth', 'Output filename must have .pth extension'

		# Create dictionary of training variables
		train_dict = {'best' : self.best, 'train_loss_store': self.train_loss_store,
					  'valid_loss_store' : self.valid_loss_store,
					  'full_loss_store' : self.full_loss_store,
					  'epochs' : self.epochs, 'current_step' : self.current_step,
					  'last_decay_epoch' : self.last_decay_epoch,
					  'learning_rate' : self.learning_rate,
					  'cost_weights' : self.cost_weights,
					  'dataset_name' : self.dataset_name}

		# Save network parameters, optimizer state, and training variables
		# torch.save({'net' : self.state_dict(), 'opt' : self.optimizer.state_dict(), 'train' : train_dict, 'time_series' : variables},
		# 		   save_loc+output_filename)

		torch.save({'net' : self.state_dict(), 'opt' : self.optimizer.state_dict(), 'train' : train_dict},
		   save_loc+output_filename)

	#------------------------------------------------------------------------------
	#------------------------------------------------------------------------------

	def _set_params(self, params):
		for k in params.keys():
			self.__setattr__(k, params[k])

	#------------------------------------------------------------------------------
	#------------------------------------------------------------------------------

	def _update_params(self, count,prev_params, new_params):
		if new_params:
			params = update_param_dict(prev_params, new_params)
		else:
			params = prev_params
		self._set_params(params)
		self.print_params(count,params)

	def print_params(self,cv,params):
		save_loc = '%s/models/%s/%s/'%('./projects/def-kjerbi/sirish01', self.dataset_name, self.run_name)
		if not os.path.isdir(save_loc):
			os.makedirs(save_loc)
		with open(save_loc + 'model' + str(cv) + '.txt','w') as f:
			for k in params.keys():
				f.write(str(k) + ':' + str(params[k]) +'\n')

'''
model run
'''
device = ("cuda" if torch.cuda.is_available() else "cpu")
outputpath = './projects/def-kjerbi/sirish01'
from sklearn.model_selection import KFold
from numpy.random import permutation
indices = np.arange(20)
print(indices)
kf = KFold(n_splits=3)
for cv, (train_index, test_index) in enumerate(kf.split(indices)):
	print(train_index,test_index)
	network = LFADS(count = cv,inputs_dim = 128,output_dim = 105,T = 50,value1 = 125,value2 = 10,model_hyperparams= None,device = device,save_variables = True).to(device)
	network.fit(input_data,output_data,train_index,test_index,cv,batch_size = 64,max_epochs=30,use_tensorboard = False,output = outputpath)


#!/usr/bin/python
# -*- coding: UTF-8 -*-

#   data-analysis
#   basicSVM.py

"""
    Description: 基础版SVM
    author : zhangjingbo
    Date:    2018/12/20
"""
import os
import numpy as np
import pickle
import matplotlib.pylab as plt
import random

class basicSVM:
    def __init__(self, testSet='testSet'):
        self.testSet = testSet
        if not os.path.isfile(testSet):
            self.file2vect()
            self.save_data()
        self.featureMat = self.load_data()

    def file2vect(self):
        featureMat = []
        try:
            with open(self.testSet + '.txt') as f:
                for s in f:
                    tmp = s.rstrip('\n').split('\t')
                    featureMat.append(tmp)
        except FileNotFoundError as e:
            print(e.filename)
        self.featureMat = np.mat(featureMat).astype(float)

    def load_data(self):
        with open(self.testSet, 'rb') as f:
            featureMat = pickle.load(f)
        return featureMat

    def save_data(self):
        print(self.testSet)
        with open(self.testSet, 'wb') as f:
            pickle.dump(self.featureMat, f)

    def plot_data(self,w,b):
        plt.figure(1)
        plt.title('dataSet plot')
        plt.xlabel('x1')
        plt.ylabel('x2')
        colorVect = []
        print(self.featureMat.shape)
        for i in range(len(self.featureMat)):
            if self.featureMat[i, -1] == 1:
                colorVect.append('red')
            else:
                colorVect.append('blue')
        plt.scatter(self.featureMat[:, 0].flatten().A[0], self.featureMat[:, 1].flatten().A[0], color=colorVect, s=15,
                    alpha=0.5)
        x1Vect = np.arange(-1.0, 10.0, 0.1)
        x2Vect = []
        w1 = w[0,0]
        w2 = w[1,0]
        for x in x1Vect:
            x2 = (-b[0,0]-w1*x)/w2
            x2Vect.append(x2)
        plt.plot(x1Vect, x2Vect, 'k-')
        plt.show()

    def select_randJ(self, i, m):
        """
        随机选择一个整数
        Args:
            i  第一个alpha的下标
            m  所有alpha的数目
        Returns:
            j  返回一个不为i的随机数，在0~m之间的整数值
        """
        j = i
        while j == i:
            j = int(random.uniform(0, m))
        return j

    def clipAlpha(self, aj, H, L):
        """clipAlpha(调整aj的值，使aj处于 L<=aj<=H)
        Args:
            aj  目标值
            H   最大值
            L   最小值
        Returns:
            aj  目标值
        """
        if aj > H:
            aj = H
        if L > aj:
            aj = L
        return aj

    def sequential_minimal_optimization(self, C, toler, maxIter):
        """
        无核函数

        Args:
            C   惩罚因子
            toler   松弛因子
            maxIter 退出前最大的循环次数
        Returns:
            b       模型的常量值
            alphas  拉格朗日乘子
        """
        m, n = self.featureMat.shape
        featureData = self.featureMat[:, 0:-1]
        labelData = self.featureMat[:, -1]
        b = 0
        iter = 0
        alphas = np.mat(np.zeros([m, 1]))
        while iter < maxIter:
            alphaPairsChanged = 0
            for i in range(m):
                fxi = np.multiply(alphas, labelData).T * featureData * featureData[i, :].T + b
                Ei = fxi - labelData[i]
                # 判断KNN条件
                if (labelData[i] * Ei < -toler and alphas[i] < C) or (labelData[i] * Ei > toler and alphas[i] > 0):
                    alphaIold = alphas[i].copy()
                    j = self.select_randJ(i, m)
                    alphaJold = alphas[j].copy()
                    fxj = np.multiply(alphas, labelData).T * featureData * featureData[j, :].T + b
                    Ej = fxj - labelData[j]
                    if labelData[i] == labelData[j]:
                        L = max(0, alphas[j] + alphas[i] - C)
                        H = min(C, alphas[j] + alphas[i])
                    else:
                        L = max(0, alphas[j] - alphas[i])
                        H = min(C, alphas[j] - alphas[i] + C)
                    if L == H:
                        print("L==H")
                        continue
                    # eta 反应了i和j之间的距离
                    eta = featureData[i, :] * featureData[i, :].T + featureData[j, :] * featureData[j, :].T - 2.0 * (
                        featureData[i, :] * featureData[j, :].T)
                    if eta <= 0:
                        print("eta<=0")
                        continue
                    alphas[j] += labelData[j] * (Ei - Ej) / eta
                    alphas[j] = self.clipAlpha(alphas[j], H, L)
                    if abs(alphas[j] - alphaJold) < 0.00001:
                        continue
                    alphas[i] += (alphaJold - alphas[j]) * labelData[i] * labelData[j]
                    bi = b - (Ei + labelData[i] * (featureData[i, :] * featureData[i, :].T) * (alphas[i] - alphaIold) + \
                              labelData[j] * (featureData[i, :] * featureData[j, :].T) * (alphas[j] - alphaJold))
                    bj = b - (Ej + labelData[i] * (featureData[i, :] * featureData[i, :].T) * (alphas[i] - alphaIold) + \
                              labelData[j] * (featureData[i, :] * featureData[j, :].T) * (alphas[j] - alphaJold))
                    if (0 < alphas[i]) and (C > alphas[i]):
                        b = bi
                    elif (0 < alphas[j]) and (C > alphas[j]):
                        b = bj
                    else:
                        b = (bi + bj) / 2.0
                    alphaPairsChanged += 1
                    print("iter: %d i:%d, pairs changed %d" % (iter, i, alphaPairsChanged))
                    # 在for循环外，检查alpha值是否做了更新，如果在更新则将iter设为0后继续运行程序
                    # 知道更新完毕后，iter次循环无变化，才推出循环。
            if (alphaPairsChanged == 0):
                iter += 1
            else:
                iter = 0
            print("iteration number: %d" % iter)
        return b, alphas

    def calcWs(self, alphas, dataArr, classLabels):
        """
        基于alpha计算w值
        Args:
            alphas        拉格朗日乘子
            dataArr       feature数据集
            classLabels   目标变量数据集

        Returns:
            wc  回归系数
        """
        X = np.mat(dataArr)
        labelMat = np.mat(classLabels)
        m, n = X.shape
        w = np.zeros((n, 1))
        for i in range(m):
            w += np.multiply(alphas[i] * labelMat[i], X[i, :].T)
        return w

if __name__ == '__main__':
    bsvm = basicSVM()
    b, alphas = bsvm.sequential_minimal_optimization(0.1, 0.01, 10)
    print(alphas)
    w = bsvm.calcWs(alphas,bsvm.featureMat[:, 0:-1],bsvm.featureMat[:, -1])
    bsvm.plot_data(w,b)


import io
import numpy as np
import collections
# import faiss
import scipy.spatial
import torch

def load_vectors(fname, maxload=200000, norm=True, center=False, verbose=True):
    if verbose:
        print("Loading vectors from %s" % fname)
    fin = io.open(fname, 'r', encoding='utf-8', newline='\n', errors='ignore')
    n, d = map(int, fin.readline().split())
    if maxload > 0:
        n = min(n, maxload)
    x = np.zeros([n, d])
    words = []
    for i, line in enumerate(fin):
        if i >= n:
            break
        tokens = line.rstrip().split(' ')
        words.append(tokens[0])
        v = np.array(tokens[1:], dtype=float)
        x[i, :] = v
    if norm:
        x /= np.linalg.norm(x, axis=1)[:, np.newaxis] + 1e-8
    if center:
        x -= x.mean(axis=0)[np.newaxis, :]
        x /= np.linalg.norm(x, axis=1)[:, np.newaxis] + 1e-8
    if verbose:
        print("%d word vectors loaded" % (len(words)))
    return words, x

def idx(words):
    w2i = {}
    for i, w in enumerate(words):
        if w not in w2i:
            w2i[w] = i
    return w2i

def save_vectors(fname, x, words):
    n, d = x.shape
    fout = io.open(fname, 'w', encoding='utf-8')
    fout.write(u"%d %d\n" % (n, d))
    for i in range(n):
        fout.write(words[i] + " " + " ".join(map(lambda a: "%.4f" % a, x[i, :])) + "\n")
    fout.close()

def save_matrix(fname, x):
    n, d = x.shape
    fout = io.open(fname, 'w', encoding='utf-8')
    fout.write(u"%d %d\n" % (n, d))
    for i in range(n):
        fout.write(" ".join(map(lambda a: "%.4f" % a, x[i, :])) + "\n")
    fout.close()

def procrustes(X_src, Y_tgt):
    U, s, V = np.linalg.svd(np.dot(Y_tgt.T, X_src))
    return np.dot(U, V)

def select_vectors_from_pairs(x_src, y_tgt, pairs):
    n = len(pairs)
    d = x_src.shape[1]
    x = np.zeros([n, d])
    y = np.zeros([n, d])
    for k, ij in enumerate(pairs):
        i, j = ij
        x[k, :] = x_src[i, :]
        y[k, :] = y_tgt[j, :]
    return x, y

def load_lexicon(filename, words_src, words_tgt, verbose=True):
    f = io.open(filename, 'r', encoding='utf-8')
    lexicon = collections.defaultdict(set)
    idx_src , idx_tgt = idx(words_src), idx(words_tgt)
    vocab = set()
    for line in f:
        word_src, word_tgt = line.split()
        if word_src in idx_src and word_tgt in idx_tgt:
            lexicon[idx_src[word_src]].add(idx_tgt[word_tgt])
        vocab.add(word_src)
    if verbose:
        coverage = len(lexicon) / float(len(vocab))
        print("Coverage of source vocab: %.4f" % (coverage))
    return lexicon, float(len(vocab))

def load_pairs(filename, idx_src, idx_tgt, verbose=True):
    f = io.open(filename, 'r', encoding='utf-8')
    pairs = []
    tot = 0
    for line in f:
        a, b = line.rstrip().split(' ')
        tot += 1
        if a in idx_src and b in idx_tgt:
            pairs.append((idx_src[a], idx_tgt[b]))
    if verbose:
        coverage = (1.0 * len(pairs)) / tot
        print("Found pairs for training: %d - Total pairs in file: %d - Coverage of pairs: %.4f" % (len(pairs), tot, coverage))
    return pairs

def compute_nn_accuracy(x_src, x_tgt, lexicon, bsz=100, lexicon_size=-1):
    if lexicon_size < 0:
        lexicon_size = len(lexicon)
    idx_src = list(lexicon.keys())
    acc = 0.0
    x_src /= np.linalg.norm(x_src, axis=1)[:, np.newaxis] + 1e-8
    x_tgt /= np.linalg.norm(x_tgt, axis=1)[:, np.newaxis] + 1e-8
    for i in range(0, len(idx_src), bsz):
        e = min(i + bsz, len(idx_src))
        scores = np.dot(x_tgt, x_src[idx_src[i:e]].T)
        pred = scores.argmax(axis=0)
        for j in range(i, e):
            if pred[j - i] in lexicon[idx_src[j]]:
                acc += 1.0
    return acc / lexicon_size

def compute_csls_accuracy(x_src, x_tgt, lexicon, lexicon_size=-1, k=10, bsz=1024):
    if lexicon_size < 0:
        lexicon_size = len(lexicon)
    idx_src = list(lexicon.keys())

    x_src /= np.linalg.norm(x_src, axis=1)[:, np.newaxis] + 1e-8
    x_tgt /= np.linalg.norm(x_tgt, axis=1)[:, np.newaxis] + 1e-8

    sr = x_src[list(idx_src)]
    sc = np.dot(sr, x_tgt.T)
    similarities = 2 * sc
    sc2 = np.zeros(x_tgt.shape[0])
    for i in range(0, x_tgt.shape[0], bsz):
        j = min(i + bsz, x_tgt.shape[0])
        sc_batch = np.dot(x_tgt[i:j, :], x_src.T)
        dotprod = np.partition(sc_batch, -k, axis=1)[:, -k:]
        sc2[i:j] = np.mean(dotprod, axis=1)
    similarities -= sc2[np.newaxis, :]

    nn = np.argmax(similarities, axis=1).tolist()
    correct = 0.0
    for k in range(0, len(lexicon)):
        if nn[k] in lexicon[idx_src[k]]:
            correct += 1.0
    return correct / lexicon_size

# new adding
def load_alignment_pair(file_name):
    alignment_pair = []
    c = 0
    for line in open(file_name, 'r'):
        e1, e2 = line.split()
        alignment_pair.append((int(e1), int(e2)))
    return alignment_pair

def my_dist_func(L, R, k=100):
    dim = len(L[0])
    torch.cuda.empty_cache()
    index = faiss.IndexFlat(dim)
    index = faiss.index_cpu_to_all_gpus(index)
    index.add(R)
    D, I = index.search(L, k)
    print("FAISS complete")
    return I

#
# def get_hits(em1, em2, test_pair, top_k=(1, 5, 10, 50, 100), partition=1, src_nodes=None, trg_nodes=None):
#     em1 = em1.cpu().detach().numpy()
#     em2 = em2.cpu().detach().numpy()
#
#     # em1 = em1 / np.linalg.norm(em1, axis=-1, keepdims=True)
#     # em2 = em2 / np.linalg.norm(em2, axis=-1, keepdims=True)
#     def filter_pair(pair, src, trg):
#         if src is None or trg is None:
#             return pair
#         src = set(src)
#         trg = set(trg)
#         return list(filter(lambda x: x[0] in src and x[1] in trg, pair))
#
#     batch_size = len(test_pair) // partition
#     print(batch_size)
#     total_size = 0
#     top_lr = [0] * len(top_k)
#     for x in range(partition):
#         left = x * batch_size
#         right = left + batch_size if left + batch_size < len(test_pair) else len(test_pair)
#         filtered = filter_pair(test_pair[left:right], src_nodes, trg_nodes)
#         print(len(filtered))
#         if len(filtered) == 0:
#             continue
#         total_size += len(filtered)
#         Lvec = np.array([em1[e1] for e1, e2 in filtered])
#         Rvec = np.array([em2[e2] for e1, e2 in filtered])
#         ranks = my_dist_func(Lvec, Rvec)
#         for i in range(Lvec.shape[0]):
#             rank = ranks[i]
#             rank_index = np.where(rank == i)[0][0] if i in rank else 1000
#             for j in range(len(top_k)):
#                 if rank_index < top_k[j]:
#                     top_lr[j] += 1
#     print('For each left:')
#     print('Total size=', total_size)
#     for i in range(len(top_lr)):
#         print('Hits@%d: %.2f%%' % (top_k[i], top_lr[i] / (total_size + 1e-8) * 100))
#
#     return top_k, top_lr, total_size

def get_hits_slow(em1, em2, test_pair, top_k=(1, 10)):

    Lvec = np.array([em1[e1] for e1, e2 in test_pair])
    Rvec = np.array([em2[e2] for e1, e2 in test_pair])
    sim = scipy.spatial.distance.cdist(Lvec, Rvec, metric='cityblock')
    top_lr = [0] * len(top_k)
    for i in range(Lvec.shape[0]):
        rank = sim[i, :].argsort()
        rank_index = np.where(rank == i)[0][0]
        for j in range(len(top_k)):
            if rank_index < top_k[j]:
                top_lr[j] += 1
    top_rl = [0] * len(top_k)
    for i in range(Rvec.shape[0]):
        rank = sim[:, i].argsort()
        rank_index = np.where(rank == i)[0][0]
        for j in range(len(top_k)):
            if rank_index < top_k[j]:
                top_rl[j] += 1
    print('For each left:')
    for i in range(len(top_lr)):
        print('Hits@%d: %.2f%%' % (top_k[i], top_lr[i] / len(test_pair) * 100))
    print('For each right:')
    for i in range(len(top_rl)):
        print('Hits@%d: %.2f%%' % (top_k[i], top_rl[i] / len(test_pair) * 100))


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from flask import *
import os, time, random, json, socket
from utils.chars import saveCharacter, switchCharacter, saveUnsavedPlayers
app = Flask(__name__)
app.secret_key = os.urandom(24)

@app.route('/save/<identifier>/<disconnect>')
def save(identifier, disconnect):
	if disconnect == 'false':
		disconnect = False
	elif disconnect == 'true':
		disconnect = True

	print('Saving {} ...'.format(identifier))
	millis = int(round(time.time() * 1000))
	resp = saveCharacter(identifier, disconnect)
	millis2 = int(round(time.time() * 1000))
	res = millis2 - millis

	if resp['success']:
		resp['time'] = str(res) + ' ms'
		print("Saved " + identifier)
	else:
		print("Error saving " + identifier)

	return jsonify(resp)

@app.route("/switch/<identifier>/<networkID>")
def switch(identifier, networkID):
	networkID = int(networkID)
	millis = int(round(time.time() * 1000))
	resp = switchCharacter(identifier, networkID)
	millis2 = int(round(time.time() * 1000))
	res = millis2 - millis
	resp['time'] = str(res) + ' ms'

	return jsonify(resp)

@app.route('/saveUnsaved')
def saveUnsaved():
	millis = int(round(time.time() * 1000))
	resp = saveUnsavedPlayers()
	millis2 = int(round(time.time() * 1000))
	res = millis2 - millis
	resp['time'] = str(res) + ' ms'

	return jsonify(resp)

if __name__ == "__main__":
	app.run(host="localhost", port=5004, threaded=True)


import cv2
import numpy as np

img = np.zeros((600, 600, 3), np.uint8)

cv2.line(img, (0, 0), (img.shape[1], img.shape[0]), (0, 255, 0), 8)
# cv2.line(影像, 开始座标, 结束座标, 颜色, 线条宽度)
# 在图像上画线
# img.shape[0] = 600 、 img.shape[1] = 600 、 img.shape[2] = 3

cv2.rectangle(img, (0, 0), (400, 300), (0, 0, 255), cv2.FILLED)
# cv2.rectangle(影像, 顶点座标, 对向顶点座标, 颜色, 线条宽度)

cv2.circle(img, (300, 400), 30, (255, 0, 0), cv2.FILLED)
# cv2.circle(影像, 圆心座标, 半径, 颜色, 线条宽度)
# 线条宽度用负值，则会变成实心，也可以用函式来表示

cv2.ellipse(img, (500, 200), (25, 55), 45, 0, 360, (0, 0, 255), 5)
# cv2.ellipse(影像, 中心座标, 轴长, 旋转角度, 起始角度, 结束角度, 颜色, 线条宽度)

cv2.putText(img, 'Eris', (100, 500), cv2.FONT_HERSHEY_PLAIN, 2, (255, 255, 255), 2 , cv2.LINE_AA )
# cv2.putText(影像, 文字, 座标, 字型, 大小, 颜色, 线条宽度, 线条种类)
# 线条种类可以省略不写，预设值为cv2.LINE_8，也可以设定反锯齿的线条效果:cv2.LINE_AA

cv2.imshow('img', img)
cv2.waitKey(0)


def identify_locations(outputs):
    overlaps = [] # overlapping paths for part 1
    path = [] # all scaffold locations, for part 2

    for i in range(len(outputs)):
        for j in range(len(outputs[i])):
            characters = []

            #center
            characters.append(outputs[i][j]=='#')

            try:
                # north
                assert i-1>=0
                characters.append(outputs[i-1][j]=='#')
            except:
                characters.append(False)
            try:
                # south
                assert i+1<=len(outputs)
                characters.append(outputs[i+1][j]=='#')
            except:
                characters.append(False)
            try:
                # east
                assert j+1<=len(outputs[i])
                characters.append(outputs[i][j+1]=='#')
            except:
                characters.append(False)
            try:
                # west
                assert j-1>=0
                characters.append(outputs[i][j-1]=='#')
            except:
                characters.append(False)

            if outputs[i][j] not in['#','.']:
                start = (i,j)

            if sum(characters)==5:
                # it's 2 paths overlapping!
                overlaps.append((i,j))

            # store everything in path
            if characters[0]:
                path.append((i,j))

    return overlaps, path, start

def build_instructions(path, start):
    directions = [[0,1], [1,0], [0,-1], [-1,0]]

    # start drone facing down (inverted coordinates)
    start_direction = directions[3]

    # keep track of all the positions the drone has visited
    already_visited = []

    # initialize drone's position and direction
    current_y, current_x = start
    direction = start_direction

    # start the instruction list ([direction_0, length_0, direction_1, length_1, ...])
    instructions = [direction, 0]

    while len(set(path)-set(already_visited))>0:
        try:
            next_location = path.index((current_y+direction[0], current_x+direction[1]))
            # ^ if this passed, we can keep going straight

            # update distance (instructions[-1])
            instructions[-1] += 1

            # update locations
            current_y, current_x = path[next_location]
            already_visited.append((current_y,current_x))

        except:
            # need to turn drone
            if instructions[-1]==0:
                # drone hasn't moved in current direction - no need to store it
                instructions = instructions[:-2]

            try:
                # turn left 90 degrees
                direction = directions[directions.index(direction)-1]
                path.index((current_y+direction[0], current_x+direction[1]))
                instructions.append('L')
                instructions.append(direction)
                instructions.append(0)

                # instructions now looks like [...., [new_y_slope,new_x_slope], 0]
                continue
            except:
                try:
                    # turn right 90 degrees
                    direction = directions[directions.index(direction)-2]
                    path.index((current_y+direction[0], current_x+direction[1]))
                    instructions.append('R')
                    instructions.append(direction)
                    instructions.append(0)

                    # instructions now looks like [...., [new_y_slope,new_x_slope], 0]
                    continue
                except:
                    break

    return ','.join([str(x) for x in instructions if type(x)!=list])

def functionize(instructions):
    options = instructions.split(',')
    options = [",".join(options[i:i+2]) for i in range(0, len(options), 2)]

    # assemble set of all patterns in instructions, that could be made into function
    function_options = []
    for i in range(len(options)):
        for length in range(1,len(options)-i):
            slic = ','.join(options[i:i+length])
            if slic not in function_options:
                function_options.append(slic)

    # iterate through options, stop when we find one function combo that works
    from itertools import permutations

    for a,b,c in permutations(function_options, 3):
        cpy = instructions.replace(a, 'A').replace(b,'B').replace(c,'C')
        if 'R' not in cpy and 'L' not in cpy:
            break

    main = [ord(x) for x in cpy+'\n']
    a = [ord(x) for x in a+'\n']
    b = [ord(x) for x in b+'\n']
    c = [ord(x) for x in c+'\n']

    # check function lengths
    assert len(main)<=21 and len(a)<=21 and len(b)<=21 and len(c)<=21

    return main, a, b, c

def main():
    # read in data
    with open('day17.txt', 'r') as f:
        data = np.array(f.read().strip().split(','), dtype=np.int64)
    f.close()

    from intComputer import IntcodeComputer
    brain = IntcodeComputer(dt=data)

    # assemble map of scaffolds
    outputs = ''
    while True:
        output = brain.process_code()
        if output is not None:
            outputs += chr(output)
        else:
            break

    # split by newline character to form list of strings
    outputs = [output for output in outputs.split('\n') if len(output)>0]

    overlaps, path, start = identify_locations(outputs)

    # part 1
    print(sum([x*y for x,y in overlaps]))

    instructions = build_instructions(path, start)
    main, a, b, c = functionize(instructions)

    # wake up robot
    brain.dt[0] = 2
    # add calculated inputs to move the bot
    brain.inputs = main+a+b+c+[ord('n'), ord('\n')]

    # run robot until program terminates
    vals = [brain.process_code()]
    while vals[-1] is not None:
        vals.append(brain.process_code())
    # part 2
    print(vals[-2])

if __name__=='__main__':
    import numpy as np
    main()


import pandas as pd
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from torch.utils.data import DataLoader, TensorDataset

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

# Load the dataset
df = pd.read_csv("diabetes.csv")
column_names = df.columns.tolist()

X = df[df.columns[:-1]].values
y = df[df.columns[-1]].values
#scaler = StandardScaler()

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=42)

X_train = torch.FloatTensor(X_train)
y_train = torch.LongTensor(y_train)
X_val = torch.FloatTensor(X_val)
y_val = torch.LongTensor(y_val)
X_test = torch.FloatTensor(X_test)
y_test = torch.LongTensor(y_test)

# Step 3: Define your neural network model
class DiabetesModel(nn.Module):
    def __init__(self):
        super(DiabetesModel, self).__init__()
        self.fc1 = nn.Linear(8, 64)
        self.relu = nn.ReLU()
        self.fc2 = nn.Linear(64, 2)  # 2 output classes (0 or 1)

    def forward(self, x):
        x = self.fc1(x)
        x = self.relu(x)
        x = self.fc2(x)
        return x

# Step 4: Train the model on the training set

model = DiabetesModel()
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.001)
train_dataset = TensorDataset(X_train, y_train)
train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)
epochs = 2000

# Lists to store training and validation loss and accuracy
train_losses = []
train_accuracies = []
val_losses = []
val_accuracies = []

for epoch in range(epochs):
    model.train()  # Set the model to training mode
    total_loss = 0
    correct = 0
    total = 0

    for inputs, labels in train_loader:
        optimizer.zero_grad()
        outputs = model(inputs)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()

        total_loss += loss.item()
        _, predicted = torch.max(outputs, 1)
        total += labels.size(0)
        correct += (predicted == labels).sum().item()

    # Calculate training loss and accuracy for the epoch
    train_loss = total_loss / len(train_loader)
    train_accuracy = correct / total
    train_losses.append(train_loss)
    train_accuracies.append(train_accuracy)

    print(f'Epoch [{epoch + 1}/{epochs}] Training Loss: {train_loss:.4f} Training Accuracy: {train_accuracy * 100:.2f}%')

# Step 5: Evaluate the model on the validation set
model.eval()  # Set the model to evaluation mode
with torch.no_grad():
    y_pred = model(X_val)
    _, predicted = torch.max(y_pred, 1)
    val_loss = criterion(y_pred, y_val)
    val_accuracy = (predicted == y_val).sum().item() / y_val.size(0)
    val_losses.append(val_loss.item())
    val_accuracies.append(val_accuracy)

print(f'Validation Loss: {val_loss.item():.4f} Validation Accuracy: {val_accuracy * 100:.2f}%')

# Step 6: Evaluate the model on the test set
model.eval()  # Set the model to evaluation mode
with torch.no_grad():
    y_pred = model(X_test)
    _, predicted = torch.max(y_pred, 1)
    test_loss = criterion(y_pred, y_test)
    test_accuracy = (predicted == y_test).sum().item() / y_test.size(0)

print(f'Test Loss: {test_loss.item():.4f} Test Accuracy: {test_accuracy * 100:.2f}%')

# Save the trained model
torch.save(model.state_dict(), 'diabetes_model.pt')
print("Model saved.")

# # Load the saved model
# loaded_model = DiabetesModel()
# loaded_model.load_state_dict(torch.load('diabetes_model.pt'))
# loaded_model.to(device)
# loaded_model.eval()
# print("Loaded Model: ",loaded_model)

def predict_diabetes(input_data):

    model.eval()  # Set the model to evaluation mode

    # Convert input_data to a PyTorch tensor
    input_data_tensor = torch.FloatTensor(input_data)

    # Ensure input_data has the same shape as the model expects (8 features)
    if input_data_tensor.shape != (8,):
        raise ValueError("Input data must have 8 features.")

    with torch.no_grad():
        # Forward pass to get predictions
        predictions = model(input_data_tensor)

        # Assuming class 1 corresponds to index 1
        sigmoid = torch.nn.Sigmoid()
        probability_class_1 = sigmoid(predictions[1]).item()

        # Get the predicted class (0 or 1) by rounding the probability
        predicted_class = torch.round(torch.tensor(probability_class_1))
        print(predicted_class)
        result = 'Diabetic' if predicted_class == 1 else 'Not Diabetic'

    return result

# # Example new inputs for prediction
# new_inputs = [6,148,72,35,0,33.6,0.627,50]
# # non - 0
# # yes - 1
# # Predict the outcome using the loaded model
# predicted_outcome = predict_diabetes(new_inputs)
# print(f"According to your data you are: {predicted_outcome}")


from collections import deque

N = int(input())
adj = [[] for _ in range(N + 1)]
visit = [[False] * (N + 1) for _ in range(N + 1)]
dist = [[0] * (N + 1) for _ in range(N + 1)]
minScore = 1e9
answer = []

while True:
    a, b = map(int, input().split())
    if a == -1 and b == -1:
        break
    adj[a].append(b)
    adj[b].append(a)

for i in range(1, N + 1):
    queue = deque()
    queue.append(i)
    visit[i][i] = True
    cnt = 0

    while queue:
        size = len(queue)
        for _ in range(size):
            now = queue.popleft()
            dist[i][now] = cnt

            for next in adj[now]:
                if visit[i][next]:
                    continue
                visit[i][next] = True
                queue.append(next)
        cnt += 1

for i in range(1, N + 1):
    minScore = min(minScore, max(dist[i]))

for i in range(1, N + 1):
    if max(dist[i]) == minScore:
        answer.append(i)

print(minScore, len(answer))
print(*answer)


import sys
import warnings
import numpy as np
import matplotlib.pyplot as plt
from skimage.segmentation import (morphological_geodesic_active_contour,
                                 inverse_gaussian_gradient)
import skimage
import cv2
import os
import SimpleITK as sitk
import glob
import napari

def viewVolume(volume):
    """
        Shows the 3D image using the Napari library.
        Parameters:
            volume (numpy.ndarray): Numpy array 3D image.
        Returns:
            void
    """
    viewer = napari.view_image(volume, contrast_limits=[0, 1])
    napari.run()

def store_evolution_in(lst):
   """
   Returns a callback function to store the evolution of the level sets in
   the given list.
   """

   def _store(x):
       lst.append(np.copy(x))

   return _store

def despeckle(image,option):
    """
        Despeckles the 3D SimpleITK image using one of the options.
        The options are 'bilateral901', 'median110', 'median111'
        and 'median221'.
        Parameters:
            image (sitk.Image): 3D image that needs to be despeckled
            option: despeckling option
        Returns
            sitk.Image: The despeckled image

    """

    if option == 'bilateral901':
       image = sitk.GetArrayFromImage(image)
       for slice in range(0, image.shape[0]):
           image[slice] = cv2.bilateralFilter(image[slice], 9, 0, 1)  # cv2.medianBlur(image_1bone[slice],3)
       image = sitk.GetImageFromArray(image)

    if option == 'median110':
       image = sitk.BinaryMedian(image,(1,1,0))

    if option == 'median111':

       image = sitk.Median(image,(1,1,1))

    if option == 'median221':
       image = sitk.BinaryMedian(image, (2, 2, 1))

    return image

if __name__ == "__main__":
    try:
        bmp_folder_path_PCCT = sys.argv[1]
        thresh_val = int(sys.argv[2])
        kernel_size = 15
        if thresh_val > 110:
            kernel_size = 35

        # getting the filenames for the output
        filenames = [f for f in os.listdir(bmp_folder_path_PCCT) if
                       os.path.isfile(os.path.join(bmp_folder_path_PCCT, f)) and f.endswith('.bmp')]
        output_path_PCCT = os.path.join(bmp_folder_path_PCCT, 'bonemask')
        if not os.path.exists(output_path_PCCT):
           os.makedirs(output_path_PCCT)

        # reading the image
        image = [cv2.cvtColor(cv2.imread(file), cv2.COLOR_BGR2GRAY) for file in glob.glob(bmp_folder_path_PCCT + "\*.bmp")]
        image_original = np.asarray(image)
        shape_1slice = image_original.shape

        # converting image to SimpleITK format
        image = sitk.GetImageFromArray(image_original)

        # thresholding the image
        image = sitk.Threshold(image,lower=thresh_val,upper=255)
        thresholded_image = sitk.GetArrayFromImage(image)
        image = despeckle(image, 'bilateral901')

        # Despeckle
        image = sitk.ConnectedComponent(image, False)
        image = sitk.RelabelComponent(image, sortByObjectSize=True)
        image = sitk.GetArrayFromImage(image)

        for slice in range(0,shape_1slice[0]):
           print(str(slice))
           image_1slice = image[slice,:,:]
           output_path_file = os.path.join(output_path_PCCT, filenames[slice])
           output_image = np.zeros((shape_1slice[1], shape_1slice[2]), np.uint8)
           for bone in range(1,5):
               one_bone_image_1slice = (image_1slice == bone).astype(np.uint8)

               if thresh_val < 100:
                   one_bone_image_1slice = cv2.bilateralFilter(one_bone_image_1slice, 9, 0, 1)

               empty = np.zeros_like(one_bone_image_1slice)
               one_bone_image_1slice = cv2.normalize(one_bone_image_1slice, empty, 0, 255, cv2.NORM_MINMAX, dtype=cv2.CV_8U)

               if np.mean(one_bone_image_1slice) > 0.0001:

                   # create mask for active contours
                   kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (kernel_size, kernel_size))
                   trial_image = np.copy(one_bone_image_1slice)
                   mask_VOI = cv2.dilate(trial_image,kernel)
                   mask_VOI = cv2.bitwise_not(mask_VOI)
                   num_labels, labels_im = cv2.connectedComponents(mask_VOI)
                   mask_VOI = (labels_im == 1).astype(np.uint8)
                   empty = np.zeros_like(one_bone_image_1slice)
                   mask_VOI = cv2.normalize(mask_VOI, empty, 0, 255, cv2.NORM_MINMAX,dtype=cv2.CV_8U)
                   mask_VOI = cv2.bitwise_not(mask_VOI)

                   output = thresholded_image[slice,:,:]
                   output = cv2.bitwise_and(output, output, mask=one_bone_image_1slice)
                   image_preprocessed = cv2.bitwise_not(output)
                   image_preprocessed = skimage.img_as_float(image_preprocessed)

                   # Shrink wrap
                   # Initial level set
                   init_ls = mask_VOI

                   evolution = []
                   callback = store_evolution_in(evolution)

                   gimage = inverse_gaussian_gradient(image_preprocessed,alpha=100.0,sigma=4.0)

                   ls = morphological_geodesic_active_contour(gimage,num_iter=80,init_level_set = init_ls,
                                                              smoothing=1,balloon=-1,iter_callback=callback)

                   #code for showing the evolution of the active contours
                   # fig, axes = plt.subplots(1, 2, figsize=(16, 16))
                   # ax = axes.flatten()
                   #
                   # ax[0].imshow(image_original[slice,:,:], cmap="gray")
                   # ax[0].set_axis_off()
                   # ax[0].contour(ls, [0.5], colors='r')
                   # ax[0].set_title("Morphological GAC segmentation", fontsize=12)
                   #
                   # ax[1].imshow(image_preprocessed, cmap="gray") #ls ipv image_preprocessed
                   # ax[1].set_axis_off()
                   # contour = ax[1].contour(evolution[2], [0.5], colors='g')
                   # contour.collections[0].set_label("Iteration 2")
                   # contour = ax[1].contour(evolution[30], [0.5], colors='y')
                   # contour.collections[0].set_label("Iteration 30")
                   # contour = ax[1].contour(evolution[-1], [0.5], colors='r')
                   # contour.collections[0].set_label("Iteration 80")
                   # ax[1].legend(loc="upper right")
                   # title = "Morphological GAC evolution"
                   # ax[1].set_title(title, fontsize=12)
                   # fig.tight_layout()
                   # plt.show()

                   # add the different bones to one image
                   output_image = output_image | ls

           # write image
           out = np.zeros_like(one_bone_image_1slice)
           output_image = cv2.normalize(output_image, out, 0, 255, cv2.NORM_MINMAX, dtype=cv2.CV_8U)
           cv2.imwrite(output_path_file, output_image)

    except:
        if len(sys.argv) != 3:
            warnings.warn('Not the right amount of arguments were given.'
            'As a reminder, the input should be '
            'python Bone_Mask.py <"link to input folder"> <threshold>')
        else:
            warnings.warn('An error occurred during the STB segmentation.'
            'Please check the input. As a reminder, the input should be '
            'python Bone_Mask.py <"link to input folder"> <threshold>')


from django import forms
from .models import ReporteDiario, Funda, Defecto, PorcentajeDefecto # Asegúrate de importar los modelos necesarios
from django.forms import modelformset_factory

# class ReporteDiarioForm(forms.ModelForm):
#     class Meta:
#         model = ReporteDiario
#         fields = '__all__'

#     marca_funda = forms.ModelChoiceField(queryset=Funda.objects.all(), empty_label=None)

class ReporteDiarioForm(forms.ModelForm):
    defectos = forms.ModelMultipleChoiceField(
        queryset=Defecto.objects.all(),
        widget=forms.CheckboxSelectMultiple,
        required=False
    )

    class Meta:
        model = ReporteDiario
        fields = ['nombre_analista','turno','presentacion','lote','hora', 'fecha','clasificacion', 'cuenta_lbs', 'total_camaron','peso_neto', 'camarones_grandes', 'camarones_pequenos', 'marca_funda_manual', 'peso_bruto', 'peso_sin_funda_manual','porcentaje_glaseo','uniformidad','observaciones','correctivos']

    def __init__(self, *args, **kwargs):
        super(ReporteDiarioForm, self).__init__(*args, **kwargs)
        # Configura los campos como no requeridos
        self.fields['total_camaron'].required = False
        self.fields['camarones_grandes'].required = False
        self.fields['camarones_pequenos'].required = False
        self.fields['porcentaje_glaseo'].required = False
        self.fields['uniformidad'].required = False

class FundaForm(forms.ModelForm):
    class Meta:
        model = Funda
        fields = ['marca', 'peso_funda']

class PorcentajeDefectoForm(forms.ModelForm):
    defectos = [
        ("Rosado leve", "Rosado leve"),
        ("Deshidratado", "Deshidratado"),
        ("Picado", "Picado"),
        ("Dañados o piezas", "Dañados o piezas"),
        ("Mudado", "Mudado"),
        ("Suave o quebrado", "Suave o quebrado"),
        ("Mal descabezado", "Mal descabezado"),
        ("Sin telson", "Sin telson"),
        ("Cascara", "Cascara"),
        ("Corte profundo", "Corte profundo"),
        ("Falta de corte", "Falta de corte"),
        ("Intestino / vena", "Intestino / vena"),
        ("Semi crudo", "Semi crudo"),
        ("Pequeño", "Pequeño"),
        ("Otros", "Otros"),
    ]

    defecto = forms.ChoiceField(
        choices=defectos,
        widget=forms.Select(attrs={'class': 'form-control'})
    )
    class Meta:
        model = PorcentajeDefecto
        fields = ['defecto', 'cantidad', 'porcentaje']


import argparse
import logging
import os
import os.path as osp
import time
import re
from collections import deque

import torch
import torch.nn.functional as F
import torch_geometric.transforms as T
from torch_geometric.data import DataLoader

from DatasetLoader.loader import LoadDataset
from Utils.helper import time_to_hms_string
from Utils.generics import saveModelCheckpoint, loadFromCheckpoint
from Utils.lr_schedulers import limitedExponentialDecayLR as customExpDecayLambda
from Normalization.normalization import Normalize
from Sampling.poisson_disk_sampling import PoissonDiskSampling

from Models.PointNet2.pointnet2_cls_ssg import PointNet2Class as PointNet2
from Models.PointNet2MSG.pointnet2_cls_msg import PointNet2MSGClass as PointNet2MSG
from Models.PointNet2MRG.pointnet2_cls_mrg import PointNet2MRGClass as PointNet2MRG
from Models.PointNet2MRGSortPool.pointnet2_cls_mrg_sort_pool import PointNet2MRGSortPoolClass as PointNet2MRGSortPool
from Models.PointNet2MSGSortPool.pointnet2_cls_msg_sort_pool import PointNet2MSGSortPoolClass as PointNet2MSGSortPool
from Models.PointNet2MSGFPSortPool.pointnet2_cls_msgfp_sort_pool import PointNet2MSGFPSortPoolClass as PointNet2MSGFPSortPool
from Models.PointNet2MRGLight.pointnet2_cls_mrg_light import PointNet2MRGLightClass as PointNet2MRGLight
from Models.PointNetVanilla.pointnet_cls_vanilla import PointNetVanillaClass  as PointNetVanilla
from Models.PointNetInputEnhanced.pointnet_cls_ie import PointNetInputEnhanced  as PointNetInputEnhanced

parser = argparse.ArgumentParser()
parser.add_argument('--model', default=None, help='Model name (PointNet2, PointNet2MSG, PointNet2MSGSortPool, PointNet2MRG, PointNet2MRGSortPool, PointNet2MSGFPSortPool)')
parser.add_argument('--num_point', type=int, default=1024, help='Point Number sampled from each object [default: 1024]')
parser.add_argument('--epoch', type=int, default=251, help='Epoch to run [default: 251]')
parser.add_argument('--batch_size', type=int, default=32, help='The size of the batch to use [default: 16]')
parser.add_argument('--weight_decay', type=float, default=1e-5, help='The weight decay to use for the training, the original PointNet2 1e-5 [default:1e-5]')
parser.add_argument('--lr_decay', type=float, default=0.7, help='The weight decay used as to start the exponential LR decay, the step is defined from the Exponential class of pytorch.[default:0.7]')
parser.add_argument('--lr_decay_step', type=float, default=2e5, help='Learning rate decay step [default: 2e5]');
parser.add_argument('--train_dataset', default='ModelNet40', help='The name of the dataset to use to train the model (ModelNet10, ModelNet40, ShapeNet)[default: ModelNet40]')
parser.add_argument('--test_dataset', default='ModelNet40', help='The name of the dataset to use to test the model (ModelNet10, ModelNet40, ShapeNet)[default: ModelNet40]')
parser.add_argument('--checkpoint', default=None, help='The path to the checkpoint file from which continue the train')
parser.add_argument('--log_file', default='./train_log.txt', help='The file where to print the logs [default: ./train_log.txt]')
parser.add_argument('--use_normals', type=bool, default=False, help='Specufy if train the model using point normals from the data sets[default: False]')
parser.add_argument('--sort_pool_k', type=int, default=32, help='The number of point the sort_pool should keep <only needed with the sort_pool model> [default: 32]')
parser.add_argument('--sampling_method_train', default='ImportanceSampling', help='The type of sampling to use. [PoissonDiskSampling or ImportanceSampling]')
parser.add_argument('--sampling_method_test', default='ImportanceSampling', help='The type of sampling to use. [PoissonDiskSampling or ImportanceSampling]')
ARGS = parser.parse_args()

BATCH_SIZE = ARGS.batch_size
NUM_POINT = ARGS.num_point
EPOCH = ARGS.epoch
DATASET_TRAIN = ARGS.train_dataset
DATASET_TEST = ARGS.train_dataset
MODEL_NAME = ARGS.model
CHECKPOINT = ARGS.checkpoint
WEIGHT_DECAY = ARGS.weight_decay
LR_DECAY = ARGS.lr_decay
LR_DECAY_STEP = ARGS.lr_decay_step
LOG_FILE = ARGS.log_file
USE_NORMALS = ARGS.use_normals
N_FEATURES = 6 if ARGS.use_normals else 3
SORT_POOL_K = ARGS.sort_pool_k
SAMPLING_TRAIN = ARGS.sampling_method_train
SAMPLING_TEST = ARGS.sampling_method_test

def train(model, train_loader, optimizer, device):
    model.train()

    for data in train_loader:
        data = data.to(device)
        if data.norm is not None:
            data.x = data.norm
        optimizer.zero_grad()
        test_values = data.category if DATASET_TRAIN == 'ShapeNet' else data.y
        loss = F.nll_loss(model(data), test_values)
        loss.backward()
        optimizer.step()

def test(model, test_loader, device):
    model.eval()

    correct = 0
    for data in test_loader:
        data = data.to(device)
        if data.norm is not None:
            data.x = data.norm
        test_result = data.category if DATASET_TEST == 'ShapeNet' else data.y
        with torch.no_grad():
            pred = model(data).max(1)[1]
        correct += pred.eq(test_result).sum().item()
    return correct / len(test_loader.dataset)

def setupTrain(model_name, n_features, class_number, device, w_decay, lr_decay, batch_size, decay_step):
    model = getModel(model_name, n_features, class_number).to(device)
    optimizer = torch.optim.Adam(model.parameters(), lr=0.001, weight_decay=w_decay)
    start_epoch = 1

    scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer,
                    lr_lambda=customExpDecayLambda(lr_decay, batch_size, decay_step))

    return model, optimizer, start_epoch , scheduler

def getModel(name, input_features, class_count):
    model = None
    if (MODEL_NAME == 'PointNetVanilla'):
        model = PointNetVanilla(class_count, nfeatures=input_features, nPoints=NUM_POINT)

    elif (MODEL_NAME == 'PointNetInputEnhanced'):
        model = PointNetInputEnhanced(class_count, nfeatures=input_features, batch_size=BATCH_SIZE, nPoints=NUM_POINT)

    elif (MODEL_NAME == 'PointNet2'):
        model = PointNet2(class_count, bn_momentum=0.1)

    elif (MODEL_NAME == 'PointNet2MSG'):
        model = PointNet2MSG(class_count, nfeatures=input_features)

    elif (MODEL_NAME == 'PointNet2MRG'):
        model = PointNet2MRG(class_count, nfeatures=input_features)

    elif (MODEL_NAME == 'PointNet2MRGLight'):
        model = PointNet2MRGLight(class_count, n_features=input_features)

    elif (MODEL_NAME == 'PointNet2MSGSortPool'):
        model = PointNet2MSGSortPool(class_count, n_feature=input_features, sort_pool_k=SORT_POOL_K)

    elif (MODEL_NAME == 'PointNet2MSGFPSortPool'):
        model = PointNet2MSGFPSortPool(class_count, n_feature=input_features, sort_pool_k=SORT_POOL_K)

    elif (MODEL_NAME == 'PointNet2MRGSortPool'):
        model = PointNet2MRGSortPool(class_count, n_features=input_features, sort_pool_k=SORT_POOL_K)

    return model

def getSampler(name, dataset_name):
    transform = None

    if(dataset_name == 'ShapeNet'):
        transform = T.FixedPoints(NUM_POINT)

    elif(name == 'ImportanceSampling'):
        transform = T.SamplePoints(NUM_POINT, remove_faces=True, include_normals=USE_NORMALS)

    elif (name == 'PoissonDiskSampling'):
        transform = PoissonDiskSampling(NUM_POINT, remove_faces=True)

    return transform

if __name__ == '__main__':
    logging.basicConfig(filename=LOG_FILE, level=logging.INFO)

    modelWeightsPath = osp.join(
        osp.dirname(osp.realpath(__file__)), '.', 'weights/', MODEL_NAME + '_cls/')
    if not os.path.exists(modelWeightsPath):
        os.makedirs(modelWeightsPath)
        os.chmod(modelWeightsPath, 0o777)

    # DATASET preprocessing
    if(USE_NORMALS and ('Poisson' in DATASET_TRAIN or 'Poisson' in DATASET_TEST)):
        print("Poisson sampled data do not support normals(at the moment).")
        exit(1)

    pre_transform = Normalize(),
    transform_train = getSampler(SAMPLING_TRAIN, DATASET_TRAIN)  # POINT SAMPLING
    transform_test = getSampler(SAMPLING_TEST, DATASET_TEST)
    (train_dataset, test_dataset) = LoadDataset(
        DATASET_TRAIN, DATASET_TEST,
        transform_train=transform_train, pre_transform_train=pre_transform,
        transform_test=transform_test, pre_transform_test=pre_transform,
    )

    train_loader = DataLoader(
        train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=6)
    test_loader = DataLoader(
        test_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=6)

    # TRAIN SETUP
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    last_10_loss_values = deque([], 10)

    (model, optimizer, start_epoch, optimizer_scheduler) = setupTrain(
        model_name = MODEL_NAME,
        n_features = N_FEATURES,
        class_number = train_dataset.num_classes,
        w_decay = WEIGHT_DECAY,
        lr_decay = LR_DECAY,
        decay_step=LR_DECAY_STEP,
        batch_size = BATCH_SIZE,
        device = device
	)

    train_start_time = time.time()

    if (CHECKPOINT != None):
        (model, optimizer, optimizer_scheduler, start_epoch, train_checkpoint_time) = loadFromCheckpoint(
            CHECKPOINT, model, optimizer, optimizer_scheduler, device)
        train_start_time-=train_checkpoint_time

    # TRAIN CYCLE
    for epoch in range(start_epoch, start_epoch + EPOCH):
        train(model, train_loader, optimizer, device)
        test_acc = test(model, test_loader, device)
        optimizer_scheduler.step()

        # the older item are automatically removed from the deque when full
        last_10_loss_values.append(test_acc)
        loss_avg = sum(last_10_loss_values) / float(len(last_10_loss_values))
        current_train_time = time.time() - train_start_time

        epoch_to_print = '{} :: Epoch: {:03d}, Test: {:.4f}, Last 10 AVG: {:.4f}, LR: {:.8f}'.format(
            time_to_hms_string(current_train_time) ,epoch, test_acc, loss_avg, optimizer_scheduler.get_lr()[0])

        print(epoch_to_print)
        logging.info(epoch_to_print)

        # TODO add better system to save checkpoint, and maybe a better naming (with start and end feature count)
        if (epoch % 10 == 0):

            saveModelCheckpoint(
                modelWeightsPath, model, optimizer, test_acc, epoch, optimizer_scheduler, loss_avg, current_train_time)

    # Final trained Model store for inference
    inferenceReadyModel = osp.join( modelWeightsPath, 'trained/model_inference.pt')

    torch.save(model.state_dict(), inferenceReadyModel)


import os
import multiprocessing as mp

file_list = os.listdir("2-GeneFamiliesDivTimeAligned/")

def to_run(fasta_file):
	os.chdir("2-GeneFamiliesDivTimeAligned/")
	os.system("iqtree2 -s " + str(fasta_file) + " -nt 1")

pool = mp.Pool(15)
pool.map(to_run, file_list)


from fastapi import APIRouter, Depends
from fastapi.exceptions import HTTPException
from config.db import db
from templates import webhook_template
from functions import auth
from models.models import User

app = APIRouter()

@app.get("/admin/users")
async def getAllUsers(admin: User = Depends(auth.get_current_user)):
    """
    Get all users from database
    """
    try:
        if not admin["admin"]:
            return {
                'error':"This route is only for admins"
            }

        users = db.user.find()
        usersList = []
        for user in users:
            user["_id"] = str(user["_id"])
            usersList.append(user)
        return usersList
    except:
        raise HTTPException(status_code=500, detail={
            "error":"Something went wrong"
        })

@app.get("/admin/user/{email}", description="Get specific user by email")
async def getUserByEmail(email, admin: User = Depends(auth.get_current_user)):
    if not admin["admin"]:
        return {
            "error":"This route is only for admins"
        }

    user = db.user.find_one({
        "email":email
    })
    if not user:
        raise HTTPException(status_code=404, detail={
            "error":"User not found"
        })
    user["_id"] = str(user["_id"])
    return user

@app.delete("/admin/user/{email}",description="Delete's an user by it's email")
async def deleteUserByEmail(email, admin: User = Depends(auth.get_current_user)):
    if not admin["admin"]:
        return {
            "error":"This route is only for admins"
        }

    user = db.user.find_one_and_delete({
        "email":email
    })
    if not user:
        raise HTTPException(status_code=404, detail={
            "error":"User not found"
        })
    user["_id"] = str(user["_id"])
    return user

@app.post("/webhook")
async def test_webhook(admin: User = Depends(auth.get_current_user)):
    if not admin["admin"]:
        return {
            "error":"This route is only for admins"
        }
    return webhook_template.send_test_webhook()


import logging
logger = logging.getLogger("live_control")

from django.shortcuts import render_to_response
from django.http import HttpResponse, HttpResponseRedirect
from django.template import loader
from django.contrib.auth.decorators import login_required

from django.template import RequestContext
from django.core.urlresolvers import reverse

from controller_setup.models import Document
from controller_setup.forms import DocumentForm
from controller_setup.management.commands.templates import main as generate_templates

# Create your views here.
from .models import Template, LinkUserToPlant
from .tools_ssh import start_plant, restart_plant, stop_plant, status_plant, sync_plant

@login_required(login_url='/login/')
def index(request):
    template = loader.get_template('controller_setup/index.html')
    return HttpResponse(template.render(context=None, request=request))

def done(request, go_back=None, string_list=None):
    template = loader.get_template('controller_setup/done.html')
    context = {
        'go_back': go_back,
        'string_list': string_list,
    }
    return HttpResponse(template.render(context, request))

@login_required(login_url='/login/')
def control(request):
    links = LinkUserToPlant.objects.all()
    template = loader.get_template('controller_setup/control.html')
    context = {
        'links': links,
    }
    return HttpResponse(template.render(context, request))

@login_required(login_url='/login/')
def command(request, command, link_pk):
    link = LinkUserToPlant.objects.get(pk=link_pk)
    response = command(link)
    logger.info("for command %s, response %s" % (command, response))
    try:
        response = map(str, response)
    except:
        response = [str(response)]
    return done(request, go_back='control', string_list=response)

def start(request, link_pk):
    return command(request, start_plant, link_pk)
def restart(request, link_pk):
    return command(request, restart_plant, link_pk)
def stop(request, link_pk):
    return command(request, stop_plant, link_pk)
def status(request, link_pk):
    return command(request, status_plant, link_pk)

def sync(request, link_pk):
    return command(request, sync_plant, link_pk)

@login_required(login_url='/login/')
def templates(request):
    templates = Template.objects.all()
    template = loader.get_template('controller_setup/templates.html')
    context = {
        'templates': templates,
    }
    return HttpResponse(template.render(context, request))

@login_required(login_url='/login/')
def upload(request):
    # Handle file upload
    if request.method == 'POST':
        form = DocumentForm(request.POST, request.FILES)
        if form.is_valid():
            newdoc = Document(docfile = request.FILES['docfile'])
            newdoc.save()

            # Redirect to the document list after POST
            return HttpResponseRedirect(reverse('controller_setup.views.upload'))
    else:
        form = DocumentForm() # A empty, unbound form

    # Load documents for the list page
    documents = Document.objects.all()

    # Render list page with the documents and the form
    return render_to_response(
        'controller_setup/upload.html',
        {'documents': documents, 'form': form},
        context_instance=RequestContext(request)
    )

@login_required(login_url='/login/')
def autotemplates(request):
    return done(request, go_back='templates', string_list=generate_templates())


from django.test import TestCase, Client
from django.urls import reverse
from review.models import comments
from account.models import User, Business

class TestViews(TestCase):

    '''Login required - cant test all functunals'''
    def setUp(self):
        # Create objects
        self.client = Client()
        self.commentObj = comments.objects.create(
            user = User.objects.create(id = 199),
            business_comments = Business.objects.create(user = User.objects.get(id = 199),),
            subject = 'subject-spam',
            comment = 'comment-spam',
            rate = 3
        )

    def test_commentt(self):
        # Check if user and business_comments are the same
        self.assertEqual(
            self.commentObj.user,
            self.commentObj.business_comments.user
            )


import pygame
from game import Game

SCREEN_WIDTH = 1024
SCREEN_HEIGHT = 576

FPS = 60

def main():

    pygame.init()

    screen = pygame.display.set_mode((SCREEN_WIDTH,SCREEN_HEIGHT))

    pygame.display.set_caption("Maze Escape-3D")

    done = False

    clock = pygame.time.Clock()

    game = Game()

    while not done:
        done = game.process_events()
        game.display_frame(screen)
        clock.tick(FPS)
    pygame.quit()

if __name__ == '__main__':
    main()


from decouple import config

if __name__ == "__main__":
    DEBUG = config('DEBUG', default=False, cast=bool)
    print('DEBUG', DEBUG)
    '''
    environment variables have precedence over config files.
    os.environ['PATH'] = '/my_path'
    '''
    PATH = config('PATH', cast=str)
    print('PATH', PATH)


from uuid import UUID

from django.urls import reverse

import pytest
from rest_framework import status
from rest_framework.test import APIClient

from tests.factories import RecipeFactory

pytestmark = [pytest.mark.django_db]

def get_recipe_detail_url(recipe_id: str) -> str:
    """Return a recipe detail url."""
    return reverse('api:recipes-detail', args=[recipe_id])

def test_update_recipe_successful(recipe_factory: RecipeFactory, api_user_client: APIClient) -> None:
    """Test retrieve recipe is successful.

    Args:
        recipe_factory: ingredient factory;
        api_user_client: django rest framework api client.
    """
    user, api_client = api_user_client
    recipe = recipe_factory.create(author=user)

    payload = {
        'id': recipe.id,
        'name': 'recipe',
        'text': 'good recipe',
        'tags': [tag.id for tag in recipe.tags.all()],
        'ingredients': [
            {
                'id': ingredient.id,
                'amount': 50,
            }
            for ingredient in recipe.ingredients.all()
        ],
        'cooking_time': 50,
    }

    response = api_client.patch(get_recipe_detail_url(recipe_id=recipe.id), payload, format='json')

    assert response.status_code == status.HTTP_200_OK
    assert response.data['name'] == 'recipe'


""""
This script performs the first preprocessing step: Vocals Isolation.
It uses the MadTwinNet (https://github.com/dr-costas/mad-twinnet)
as a source-separation algorithm, yielding close to SOTA results
at the moment of writing this code.
"""

def isolate_vocals(
        d00sr01_f1,
        weights_d0,
        rnn_enc_output_dim,
        original_input_dim,
        reduced_dim,
        context_length,
        window_size,
        fft_size,
        hop_size,
        seq_length,
        batch_size,
        top_db,
):
    """
    Isolates the vocals from an arbitrary music recording.

        Parameters
        ----------
        d00sr01_f1 : tuple
            The tuple containing the input music recording
        weights_d0 : int
            The already loaded in RAM weights of the neural network
        rnn_enc_output_dim : int
            The output dimension of the RNN encoder
        original_input_dim : int
            The original input dimension of the RNN encoder
        reduced_dim : int
            The reduced input dimension of the RNN encoder
        context_length : int
            The length of the scope considered by the RNN encoder
        window_size : int
            The size of the Hamming window applied to mitigate
            the leakage effect when performing the FFT
        fft_size : int
            The size of the FFT window used to compute the
            STFT and extract the magnitudes of the input audios
        hop_size : int
            The number of samples between each successive FFT window
        seq_length : int
            The lenght of the input audio fed to the neural network
        batch_size : int
            The size of the batches fed to the neural network
        top_db : int
            The threshold (in dB) above which
    """

    import os
    import time

    import numpy as np

    import torch
    torch.set_grad_enabled(False)

    from utils.data_utils import data_generator_testing, data_process_results_testing
    from utils.settings import debug, usage_output_string_per_example, \
        usage_output_string_total
    from madtwinnet.modules import RNNEnc, RNNDec, FNNMasker, FNNDenoiser

    # extract sampling rate and filename from the input tuple
    sr = d00sr01_f1[0][1]
    filename = d00sr01_f1[1]

    if debug:
        print('\n-- Cannot proceed in debug mode.',
              'Please set debug=False at the settings file.')
        print('-- Exiting.')
        exit(-1)
    print('\nNow MaD TwinNet will extract the voice from the provided files')

    # set the code to run on a GPU if present
    device = 'cuda' if not debug and torch.cuda.is_available() else 'cpu'

    # masker modules
    rnn_enc = RNNEnc(reduced_dim, context_length, debug)
    rnn_dec = RNNDec(rnn_enc_output_dim, debug)
    fnn = FNNMasker(rnn_enc_output_dim, original_input_dim, context_length)

    # denoiser modules
    denoiser = FNNDenoiser(original_input_dim)

    rnn_enc.load_state_dict(weights_d0[0])
    rnn_enc.to(device).eval()

    rnn_dec.load_state_dict(weights_d0[1])
    rnn_dec.to(device).eval()

    fnn.load_state_dict(weights_d0[2])
    fnn.to(device).eval()

    denoiser.load_state_dict(weights_d0[3])
    denoiser.to(device).eval()

    mix, mix_magnitude, mix_phase = data_generator_testing(
        window_size=window_size,
        fft_size=fft_size,
        hop_size=hop_size,
        seq_length=seq_length,
        context_length=context_length,
        batch_size=batch_size,
        d00sr01_f1=d00sr01_f1
    )

    print('Let\'s go!\n')
    total_time = 0

    s_time = time.time()
    # initialize vocals array
    vocals_mag = np.zeros(
        (
            mix_magnitude.shape[0],
            seq_length - context_length * 2,
            window_size
        ),
        dtype=np.float32
    )

    # inference loop
    for batch in range(int(mix_magnitude.shape[0] / batch_size)):
        b_start = batch * batch_size
        b_end = (batch + 1) * batch_size

        # forward pass
        v_in = torch.from_numpy(mix_magnitude[b_start:b_end, :, :]).to(device)
        tmp_voice_predicted = rnn_enc(v_in)
        tmp_voice_predicted = rnn_dec(tmp_voice_predicted)
        tmp_voice_predicted = fnn(tmp_voice_predicted, v_in)
        tmp_voice_predicted = denoiser(tmp_voice_predicted)

        # get the magnitude of the resulting vocals
        vocals_mag[b_start:b_end, :, :] = tmp_voice_predicted.data.cpu().numpy()

    # recover the audio from the magnitudes
    _, vocals_wav_split = data_process_results_testing(
        voice_predicted=vocals_mag,
        window_size=window_size,
        mix_magnitude=mix_magnitude,
        mix_phase=mix_phase,
        hop=hop_size,
        context_length=context_length,
        top_db=top_db
    )

    e_time = time.time()

    # report time taken
    print(
        usage_output_string_per_example.format(
        f=os.path.basename(filename),
        t=e_time - s_time
        )
    )
    total_time += e_time - s_time
    print(usage_output_string_total.format(t=total_time))

    return (None, None, vocals_wav_split, sr), filename

# since PySpark needs an iter() object, we create a wrapper method for that purpose
def P_isolate_vocals(
        P_d00sr01_f1, weights_d0,
        rnn_enc_output_dim,
        original_input_dim,
        reduced_dim,
        context_length,
        window_size,
        fft_size,
        hop_size,
        seq_length,
        batch_size,
        top_db
):

    P_out = []

    for d00sr01_f1 in P_d00sr01_f1:
        P_out.append(
            isolate_vocals(
                d00sr01_f1,
                weights_d0,
                rnn_enc_output_dim,
                original_input_dim,
                reduced_dim,
                context_length,
                window_size,
                fft_size,
                hop_size,
                seq_length,
                batch_size,
                top_db
            )
        )

    return iter(P_out)


import streamlit as st
import pandas as pd
import gig_scraper
def local_css(file_name):
    with open(file_name) as f:
        st.markdown(f'<style>{f.read()}</style>', unsafe_allow_html=True)

def remote_css(url):
    st.markdown(f'<link href="{url}" rel="stylesheet">', unsafe_allow_html=True)

def icon(icon_name):
    st.markdown(f'<i class="material-icons">{icon_name}</i>', unsafe_allow_html=True)
@st.cache
def get_df(v):
    try:
	    tor_process=gig_scraper.start_tor()
    except OSError:
	    pass
    return(gig_scraper.craigslist_gig_scrape(v))

def next(df):
    if st.checkbox("Remove Duplicate Titles"):
            df=df.drop_duplicates(subset=['titles'],keep="first")
            df=df.reset_index()
            #df=df.drop(['timestamp'], axis = 1)
            st.write(df)
            icon("search")
            selected = st.text_input("", "Search...")
            if st.button("OK"):
                df=df[df['titles'].str.contains(selected)]
                L=df.to_html(escape=False)
                st.write(L, unsafe_allow_html=True)
def main():
    #local_css("style.css")
    st.title('Craigslist Gig Scraper')

    remote_css('https://fonts.googleapis.com/icon?family=Material+Icons')
    values=("/d/computer-gigs/search/cpg","/d/creative-gigs/search/crg","/d/crew-gigs/search/cwg","/d/domestic-gigs/search/dmg","/d/event-gigs/search/evg","/d/labor-gigs/search/lbg","/d/talent-gigs/search/tlg","/d/writing-gigs/search/wrg")
    options=("Computer","Creative","Crew","Domestic","Event","Labor","Talent","Writing")
    value=st.sidebar.selectbox("Gig Type",options)
    if st.sidebar.checkbox("Start"):
	    v=values[options.index(value)]
	    df=get_df(v)
	    df
	    next(df)
if __name__ == '__main__':
	main()


import spacy
nlp = spacy.load("en_core_web_lg")
class FileCleaning:
    def __init__(self,filename):
        self.filename = filename

    def filtering(self):
        with open(self.filename) as f:
            lines = f.readlines()
        for line in range(0,len(lines)):
            lines[line] = lines[line].strip().strip("-")
        cancel = []
        changes = []
        current = ""
        for line in lines:
            if line=="CHANGES":
                current = "changes"
            elif line=="CANCELLATIONS":
                current = "cancel"
            if current == "changes":
                changes.append(line)
            elif current == "cancel":
                cancel.append(line)
            else:
                continue
        cancel = '\n'.join(cancel).replace('HR', 'HOURS')
        cancel = cancel.replace('HRS', "HOURS")
        changes = '\n'.join(changes).replace('HR', 'HOURS')
        changes = changes.replace('HRS', "HOURS")
        return [cancel,changes]

    def cancellation_split(self):
        file_rules = self.filtering()[0]
        rules = file_rules.split("\n\n")
        return rules

    def changes_split(self):
        file_rules = self.filtering()[1]
        rules = file_rules.split("\n\n")
        return rules

    def decision_making_cancellation(self):
        decision_1 = nlp("REFUNDABLE")
        decision_2 = nlp("NON REFUNDABLE")
        decision_rules = self.cancellation_split()
        result = nlp(str(decision_rules[0]))
        if decision_1.similarity(result) > decision_2.similarity(result):
            return 1
        else:
            return 0

    def decision_making_changes(self):
        decision_1 = nlp("CHANGES PERMITTED")
        decision_2 = nlp("CHNAGES NOT PERMITTED")
        decision_rules = self.changes_split()
        for rule in decision_rules:
            doc = nlp(str(rule))
            if decision_1.similarity(doc) < decision_2.similarity(doc):
                print(doc, " => ", "CHNAGES PERMITTED\n\n")

            else:
                print(doc, " => ", "NO CHNAGES PERMITTED\n\n")

# filerules = FileCleaning("DELBOM-AI-INR.txt")
# rules = filerules.changes_split()[0]
# print(rules)