'''
	Use early stopping with this condition , if the testset accuracy doesnt increase for a certain number of epochs = stop(store max accuracy and compare)

	References Used :
		https://www.coursera.org/learn/machine-learning/supplement/pjdBA/backpropagation-algorithm
		http://neuralnetworksanddeeplearning.com/chap1.html

	IMPORTANT : for speedup its necessary that  the input data is in the required format, 1 column vector otherwise it has to be converted into that form again and again in each epoch which is a redundant operation and takes a lot of time
				(this was a huge speedup from 250s to 50 s for each epoch)
				converted formats for input of train_data,test_data and valid_data and output of train_data

				Second speed up(in setup part) : use np.reshape(x,(784,1)) when you know the size of x instead of np.vstack, reshape is lot faster: np.vstack has a completely different purpose, it is used to add a row to an array

	The only difference in cross entropy and mean square error is that in mse in (INITIALLY for last layer weights)delta = np.subtract( self.activations[-1],output)*der_act(here derivative is multiplied so learning(i.e weight update) slows down as derivative becomes 0) whereas in ce #delta = np.subtract( self.activations[-1],output) --- > refer : http://neuralnetworksanddeeplearning.com/chap3.html#the_cross-entropy_cost_function
	Ref : https://github.com/cazala/synaptic/issues/33

	Caution : At some points ,code gives :  RuntimeWarning: overflow encountered in exp
  return 1.0 / (1.0 + np.exp(-z) ) ---> This is probably because the value of -z becomes too negative and hence it creates problem for exp function, try to scale down hyperparameters to avoid this
	MORE INSIGHT ON THIS ERROR : https://stats.stackexchange.com/questions/210726/neural-network-can-only-follow-increasing-function

	Step Size = Learning Rate , EXTRA Addition : L2 Regularisation(Proabably it helps in correcting the Runtime error due to over shooting of weights)(BUT RESULTS WERE BETTER WITHOUT ANY REGULARISATION)
	DEFN : An epoch means one iteration over all of the training data. For instance if you have 20,000 images and a batch size of 100 then the epoch should contain 20,000 / 100 = 200 steps.

'''

import numpy as np
#Python 3 , in Python 2 the library is named cPickle
import pickle
import gzip
import time
import math
import os

import operator
import random
random.seed(1234)
np.random.seed(4567)

###################################	Code For Argument Parse	#######################################################

import argparse

loss_function = {'ce': 0, 'sq': 1}
optimization_algorithm = {'gd': 0,'momentum': 1,'nag': 2, 'adam': 3}
activation_function  = {'sigmoid': 0, 'tanh': 1}

parser = argparse.ArgumentParser()
parser.add_argument('--lr', help='initial learning rate for gradient descent algorithms',default = 0.25, type = float)
parser.add_argument("--momentum", help="momentum to be used by momentum based algorithms",default = 0, type = float)
parser.add_argument('--num_hidden', help='number of hidden layers',default = 1, type = int)
parser.add_argument('--sizes', help='list of neurons in each hidden layer',default = [30], type = int, nargs = '*' )
parser.add_argument('--activation', help='choice of activation function',default = 'sigmoid', choices = activation_function)
parser.add_argument('--loss', help=' loss function to be used by optimization algorithm ',default = 'ce',choices = loss_function)
parser.add_argument('--opt', help='optimization algorithm',default = 'gd', choices=['gd','momentum','nag', 'adam'])
parser.add_argument('--batch_size', help='batch size to be used',default = 100, choices = [1] + [i for i in range(5,50005,5)], type = int)
parser.add_argument('--anneal', help='to half the learning rate when validation loss decreases in an epoch',default = False , type = bool)
parser.add_argument('--save_dir', help='dir in which pickled model(weights and biases of network) will be saved',default = os.path.abspath(os.curdir))
parser.add_argument('--expt_dir', help='dir in which log files will be stores',default = os.path.abspath(os.curdir))
parser.add_argument('--mnist', help='path to mnist data in pcikled format',default = os.path.abspath(os.curdir))
parser.add_argument('--epoch', help='maximum number of epochs',default = 5 ,type = int)

args = parser.parse_args()

args.loss = loss_function[args.loss]
args.opt = optimization_algorithm[args.opt]
args.activation = activation_function[args.activation]

###################################################################################################################
file_names = ['/log_loss_train.txt','/log_err_train.txt' , '/log_loss_valid.txt', '/log_err_valid.txt', '/log_loss_test.txt' , '/log_err_test.txt']

openfiles = []
for i in range(6):
	file_path = args.save_dir + file_names[i]
	file = open(file_path, 'a')
	openfiles.append(file)

############################################## Class ##################################################################33

class NN(object):

	'''
		INIT fuction, nodes in layers is an aray containing the number of neurons in each layer
		nodesInLayers = [784,15,10]
	'''
	def __init__(self, nodesInLayers, momentum, activation_function_value, loss_function_value ):  # WHY USING MOMENTUM OF VALUE 1 GAVE SATURATION AROUND 900 correct detections
			# the value of momentum makes things a bit uncertain, keep it below 0.9

		self.layers = len(nodesInLayers)
		self.sizes = nodesInLayers 		#total number of units(neurons) in each layer, excluding the biases
		self.bias = [np.random.randn(y,1) for y in nodesInLayers[1:] ]	 # bias node of current level will have weights for each neuron(node/unit) of the next layer
		self.weights = [np.random.randn(y,x) for x,y in zip(nodesInLayers,nodesInLayers[1:])] # weights matrix for all nodes between successive layers(except bias weights)
		self.activations = []
		self.activation_function_value = activation_function_value #( 0 mean sigmoid, 1 mean tanh)
		self.loss_function_value = loss_function_value #( 0 means cross entropy , 1 means mean square error)
		self.momentum_coefficient = momentum

		#Load Data
		f = gzip.open(args.mnist + '/mnist.pkl.gz', 'rb')
		self.train_set, self.valid_set, self.test_set = pickle.load(f, encoding='latin1')
		# self.train_set = self.valid_set
		f.close()

		print('Data Values for arguments')
		print('nodesInLayers = ',nodesInLayers)
		print('mometum = ',momentum)
		print('activation function = ',activation_function_value)
		print('loss function = ',loss_function_value)
		print('Anneal = ',args.anneal)

	def setup(self):
		self.test_set = self.format_data(self.test_set)
		self.valid_set = self.format_data(self.valid_set)
		self.train_set = self.format_data_train(self.train_set)
		# self.train_set = self.train_set[:10000]

	def format_data(self,data_set):
		n = len(data_set[0])
		x,y = data_set			#data_set is a tuple, x contains 2d image pixel representaion , y contains integer for that representation
		data_set = [(np.reshape(a,(784,1)),b) for a,b in zip(x,y) ]
		return data_set

	def format_data_train(self,data_set):
		n = len(data_set[0])
		x,y = data_set			#data_set is a tuple, x contains 2d image pixel representaion , y contains integer for that representation
		data_set = [(np.reshape(a,(784,1)), np.reshape(self.vectorized_result(b) , (10,1) ) ) for a,b in zip(x,y) ]
		return data_set

	def vectorized_result(self,j):
		"""Return a 10-dimensional unit vector with a 1.0 in the jth
		position and zeroes elsewhere.  This is used to convert a digit
		(0...9) into a corresponding desired output from the neural
		network(one hot encoding)."""
		e = np.zeros((10, 1))
		e[j] = 1.0
		return e

	def activation_function(self,z, der = False):
		if der == True :
			if self.activation_function_value == 0 :
				return self.sigmoid(z,True)
			else:
				return self.tanh(z,True)

		elif self.activation_function_value == 0 :
			return self.sigmoid(z)
		else:
			return self.tanh(z)

	def sigmoid(self,z,der = False):
		if(der == True) :
			return z * (1 - z)
		else :
			return 1.0 / (1.0 + np.exp(-z) )

	def tanh(self,z,der = False):
		if(der == True) :
			return (1 - (z*z) )
		else :
			pos_z = np.exp(z)
			neg_z = np.exp(-z)
			return (pos_z - neg_z)/ ( pos_z + neg_z)

	''' Gradient Descent
		It's actually Momentum, but converts to SGD when momentum coefficient = 0
	'''
	def SGD(self,epoch,eta,mini_batch_size,lmbda = 0):
		# eta is learning rate

		#print("bias\n",self.bias[-1])
		self.learning_rate = eta

		t = 0
		prev_min = 1000000000 			  # infinite value, should use infinity designed inside python, this is just a workaround

		while t < epoch:
			trainining_loss = 0
			# number of examples in training data
			n = len(self.train_set)
			velocity_weight = [np.zeros(w.shape) for w in self.weights]
			velocity_bias = [np.zeros(b.shape) for b in self.bias]

			start_time = time.time()

			# random.shuffle(self.train_set)

			mini_batches = [self.train_set[k:k+mini_batch_size]
				for k in range(0, n, mini_batch_size)]

			min_validation_loss_in_epoch = 1000000000

			for step,mini_batch in enumerate(mini_batches):
				delta_cost = [np.zeros(w.shape) for w in self.weights]  # Matrix to store Patrial Derivatices of cost function w.r.t weights
				delta_bias_cost = [np.zeros(b.shape) for b in self.bias]

				for i in range( len(mini_batch) ):

					x,y = mini_batch[i][0], mini_batch[i][1]

					#Forward Propagation
					self.forwardprop(x)

					#Back Propagation
					delta_cost_temp,delta_bias_cost_temp = self.backprop(x,y)
					delta_cost = [dc + dct for dc,dct in zip(delta_cost,delta_cost_temp)]
					delta_bias_cost = [dbc + dbct for dbc,dbct in zip(delta_bias_cost,delta_bias_cost_temp)]

					if self.activation_function_value == 0 :
						trainining_loss = trainining_loss - np.argmax(y)*math.log(self.activations[-1][np.argmax(y)])
					else:
						trainining_loss += 0.5*np.sum(np.square(y - self.activations[-1]))	#mean square error loss

				velocity_weight = np.multiply(self.momentum_coefficient , velocity_weight )  # here old value of velocity is used
				velocity_bias = np.multiply(self.momentum_coefficient , velocity_bias )

				velocity_weight = velocity_weight  +  (eta/mini_batch_size) * np.array(delta_cost)		# here old value is used to calculate new value
				velocity_bias = velocity_bias +  (eta/(mini_batch_size)) * np.array( delta_bias_cost)

				# ADDING L2 REGULARISATION
				self.weights = np.subtract(np.multiply( 1 - (eta * lmbda/n), self.weights), velocity_weight)

				self.bias = np.subtract(self.bias,  velocity_bias)

				# TEST OF ANNEALING

				#STORING VALUES
				if(step + 1) % 100 == 0:

					#Store error and loss values for each data set
					self.print_log(t,step+1)

					# # Store Weights and biases
					self.store_weights()

			if(args.anneal):
				_,loss = self.evaluate(self.valid_set)
				if min_validation_loss_in_epoch > loss :
					min_validation_loss_in_epoch = loss

			if min_validation_loss_in_epoch - prev_min > 1:
				print('Minimum validation loss in last epoch is greater than minimum validation loss in this epoch --- Annealing Learning Rate')
				self.learning_rate = self.learning_rate/2
				print('New LR = ',self.learning_rate)
				continue

			print("On Test Set : accuracy = %d/10000 & loss = %f" %(self.evaluate() ))

			# USE THIS TO FIND OUT ETA(LEARNING RATE / STEP SIZE)
			print("Training Cost/Loss = ",(1/n)*trainining_loss + (lmbda/(2*n)) * (sum(np.sum(np.multiply(self.weights[i],self.weights[i]) ) for i in range(self.layers-1)  ) ) )

			# print("On Train Set : accuracy = %d/10000 & loss = %f" %(self.evaluate(self.train_set) ))
			print("time for epoch no.",(t+1),"=",time.time()-start_time )

			prev_min = min_validation_loss_in_epoch
			t = t + 1

	def NAG(self,epoch,eta,mini_batch_size,lmbda=0):	#Nesterov Accelerated Gradient
		# eta is learning rate

		#print("bias\n",self.bias[-1])

		self.learning_rate = eta
		t=0
		prev_min = 1000000000
		while t < epoch:
			trainining_loss = 0
			n = len(self.train_set)
			start_time = time.time()

			random.shuffle(self.train_set)

			mini_batches = [self.train_set[k:k+mini_batch_size]
				for k in range(0, n, mini_batch_size)]

			velocity_weight = [np.zeros(w.shape) for w in self.weights]
			velocity_bias = [np.zeros(b.shape) for b in self.bias]

			trainining_loss = 0
			min_validation_loss_in_epoch = 1000000000

			for step,mini_batch in enumerate(mini_batches):
				delta_cost = [np.zeros(w.shape) for w in self.weights]  # Matrix to store Patrial Derivatices of cost function w.r.t weights
				delta_bias_cost = [np.zeros(b.shape) for b in self.bias]

				velocity_weight = np.multiply(self.momentum_coefficient , velocity_weight )  # here old value of velocity is used
				velocity_bias = np.multiply(self.momentum_coefficient , velocity_bias )

				#Update weights before finding gradient(Weight_Look_ahead)
				self.weights = np.subtract(self.weights, velocity_weight)
				self.bias = np.subtract(self.bias,  velocity_bias)

				for i in range( len(mini_batch) ):

				#	print(i)
					x,y = mini_batch[i][0], mini_batch[i][1]

					#Forward Propagation
					self.forwardprop(x)
				#	print("after forward prop")

					#Back Propagation
					delta_cost_temp,delta_bias_cost_temp = self.backprop(x,y)
					delta_cost = [dc + dct for dc,dct in zip(delta_cost,delta_cost_temp)]
					delta_bias_cost = [dbc + dbct for dbc,dbct in zip(delta_bias_cost,delta_bias_cost_temp)]
					if self.activation_function_value == 0 :
						trainining_loss = trainining_loss - np.argmax(y)*math.log(self.activations[-1][np.argmax(y)])
					else:
						trainining_loss += 0.5*np.sum(np.square(y - self.activations[-1]))	#mean square error loss

				# REVERT BACK TO ORIGINAL Weights after FINDING GRADIENT
				self.weights = np.add(self.weights, velocity_weight)
				self.bias = np.add(self.bias,  velocity_bias)

				velocity_weight = velocity_weight  + (eta/(mini_batch_size)) * np.array(delta_cost)		# here old value is used to calculate new value
				velocity_bias = velocity_bias + (eta/(mini_batch_size)) * np.array( delta_bias_cost)

				self.weights = np.subtract(np.multiply( 1 - (eta * lmbda/n), self.weights),  velocity_weight)

				self.bias = np.subtract(self.bias,velocity_bias)

				#STORING VALUES
				if(step + 1) % 100 == 0:

					#Store error and loss values for each data set
					# self.print_log(t,step+1)

					# Store Weights and biases
					self.store_weights()

			if(args.anneal):
				_,loss = self.evaluate(self.valid_set)
				if min_validation_loss_in_epoch > loss :
					min_validation_loss_in_epoch = loss

			if min_validation_loss_in_epoch - prev_min > 1:
				print('Minimum validation loss in last epoch is greater than minimum validation loss in this epoch --- Annealing Learning Rate')
				self.learning_rate = self.learning_rate/2
				continue

			print("On Test Set : accuracy = %d/10000 & loss = %f" %(self.evaluate() ))

			# USE THIS TO FIND OUT ETA(LEARNING RATE / STEP SIZE)
			print("Traingin Cost/Loss = ",(1/n)*trainining_loss + (lmbda/(2*n)) * (sum(np.sum(np.multiply(self.weights[i],self.weights[i]) ) for i in range(self.layers-1)  ) ) )

			# print("On Train Set : accuracy = %d/10000 & loss = %f" %(self.evaluate(self.train_set) ))
			print("time for epoch no.",(t+1),"=",time.time()-start_time )

			prev_min = min_validation_loss_in_epoch
			t = t + 1

	def ADAM(self,epoch,eta,mini_batch_size,lmbda = 0, beta1 = 0.9, beta2 = 0.999, epsilon = 1e-8):
		# eta is learning rate
		self.learning_rate = eta
		#print("bias\n",self.bias[-1])
		t=0
		prev_min = 1000000000

		while t < epoch:
			trainining_loss = 0
			min_validation_loss_in_epoch = 1000000000
			# number of examples in training data
			n = len(self.train_set)

			start_time = time.time()

			random.shuffle(self.train_set)

			mini_batches = [self.train_set[k:k+mini_batch_size]
				for k in range(0, n, mini_batch_size)]

			final_grad = [np.zeros(w.shape) for w in self.weights]
			velocity_weight = [np.zeros(w.shape) for w in self.weights]
			beta1_exp = 1.0   # increasing precision is important as we are getting overflow , but it also slows down computation, it was faster t increase batch size than precision
			beta2_exp = 1.0

			for step,mini_batch in enumerate(mini_batches):
				delta_cost = [np.zeros(w.shape) for w in self.weights]  # Matrix to store Patrial Derivatices of cost function w.r.t weights
				delta_bias_cost = [np.zeros(b.shape) for b in self.bias]

				for i in range( len(mini_batch) ):

				#	print(i)
					x,y = mini_batch[i][0], mini_batch[i][1]

					#Forward Propagation
					self.forwardprop(x)
				#	print("after forward prop")

					#Back Propagation
					delta_cost_temp,delta_bias_cost_temp = self.backprop(x,y)
					delta_cost = [dc + dct for dc,dct in zip(delta_cost,delta_cost_temp)]
					delta_bias_cost = [dbc + dbct for dbc,dbct in zip(delta_bias_cost,delta_bias_cost_temp)]

					if self.activation_function_value == 0 :
						trainining_loss = trainining_loss - np.argmax(y)*math.log(self.activations[-1][np.argmax(y)])
					else:
						trainining_loss += 0.5*np.sum(np.square(y - self.activations[-1]))	#mean square error loss

				beta1_exp *= beta1
				beta2_exp *= beta2
				delta_bias_cost = (1/mini_batch_size) * np.array(delta_bias_cost)
				delta_cost = np.array(delta_cost)
				velocity_weight = np.array(velocity_weight)
				# velocity_weight = np.array(velocity_weight)
				delta_cost = (1/(mini_batch_size) ) * np.array(delta_cost)

				final_grad = beta1 * np.array(final_grad )+ (1 - beta1)*np.array(delta_cost)
				velocity_weight = beta2 * velocity_weight + (1 - beta2)*(np.multiply(delta_cost,delta_cost))

				final_grad = np.array(final_grad)/(1-beta1_exp)
				velocity_weight = np.array(velocity_weight )/(1-beta2_exp)

				# print(np.sqrt(velocity_weight[i] for i in range(len(velocity_weight[0]))) )
				# ADDING L2 REGULARISATION
				for i in range(len(self.weights)) :
					self.weights[i] = np.subtract(np.multiply( 1 - (eta * lmbda/n), self.weights[i]), (eta/(np.sqrt(velocity_weight[i]) + epsilon))*final_grad[i] )

				self.bias = np.subtract(self.bias, (eta) *np.array( delta_bias_cost) )

				#STORING VALUES
				if(step + 1) % 100 == 0:

					#Store error and loss values for each data set
					# self.print_log(t,step+1)
					pass
					# # Store Weights and biases
					# self.store_weights()

			if(args.anneal):
				_,loss = self.evaluate(self.valid_set)
				if min_validation_loss_in_epoch > loss :
					min_validation_loss_in_epoch = loss

			print("accuracy = %d/10000 & loss = %f" %(self.evaluate() ))
			print("accuracy = %d/10000 & loss = %f" %(self.evaluate(self.test_set) ))
			print("Traingin Cost/Loss = ",(1/n)*trainining_loss + (lmbda/(2*n)) * (sum(np.sum(np.multiply(self.weights[i],self.weights[i]) ) for i in range(self.layers-1)  ) ) )
			print("time for epoch no.",(t+1),"=",time.time()-start_time )

			if min_validation_loss_in_epoch - prev_min > 1:
				print('Minimum validation loss in last epoch is greater than minimum validation loss in this epoch --- Annealing Learning Rate')
				self.learning_rate = self.learning_rate/2
				print('lr = ',self.learning_rate)
				prev_min = min_validation_loss_in_epoch
				continue

			prev_min = min_validation_loss_in_epoch
			t = t + 1

	def store_weights(self):
		path_weights = args.save_dir + '/weights.npy'
		file = open(path_weights,'a')   # Trying to create a new file or open one
		file.close()
		path_bias = args.save_dir + '/bias.npy'
		file = open(path_bias,'a')   # Trying to create a new file or open one
		file.close()
		np.save(path_weights,self.weights)
		np.save(path_bias,self.bias)

	def print_log(self,current_epoch,current_step):

		error, loss = self.evaluate(self.train_set)
		openfiles[0].write('Epoch %d, Step %d, Loss: %f, lr: %f\n'%(current_epoch,current_step,loss, self.learning_rate))
		openfiles[1].write('Epoch %d, Step %d, Error: %.2f, lr: %f\n'%(current_epoch,current_step,round((50000 - error)/500,2), self.learning_rate))

		error, loss  = self.evaluate(self.valid_set)
		openfiles[2].write('Epoch %d, Step %d, Loss: %f, lr: %f\n'%(current_epoch,current_step,loss, self.learning_rate))
		openfiles[3].write('Epoch %d, Step %d, Error: %.2f, lr: %f\n'%(current_epoch,current_step,round((10000 - error)/100,2), self.learning_rate))

		error, loss = self.evaluate(self.test_set)
		openfiles[4].write('Epoch %d, Step %d, Loss: %f, lr: %f\n'%(current_epoch,current_step,loss, self.learning_rate))
		openfiles[5].write('Epoch %d, Step %d, Error: %.2f, lr: %f\n'%(current_epoch,current_step,round((10000 - error)/100,2), self.learning_rate))

		# file_path = args.save_dir + file_names[0]
		# with open(file_path, 'a') as f:
		# 	print('Epoch %d, Step %d, Loss: %f, lr: %f'%(current_epoch,current_step,loss, self.learning_rate) ,file=f )
		# file_path = args.save_dir + file_names[1]
		# with open(file_path, 'a') as f:
		# 	print('Epoch %d, Step %d, Error: %.2f, lr: %f'%(current_epoch,current_step,round((50000 - error)/500,2), self.learning_rate),file=f )

		# file_path = args.save_dir + file_names[2]
		# with open(file_path, 'a') as f:
		# 	print('Epoch %d, Step %d, Loss: %f, lr: %f'%(current_epoch,current_step,loss, self.learning_rate) ,file=f )
		# file_path = args.save_dir + file_names[3]
		# with open(file_path, 'a') as f:
		# 	print('Epoch %d, Step %d, Error: %.2f, lr: %f'%(current_epoch,current_step,round((10000 - error)/100,2), self.learning_rate),file=f )

		# file_path = args.save_dir + file_names[4]
		# with open(file_path, 'a') as f:
		# 	print('Epoch %d, Step %d, Loss: %f, lr: %f'%(current_epoch,current_step,loss, self.learning_rate) ,file=f )
		# file_path = args.save_dir + file_names[5]
		# with open(file_path, 'a') as f:
		# 	print('Epoch %d, Step %d, Error: %.2f, lr: %f'%(current_epoch,current_step,round((10000 - error)/100,2), self.learning_rate),file=f )

# def SGD(self,epoch,eta,mini_batch_size):
# 		# eta is learning rate

# 		#print("bias\n",self.bias[-1])
# 		self.epoch = epoch
# 		index = 0

# 		for t in range(self.epoch):

# 			n = len(self.train_set)

# 			start_time = time.time()

# 			random.shuffle(self.train_set)

# 			mini_batches = [self.train_set[k:k+mini_batch_size]
# 				for k in range(0, n, mini_batch_size)]

# 			for mini_batch in mini_batches:
# 				delta_cost = [np.zeros(w.shape) for w in self.weights]  # Matrix to store Patrial Derivatices of cost function w.r.t weights
# 				delta_bias_cost = [np.zeros(b.shape) for b in self.bias]
# 			# for num_iter in range(len(self.train_set[0]) // batch_size):
# 			# 	listL=[]
# 			# 	listR=[]
# 			# 	for k in range(batch_size):
# 			# #		index = randint(0,len(self.train_set[0] - 1) )  // This method was not good as some training examples might always get skipped, this will make the hypotheses fuction inacurate

# 			# 		listL.append(self.train_set[0][index])
# 			# 		listR.append(self.train_set[1][index])
# 			# 		index = (index + 1) % len(self.train_set[0])
# 			# 		# mini_batch[0][k] =   self.train_set[0][index]
# 			# 		# mini_batch[1][k] =   self.train_set[1][index]
# 			# 	mini_batch = (listL,listR)

# 				for i in range( len(mini_batch) ):

# 				#	print(i)
# 					x,y = mini_batch[i][0], mini_batch[i][1]

# 					#Forward Propagation
# 					self.forwardprop(x)
# 				#	print("after forward prop")

# 					#Back Propagation
# 					delta_cost_temp,delta_bias_cost_temp = self.backprop(x,y)
# 					delta_cost = [dc + dct for dc,dct in zip(delta_cost,delta_cost_temp)]
# 					delta_bias_cost = [dbc + dbct for dbc,dbct in zip(delta_bias_cost,delta_bias_cost_temp)]

# 				self.weights = np.subtract(self.weights, (eta/(mini_batch_size)) *np.array(delta_cost) )
# 				self.bias = np.subtract(self.bias, (eta/(mini_batch_size)) *np.array( delta_bias_cost) )

# 				#print("bias\n",delta_bias_cost)
# 			print(self.evaluate())
# 			print("time for epoch no.",(t+1),"=",time.time()-start_time )

	def forwardprop(self,input):
		self.activations = []

		self.activations.append(input)
		for i in range(self.layers - 1):
			activation = self.activation_function(np.add( np.dot(self.weights[i],self.activations[i]) , self.bias[i]) )
			self.activations.append(activation)

	def backprop(self,input,output):

		''' Finding Partial Derivatives of Cost Function(der_cost or CAPITAL DELTA) '''
		der_cost = [np.zeros(w.shape) for w in self.weights]  # Matrix to store Patrial Derivatices of cost function w.r.t weights
		der_cost_bias = [np.zeros(b.shape) for b in self.bias]
		der_act = self.activation_function(self.activations[-1],True)

		if self.loss_function_value == 1 :
			delta = np.multiply(np.subtract( self.activations[-1],output),der_act)	  # activations[-1] gives last element of activation that is output layer values
			# print('derivative = ',der_act)
		else :
			delta = np.subtract( self.activations[-1],output)

		for t in range(self.layers-1,0,-1):
			der_cost_bias[t-1] = delta
			der_cost[t-1] =   np.dot(delta, np.transpose(self.activations[t-1])) 	# this delta is of layer t+1, that for the first case of the output layer , there is no delta for input layer there there is not error in activation value of input layer which is the input values itself

			#check this change
			der_act = self.activation_function(self.activations[t-1],True)			# derivative of activation vector of layer
			delta = np.multiply ( np.dot( np.transpose(self.weights[t-1]), delta) , der_act )		 # (small)delta of previous layer

		return (der_cost,der_cost_bias)

	def evaluate(self,evaluation_set = None):
		"""Return the number of test inputs for which the neural
		network outputs the correct result. Note that the neural
		network's output is assumed to be the index of whichever
		neuron in the final layer has the highest activation."""

		if evaluation_set == None:
			evaluation_set = self.valid_set

		# random.shuffle(evaluation_set)
		evaluation_set_loss = 0
		test_results = []
		for i in range(len(evaluation_set)):
			if(evaluation_set == self.train_set):
				x,y = evaluation_set[i][0],np.argmax(evaluation_set[i][1])
			else:
				x,y = evaluation_set[i][0],evaluation_set[i][1]
			self.forwardprop(x)

			test_results.append( (np.argmax(self.activations[-1]), y )	)	#argmax returns indices

			if self.activation_function_value == 0 :
				evaluation_set_loss = evaluation_set_loss - y*math.log(self.activations[-1][y])
			else:
				evaluation_set_loss += 0.5*(np.square(y - self.activations[-1][y]))	#mean square error loss

		evaluation_set_loss = (1/len(evaluation_set) ) * evaluation_set_loss

		return (sum(int(x == y) for (x, y) in test_results), evaluation_set_loss)

def start():
	nodesInLayers =[784] + args.sizes + [10]

	net = NN(nodesInLayers,args.momentum, args.activation, args.loss)

	net.setup()
	opt_algo = args.opt

	epoch = args.epoch

	if opt_algo == 0:		# JUST FOR safe check
		args.momentum = 0

	if opt_algo == 3:
		net.ADAM(epoch, args.lr, args.batch_size)
	elif opt_algo == 2:
		net.NAG(epoch, args.lr, args.batch_size)
	else:
		net.SGD(epoch, args.lr, args.batch_size)		# for 0 and 1 , 0 = SGD , 1 = MOMENTUM

	for i in range(6):
		openfiles[i].close()

	file = open(args.expt_dir + '/valid_predictions.txt','w')
	for x,_ in net.valid_set:
		net.forwardprop(x)
		file.write( '%d \n' %(np.argmax(net.activations[-1])) )
	file.close()

	file = open(args.expt_dir + '/test_predictions.txt','w')
	for x,_ in net.test_set:
		net.forwardprop(x)
		file.write( '%d \n' %(np.argmax(net.activations[-1])) )
	file.close()

if __name__ == "__main__":

	start()


import imageio
import matplotlib.pyplot as plt
from skimage.segmentation import mark_boundaries
from skimage.color import label2rgb, rgb2gray
import numpy as np

im = imageio.imread('Elbphilharmonie.jpg')
# k-means
im_gray = rgb2gray(im)

# Flatten the image to obtain list of points
im_gray_flat = im_gray.flatten()

# parameters
k = 100
change_threshold = 0.0001 # tolerance for changes in cluster centers to detect changes

# 1. randomly choose cluster centers
centers = np.random.choice(im_gray_flat, size=k)

# we want to use NumPy broadcasting in the loop below, so let's reshape the arrays to be able to do that
im_gray_flat = im_gray_flat.reshape((im_gray_flat.size, 1)) # to [N, 1]
centers = centers.reshape((1,k)) # to [1, k]

changed = True
while changed:
	# 2. calculate cluster assignments
	dist = np.abs(im_gray_flat - centers) # result is [N, k] thanks to broadcasting
	assignments = np.argmin(dist, axis=1)

	# 3. update cluster centers, see if there are changes
	new_centers = np.zeros((1,k))
	changed = False
	for i in range(k):
		assigned_to_cluster_i = assignments == i # boolean array
		new_centers[0,i] = np.average(im_gray_flat[assigned_to_cluster_i])
		if np.abs(new_centers[0,i] - centers[0,i]) > change_threshold:
			changed = True

		# assign new centers
		centers = new_centers

# Visualization
assignments_img = assignments.reshape(im_gray.shape)
labels_kmeans_colored = label2rgb(assignments_img, im, kind='overlay')

fig, ax = plt.subplots(ncols=2, figsize=(12,12))
ax[0].imshow(im_gray, cmap='gray')
ax[1].imshow(labels_kmeans_colored)
plt.savefig('kmeans_elbphi.png', bbox_inches='tight')
plt.show()


# coding: utf-8

# In[1]:

import random
import sys

# In[2]:

def prep_data(*files):			#loading the data from the files
	data = []

	for file in files:
		in_file = open(file, 'rt')
		text = in_file.readlines()
		in_file.close()

		for line in text:
			#spliting text by whitespaces
			sentence = line.split()

			#shrinking vocab size by lowercase
			sentence = [word.lower() for word in sentence]

			#using string library to remove punctuations
			#as used by Jason Brownlee
			import string
			table = str.maketrans('', '', string.punctuation)
			sentence = [word.translate(table) for word in sentence]
			data.append(sentence)

	return data

# In[8]:

class NaiveBayes:

	def __init__(self,files, percentile):
		self.data1 = None
		self.training_set = None
		self.test_set = None
		self.document = None
		self.class_list = None
		self.log_priors = None
		self.loglikelihood = None
		self.voc_doc = None
		self.predictions = None
		self.accuracy = None
		self.files = files
		self.split_percentile = percentile

	def load_data(self):			#loading the data from the files
		data = []

		for file in self.files:
			in_file = open(file, 'rt')
			text = in_file.readlines()
			in_file.close()

			for line in text:
				#spliting text by whitespaces
				sentence = line.split()

				#shrinking vocab size by lowercase
				sentence = [word.lower() for word in sentence]

				#using string library to remove punctuations
				#as used by Jason Brownlee
				import string
				table = str.maketrans('', '', string.punctuation)
				sentence = [word.translate(table) for word in sentence]
				data.append(sentence)

		self.data1 = data

	def spliting_dataset(self):			#spliting data into training and test data

		copy = self.data1
		training_data_size = int(len(copy) * self.split_percentile)
		training_set = []

		while len(training_set) < training_data_size:
			index = random.randrange(len(copy))
			training_set.append(copy.pop(index))

		#after appending to the training set, the remaining items in data will be used for testing
		#data being returned is the test data
		self.training_set = training_set
		self.test_set = copy

	def spliting_by_class(self):

		class_separation = {}
		for i in range(len(self.data1)):
			sentence = self.data1[i]
			if sentence[-1] not in class_separation:
				class_separation[sentence[-1]] = []
			class_separation[sentence[-1]].append(sentence[:-1])

		self.document = class_separation
		self.class_list = class_separation.keys()

	def train_naive_bayes(self):			#training naive bayes
														#training document passed is a dictionary

		log_priors = []
		loglikelihood =  {}
		num_doc = sum([len(self.document[tense]) for tense in self.class_list])

		# Creating vocabulary of all words in document
		voc_doc = set()

		for class_given in self.class_list:
			for tenses in self.document[class_given]:
				for word in tenses:
					voc_doc.add(word)

		import math
		from collections import Counter

		for class_given in self.class_list:
			loglikelihood[class_given] = []
			num_class = len(self.document[class_given])

			logprior_class_given = math.log(num_class/num_doc)

			log_priors.append([class_given, round(logprior_class_given,4)])

			#creating list of words in given class
			bigdoc_class = []
			for tenses in self.document[class_given]:
				for word in tenses:
					bigdoc_class.append(word)

			wordcount = Counter(bigdoc_class)

			for word in voc_doc:
				count_in_class_given = wordcount[word]

				loglikelihood_in_class_given_numerator = count_in_class_given + 1
				loglikelihood_in_class_given_denominator = len(bigdoc_class) + len(voc_doc)
				loglikelihood_in_class_given = math.log(loglikelihood_in_class_given_numerator/ loglikelihood_in_class_given_denominator)

				# holding loglikelihoods of each word in each class in a list with index same as class value
				loglikelihood[class_given].append([word, round(loglikelihood_in_class_given,4)])

		self.log_priors = log_priors
		self.loglikelihood = loglikelihood
		self.voc_doc = voc_doc

	def retrieve_loglikelihood(self, loglikelihood_dict,word,class_given):
		likelihood = 0
		list = loglikelihood_dict[class_given]
		for bond in list:
			if bond[0] == word:
				likelihood = bond[1]
		return likelihood

	def retrieve_logprior(self, logprior_list, class_given):
		prior = 0
		for list in logprior_list:
			if list[0] == class_given:
				prior = list[1]
		return prior

	def train(self):
		self.load_data()

		self.spliting_dataset()

		self.spliting_by_class()

		self.train_naive_bayes()

	def getTestSet(self):
		return self.test_set

	def getPredictions(self):
		return self.predictions

	def getAccuracy(self):
		return self.accuracy

	def test_naive_bayes(self, test_set):
		argmax_by_class = []
		predictions = []
		for class_given in self.class_list:
			for tenses in test_set:
				sum_class_given = self.retrieve_logprior(self.log_priors, class_given)
				#print(tenses)
				for word in tenses[:-1]:
					if word in self.voc_doc:
						sum_class_given += self.retrieve_loglikelihood(self.loglikelihood,word,class_given)

				argmax_by_class.append([class_given,sum_class_given])

		#retrieving best predicted class
		for i in range(int((len(argmax_by_class)/2))):
			if argmax_by_class[i][1] > argmax_by_class[i+int((len(argmax_by_class)/2))][1]:
				predictions.append(argmax_by_class[i][0])
			else:
				predictions.append(argmax_by_class[i+int((len(argmax_by_class)/2))][0])

		self.predictions = predictions

	def check_accuracy(self, test_set):
		correct = 0
		for i in range(len(test_set)):
			if test_set[i][-1] == self.predictions[i]:
				correct += 1
		self.accuracy = round((correct/float(len(test_set))) * 100.0, 2)

if __name__ == '__main__':
    model = NaiveBayes(['amazon_cells_labelled.txt', 'yelp_labelled.txt', 'imdb_labelled.txt'], 0.9)
    model.train()
    test_file = sys.argv[1]
    test_set = prep_data(test_file)
    model.test_naive_bayes(test_set)
    predictions = model.getPredictions()

    out_file = open('results_file.txt', 'w')
    for prediction in predictions:
        out_file.write(prediction + "\n")

    model.check_accuracy(test_set)
    accuracy = model.getAccuracy()
    out_file.write(str(accuracy) + "% accuracy")

    out_file.close()

{'string': 'string', 'math': 'math', 'Counter': 'collections.Counter'}


import tensorflow as tf
from keras.backend.tensorflow_backend import set_session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.gpu_options.visible_device_list = "0"
set_session(tf.Session(config=config))
import my_s2s_v2 as my
from keras import backend as K
import sys
import os

orig_stdout = sys.stdout
f = open('Both_fra_test.txt', 'a')
sys.stdout = f
'''
list models: [{plain, rev}latent_dim, act_func, loss_func, latent_dim]
'''

act_functions = ["linear"]
losses = ["logcosh"]
paths = ["deu.txt"]
ways = ["for", "rev"]
dat = my.data_prep("deu.txt", True)
dat.as_wrdvec()
for act in act_functions:
    for loss in losses:
        for data in paths:
            for way in ways:
                if not os.path.isfile("{}_e{}_wordvec_{}_{}.list".format(way, data.split(".")[0][:-1], act, loss)):
                    mod = my.seq2seq(dat)
                    if way == "for":
                        mod.twolayer(act, loss, 900)
                    else:
                        mod.reverse(act, loss, 900)
                    if data == "fra.txt":
                        pre = "fr"
                    else:
                        pre = "de"
                    mod.loadwts("{}_e{}_wordvec_{}_{}.h5".format(way, pre, act, loss))
                    print("\n\n{}_e{}_wordvec_{}_{}.h5".format(way, pre, act, loss))
                    a, i = mod.train_acc()
                    print("{}_e{}_wordvec_{}_{}.h5\n\n".format(way, pre, act, loss))
                    K.clear_session()
                else:
                    print("Skipped")


import itertools

class Solution:
    def uniquePaths(self, m: int, n: int) -> int:
        list_ = [0]*(m-1) + [1]*(n-1)
        permutation = itertools.permutations(list_)

        return len(set(permutation))

sol = Solution()
sol.uniquePaths(m=7, n=3)


import pandas as pd
import numpy as np

import matplotlib.pyplot as plt
import seaborn as sns

from sklearn.preprocessing import MaxAbsScaler

from sklearn.feature_selection import VarianceThreshold
from sklearn.feature_selection import SelectFromModel
from sklearn.ensemble import ExtraTreesClassifier

from sklearn.cross_validation import train_test_split
from sklearn.ensemble import RandomForestClassifier

from sklearn.metrics import roc_curve
from sklearn.metrics import auc
#-----------------------------------Load Data-----------------------------#
#加载数据
#<param>path</param>
#<return>data</return>
def load_data(path):
	return pd.read_csv(path)

#-----------------------------------Data Explore-----------------------------#
#查看特征离散连续分布
def query_lslx(data_X,type_num):
	data_X_type = []
	for column in data_X.columns:
		data_X_type_item = []
		data_X_type_item.append(column)
		data_X_type_item.append(len(data_X[column].value_counts()))
		data_X_type.append(data_X_type_item)
	data_X_type = pd.DataFrame(data=data_X_type,columns=['Feature','Number'])
	print(data_X_type.sort_values(by='Number',ascending=False))

	lx_num = data_X_type[data_X_type['Number']>type_num].shape[0]
	ls_num = data_X_type[data_X_type['Number']<=type_num].shape[0]
	# data_X_type = pd.DataFrame({'LX':lx_num,'LS':ls_num})
	data_X_type = {'LX':lx_num,'LS':ls_num}
	return data_X_type

#数据探索
#<param>data</param>
#<return>None</return>
#1.【整体】整体查看数据
#2.【标签】查看标签分布
#3.【特征】特征字符数值类型情况、特征离散连续类型情况、特征空值情况
#4.【特征】特征信息量情况、特征大信息量分布情况
#5.【特征与特征、特征与标签】特征之间相关情况、特征与标签相关性情况
def data_explore(data):
	#1
	print(data.shape)
	print(data.head(5))

	#2
	data_y = data['TARGET']
	print(data_y.value_counts())

	#3
	data_X = data.drop(['TARGET'],axis = 1)
	print(data_X.dtypes.value_counts())

	data_x_lslx = query_lslx(data_X,500)
	print(data_x_lslx)

	missing_percent = (data_X.isnull().sum()/data_X.isnull().count()).sort_values(ascending=False)
	print(missing_percent)

	#4
	data_std = data_X.apply(lambda x : x.std()).sort_values(ascending=False)
	print(data_std)

	plt.hist(data_X['delta_num_aport_var13_1y3'])
	plt.show()
	plt.hist(data_X['delta_imp_aport_var13_1y3'])
	plt.show()

	#5
	data_new = data
	data_corr = data_new.corr()
	k=10
	cols=data_corr.nlargest(k,'TARGET')['TARGET'].index
	cm=np.corrcoef(data_new[cols].values.T)
	sns.set(font_scale=1.25)
	hm=sns.heatmap(cm,cbar=True,annot=True,square=True,fmt='.2f',annot_kws={'size':10},yticklabels=cols.values,xticklabels=cols.values)
	plt.show()

	return None

#-----------------------------------Feature Prep-----------------------------#
#特征预处理
#<param>data_X</param>
#<return>data_X</return>
#1.归一化（稀疏矩阵用Maxab，非稀疏正态分布用StandardScaler，非稀疏非正太用MinMax）
#2.字符转数值型（one-hot）
#3.特征转换（例如多项式转换）
def feature_prep(data_X):
	#1
	scaler = MaxAbsScaler()
	data_X_np = scaler.fit_transform(data_X)
	data_X = pd.DataFrame(data_X_np,columns=data_X.columns)

	#2
	#经数据探索。无字符型特征，均为数值型

	#3
	#暂不做处理

	return data_X

#-----------------------------------Feature Select-----------------------------#
#特征选择
#<param>data</param>
#<return>feature columns</return>
#1.【特征】过滤单特征信息量少的特征
#2.【特征与特征】过滤特征间相关高的特征
#3.【特征与标签】过滤特征与标签相关性低的特征
def feature_select(data_X,data_y):

	#1
	f_select = VarianceThreshold(threshold=0.001)
	data_X_np = f_select.fit_transform(data_X)
	data_X = pd.DataFrame(data_X_np,columns=data_X.ix[:,f_select.get_support(indices=True)].columns)
	print(data_X.shape)

	#2
	# print(query_lslx(data_X, 200))

	#3
	f_select_model = ExtraTreesClassifier()
	f_select_model.fit(data_X,data_y)
	f_select = SelectFromModel(f_select_model,threshold='0.1*mean',prefit=True)
	data_X = data_X.ix[:,f_select.get_support(indices=True)]

	return data_X.columns

#-----------------------------------Feature Transform-----------------------------#
#特征转换
#<param>data_X</param>
#<return>data_X</return>
#1.【特征】缺失值处理
#2.【特征】异常值处理（分离散和连续）
#3.【特征】特征多项式转换
def feature_transform(data_X):
	#1
	#经数据探索，无缺失值

	#2
	data_X = data_X.drop(['ID'],axis = 1)
	data_std = data_X.apply(lambda x : x.std()).sort_values(ascending=False)
	# print(data_std)

	data_std_10 = data_std[0:10]
	for index in data_std_10.index:
		print(index,data_X[index].value_counts())

	#连续数据
	#箱式图
	sns.boxplot(pd.concat([data_X['var15'],data_X['num_var45_hace2'],data_X['num_med_var45_ult3'],data_X['num_op_var39_ult3']],axis = 1))
	plt.show()

	#离散数据
	#通过value_counts查看是否出现异常值

	#3
	#暂不处理

	return data_X

#-----------------------------------Model Select And Train Pred-----------------------------#
#模型选择和训练
#<param>train_X,train_y</param>
#<return>model</return>
def model_train(train_X,train_y,test_X):
	clf = RandomForestClassifier(n_estimators = 50,criterion ='gini',min_samples_split = 60 , max_depth = 15 ,max_features = 7)
	clf.fit(train_X,train_y)
	pred = clf.predict_proba(test_X)
	return pred[:,1]

#-----------------------------------Model Evaluate-----------------------------#
#模型评估
#<param>model,test_X,test_y</param>
#<return>auc</return>
def model_evaluate(pred,test_y):
	fpr, tpr, thresholds = roc_curve(test_y,pred)
	return auc(fpr, tpr)

#-----------------------------------Main Process-----------------------------#
#load data
train_origin_data = load_data('../data/train.csv')

#data explore
# data_explore(train_origin_data)

#feature prep
train_data_X = train_origin_data.drop(['TARGET'],axis = 1)
train_data_y = train_origin_data['TARGET']
train_data_X = feature_prep(train_data_X)

#feature select
X_columns = feature_select(train_data_X,train_data_y)
train_data_X = train_data_X[X_columns]
# print(train_data_X.head(5))

#feature_transform
train_data_X = feature_transform(train_data_X)

#model select and train pred
train_X,test_X,train_y,test_y = train_test_split(train_data_X,train_data_y,train_size = 0.7,random_state = 0)
pred_train = model_train(train_X, train_y, train_X)
pred_test = model_train(train_X, train_y, test_X)

#model evaluate
print(model_evaluate(pred_train,train_y))
print(model_evaluate(pred_test,test_y))


import os
import sqlite3

# construction of  a path to wherever the database exists
# DB_FILEPATH = "rpg_db.sqlite3.db"
DB_FILEPATH = os.path.join(os.path.dirname(__file__), "rpg_db.sqlite3")

connection = sqlite3.connect(DB_FILEPATH)
# print("CONNECTION:", connection)

cursor = connection.cursor()
# print("CURSOR", cursor)

# How many total characters?
# In Table charactercreator_character

# Commenting out the code on why they are taking a specific action
# print(len(c1.execute('SELECT * FROM charactercreator_character;').fetchall()[0][2:]))

count_characters = "SELECT COUNT(*) FROM charactercreator_character;"
print("The total number of characters is",
      cursor.execute(count_characters).fetchall())

# How many characters of each specific subclass?

count_characters_cleric = "SELECT COUNT(*) FROM charactercreator_cleric;"
print("The total number of cleric characters is",
      cursor.execute(count_characters_cleric).fetchall())

count_characters_fighter = "SELECT COUNT(*) FROM charactercreator_fighter;"
print("The total number of fighter characters is",
      cursor.execute(count_characters_fighter).fetchall())

count_characters_mage = "SELECT COUNT (*) FROM charactercreator_mage;"
print("The total number of mage characters is",
      cursor.execute(count_characters_mage).fetchall())

count_characters_necromancer = "SELECT COUNT(*) FROM charactercreator_necromancer;"
print("The total number of necromancer characters is",
      cursor.execute(count_characters_necromancer).fetchall())

count_characters_thief = "SELECT COUNT (*) FROM charactercreator_thief;"
print("The total number of thief characters is",
      cursor.execute(count_characters_thief).fetchall())

# How many total Items?

count_armory_items = "SELECT COUNT (*) FROM armory_item;"
print("The total number of armory items is",
      cursor.execute(count_armory_items).fetchall())

# How many of the Items are weapons? How many are not?

count_armory_weapons = "SELECT COUNT (*) FROM armory_weapon;"
print("The total number of armory weapons is",
      cursor.execute(count_armory_weapons).fetchall())

# Items which are not weapons

armory_not_weapons = (174 - 37)
print("The number of non-weapon armory items is:", armory_not_weapons)

# How many Items does each character have? (Return first 20 rows)

item_number = '''
SELECT cc.character_id, COUNT(ai.item_id)
FROM
charactercreator_character AS cc,
armory_item AS ai,
charactercreator_character_inventory AS cci
WHERE
cc.character_id = cci.character_id AND
ai.item_id = cci.item_id
GROUP BY cc.character_id
LIMIT 20;
'''
print(cursor.execute(item_number).fetchall())

# How many Weapons does each character have? (Return first 20 rows)

weapon_number = '''
SELECT cc.character_id, COUNT(aw.item_ptr_id)
FROM
charactercreator_character AS cc,
charactercreator_character_inventory AS cci,
armory_weapon AS aw
WHERE
cc.character_id = cci.character_id
AND
cci.item_id = aw.item_ptr_id
GROUP BY cc.character_id
LIMIT 20;
'''
print(cursor.execute(weapon_number).fetchall())

# On average,how many items does each character have?
# assume: include characters that do not have any items
# Output : two columns character_id, item_count
# row per character(302 total)

average_items_per_character = '''
SELECT AVG(item_count) as avg_items_per_char
FROM(
	SELECT
	cc.character_id,
	--cc.'name',
	--cci.*,
	--ai.*,
	count(distinct ai.item_id) as item_count
	FROM charactercreator_character cc
	LEFT JOIN charactercreator_character_inventory cci
	ON cc.character_id=cci.character_id
	LEFT JOIN armory_item ai
	ON cci.item_id=ai.item_id
	GROUP BY cc.character_id
)
'''

print("The average items per character", cursor.execute(
    average_items_per_character).fetchall())

# On average,how many weapons does each character have?
# assume: include characters that do not have any weapons
# two columns character_id, weapon_count
# row per character (302 total)

average_weapons_per_character = '''
SELECT AVG(weapon_count) as avg_weapons_per_char
FROM(
	SELECT
	cc.character_id,
	--cc.'name',
	--cci.*,
	--aw.*,
	count(distinct aw.item_ptr_id) as weapon_count
	FROM charactercreator_character cc
	LEFT JOIN charactercreator_character_inventory cci
	ON cc.character_id=cci.character_id
	LEFT JOIN armory_weapon aw
	ON cci.item_id=aw.item_ptr_id
	GROUP BY cc.character_id
)
'''
print("The average weapons per character", cursor.execute(
    average_weapons_per_character).fetchall())


import matplotlib.pyplot as plt
import pandas as pd
from sklearn.tree import DecisionTreeRegressor
import numpy as np

# Decision tree regression is very usefull with multiple variables.
# With only one variable is not very usefull

""" Data Preprocessing """
# Import Dataset
dataset = pd.read_csv('Data/Decision Tree Regression/Position_Salaries.csv')
X = dataset.iloc[:, 1:2].values
Y = dataset.iloc[:, 2].values

# Create regressor
regressor = DecisionTreeRegressor(random_state=0)
regressor.fit(X,Y)

Y_predict = regressor.predict([[6.5]])

print( Y_predict)

# Visualize regression
X_grid = np.arange(min(X), max(X), 0.01)
X_grid = X_grid.reshape(len(X_grid), 1)
plt.scatter(X, Y, color='red')
plt.plot(X_grid, regressor.predict(X_grid), color='blue')
plt.show()

Udemy - Machine Learning A-Z/2-Regression/decision_tree_regression.py


import logging

from rest_framework.authtoken.views import ObtainAuthToken
from rest_framework.response import Response
from rest_framework.authtoken.models import Token

from common.renderer import JSONRenderer
from usr.serializers import LoginSerializer

LOG = logging.getLogger('bgapi.' + __name__)

class CustomObtainAuthToken(ObtainAuthToken):
    """
    To obtain authentication token for users using their ``username`` and ``password``
    Required fields: ``username``, ``password``
    """

    #: Overriding the rest_framework's ObtainAuthToken for generating JSON response
    #: as per BuilderGiant's specification.
    # It has to be done separately as it's not part of this system. Hence can't use the default
    # render class parameter set in configuration
    renderer_classes = (JSONRenderer,)

    def post(self, request):
        serializer = self.serializer_class(data=request.data)
        serializer.is_valid(True)
        user = serializer.validated_data['user']
        user_data = {}
        profile = getattr(user, 'profile', None)
        if profile:
            profile = LoginSerializer(profile)
            user_data = profile.data

        # get or create access token
        token, created = Token.objects.get_or_create(user=user)
        # merging token and user information
        data = dict({'token': token.key}.items() + user_data.items())
        LOG.debug('Login success')
        return Response(data)

#: To bind CustomObtainAuthToken to system's URL
obtain_auth_token = CustomObtainAuthToken.as_view()


import os
from datetime import datetime

# `__file__` is relative to the current working directory
# `os.chdir()` can change current working directory, but don't change `__file__`
# so if `os.chdir()` is used, this method fails to get current directory
current_dir = os.path.dirname(os.path.realpath(__file__))
COOKIES = open(os.path.join(current_dir, 'cookie.txt'), 'r').readline().strip()

TIMESTAMP = str(datetime.now().strftime('%Y-%m-%d-%H:%M:%S'))

# 保存文件粒度
GRANULARITY_HOUR = False

# 强制爬取
FORCE_CRAWL = False

# 汇率
DOLLAR_TO_CNY = 7.0
# steam视频售卖税率
STEAM_SELL_TAX = 0.13

# filter
# 爬取历史价格的话，每个都要单独爬一次，爬取量翻了好几十倍，所以扔掉一些，只爬取某个价格区间内的饰品……
# 大致价格分位点：0 - 10000; 20 - 5000; 50 - 4000; 100 - 3400; 200 - 3000; 500 - 2200; 1000 - 1200
CRAWL_MIN_PRICE_ITEM = 100
CRAWL_MAX_PRICE_ITEM = 300
# buff最低价门槛，低于该价格的条目忽略（要不然统计出来很多封装的涂鸦……）
MIN_PRICE_THRESHOLD = 100
# 交易记录阈值，冷门物品不在考虑范围
MIN_SOLD_THRESHOLD = 10
# 太大都是steam定价虚高，实际卖不掉的……有了steam求购价之后就不需要这个了
MAX_GAP_PERCENTAGE = 0.8
# 每一项指标的输出个数
TOP_N = 50

# 文件
DATE_DAY = str(datetime.now().strftime('%Y-%m-%d'))
DATE_HOUR = str(datetime.now().strftime('%Y-%m-%d-%H'))
OUTPUT_PATH = "output"
OUTPUT_NAME_HOUR = "csgo_skins_" + DATE_HOUR
OUTPUT_NAME_DAY = "csgo_skins_" + DATE_DAY
OUTPUT_NAME = OUTPUT_NAME_HOUR if GRANULARITY_HOUR else OUTPUT_NAME_DAY

OUTPUT_FILE_NAME = os.path.join(
    OUTPUT_PATH, OUTPUT_NAME + ".csv"
)
OUTPUT_FILE_NAME_DAY = os.path.join(
    OUTPUT_PATH, OUTPUT_NAME_DAY + ".csv"
)


"""
This code is a streaming version of code written by Zulko who
created the 'pianoputer.' All the ideas are from Zulko's
version. All we do here is show how to convert code for an
array into code for a stream.

To play music, download sounddevice.

A problem encountered when switching from arrays to streams
is that code operating on an array can use metrics --- such as
maximum --- over the entire array, whereas code operating on
streams has to compute pitch shift based on the available
data up to that point. For the streaming version we assume
that the maximum over the entire array is 4096.0 (See the
last lines of stretch in which result is computed). A poor
assumption of the maximum may result in clipping or numerical
problems.

This code has both the original version (modified with max
assumed to be 4096) and the streaming version so that you
can see how one is converted into the other. speedx and
stretch are from the original version, while the method
Stretch.stretch is the streaming version.

The repository includes a short wav file called 'guitar.wav'
If you run test_pitchshift you will hear the sound shifted
to a lower pitch, then the original sound, and then the sound
shifted to a higher pitch. In each case you will first hear
the sound created by original version (modified by assuming
max is 4096) and the streaming version.

The streaming code was written by Deepak Narayanan and Mani
Chandy. The Stretch.stretch() function is based on Zulko's
code.

"""

#!/usr/bin/env python
import sys
import os
import numpy as np
from scipy.io import wavfile
from scipy.io.wavfile import read, write
import sounddevice as sd

sys.path.append(os.path.abspath("../../IoTPy/core"))
sys.path.append(os.path.abspath("../../IoTPy/agent_types"))
sys.path.append(os.path.abspath("../../IoTPy/helper_functions"))

from stream import StreamArray
from run import run
from sink import sink_window

#---------------------------------------------------------------------
# CODE FROM ZULKO, PIANOPUTER. MERELY FOR REFERENCE.
#---------------------------------------------------------------------

def speedx(sound_array, factor):
    """
    Multiplies the sound's speed by factor
    Taken from Zulko, pianoputer

    Parameters
    ----------
    sound_array: np.array
      The array that is being stretched
    factor: positive number
      The sound is speeded up when factor > 1.

    """
    indices = np.round( np.arange(0, len(sound_array), factor) )
    indices = indices[indices < len(sound_array)].astype(int)
    return sound_array[ indices.astype(int) ]

def stretch(sound_array, f, window_size, h):
    """
    Stretches the sound by a factor f.
    Taken from Zulko, pianoputer

    Parameters
    ----------
    sound_array: np.array
      The array that is being stretched
    f: positive number
      The amount of stretch when f > 1 and contraction if f < 1.
    window_size: int or float
      The sound_array is inspected by subarrays each of which is of
      size window_size.
    h: int or float
      The overlap between successive windows.
    """
    window_size = int(window_size)
    phase  = np.zeros(window_size)
    hanning_window = np.hanning(window_size)
    result = np.zeros( int(len(sound_array) /f + window_size))

    for i in np.arange(0, len(sound_array)-(window_size+h), int(h*f)):

        # two potentially overlapping subarrays
        a1 = sound_array[i: i + int(window_size)]
        a2 = sound_array[i + h: i + int(window_size) + int(h)]

        # resynchronize the second array on the first
        s1 =  np.fft.fft(hanning_window * a1)
        s2 =  np.fft.fft(hanning_window * a2)
        phase = (phase + np.angle(s2/s1)) % 2*np.pi
        a2_rephased = np.fft.ifft(np.abs(s2)*np.exp(1j*phase))

        # add to result
        i2 = int(i/f)
        result[i2 : i2 + window_size] += np.real((hanning_window*a2_rephased))

    #result = ((2**(16-4)) * result/result.max()) # normalize
    # Assume result.max() is 2**(16-4)x
    return result.astype('int16')

def pitchshift(sound_array, n, window_size=2**13, h=2**11):
    """
    Changes the pitch of a sound by n semitones.
    Taken from Zulko, pianoputer
    """
    factor = 2**(1.0 * n / 12.0)
    stretched = stretch(sound_array, 1.0/factor, window_size, h)
    return speedx(stretched[window_size:], factor)

#---------------------------------------------------------------------
# END OF CODE FROM ZULKO, PIANOPUTER
#---------------------------------------------------------------------

#---------------------------------------------------------------------
# CODE TO CONVERT OPERATION ON AN ARRAY TO OPERATION ON A STREAM.
#---------------------------------------------------------------------
def pitchshift_stream(sound_array, n, window_size=2**13, h=2**11):
    """
    Changes the pitch of a sound by n semitones.

    Notes
    -----
    This application has 2 sink_window agents and 3 streams x, y, z.
    Stretch agent: The first agent gets input x and outputs y which
    stretches the data in stream x. The stretching code is from Zulko,
    pianoputer.
    Speed up agent: The next agent gets input y and outputs z which
    speeds up y by the specified factor. This agent interpolates the
    data in y to the number of points determined by factor.

    """
    factor = 2**(1.0 * n / 12.0)
    f = 1.0/factor

    # Declare streams
    x = StreamArray('x', dtype=np.int16)
    y = StreamArray('y', dtype=np.int16)
    z = StreamArray('z', dtype=np.int16)

    # Define the stretch agent
    stretch_object = Stretch(
        in_stream=x, out_stream=y, factor=factor,
        window_size=window_size, h=h)
    sink_window(
        func=stretch_object.stretch, in_stream=x,
        window_size=window_size+h, step_size=int(h*f))

    # Define the speedup agent.
    def f(window, out_stream):
        indices = np.arange(0, window_size, factor)
        out_stream.extend(
            np.int16(np.interp(
                indices, np.arange(window_size), window)))
    sink_window(func=f, in_stream=y, window_size=window_size,
    step_size=window_size, out_stream=z)

    # Partition sound_array into sound bites. Extend the
    # input with a sequence of sound bites and run each
    # sound bite until the sound_array data is finished.
    sound_bite_size = 2**14
    for i in range(0, sound_array.size, sound_bite_size):
        # sound_bite = sound_array[i:i+sound_bite_size]
        x.extend(sound_array[i:i+sound_bite_size])
        run()
    # Process any data in sound_array that wasn't processed
    # in the for loop.
    x.extend(sound_array[i:])

    # Return the result.
    return z.recent[:z.stop]

class Stretch(object):
    """
    Parameters
    __________

    """
    def __init__(self, in_stream, out_stream, factor, window_size, h):
        self.in_stream = in_stream
        self.out_stream = out_stream
        self.factor = factor
        self.f = 1.0/factor
        self.window_size = window_size
        self.h = h
        self.phase = np.zeros(window_size)
        self.hanning_window = np.hanning(self.window_size)
        self.result = np.zeros(window_size+h)
    def stretch(self, window):
        # -----------------------------------------------------
        # From Zulko stretch()
        # a1 and a2 are two overlapping subarrays, each of size
        # window_size with an overlap of h.
        a1 = window[:self.window_size]
        a2 = window[int(self.h): self.window_size+int(self.h)]

        # resynchronize the second array on the first
        s1 = np.fft.fft(self.hanning_window * a1)
        s2 = np.fft.fft(self.hanning_window * a2)
        self.phase = (self.phase + np.angle(s2/s1)) % 2*np.pi
        a2_rephased = np.fft.ifft(np.abs(s2)*np.exp(1j*self.phase))

        # Add resynchronized second array to result, and output
        # on out_stream. Recall that the size of self.result is
        # self.window_size + self.h.
        self.result[: self.window_size] += np.real(
            (self.hanning_window*a2_rephased))
        current_output = (self.result[:self.h]).astype(np.int16)
        self.out_stream.extend(current_output)

        # Save self.result[self.h : ] for next window.
        self.result = np.roll(self.result, -self.h)
        self.result[self.window_size:] = 0.0

#---------------------------------------------------------------------
# TEST
#---------------------------------------------------------------------
def test_pitchshift():
    fps, sound_array = wavfile.read("./guitar.wav")
    tones = [-12, 0, 12]
    for tone in tones:
        print ('tone is ', tone)

        # Array code
        # transposed = pitchshift(sound_array, tone)
        # print ('Playing sound from array code')
        # sd.play(transposed, blocking=True)

        # Stream code
        transposed_stream = pitchshift_stream(sound_array, tone)
        print ('Playing sound from stream code')
        sd.play(transposed_stream, blocking=True)
        print ()

if __name__ == '__main__':
    test_pitchshift()


#!/usr/bin/python
import argparse
import os

def main():
    args = parseargs()
    download_pdb(args.pdbids)

def parseargs():
    parser = argparse.ArgumentParser()
    parser.add_argument('-p','--pdbids',nargs="+",help='The pdb id of the structure to download')
    args = parser.parse_args()
    return args

def download_pdb(pdbs):
    for pdbid in pdbs:
        try:
            command = 'wget http://www.rcsb.org/pdb/files/%s.pdb.gz'%pdbid
            os.system(command)
            os.system('gunzip %s.pdb.gz'%pdbid)
        except:
            print('error',pdbid,'could not be downloaded')

if __name__ == '__main__':
    main()


from django.urls import path
from . import views

urlpatterns = [
    path('companies/',views.companylist,name = "org_company_list"),
    path('companies/add/',views.CompanyCreate.as_view(),name="org_company_create"),
    path('companies/members/',views.members,name = "org_members_list"),
    path('companies/members/add/',views.create_members,name = "org_members_create"),

    path('org/upgrade/',views.upgrade,name = "org_upgrade"),
    path('org/payment_method/',views.payment_method,name = 'org_payment_method'),
    path('org/card',views.card,name="org_card"),
]


import pandas as pd

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from UliPlot.XLSX import auto_adjust_xlsx_column_width
import chromedriver_binary
import csv
import time
import sys
import os

# https://www.encorewire.com/products/index.html
class ScrapePlaces:
    def __init__(self):
        self.driver = webdriver.Chrome('./chromedriver')
        url = "https://www.littelfuse.com/products.aspx"
        self.driver.get(url)
        category_list = self.driver.find_element_by_class_name('product-grid').find_elements_by_tag_name('li')
        index = 0
        df = pd.DataFrame()
        for i in range(len(category_list)):
            link = category_list[i].find_element_by_tag_name('a')
            # category_name = link.text
            link.click()
            self.waiting()
            sub_category_list = self.driver.find_element_by_class_name('product-grid').find_elements_by_tag_name('li')
            for j in range(len(sub_category_list)):
                sub_link = sub_category_list[j].find_element_by_tag_name('a')
                sub_link.click()
                self.waiting()
                sub_sub_category_list_div = self.driver.find_element_by_class_name('product-grid')
                if sub_sub_category_list_div:
                    sub_sub_category_list = sub_sub_category_list_div.find_elements_by_tag_name('li')
                    for k in range(len(sub_sub_category_list)):
                        sub_sub_link = sub_sub_category_list[j].find_element_by_tag_name('a')
                        sub_sub_link.click()
                        self.waiting()
                        last_category_list = self.driver.find_elements_by_class_name('descriptive-box')
                        for l in range(len(last_category_list)):
                            last_link = last_category_list[l].find_element_by_tag_name("a")
                            self.driver.execute_script("arguments[0].click();", last_link)
                            # print(last_link.get_attribute("innerHTML"))
                            # last_link.click()
                            self.waiting()
                            product_list = self.driver.find_element_by_id('tblElecChars').find_element_by_tag_name("tbody").find_elements_by_tag_name("tr")
                            for m in range(len(product_list)):
                                product_link = product_list[m].find_element_by_tag_name("a")
                                sku = product_link.get_attribute("innerHTML")
                                print(index, sku)
                                self.driver.execute_script("arguments[0].click();", product_link)
                                self.waiting()
                                pro_name = self.driver.find_element_by_tag_name('h1').text
                                df.at[index,"url"] = self.driver.current_url
                                df.at[index,"Title"] = pro_name
                                df.at[index,"Brand"] = "littelfuse"
                                pro_description = self.driver.find_element_by_class_name('feaures-benefits-box').get_attribute("innerHTML")
                                df.at[index,"Long/Short Descriptions"] = pro_description.split("
", 1)[1].split("
",1)[0].replace("\n","").replace("\t","").replace("  ","");
                                pro_feature = ''
                                feature_list = self.driver.find_element_by_class_name('feaures-benefits-box').find_elements_by_tag_name("ul")[2].find_elements_by_tag_name("li")
                                for feature in feature_list:
                                    if pro_feature!='':
                                        pro_feature += " | "
                                    pro_feature += feature.text
                                df.at[index,"Features"] = pro_feature

                                applications = ''
                                applications_list = self.driver.find_element_by_class_name('feaures-benefits-box').find_elements_by_tag_name("ul")[1].find_elements_by_tag_name("li")
                                for app in applications_list:
                                    if applications!='':
                                        applications += " | "
                                    applications += app.text
                                df.at[index,"Applications"] = applications

                                spec_txt = ""
                                spec_list = self.driver.find_element_by_id("ElectricalCharacteristics").find_elements_by_tag_name("tbody")
                                for spec in spec_list:
                                    key = spec.find_element_by_tag_name("tr").find_element_by_tag_name("a").text
                                    value = spec.find_element_by_tag_name("tr").find_elements_by_tag_name("td")[1].text
                                    spec_txt += "\""+ key + "\":\"" + value + "\","
                                df.at[index,"Specifications"] = "{"+spec_txt[:-1]+"}"

                                breadcrumb_list = self.driver.find_element_by_class_name("breadcrumb").find_elements_by_tag_name("a")
                                breadcrumb_txt = ""
                                for breadcrumb_index in range(len(breadcrumb_list)):
                                    breadcrumb_txt += breadcrumb_list[breadcrumb_index].text+"/"
                                df.at[index,"Breadcrumb"] = breadcrumb_txt + sku
                                df.at[index,"Product SKU Number"] = "["+sku+"]"
                                img_txt = ""
                                detail_img = self.driver.find_element_by_class_name('zoomContainer').find_element_by_tag_name('img').get_attribute("src")
                                img_txt = "{\"Item_Image\":\""+detail_img+"\",\"Detailed_Image\":\""+ detail_img+"\",\"Zoom_Image\":\""+detail_img+"\"},"
                                df.at[index,"Image"] = "["+img_txt[:-1]+"]"

                                assets = ''
                                assets_datas = self.driver.find_element_by_id('tabTechRes').find_element_by_tag_name("tbody").find_elements_by_tag_name("tr")
                                for asset_data in assets_datas:
                                    key = asset_data.find_element_by_tag_name("a").get_attribute("innerHTML").replace("  ","")
                                    value = asset_data.find_element_by_tag_name("a").get_attribute("href")
                                    assets += "'"+key+"' : \""+value+"\"" + ","
                                df.at[index,"Assets"] = "{"+assets[:-1]+"}"

                                writer = pd.ExcelWriter('littelfuse.xlsx', engine='xlsxwriter')
                                df.to_excel(writer, sheet_name='Sheet1', startrow=0, index=False)
                                # auto_adjust_xlsx_column_width(df, writer, sheet_name="Sheet2", margin=0)
                                for column in df:
                                    column_width = max(df[column].astype(str).map(len).max(), len(column)) + 2
                                    column_width = min(column_width, 300)
                                    col_idx = df.columns.get_loc(column)
                                    writer.sheets['Sheet1'].set_column(col_idx, col_idx, column_width)
                                writer.save()
                                index += 1
                                self.driver.execute_script("window.history.go(-1)")
                                self.waiting()
                                product_list = self.driver.find_element_by_id('tblElecChars').find_element_by_tag_name("tbody").find_elements_by_tag_name("tr")
                            self.driver.execute_script("window.history.go(-1)")
                            self.waiting()
                            last_category_list = self.driver.find_elements_by_class_name('descriptive-box')
                        self.driver.execute_script("window.history.go(-1)")
                        self.waiting()
                        sub_sub_category_list = sub_sub_category_list_div.find_elements_by_tag_name('li')
                # else :
                #     continue
                self.driver.execute_script("window.history.go(-1)")
                self.waiting()
                sub_category_list = self.driver.find_element_by_class_name('product-grid').find_elements_by_tag_name('li')
            self.driver.execute_script("window.history.go(-1)")
            self.waiting()
            category_list = self.driver.find_element_by_class_name('product-grid').find_elements_by_tag_name('li')
    def waiting(self):
        timeout = 6# timeout , change if connection slow
        try:
            WebDriverWait(self.driver, timeout).until(EC.visibility_of_element_located((By.XPATH,"//button[@class='section-back-to-list-button']")))
            print('waiting done')
        except TimeoutException:
            # print('process item expcetion')

            pass

    def closeDriver(self):
        print("\n\nEnding Scrapper Session")
        self.driver.close()

if __name__ == "__main__":
    scrap = ScrapePlaces()
    scrap.closeDriver()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# michael a.g. aïvázis
# orthologue
# (c) 1998-2023 all rights reserved
#

"""
Instantiate a simple mutable record using raw data
"""

def test():
    import pyre.records

    class record(pyre.records.record):
        """
        A sample record
        """
        sku = pyre.records.measure()
        description = pyre.records.measure()
        cost = pyre.records.measure()

    # build a record
    r = record.pyre_mutable(data=("9-4013", "organic kiwi", .85))
    # check
    assert r.sku == "9-4013"
    assert r.description == "organic kiwi"
    assert r.cost == .85

    # make a change to the cost
    r.cost = 1

    # and verify that it was stored
    assert r.cost == 1

    # all done
    return r

# main
if __name__ == "__main__":
    # skip pyre initialization since we don't rely on the executive
    pyre_noboot = True
    # do...
    test()

# end of file


# Import dependencies
from splinter import Browser
from bs4 import BeautifulSoup as bs
import pymongo
import pandas as pd

def init_browser():
    # @NOTE: Replace the path with your actual path to the chromedriver
    executable_path = {'executable_path': 'chromedriver.exe'}
    browser = Browser('chrome', **executable_path, headless=True)

def scrape():
    browser = init_browser()
    listings = {}

    ## -- Look at news page for Mars from NASA -- ##
    url = "https://mars.nasa.gov/news/"
    browser.visit(url)
    html = browser.html

    # Create a Beautiful Soup object
    soup = bs(html, 'html.parser')

    # Collect latest news title
    news_title = soup.find_all('div', class_="content_title")

    # Collect latest new paragraph
    news_p = soup.select('ul.item_list li.slide div.article_teaser_body')

    ## -- URL of Mars Images website -- ##
    img_url = "https://www.jpl.nasa.gov/spaceimages/?search=&category=Mars"
    browser.visit(img_url)

    img_html = browser.html
    # Create a Beautiful Soup object
    img_soup = bs(img_html, 'html.parser')

    # Collect current featured Mars image
    first_img = img_soup.select_one('div.image_and_description_container div.img img.thumb')
    first_img_src = first_img['src']

    base_img_url = 'https://www.jpl.nasa.gov/spaceimages/images/largesize'
    featured_img_url = base_img_url + first_img_src

    ## -- URL of Mars Weather twitter page -- ##
    twitter_url = 'https://twitter.com/marswxreport?lang=en'
    browser.visit(twitter_url)

    twitter_html = browser.html

    # Create a Beautiful Soup object
    tweet_soup = bs(twitter_html, 'html.parser')

    # Save text from mars weather latest tweet
    mars_weather_tweet = tweet_soup.find('p', class_='TweetTextSize TweetTextSize--normal js-tweet-text tweet-text').text

    ## -- URL of Space Facts - Mars Facts -- ##
    facts_url = "https://space-facts.com/mars/"

    # Read tables from Space Facts with pandas
    facts_table = pd.read_html(facts_url)
    facts_table

    # Create DataFrame from first table found
    facts_df = facts_table[0]

    # Convert facts_df to html string
    facts_html_table = facts_df.to_html()

    # Get rid of any extra newlines to clean up table
    facts_html_table = facts_html_table.replace('\n', '')

    ## -- USGS Astrogeology website -- ##
    hemisphere_url = 'https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars'
    browser.visit(hemisphere_url)

    hemisphere_html = browser.html

    # Create Beautiful Soup object
    hemisphere_soup = bs(hemisphere_html, 'html.parser')

    # Create list to save titles/captions of mars hemispheres
    hemisphere_captions = hemisphere_soup.select("a.itemLink  h3")
    captions = []
    for attribute in hemisphere_captions:
        caption = attribute.text
        captions.append(caption)

    # Create list to save high def images of mars hemispheres
    hemisphere_urls = hemisphere_soup.select('div.item div.description a')
    urls = []

    # Perform for loop to iterrate and click through each link to the
    # high def image link and save link
    for x in hemisphere_urls:
        url = x['href']

    #   Create new url to access high def image
        enhanced_url = 'https://astrogeology.usgs.gov' + url
        browser.visit(enhanced_url)

        enhanced_html = browser.html
        enhanced_soup = bs(enhanced_html, 'html.parser')

    #   Use enhanced_soup object to find link to high def image
        almost_img = enhanced_soup.select_one('div.downloads ul li a')
        enhanced_img = almost_img['href']

    #   Add full enhanced_img_url to the urls list
        urls.append(enhanced_img)

        # Create list of dictionaries for each hemisphere caption and url
    hemisphere_1 = {'title': captions[0], 'img_urls': urls[0]}
    hemisphere_2 = {'title': captions[1], 'img_urls': urls[1]}
    hemisphere_3 = {'title': captions[2], 'img_urls': urls[2]}
    hemisphere_4 = {'title': captions[3], 'img_urls': urls[3]}

    # Put hemisphere dictionaries into a list
    hemisphere_image_urls = [hemisphere_1, hemisphere_2,
                            hemisphere_3, hemisphere_4]

    ## -- Put all scraped data into one dictionary -- ##
    scrape_dictionary = dict({'News_P': news_p,
        'News_Title': news_title,
        'Feature_Img_Url': featured_img_url,
        'Tweet': mars_weather_tweet,
        'Facts_Table': facts_html_table,
        'Imgs of Hemisphere': hemisphere_image_urls})

    return(scrape_dictionary)


from keras.layers import Input, Lambda, Dense, Flatten
from tensorflow.keras import datasets,layers,models
from keras.models import Model

from keras.applications.vgg16 import VGG16
from keras.applications.vgg16 import preprocess_input

# from keras.applications.resnet50 import ResNet50
# from keras.applications.resnet50 import preprocess_input

# from keras.applications.inception_v3 import InceptionV3
# from keras.applications.inception_v3 import preprocess_input

from keras.preprocessing import image
# from keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.utils import to_categorical
import numpy as np
import matplotlib.pyplot as plt

(trainX, trainY),(testX,testY)=datasets.cifar10.load_data()
vgg = VGG16(input_shape=[32,32,3], weights='imagenet', include_top=False)
# resnet50 = ResNet50(input_shape=[32,32,3], weights='imagenet', include_top=False)
# inceptionv3= InceptionV3(input_shape=[32,32,3], weights='imagenet', include_top=False)

for layers in vgg.layers:
  layers.trainable=False

x=Flatten()(vgg.output)
prediction=Dense(10,activation='softmax')(x)

model=Model(inputs=vgg.input,outputs=prediction)
print(model.summary())

model.compile(loss='categorical-crossentropy',optimizer='adam',metrics=['accuracy'])

trainX=trainX/255
testX=testX/255
trainY_ohe= to_categorical(trainY)
testY_ohe= to_categorical(testY)

r=model.fit(np.array(trainX),np.array(trainY_ohe),batch_size=32,epochs=5)

# dataAugmentaion = ImageDataGenerator(rotation_range = 30, zoom_range = 0.20,
# fill_mode = "nearest", shear_range = 0.20, horizontal_flip = True,
# width_shift_range = 0.1, height_shift_range = 0.1)

# # training the model
# model.fit_generator(dataAugmentaion.flow(trainX, trainY, batch_size = 32),
#  validation_data = (testX, testY),
#  epochs = 5)

# loss
plt.plot(r.history['loss'], label='train loss')
plt.legend()
plt.show()
plt.savefig('LossVal_loss')

# accuracies
plt.plot(r.history['accuracy'], label='train acc')
plt.legend()
plt.show()
plt.savefig('AccVal_acc')


import os.path
import argparse
from lark import Lark, Transformer, tree, lexer

strip_options = {
    'strip_empty_objects': False,
    'strip_comment_objects': False,
    'strip_disabled_objects': False
}

class BLKTransformer(Transformer):
    def var_value(self, s):
        if type(s[0]) == str:
            return s[0]
        elif type(s[0]) == tree.Tree:
            return ''.join(s[0].children)
        return s

    def var_name(self, s):
        if type(s) == list:
            pass
        return ''.join([value for value in s])

    def expr_end(self, s):
        return ";"

    def expr_end_optional(self, s):
        return ""

    def value_array_el(self, s):
        res = []
        for t in s:
            if type(t) == list:
                res.append(''.join(t))
            else:
                res.append(t)
        return ''.join(res)

    def value_array(self, s):
        res = []
        for t in s:
            if type(t) == tree.Tree:
                print("error in value_array?")
                exit(1)
            else:
                res.append(t)
        return ''.join(res)

    def key_type_value(self, s):
        res = []
        for t in s:
            if type(t) == list:
                res.append(t[0])
            else:
                res.append(t)
        return ''.join(res)

    def named_object(self, s):
        # better remove node, than transform it?
        res = []
        if strip_options['strip_comment_objects']:
            if s[0] == 'comment':
                return ''
        # disabled objects starts with __ in mission editor:  __unitRespawn{
        if strip_options['strip_disabled_objects']:
            if s[0].startswith('__'):
                return ''
        for t in s:
            # skip newline token
            if type(t) == lexer.Token and t.type == 'NEWLINE':
                pass
            # and empty string, from collapsed objects
            elif t == '':
                pass
            else:
                res.append(t)
        if strip_options['strip_empty_objects']:
            # there smth in object, except it's name plus braces
            if len(res) > 3:
                return ''.join(res)
            else:
                return ''
        else:
            return ''.join(res)

    def numbers_list(self, s):
        return ''.join(s)

    def values(self, s):
        return ''.join(s)

    def r_include(self, s):
        return ' '.join(s)

    def values_in_named_object(self, s):
        return ''.join(s)

def main():
    parser = argparse.ArgumentParser(description="minify blk")
    parser.add_argument('filename', help="blk file")
    parser.add_argument("-O", dest='out_filename', default=False, nargs='?', help="output file")
    # removes not all empty objects, really
    parser.add_argument('--strip_empty_objects', dest='strip_empty_objects', action="store_true",
                        default=False, help="remove empty objects")
    # only comment objects, inlined one, like `comments:t=""` not removed
    parser.add_argument('--strip_comment_objects', dest='strip_comment_objects', action="store_true",
                        default=False, help="remove comment objects")
    parser.add_argument('--strip_disabled_objects', dest='strip_disabled_objects', action="store_true",
                        default=False, help="remove disabled objects")
    parser.add_argument('--strip_all', dest='strip_all', action="store_true",
                        default=False, help="select all options")
    parse_result = parser.parse_args()

    filename = parse_result.filename

    out_filename = ''
    if parse_result.out_filename:
        out_filename = parse_result.out_filename
    else:
        f_path, f_ext = os.path.splitext(filename)
        out_filename = f_path + '.min' + f_ext

    if parse_result.strip_all:
        for option in strip_options:
            strip_options[option] = True

    if parse_result.strip_empty_objects:
        strip_options['strip_empty_objects'] = True

    if parse_result.strip_comment_objects:
        strip_options['strip_comment_objects'] = True

    if parse_result.strip_disabled_objects:
        strip_options['strip_disabled_objects'] = True

    # get size, as we get it wrong from text opened file
    parsed_file_size = os.path.getsize(filename)
    with open(filename, mode='r', encoding="utf8") as f:
        data = f.read()

    blk_parser = Lark(open('blk.lark').read(), parser='lalr',
                      transformer=BLKTransformer(), keep_all_tokens=True)

    parsed_data = blk_parser.parse(data)
    with open(out_filename, 'w', encoding="utf8") as f:
        f.write(parsed_data)
    print("minified {} from {} to {}, with rate {:.2}".format(filename,
                                                              parsed_file_size, len(parsed_data),
                                                              len(parsed_data) / parsed_file_size))

if __name__ == '__main__':
    main()


import requests
import json
from django.conf import settings

class DCSendGridWrapper:

    BASE_URL = "https://api.sendgrid.com/v3"
    SENDGRID_LISTS = {
        "main_list": 1246575,
        "election_reminders": 1342244,
    }

    # list of properties we shouldn't pass to SendGrid
    # because we don't have a custom field for it
    FIELD_BLACKLIST = ["source_url"]

    def __init__(self):
        key = getattr(settings, "SENDGRID_API_KEY", "")
        self.headers = {"Authorization": "Bearer " + key}

    def get_users_payload(self, users):
        payload = []
        for user in users:
            data = {}
            for key in user:
                if key not in self.FIELD_BLACKLIST:
                    data[key] = user[key]
            payload.append(data)
        return payload

    def add_users(self, payload):
        url = self.BASE_URL + "/contactdb/recipients"
        r = requests.post(url, data=json.dumps(payload), headers=self.headers)
        r.raise_for_status()
        return r.json()

    def get_contact_lists(self):
        url = self.BASE_URL + "/contactdb/lists"
        r = requests.get(url, headers=self.headers)
        # if we get a non-2xx status code, just raise it
        r.raise_for_status()
        return r.json()

    def add_users_to_lists(self, payload, mailing_list):
        url = self.BASE_URL + "/contactdb/lists/{list_id}/recipients".format(
            list_id=mailing_list
        )
        r = requests.post(url, data=json.dumps(payload), headers=self.headers)
        # if we get a non-2xx status code, just raise it
        r.raise_for_status()
        # if success response payload wil be empty so there is no return val

class SendGridAPIError(ValueError):
    pass


from django.conf import settings
from django.db import connection
from south import migration
from south.migration.base import Migrations
from south.management.commands.migrate import Command as MigrateCommand
from tenant_schemas.management.commands import SyncCommon
from tenant_schemas.utils import get_tenant_model, get_public_schema_name

class Command(SyncCommon):
    help = "Migrate schemas with South"
    option_list = MigrateCommand.option_list + SyncCommon.option_list

    def handle(self, *args, **options):
        super(Command, self).handle(*args, **options)

        if self.sync_public:
            self.migrate_public_apps()
        if self.sync_tenant:
            self.migrate_tenant_apps(self.schema_name)

    def _set_managed_apps(self, included_apps, excluded_apps):
        """ while sync_schemas works by setting which apps are managed, on south we set which apps should be ignored """
        ignored_apps = []
        if excluded_apps:
            for item in excluded_apps:
                if item not in included_apps:
                    ignored_apps.append(item)

        for app in ignored_apps:
            app_label = app.split('.')[-1]
            settings.SOUTH_MIGRATION_MODULES[app_label] = 'ignore'

        self._clear_south_cache()

    def _save_south_settings(self):
        self._old_south_modules = None
        if hasattr(settings, "SOUTH_MIGRATION_MODULES") and settings.SOUTH_MIGRATION_MODULES is not None:
            self._old_south_modules = settings.SOUTH_MIGRATION_MODULES.copy()
        else:
            settings.SOUTH_MIGRATION_MODULES = dict()

    def _restore_south_settings(self):
        settings.SOUTH_MIGRATION_MODULES = self._old_south_modules

    def _clear_south_cache(self):
        for mig in list(migration.all_migrations()):
            delattr(mig._application, "migrations")
        Migrations._clear_cache()

    def _migrate_schema(self, tenant):
        connection.set_tenant(tenant, include_public=True)
        MigrateCommand().execute(*self.args, **self.options)

    def migrate_tenant_apps(self, schema_name=None):
        self._save_south_settings()

        apps = self.tenant_apps or self.installed_apps
        self._set_managed_apps(included_apps=apps, excluded_apps=self.shared_apps)

        if schema_name:
            self._notice("=== Running migrate for schema: %s" % schema_name)
            connection.set_schema_to_public()
            tenant = get_tenant_model().objects.get(schema_name=schema_name)
            self._migrate_schema(tenant)
        else:
            all_tenants = get_tenant_model().objects.exclude(schema_name=get_public_schema_name())
            if not all_tenants:
                self._notice("No tenants found")

            for tenant in all_tenants:
                Migrations._dependencies_done = False  # very important, the dependencies need to be purged from cache
                self._notice("=== Running migrate for schema %s" % tenant.schema_name)
                self._migrate_schema(tenant)

        self._restore_south_settings()

    def migrate_public_apps(self):
        self._save_south_settings()

        apps = self.shared_apps or self.installed_apps
        self._set_managed_apps(included_apps=apps, excluded_apps=self.tenant_apps)

        self._notice("=== Running migrate for schema public")
        MigrateCommand().execute(*self.args, **self.options)

        self._clear_south_cache()
        self._restore_south_settings()


# -*- coding: utf-8 -*-
from sklearn import svm
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn import metrics
import warnings
import jieba
import os
import numpy as np

#去除各种警告
def warn_cancel():
    warnings.filterwarnings('ignore')

#将训练数据和测试数据切分为训练集和标签，以及测试集和标签
def splitset(trainset, testset):
    train_words = []
    train_tags = []
    test_words = []
    test_tags = []
    for i in trainset:
        i = i.strip()
        train_words.append(i[:-2])
        #-2为冒号的位置
        train_tags.append(int(i[-1]))
    for i in testset:
        i = i.strip()
        test_words.append(i[:-2])
        test_tags.append(int(i[-1]))
    return train_words, train_tags, test_words, test_tags

#将训练集和测试集中的特征词提取出来，并转为稀疏矩阵
def tfvectorize(train_words, test_words):
    warn_cancel()
    v = TfidfVectorizer()
    train_data = v.fit_transform(train_words)
    #v.get_feature_names(),可获取特征词
    test_data = v.transform(test_words)
    return train_data, test_data

#得到准确率
#test,re
def evaluate(actual, pred):
    warn_cancel()
    m_precision = metrics.precision_score(actual, pred)
    print('precision:{0:.3f}'.format(m_precision))

# 创建svm分类器
def train_clf(train_data, train_tags):
    warn_cancel()
    clf = svm.SVC(C=10.0,cache_size=200, class_weight=None, coef0=0.0,
    decision_function_shape=None, degree=3, gamma='auto', kernel='rbf',
    max_iter=-1, probability=False, random_state=None, shrinking=True,
    tol=0.001, verbose=False)
    clf.fit(train_data, train_tags)
    return clf

if __name__ == '__main__':
    tr = open('trainset.txt',encoding='utf-8').readlines()
    te = open('testset.txt',encoding='utf-8').readlines()
    train_words, train_tags, test_words, test_tags = splitset(tr, te)
    train_data, test_data = tfvectorize(train_words, test_words)
    clf = train_clf(train_data, train_tags)
    re = clf.predict(test_data)
    #re即为预测的测试集的分类
    #print(re)
    evaluate(test_tags, re)