import matplotlib.pyplot as plt
import numpy as np

#---------------------------------------------------------------------------
#---------------------------------------------------------------------------

def periodogram6(ax, x, y, vlines, plot_min_t=1, max_f=1, spp=100):

	from scipy.signal import find_peaks
	from astropy.timeseries import LombScargle

	time_span = (max(x) - min(x))
	min_f   = 1/time_span

	frequency, power = LombScargle(x, y).autopower(minimum_frequency=min_f,
												   maximum_frequency=max_f,
												   samples_per_peak=spp)

	plot_x = 1/frequency
	idxx = (plot_x>plot_min_t) & (plot_x<100)
	height = max(power[idxx])*0.9
	ax.plot(plot_x[idxx], power[idxx], 'k-', alpha=0.7)
	peaks, _ = find_peaks(power[idxx], height=height)
	ax.plot(plot_x[idxx][peaks], power[idxx][peaks], "ro")

	for n in range(len(plot_x[idxx][peaks])):
		ax.text(plot_x[idxx][peaks][n], power[idxx][peaks][n], '%.1f' % plot_x[idxx][peaks][n], fontsize=10)

	for xc in vlines:
		ax.axvline(x=xc, color='r', linestyle='-', lw=2, alpha=0.3)

	ax.set_xlim([plot_min_t,100])
	ax.set_ylim([0,1.7*height])
	ax.set_xscale('log')

#---------------------------------------------------------------------------
#---------------------------------------------------------------------------

def plot_all(k_mode, t, rv, erv, ind, eind, ts_xlabel, rv_xlabel, pe_xlabel, ind_yalbel, file_name, height_ratio=0.7, vlines=[], HARPS=True):

	'''
	e.g.
		k_mode 		= 11
		t 			= bjd_daily
		rv 			= rv_daily
		erv 		= erv_daily
		ind 		= ΔRV_k
		eind 	 	= eRV_FT_k
		ts_xlabel 	= 'BJD - 2400000 [d]'
		rv_xlabel 	= '$RV_{HARPS}$'
		pe_xlabel 	= 'Period [days]'
		ind_yalbel	= 'A'
		file_name 	= 'time-series_and_shift_correlation.pdf'

	'''

	def new_periodogram(x, y, dy, vlines, height_ratio=height_ratio, plot_min_t=2, max_f=1, spp=100):

		from scipy.signal import find_peaks
		from astropy.timeseries import LombScargle

		time_span = (max(x) - min(x))
		min_f   = 1/time_span

		frequency, power = LombScargle(x, y, dy).autopower(minimum_frequency=min_f,
													   maximum_frequency=max_f,
													   samples_per_peak=spp)

		plot_x = 1/frequency
		idxx = (plot_x>plot_min_t) & (plot_x<time_span/2)
		height = max(power[idxx])*height_ratio
		ax.plot(plot_x[idxx], power[idxx], 'k-', alpha=0.5)
		peaks, _ = find_peaks(power[idxx], height=height)
		ax.plot(plot_x[idxx][peaks], power[idxx][peaks], "ro")

		for n in range(len(plot_x[idxx][peaks])):
			ax.text(plot_x[idxx][peaks][n], power[idxx][peaks][n], '%.1f' % plot_x[idxx][peaks][n], fontsize=10)

		ax.set_xlim([plot_min_t,time_span/2])
		ax.set_ylim([0, 1.25*max(power[idxx])])

		if vlines!=[]:
			for xc in vlines:
				ax.axvline(x=xc, color='r', linestyle='-', lw=2, alpha=0.2)
		if HARPS == True:
			ax.axvspan(200, 220, facecolor='r', alpha=0.2)

		ax.set_xscale('log')

	from sklearn.linear_model import LinearRegression

	# set up the plotting configureations
	alpha1, alpha2 = [0.5,0.2]
	widths 	= [7,1,7]
	heights = [1]*(k_mode+1)
	gs_kw 	= dict(width_ratios=widths, height_ratios=heights)
	plt.rcParams.update({'font.size': 12})
	fig6, f6_axes = plt.subplots(figsize=(16, k_mode+1), ncols=3, nrows=k_mode+1, constrained_layout=True,
	                             gridspec_kw=gs_kw)

	# plots
	for r, row in enumerate(f6_axes):
		for c, ax in enumerate(row):

			# time-series
			if c==0:
				if r==0:
					ax.errorbar(t, rv-np.mean(rv), erv, marker='.', ms=5, color='black', ls='none', alpha=alpha1)
					ax.set_title('Time-series')
					ax.set_ylabel(rv_xlabel)
				else:
					ax.errorbar(t, ind[r-1,:], eind[r-1,:],  marker='.', ms=5, color='black', ls='none', alpha=alpha1)
					ax.set_ylabel(ind_yalbel + '$_{' + str(r) + '}$')
				if r!=k_mode:
					ax.tick_params(labelbottom=False)
				else:
					ax.set_xlabel(ts_xlabel)

			if c==1:
				if r==0:
					reg = LinearRegression().fit(rv.reshape(-1, 1), rv.reshape(-1, 1))
					score = reg.score(rv.reshape(-1, 1), rv.reshape(-1, 1))
					adjust_R2 = 1-(1-score)*(len(t)-1)/(len(t)-1-1)
					title = r'$\bar{R}$' + r'$^2$'
					ax.set_title(title + ' = {:.2f}'.format(adjust_R2))
					ax.plot(rv-np.mean(rv), rv-np.mean(rv), 'k.', alpha = alpha2)
				if r>0:
					reg = LinearRegression().fit(rv.reshape(-1, 1), ind[r-1,:].reshape(-1, 1))
					score = reg.score(rv.reshape(-1, 1), ind[r-1,:].reshape(-1, 1))
					adjust_R2 = 1-(1-score)*(len(t)-1)/(len(t)-1-1)
					title = r'$\bar{R}$' + r'$^2$'
					ax.set_title(title + ' = {:.2f}'.format(adjust_R2))
					ax.plot(rv-np.mean(rv), ind[r-1,:], 'k.', alpha = alpha2)
				if r!=k_mode:
					ax.tick_params(labelbottom=False)
				else:
					ax.set_xlabel(rv_xlabel)
				ax.yaxis.tick_right()

			if c==2:
				if r==0:
					new_periodogram(t, rv, erv, vlines)
					ax.set_title('Periodogram')
				if r>0:
					new_periodogram(t, ind[r-1,:], eind[r-1,:], vlines)
				if r!=k_mode:
					ax.tick_params(labelbottom=False)
				if r==k_mode:
					ax.set_xlabel(pe_xlabel)

	fig6.align_ylabels(f6_axes[:, 0])
	plt.savefig(file_name)
	plt.show()
	plt.close('all')

#---------------------------------------------------------------------------
#---------------------------------------------------------------------------

def plot_all_but_corr(k_mode, t, ind, eind, height_ratio, ts_xlabel, pe_xlabel, ind_yalbel, file_name):

	'''
	e.g.
		k_mode 		= 11
		t 			= bjd_daily
		ind 		= shift_function
		eind 	 	= err_shift_spectrum
		ts_xlabel 	= 'BJD - 2400000 [d]'
		pe_xlabel 	= 'Period [days]'
		ind_yalbel	= 'A'
		file_name 	= 'time-series_and_shift_correlation.png'

	'''
	def new_periodogram(x, y, dy, height_ratio=height_ratio, plot_min_t=2, max_f=1, spp=100):

		from scipy.signal import find_peaks
		from astropy.timeseries import LombScargle

		time_span = (max(x) - min(x))
		min_f   = 1/time_span

		frequency, power = LombScargle(x, y, dy).autopower(minimum_frequency=min_f,
													   maximum_frequency=max_f,
													   samples_per_peak=spp)

		plot_x = 1/frequency
		idxx = (plot_x>plot_min_t) & (plot_x<time_span/2)
		height = max(power[idxx])*height_ratio
		ax.plot(plot_x[idxx], power[idxx], 'k-', alpha=0.5)
		peaks, _ = find_peaks(power[idxx], height=height)
		ax.plot(plot_x[idxx][peaks], power[idxx][peaks], "ro")

		for n in range(len(plot_x[idxx][peaks])):
			ax.text(plot_x[idxx][peaks][n], power[idxx][peaks][n], '%.1f' % plot_x[idxx][peaks][n], fontsize=10)

		ax.set_xlim([plot_min_t,time_span/2])
		ax.set_ylim([0, 1.25*max(power[idxx])])

		ax.set_xscale('log')

	from sklearn.linear_model import LinearRegression

	# set up the plotting configureations
	alpha1, alpha2 = [0.5,0.2]
	widths 	= [7,7]
	heights = [1]*k_mode
	gs_kw 	= dict(width_ratios=widths, height_ratios=heights)
	plt.rcParams.update({'font.size': 12})
	fig6, f6_axes = plt.subplots(figsize=(16, k_mode), ncols=2, nrows=k_mode, constrained_layout=True,
	                             gridspec_kw=gs_kw)

	# plots
	for r, row in enumerate(f6_axes):
		for c, ax in enumerate(row):

			# time-series
			if c==0:
				ax.errorbar(t, ind[r,:], eind[r,:],  marker='.', ms=5, color='black', ls='none', alpha=alpha1)
				if len(ind_yalbel)==1:
					ax.set_ylabel(ind_yalbel[0] + '$_{' + str(r+1) + '}$')
				else:
					ax.set_ylabel(ind_yalbel[r])
				if r==0:
					ax.set_title('Time-series')
				if r!=(k_mode-1):
					ax.set_xticks([])
				if r==(k_mode-1):
					ax.set_xlabel(ts_xlabel)

			if c==1:
				new_periodogram(t, ind[r,:], eind[r,:])
				if r==0:
					ax.set_title('Periodogram')
				if r!=(k_mode-1):
					ax.set_xticks([])
				if r==(k_mode-1):
					ax.set_xlabel(pe_xlabel)

	fig6.align_ylabels(f6_axes[:, 0])
	plt.savefig(file_name)
	plt.show()
	plt.close('all')

#---------------------------------------------------------------------------
#---------------------------------------------------------------------------

def weighted_pca(X, X_err, n_pca=None, nor=False):
	'''
	X.shape 	= (n_samples, n_features)
	X_err.shape = (n_samples, n_features)
	nor: normalization = True / False
	'''
	from wpca import WPCA

	weights = 1/X_err
	kwds 	= {'weights': weights}

	# subtract the weighted mean for each measurement type (dimension) of X
	X_new 	= np.zeros(X.shape)
	mean 	= np.zeros(X.shape[1])
	std 	= np.zeros(X.shape[1])

	for i in range(X.shape[1]):
		mean[i] = np.average(X[:,i], weights=weights[:,i])
		std[i] 	= np.average((X[:,i]-mean[i])**2, weights=weights[:,i])**0.5

	for i in range(X.shape[1]):
		if nor == True:
			X_new[:,i] 		= (X[:,i] - mean[i]) / std[i]
			weights[:,i]	= weights[:,i] * std[i] 	# may need to include the Fourier power later
		else:
			X_new[:,i] = X[:,i] - mean[i]

	if n_pca==None:
		n_pca = X.shape[1]

	# Compute the PCA vectors & variance
	pca 		= WPCA(n_components=n_pca).fit(X_new, **kwds)
	pca_score 	= pca.transform(X_new, **kwds)
	P 			= pca.components_

	# The following computes the errorbars
	# The transpose is only intended to be consistent with the matrix format from Ludovic's paper
	X_wpca 	= X_new.T
	C 		= np.zeros(X_wpca.shape)
	err_C 	= np.zeros(X_wpca.shape)

	W 		= weights.T
	P 		= P.T

	for i in range(X_wpca.shape[1]):
		w = np.diag(W[:, i]) ** 2
		C[:, i] = np.linalg.inv(P.T @ w @ P) @ (P.T @ w @ X_wpca[:, i])

		# the error is described by a covariance matrix of C
		Cov_C = np.linalg.inv(P.T @ w @ P)
		diag_C = np.diag(Cov_C)
		err_C[:, i] = diag_C ** 0.5

	P = pca.components_
	err_pca_score = err_C.T

	#---------#
	# results #
	#---------#
	cumulative_variance_explained = np.cumsum(
		pca.explained_variance_ratio_) * 100  # look again the difference calculated from the other way
	print('Cumulative variance explained vs PCA components')
	for i in range(len(cumulative_variance_explained)):
		print(i+1, '\t', '{:.3f}'.format(cumulative_variance_explained[i]))

	for i in range(len(cumulative_variance_explained)):
		if cumulative_variance_explained[i] < 89.5:
			n_pca = i
	n_pca += 2
	if cumulative_variance_explained[0] > 89.5:
		n_pca = 1

	print('{:d} pca scores account for {:.2f}% variance explained'.format(n_pca,
			cumulative_variance_explained[n_pca - 1]))

	print('Standard deviations of each component and the midean uncertainty are\n',
		  np.around(np.std(C[0:n_pca, :], axis=1), decimals=1), '\n',
		  np.around(np.median(err_C[0:n_pca, :], axis=1), decimals=1))

	return P, pca_score, err_pca_score, n_pca

#---------------------------------------------------------------------------
#---------------------------------------------------------------------------

def long_short_divide(x, y, yerr, r):
	'''
	x 		= bjd_daily
	y 		= shift_function[i,:]
	yerr 	= err_shift_spectrum[i,:]
	'''

	import george
	from george import kernels

	kernel 	= np.var(y) * kernels.Matern52Kernel(r**2)
	gp 		= george.GP(kernel)
	gp.compute(x, yerr)

	y_pred, _ 	= gp.predict(y, x, return_var=True)
	long_term 	= y_pred
	short_term 	= y - y_pred

	return gp, short_term, long_term

#---------------------------------------------------------------------------
#---------------------------------------------------------------------------

def lasso_lambda(X, Y, Weight, alphas, n_folds=5, N=100, title='', file_name=None):

	from sklearn.linear_model import Lasso
	from sklearn.model_selection import GridSearchCV
	from sklearn.utils import shuffle
	from datetime import datetime
	from alive_progress import alive_bar

	start_time  = datetime.now()

	alpha_size 	= len(alphas)
	array_size 	= len(Y)

	alpna_n 		= np.zeros(N)
	# coef 			= np.zeros((36,N))
	scores_test 	= np.zeros((alpha_size, N))
	scores_train 	= np.zeros((alpha_size, N))
	scores_test_std = np.zeros((alpha_size, N))
	scores_train_std= np.zeros((alpha_size, N))

	tuned_parameters 	= [{"alpha": alphas}]

	with alive_bar(N) as bar:

		for n in range(N):
			idx_random 	= shuffle(np.arange(array_size))
			X 			= X[idx_random,:]
			Y 			= Y[idx_random]
			sw 			= Weight[idx_random]

			lasso 		= Lasso()
			clf 		= GridSearchCV(lasso, tuned_parameters, cv=n_folds, scoring='neg_root_mean_squared_error', refit=True, return_train_score=True)
			clf.fit(X, Y, sample_weight=sw)
			# coef[:,n] 	= clf.best_estimator_.coef_
			alpna_n[n] 	= clf.best_estimator_.alpha

			scores_test[:,n] 		= -clf.cv_results_["mean_test_score"]
			scores_train[:,n] 		= -clf.cv_results_["mean_train_score"]
			scores_test_std[:,n]	= clf.cv_results_["std_test_score"] / np.sqrt(n_folds)
			scores_train_std[:,n] 	= clf.cv_results_["std_train_score"] / np.sqrt(n_folds)

			bar()

	scores_test_mean 		= np.mean(scores_test, axis=1)
	scores_train_mean 		= np.mean(scores_train, axis=1)
	scores_test_std_mean 	= np.mean(scores_test_std, axis=1)
	scores_train_std_mean 	= np.mean(scores_train_std, axis=1)

	end_time = datetime.now()
	print('Duration: {}'.format(end_time - start_time))

	# plots
	if file_name != None:
		plt.figure().set_size_inches(8, 6)
		plt.rcParams['font.size'] = '16'
		plt.gcf().subplots_adjust(left=0.15)

		# plot error lines showing +/- std. errors of the scores
		plt.title(title)
		plt.semilogx(alphas, scores_test_mean, 'r', label='testing')
		plt.semilogx(alphas, scores_train_mean, 'b', label='training')
		plt.semilogx(alphas, scores_test_mean + scores_test_std_mean, "r--")
		plt.semilogx(alphas, scores_test_mean - scores_test_std_mean, "r--")
		plt.semilogx(alphas, scores_train_mean + scores_train_std_mean, "b--")
		plt.semilogx(alphas, scores_train_mean - scores_train_std_mean, "b--")
		plt.fill_between(alphas, scores_test_mean + scores_test_std_mean, scores_test_mean - scores_test_std_mean, color='r', alpha=0.2)
		plt.fill_between(alphas, scores_train_mean + scores_train_std_mean, scores_train_mean - scores_train_std_mean, color='b', alpha=0.2)

		xx 	= np.median(alpna_n)
		plt.axvline(xx, color='k', alpha=0.5)
		plt.text(xx, scores_test_mean[alphas==xx], r'$\hat\lambda_1$ = {:.3f}'.format(xx))

		# determine a more robust lambda using the 1-sigma rule
		scores_1sigma = (scores_test_mean+scores_test_std_mean)[alphas==xx][0]
		for i in range(alpha_size-1):
			if (scores_test_mean[i] < scores_1sigma) & (scores_test_mean[i+1] > scores_1sigma):
				alpha_1sigma = alphas[i]
		plt.axvline(alpha_1sigma, color='k', alpha=0.5)
		plt.axhline(scores_1sigma, linestyle="--", color=".5")
		plt.text(alpha_1sigma, scores_1sigma, r'$\hat\lambda_2$ = {:.3f}'.format(alpha_1sigma))
		plt.ylabel("Residual WRMS [m/s]")
		plt.xlabel(r"$\lambda$")
		plt.axhline(np.min(scores_test_mean), linestyle="--", color=".5")
		plt.xlim([alphas[0], alphas[-1]])
		plt.legend()

		plt.savefig(file_name + '.pdf')
		plt.show()
		plt.close()

	return scores_test_mean, scores_train_mean, scores_test_std_mean, scores_train_std_mean, alpna_n

#---------------------------------------------------------------------------
#---------------------------------------------------------------------------

def mlr(feature_matrix, target_vector, etarget_vector, alpha, lag='True', day=5, feature_matrix2=None):

	'''
		Multiple linear regression
		no lag involved
	'''

	from sklearn import linear_model
	from sklearn.linear_model import Lasso

	regr 	= linear_model.LinearRegression()
	lasso 	= Lasso(alpha=alpha, max_iter=10000).fit(feature_matrix, target_vector, sample_weight=1/etarget_vector**2)

	y_hat 			= lasso.predict(feature_matrix)
	_, w_std_all 	= weighted_avg_and_std(target_vector, 1/etarget_vector**2)
	_, res_wrms 	= weighted_avg_and_std((y_hat - target_vector), weights=1/etarget_vector**2)
	_, model_wrms 	= weighted_avg_and_std(y_hat, weights=1/etarget_vector**2)
	score 			= lasso.score(feature_matrix, target_vector, sample_weight=1/etarget_vector**2)
	n, p 			= len(target_vector), feature_matrix.shape[1]
	adjust_R2 		= 1-(1-score)*(n-1)/(n-p-1)
	score 			= adjust_R2

	if lag=='False':
		k_feature2 	= feature_matrix.shape[1]
	else:
		k_feature2 	= int(feature_matrix.shape[1]/(day+1))
		k_feature 	= int(k_feature2/2)

	w_std = np.zeros(k_feature2)
	for i in range(len(w_std)):
		if lag=='True':
			_, w_std[i] = weighted_avg_and_std(feature_matrix2[:,i], 1/etarget_vector**2)
		else:
			_, w_std[i] = weighted_avg_and_std(feature_matrix[:,i], 1/etarget_vector**2)
	print('Weighted rms is reduced from {:.2f} to {:.2f} (-{:.0f}%); \n\
		Modelled RV weigthed rms = {:.2f};\n\
		Adjusted R squared = {:.3f}.'
			.format(w_std_all, res_wrms, (1-res_wrms/w_std_all)*100, model_wrms, score))

	if lag=='False':

		import pandas as pd
		df = pd.DataFrame({	'coefficients'		: lasso.coef_,
							'std'				: w_std,
							'variance'			: (lasso.coef_*w_std)**2,
							'variance percentage': (lasso.coef_*w_std)**2 / sum((lasso.coef_*w_std)**2) * 100
							})
		print(df.round({'coefficients'	: 2,
						'std'		: 2,
						'variance'		: 2,
						'variance percentage': 1
						}))

		return y_hat, w_std_all, res_wrms, score, df

	else:

		coeff_matrix = np.zeros((day*2+1, k_feature2))
		for i in range(k_feature):
			coeff_matrix[:,i] = lasso.coef_[i:k_feature*(2*day+1):k_feature]
		coeff_matrix[day,k_feature:]=lasso.coef_[k_feature*(2*day+1):]

		variance_matrix = np.zeros(coeff_matrix.shape)
		for i in range(day*2+1):
			variance_matrix[i,:] = coeff_matrix[i,:]*w_std
		variance_matrix = variance_matrix**2
		variance_matrix = variance_matrix / variance_matrix.sum() * 100

		return y_hat, w_std_all, res_wrms, score, variance_matrix

#---------------------------------------------------------------------------
#---------------------------------------------------------------------------

def weighted_avg_and_std(values, weights):
    """
    Return the weighted average and standard deviation.

    values, weights -- Numpy ndarrays with the same shape.
    """
    average = np.average(values, weights=weights)
    variance = np.average((values-average)**2, weights=weights)

    return (average, np.sqrt(variance))

#---------------------------------------------------------------------------
#---------------------------------------------------------------------------

def imshow_matrix(coeff_array, score, res_wrms, alpha, k_max, file_name):

	from matplotlib import colors

	'''
		file_name = fwhm_bis_coef
		file_name = fwhm_bis_multi_coef
		file_name = pca_coef
	'''

	day = int((coeff_array.shape[0]-1)/2)
	x = np.arange(day * 2 + 1) - day
	y = np.arange(coeff_array.shape[1]) + 1

	from mpl_toolkits.axes_grid1 import make_axes_locatable
	fig = plt.figure(figsize=(len(x)/1.5+2, len(y)/1.5+1), frameon=False)
	title = r'$\bar{R}$' + r'$^2$'
	plt.title(title + ' = {:.3f}'.format(score)
			  + '   Residual WRMS = {:.2f} m/s'.format(res_wrms))
	plt.xlabel('Lag [days]')
	ax = plt.gca()

	if file_name == 'lasso_coef':
		im2 = plt.imshow(np.transpose(coeff_array),  vmin=-5, vmax=5, cmap=plt.cm.bwr, alpha=.9)
	else:
		im2 = plt.imshow(np.transpose(coeff_array),  cmap=plt.cm.get_cmap('Reds'), alpha=.9, norm=colors.LogNorm(vmin=0.1, vmax=100))

	# Loop over data dimensions and create text annotations.
	for i in range(coeff_array.shape[1]):
		for j in range(day*2+1):
			if coeff_array[j, i] != 0:
				text = ax.text(j, i, '{:.1f}'.format(coeff_array[j, i]),
							   ha="center", va="center", color="k")

	ax.set_xticks(np.arange(len(x)))
	ax.set_yticks(np.arange(len(y)))
	ax.set_xticklabels(x)
	if file_name == 'fwhm_bis_coef':
		ax.set_yticklabels(['FWHM', 'BIS'])
	if file_name == 'Figure/fwhm_bis_multi_coef':
		ax.set_yticklabels(['S-FWHM', 'S-BIS', 'L-FWHM', 'L-BIS'])
	if file_name == 'pca_coef':
		ax.set_yticklabels([r'PC$_1$', r'PC$_2$', r'PC$_3$'])
	if file_name == 'Figure/fiesta_multi_coef':
		ax.set_yticklabels([r'S-PC$_1$', r'S-PC$_2$', r'S-PC$_3$', r'L-PC$_1$', r'L-PC$_2$', r'L-PC$_3$'])

	divider = make_axes_locatable(ax)
	cax = divider.append_axes("right", size="5%", pad=0.05)

	plt.colorbar(im2, cax=cax, label='Variance percentage')
	plt.savefig(file_name + '_{:.2f}_{:d}_{:d}'.format(alpha, day, k_max) +'.pdf')
	plt.show()
	plt.close()

#---------------------------------------------------------------------------
#---------------------------------------------------------------------------

def scatter_hist(x, y, xerr, yerr, ax, ax_histx, ax_histy):

    ax_histx.tick_params(axis="x", labelbottom=False)
    ax_histy.tick_params(axis="y", labelleft=False)

    # the scatter plot:
    ax.errorbar(x, y, xerr, yerr, c='black', marker='o', ls='none', alpha=0.2)
    ax.set_xlabel('Model 3 residual [m/s]')
    ax.set_ylabel('Model 6 residual [m/s]')

    # now determine nice limits by hand:
    binwidth = 0.25
    xymax = max(np.max(np.abs(x)), np.max(np.abs(y)))
    lim = (int(xymax/binwidth) + 1) * binwidth

    bins = np.arange(-lim, lim + binwidth, binwidth)
    ax_histx.hist(x, bins=bins, color='black', alpha=0.5)
    ax_histy.hist(y, bins=bins, orientation='horizontal', color='black', alpha=0.5)

#---------------------------------------------------------------------------
#---------------------------------------------------------------------------

def plot_soap_config(FEATURE):

    N_sample  = 8
    lat       = np.linspace(10, 80, num=N_sample)
    RV_gauss_all = np.zeros((N_sample, 100))
    ΔA_k_all = np.zeros((N_sample, 5, 100))
    Δv_k_all = np.zeros((N_sample, 5, 100))

    for i in range(N_sample):
        print('\nlatitude = ' + str(lat[i]))
        folder_name = 'CCF_PROT=25.05_i=90.00_lon=(180.0,0.0,0.0,0.0)_lat=(' + '%.1f' % lat[i] +  ',0.0,0.0,0.0)_size=(0.1000,0.0000,0.0000,0.0000)'
        DIR     = 'Data/' + FEATURE + '_lat/' + folder_name
        FILE    = sorted(glob.glob(DIR + '/fits/*fits'))
        N       = len(FILE)

        for n in range(N):
            hdulist     = fits.open(FILE[n])
            CCF[:, n]   = hdulist[0].data[v_idx]

        v_k, A_k, RV_gauss = FIESTA(V_grid_SOAP, CCF, eCCF=CCF*0, template=CCF[:,0], k_max=5)
        v_k *= 1000
        RV_gauss = (RV_gauss - RV_gauss[0]) * 1000
        RV_gauss_all[i,:] = RV_gauss

        ΔA_k_all[i,:,:] = np.array([A_k[:,kk] - A_k[:,0] for kk in range(100)]).T

        for k in range(v_k.shape[0]):
            v_k[k,:] -= RV_gauss
        Δv_k_all[i,:,:] = v_k

    #
    colors = cm.seismic(np.linspace(0, 1, RV_gauss_all.shape[0]))

    plt.rcParams.update({'font.size': 12})
    fig, axes = plt.subplots(5,2,figsize=(8, 6))
    plt.subplots_adjust(left = 0.15, hspace=0.1, wspace=0.35, top = 0.95, right = 0.95)

    phase = np.arange(100)/100
    idx = (phase<=0.8) & (phase>=0.2)

    for i in range(5):

        for j in range(N_sample):
            axes[i,0].plot(phase[idx], ΔA_k_all[j,i,idx], alpha=0.8, color=colors[7-j], label=r'$%d\degree$' %lat[j])
            axes[i,1].plot(phase[idx], Δv_k_all[j,i,idx], alpha=0.8, color=colors[7-j])
        axes[i,0].set_ylabel(r'$k$=%d' %(i+1))
        axes[i,1].set_ylabel('[m/s]')
        axes[0,0].set_title(FEATURE + r' $\Delta A_k$')
        axes[0,1].set_title(FEATURE + r' $\Delta RV_k$')

        if i == 1:
            axes[i,0].legend(fontsize=6, loc=3)

        for l in range(2):
            if i!=4:
                axes[i,l].set_xticks([])
            else:
                axes[i,l].set_xlabel('Rotation phase')

    fig.align_ylabels(axes[:,0])
    fig.align_ylabels(axes[:,1])
    plt.savefig('Figure/FIESTA-' + FEATURE + '_latitude.pdf')
    plt.show()

    RV_gauss = RV_gauss.reshape(-1, 1)
    plt.rcParams.update({'font.size': 16})
    fig, axes = plt.subplots(figsize=(20, 4))
    fig.tight_layout()
    plt.subplots_adjust(left=0.06, bottom=0.16, right=0.98, top=0.9, wspace=0.4, hspace=0.4)

    for j in range(5):
        ax = plt.subplot(1, 5, j+1)
        for i in range(RV_gauss_all.shape[0]):
            ax.plot(RV_gauss_all[i,:], Δv_k_all[i,j,:], '.-', alpha=0.8, color=colors[7-i], label=r'$%d\degree$' %lat[i])
            ax.set_xlabel(r'$RV_{Gaussian}$ [m/s]')
            ax.set_ylabel(r'$\Delta RV_{}$ [m/s]'.format(j+1))
            if FEATURE == 'spot':
                if j==0:
                    ax.legend(prop={'size': 10})
            if j==2:
                ax.set_title('Solar '+ FEATURE)
    plt.savefig('Figure/FIESTA-' + FEATURE + '_latitude_correlation.pdf')
    plt.show()


import pandas as pd
import numpy as np
from re import compile as rcompile
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from tensorflow.keras.layers import Input, Dense
from tensorflow.keras.models import Model, load_model
from tensorflow.keras.optimizers import Adam, Nadam
from sklearn.neighbors import NearestNeighbors
from joblib import load
from zipfile import ZipFile
from os.path import dirname

DIR = dirname(__file__)

rex = rcompile('[^a-zA-Z 0-9]')

tokenize = lambda x: rex.sub('', x.lower().replace(',', ' '))

MODELS_DIR = DIR + '/../../models/'
DATA_DIR = DIR + '/../../data/'

ENCODER = 'encoder.h5'
FG_ENCODER = 'fg_encoder.h5'

ENCODER_PATH = MODELS_DIR + ENCODER + '.zip'
ENCODED_DTM = MODELS_DIR + 'encoded_dtm.pkl'
TFIDF = MODELS_DIR + 'tfidf.pkl'

FG_ENCODER_PATH = MODELS_DIR + FG_ENCODER
FG_ENCODED_DF = MODELS_DIR + 'fg_encoded_df.pkl'
GENRES_TFIDF = MODELS_DIR + 'genres_tfidf.pkl'
SCALER = MODELS_DIR + 'scaler.pkl'

TRACKS = DATA_DIR + 'tracks_genres_lyrics_en.csv.zip'

class FindSongs(object):
    def __init__(self):

        with ZipFile(ENCODER_PATH, 'r') as zipObj:
            zipObj.extractall()

        self.encoder = load_model(ENCODER)
        self.tfidf = load(TFIDF)
        self.encoded_dtm = load(ENCODED_DTM)
        # Fit on DTM
        self.nn = NearestNeighbors(n_neighbors=5, algorithm='ball_tree')
        self.nn.fit(self.encoded_dtm)

        self.features = [
            'popularity', 'duration_ms', 'explicit', 'danceability',
            'energy', 'key', 'loudness', 'mode', 'speechiness',
            'acousticness', 'instrumentalness', 'liveness', 'valence',
            'tempo', 'time_signature'
        ]
        self.fg_encoder = load_model(FG_ENCODER_PATH)
        self.fg_encoded_df = load(FG_ENCODED_DF)
        self.genres_tfidf = load(GENRES_TFIDF)
        self.scaler = load(SCALER)
        self.fg_nn = NearestNeighbors(n_neighbors=5, algorithm='ball_tree')
        self.fg_nn.fit(self.fg_encoded_df)

        self.tracks_df = pd.read_csv(TRACKS)

        self.tracks_df.drop(columns=['Unnamed: 0'], inplace=True)
        self.tracks_df = self.tracks_df[self.tracks_df.genres.isna() == False]

    def find_song_entries(self, x):
        vec = self.tfidf.transform([tokenize(x)]).todense()
        encoded_vec = self.encoder.predict(vec)
        entries = self.nn.kneighbors(encoded_vec)[1][0].tolist()
        entries = self.tracks_df.iloc[entries].popularity.sort_values(ascending=False).index.tolist()

        return self.tracks_df.loc[entries]

    def find_song_entry(self, x, best_choice=True):
        df = self.find_song_entries(x)

        choice = df.index.tolist()
        if best_choice:
            choice = choice[0]

        return df.loc[choice]

    def get_recommendations(self, x):

        gvec = self.genres_tfidf.transform([tokenize(x.genres)]).todense()
        fvec = self.scaler.transform([x[self.features]])
        vec = [fvec.tolist()[0] + gvec.tolist()[0]]
        encoded_vec = self.fg_encoder.predict(vec)
        entries = self.fg_nn.kneighbors(encoded_vec)[1][0].tolist()
        entries = self.tracks_df.iloc[entries].popularity.\
            sort_values(ascending=False).index.tolist()

        return self.tracks_df.loc[entries]


# Copyright 2018 The TensorFlow Probability Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

# Dependency imports
import numpy as np
from scipy import interpolate as scipy_interpolate
import tensorflow as tf

import tensorflow_probability as tfp

from tensorflow.python.eager import context
from tensorflow.python.framework import test_util

@test_util.run_all_in_graph_and_eager_modes
class InterpRegular1DGridTest(tf.test.TestCase):
  """Test for tfp.math.interp_regular_1d_grid."""

  def test_on_1d_array_nan_fill_value(self):
    x_min = 0.
    x_max = 1.
    dtype = np.float32
    num_pts = 4

    implied_x_ref = np.linspace(x_min, x_max, num_pts, dtype=dtype)
    y_ref = 2 * implied_x_ref

    # First and last values of x are outside the [x_min, x_max] range.
    x = np.array([-1., 0.1, 0.2, 0.4, 2.]).astype(dtype)
    y_expected = 2 * x
    y_expected[0] = np.nan  # Will fill this value for points < x_min.
    y_expected[-1] = np.nan  # Will fill this value for points > x_max.

    with self.test_session():
      y = tfp.math.interp_regular_1d_grid(
          x, x_min, x_max, y_ref, fill_value=np.nan)
      self.assertAllEqual(y_expected.shape, y.shape)
      y_ = self.evaluate(y)
      self.assertAllClose(y_, y_expected, atol=0, rtol=1e-6)

  def test_on_1d_array_different_below_above_fill_values(self):
    # Simple debuggable test
    x_min = 0.
    x_max = 1.
    dtype = np.float32
    num_pts = 4

    implied_x_ref = np.linspace(x_min, x_max, num_pts, dtype=dtype)
    y_ref = 2 * implied_x_ref

    # First and last values of x are outside the [x_min, x_max] range.
    x = np.array([-1., 0.1, 0.2, 0.4, 2.]).astype(dtype)
    y_expected = 2 * x
    y_expected[0] = -111  # Will fill this value for points < x_min.
    y_expected[-1] = 111  # Will fill this value for points > x_max.

    with self.test_session():
      y = tfp.math.interp_regular_1d_grid(
          x, x_min, x_max, y_ref, fill_value_below=-111., fill_value_above=111.)
      self.assertAllEqual(y_expected.shape, y.shape)
      y_ = self.evaluate(y)
      self.assertAllClose(y_, y_expected, atol=0, rtol=1e-6)

  def test_on_1d_array_constant_extension_fill_value(self):
    # Simple debuggable test
    x_min = 0.
    x_max = 1.
    dtype = np.float32
    num_pts = 4

    implied_x_ref = np.linspace(x_min, x_max, num_pts, dtype=dtype)
    y_ref = 2 * implied_x_ref

    # First and last values of x are outside the [x_min, x_max] range.
    x = np.array([-1., 0.1, 0.2, 0.4, 2.]).astype(dtype)
    y_expected = 2 * x
    y_expected[0] = y_ref[0]  # Will fill this value for points < x_min.
    y_expected[-1] = y_ref[-1]  # Will fill this value for points > x_max.

    with self.test_session():
      y = tfp.math.interp_regular_1d_grid(
          x, x_min, x_max, y_ref, fill_value='constant_extension')
      self.assertAllEqual(y_expected.shape, y.shape)
      y_ = self.evaluate(y)
      self.assertAllClose(y_, y_expected, atol=0, rtol=1e-6)

  def test_on_1d_array_extrapolate_fill_value(self):
    # Simple debuggable test
    x_min = 0.
    x_max = 2.
    dtype = np.float32
    num_pts = 3

    implied_x_ref = np.linspace(x_min, x_max, num_pts, dtype=dtype)
    y_ref = 2 * implied_x_ref

    # First and last values of x are outside the [x_min, x_max] range.
    x = np.array([-1., 0.1, 0.2, 0.4, 2.]).astype(dtype)
    y_expected = 2 * x
    y_expected[0] = -2.  # Linear extrapolation.
    y_expected[-1] = 4.  # Linear extrapolation.

    with self.test_session():
      y = tfp.math.interp_regular_1d_grid(
          x, x_min, x_max, y_ref, fill_value='extrapolate')
      self.assertAllEqual(y_expected.shape, y.shape)
      y_ = self.evaluate(y)
      self.assertAllClose(y_, y_expected, atol=0, rtol=1e-6)

  def test_on_1d_array_log_spaced_grid(self):
    x_min = 1.
    x_max = 100000.
    num_pts = 10

    # With only 10 interpolating points between x_ref = 1 and 100000,
    # and y_ref = log(x_ref), we better use a log-spaced grid, or else error
    # will be very bad.
    implied_x_ref = tf.exp(tf.linspace(tf.log(x_min), tf.log(x_max), num_pts))
    y_ref = tf.log(implied_x_ref)

    x = tf.linspace(x_min + 0.123, x_max, 20)
    y_expected = tf.log(x)

    with self.test_session():
      y = tfp.math.interp_regular_1d_grid(
          x, x_min, x_max, y_ref, grid_regularizing_transform=tf.log)
      self.assertAllEqual(y_expected.shape, y.shape)
      y_ = self.evaluate(y)
      # Super duper accuracy!  Note accuracy was not good if I did not use the
      # grid_regularizing_transform.
      self.assertAllClose(y_, y_expected, atol=0, rtol=1e-6)

  def test_matrix_interpolation(self):
    # Interpolate a matrix-valued function of one variable.
    # Use only two interpolation points.
    mat_0 = np.array([[1., 0.], [0., 1.]])
    mat_1 = np.array([[0., -1], [1, 0]])
    y_ref = np.array([mat_0, mat_1])

    # Get three output matrices at once.
    x = np.array([0., 0.5, 1.])
    y = tfp.math.interp_regular_1d_grid(
        x, x_ref_min=0., x_ref_max=1., y_ref=y_ref, axis=0)
    self.assertAllEqual((3, 2, 2), y.shape)
    y_ = self.evaluate(y)
    self.assertAllClose(y_[0], mat_0)
    self.assertAllClose(y_[1], 0.5 * mat_0 + 0.5 * mat_1)
    self.assertAllClose(y_[2], mat_1)

    # Get one single output matrix
    y_0p5 = tfp.math.interp_regular_1d_grid(
        x=0.5, x_ref_min=0., x_ref_max=1., y_ref=y_ref, axis=0)
    self.assertAllClose(self.evaluate(y_0p5), 0.5 * mat_0 + 0.5 * mat_1)

  def _check_sinusoid(self,
                      x_shape,
                      y_ref_shape,
                      axis,
                      fill_value,
                      dtype=np.float32):
    """Check correctness on a sinusoidal function."""
    positive_axis = axis if axis >= 0 else axis + len(y_ref_shape)

    x_ref_min = 0.
    x_ref_max = 2 * np.pi

    # Create implied_x_ref, which has same shape = y_ref_shape.
    implied_x_ref_1d = np.linspace(
        x_ref_min, x_ref_max, y_ref_shape[axis], dtype=dtype)
    implied_x_ref = implied_x_ref_1d.copy()
    for _ in y_ref_shape[positive_axis + 1:]:
      implied_x_ref = implied_x_ref[..., np.newaxis]
    for _ in y_ref_shape[:positive_axis]:
      implied_x_ref = implied_x_ref[np.newaxis, ...]
    implied_x_ref = implied_x_ref + np.zeros(y_ref_shape, dtype=dtype)

    y_ref = np.sin(implied_x_ref)

    # make sure some values are not in [x_ref_min, x_ref_max]
    x = np.linspace(
        x_ref_min - 0.5, x_ref_max + 0.5, np.prod(x_shape),
        dtype=dtype).reshape(x_shape)

    with self.cached_session():
      y = tfp.math.interp_regular_1d_grid(
          x,
          x_ref_min,
          x_ref_max,
          y_ref,
          fill_value=fill_value,
          axis=axis)

      expected_out_shape = (
          y_ref.shape[:positive_axis] + x_shape +
          y_ref.shape[positive_axis + 1:])
      self.assertAllEqual(expected_out_shape, y.shape)

      sp_func = scipy_interpolate.interp1d(
          implied_x_ref_1d,
          y_ref,
          kind='linear',
          axis=axis,
          bounds_error=False,  # Allow values outside range.
          fill_value=fill_value)
      y_expected = sp_func(x)
      self.assertAllClose(self.evaluate(y), y_expected, atol=1e-5, rtol=0)

  def test_axis_1_shape_2_50_3_4_fvextrapolate_32bit(self):
    self._check_sinusoid(
        x_shape=(2, 7),
        y_ref_shape=(2, 50, 3, 4),
        axis=1,
        fill_value='extrapolate',
        dtype=np.float32)

  def test_axis_1_shape_2_50_3_4_fvscalar_64bit(self):
    self._check_sinusoid(
        x_shape=(),
        y_ref_shape=(2, 50, 3, 4),
        axis=1,
        fill_value=np.nan,
        dtype=np.float32)

  def test_axis_n2_shape_2_3_50_4_fvextrapolate_64bit(self):
    self._check_sinusoid(
        x_shape=(10,),
        y_ref_shape=(2, 3, 50, 4),
        axis=-2,
        fill_value='extrapolate',
        dtype=np.float32)

  def test_axis_n1_shape_2_3_4_50_fvextrapolate_32bit(self):
    self._check_sinusoid(
        x_shape=(1, 8),
        y_ref_shape=(2, 3, 4, 50),
        axis=-1,
        fill_value='extrapolate',
        dtype=np.float32)

  def test_axis_3_shape_2_3_4_50_fvscalar_32bit(self):
    self._check_sinusoid(
        x_shape=(5,),
        y_ref_shape=(2, 3, 4, 50),
        axis=3,
        fill_value=np.inf,
        dtype=np.float32)

  def test_gradients_and_propagation_of_nan_in_x(self):
    # If x contains NaN, this should propagate through to y, and not mess up the
    # gradients associated with finite members of x.
    # In fact, even NaN members of x result in finite (zero) gradients.

    x_min = 0.
    x_max = 1.
    dtype = np.float32
    num_pts = 4

    implied_x_ref = np.linspace(x_min, x_max, num_pts, dtype=dtype)
    y_ref = 2 * implied_x_ref

    x_ = np.array([0., 0.1, np.nan, 0.4, 1.]).astype(dtype)
    y_expected = 2 * x_

    x = tf.constant(x_)

    with self.test_session():
      y = tfp.math.interp_regular_1d_grid(x, x_min, x_max, y_ref)
      y_ = self.evaluate(y)
      self.assertAllClose(y_, y_expected, atol=0, rtol=1e-6)
      if not context.executing_eagerly():
        dy_dx_ = tf.gradients(y, x)[0].eval()
        self.assertAllClose([2., 2., 0., 2., 2.], dy_dx_)

if __name__ == '__main__':
  tf.test.main()


# -*- coding: utf-8 -*-
"""
Created on Mon Oct  7 14:20:27 2019

@author: Christian Post
"""
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.datasets import load_breast_cancer
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split, KFold
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from mpl_toolkits.mplot3d import Axes3D
from sklearn.cluster import KMeans

# Breast cancer dataset description
# https://archive.ics.uci.edu/ml/datasets/Breast+Cancer+Wisconsin+(Diagnostic)
cancer = load_breast_cancer()

features = cancer.feature_names

cancer_df = pd.DataFrame(data=cancer.data, columns=features)
cancer_df['target'] = cancer.target

# scale the data for KNN.
scaler = StandardScaler()
scaler.fit(cancer_df[features])
cancer_df[features] = scaler.transform(cancer_df[features])

# Create decision tree classifer object for feature selection
clf_params = {
        'random_state': 1,
        'n_jobs': -1,
        'n_estimators': 200,
        'max_depth': None
        }
clf = RandomForestClassifier(**clf_params)
# Train the model
clf.fit(cancer_df[features], cancer_df['target'])
# Calculate feature importances
importances = clf.feature_importances_
# Sort feature importances in descending order
indices = np.argsort(importances)[::-1]
# Rearrange feature names so they match the sorted feature importances
names = [features[i] for i in indices]

# Create plot
plt.figure(figsize=(12, 6))
# Create plot title
plt.title("Feature Importance")
# Add bars
plt.barh(range(cancer_df[features].shape[1])[::-1], importances[indices])
# Add feature names as x-axis labels
plt.yticks(range(cancer_df[features].shape[1])[::-1], names)
plt.show()

# K Means
df = pd.DataFrame(data=cancer.data, columns=cancer.feature_names)
df['target'] = cancer.target

X = df[names]
# select 3 best features
#X = df[names[:3]]
y = df['target']

# Number of clusters
kmeans = KMeans(n_clusters=2)
# Fitting the input data
kmeans = kmeans.fit(X)
# Getting the cluster labels
labels = kmeans.predict(X)
# Centroid values
C = kmeans.cluster_centers_

plt.rcParams['figure.figsize'] = (12, 8)
fig = plt.figure()
ax = Axes3D(fig)
ax.view_init(azim=60, elev=60)
ax.set_xlabel(names[0])
ax.set_ylabel(names[2])
ax.set_zlabel(names[3])
ax.scatter(X[names[0]], X[names[1]], X[names[2]], s=36, c=cancer.target)
ax.scatter(C[:, 0], C[:, 1], C[:, 2], marker='*', c='#050505', s=1000)
plt.savefig('Breast_cancer_clustering.png', bbox_inches='tight')
plt.show()

# KNN with 10 most important features
accuracies = []
for i in range(100):
    KNN = KNeighborsClassifier()
    X_train, X_valid, y_train, y_valid = train_test_split(cancer_df[names[:10]],
                                                          cancer_df['target'],
                                                          test_size=0.3,
                                                          random_state=None)
    KNN.fit(X_train, y_train)
    acc = KNN.score(X_valid, y_valid)
    accuracies.append(acc)
print(f'\nMean accuracy with KNN for validation set: {round(np.mean(accuracies), 2)}')


# -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import print_function

import io
import os
import copy
import time
import socket
import tarfile
import warnings
from contextlib import closing

import json
import docker
import pytest
import requests

from urllib3.exceptions import InsecureRequestWarning

import kubernetes
from openshift.dynamic import DynamicClient

if os.path.exists(os.path.join(os.getcwd(), 'KubeObjHelper.log')):
    os.remove(os.path.join(os.getcwd(), 'KubeObjHelper.log'))

@pytest.fixture(scope='session')
def port():
    with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
        s.bind(('', 0))
        return s.getsockname()[1]

@pytest.fixture(scope='session')
def openshift_container(request, port, pytestconfig):
    capmanager = pytestconfig.pluginmanager.getplugin('capturemanager')
    client = docker.from_env()
    openshift_version = request.config.getoption('--openshift-version')
    if openshift_version is None:
        yield None
    else:
        container = client.containers.run(
            'openshift/origin:{}'.format(openshift_version),
            'start master --listen=0.0.0.0:{}'.format(port),
            detach=True,
            ports={port: port}
        )

        try:
            # Wait for the container to no longer be in the created state before
            # continuing
            while container.status == u'created':
                capmanager.suspend()
                print("\nWaiting for container...")
                capmanager.resume()
                time.sleep(5)
                container = client.containers.get(container.id)

            # Disable InsecureRequest warnings because we can't get the cert yet
            warnings.simplefilter('ignore', InsecureRequestWarning)
            # Wait for the api server to be ready before continuing
            for _ in range(10):
                try:
                    requests.head("https://127.0.0.1:{}/healthz/ready".format(port), verify=False)
                except requests.RequestException:
                    pass
                time.sleep(1)
            warnings.simplefilter('default', InsecureRequestWarning)

            time.sleep(1)

            yield container
        finally:
            # Always remove the container
            container.remove(force=True)

@pytest.fixture(scope='session')
def kubeconfig(openshift_container, tmpdir_factory):
    # get_archive returns a stream of the tar archive containing the requested
    # files/directories, so we need use BytesIO as an intermediate step.
    if openshift_container is None:
        return None
    else:
        openshift_container.exec_run('oc login -u test -p test')
        tar_stream, _ = openshift_container.get_archive(
            '/var/lib/origin/openshift.local.config/master/admin.kubeconfig')

        tar_bytes = io.BytesIO()
        for chunk in tar_stream:
            tar_bytes.write(chunk)
        tar_bytes.seek(0)

        tar_obj = tarfile.open(fileobj=tar_bytes)
        kubeconfig_contents = tar_obj.extractfile('admin.kubeconfig').read()

        kubeconfig_file = tmpdir_factory.mktemp('kubeconfig').join('admin.kubeconfig')
        kubeconfig_file.write(kubeconfig_contents)
        return kubeconfig_file

@pytest.fixture(scope='session')
def admin_kubeconfig(openshift_container, tmpdir_factory):
    # get_archive returns a stream of the tar archive containing the requested
    # files/directories, so we need use BytesIO as an intermediate step.
    if openshift_container is None:
        return None
    else:
        openshift_container.exec_run('oc login -u system:admin')
        tar_stream, _ = openshift_container.get_archive(
            '/var/lib/origin/openshift.local.config/master/admin.kubeconfig')

        tar_bytes = io.BytesIO()
        for chunk in tar_stream:
            tar_bytes.write(chunk)
        tar_bytes.seek(0)

        tar_obj = tarfile.open(fileobj=tar_bytes)
        kubeconfig_contents = tar_obj.extractfile('admin.kubeconfig').read()

        kubeconfig_file = tmpdir_factory.mktemp('kubeconfig').join('admin.kubeconfig')
        kubeconfig_file.write(kubeconfig_contents)
        return kubeconfig_file

@pytest.fixture(scope='session')
def obj_compare():
    def compare_func(ansible_helper, k8s_obj, parameters):
        """ Assert that an object matches an expected object """
        requested = copy.deepcopy(k8s_obj)
        ansible_helper.object_from_params(parameters, obj=requested)

        ansible_helper.log('paramters:')
        ansible_helper.log(json.dumps(parameters, indent=4))
        ansible_helper.log('\n\n')

        ansible_helper.log('k8s_obj:')
        ansible_helper.log(k8s_obj.to_str())
        ansible_helper.log('\n\n')

        ansible_helper.log('from params:')
        ansible_helper.log(requested.to_str())
        ansible_helper.log('\n\n')

        match, diff = ansible_helper.objects_match(k8s_obj, requested)
        if not match:
            ansible_helper.log('\n\n')
            ansible_helper.log('Differences:')
            ansible_helper.log(json.dumps(diff, indent=4))
            ansible_helper.log('\n\n')
        assert match

    return compare_func

@pytest.fixture(scope='class')
def auth(request, kubeconfig, admin_kubeconfig, port):
    # needs_admin = request.node.cls.tasks.get('admin')
    needs_admin = True
    config = admin_kubeconfig if needs_admin else kubeconfig
    if config is not None:
        return {
            'kubeconfig': str(config),
            'host': 'https://localhost:{}'.format(port),
            'verify_ssl': False
        }

    return {}

@pytest.fixture
def openshift_version():
    return pytest.config.getoption('--openshift-version')

@pytest.fixture(scope='session')
def admin_client(admin_kubeconfig, port):
    k8s_client = kubernetes.config.new_client_from_config(str(admin_kubeconfig))
    # k8s_client.configuration.verify_ssl = False
    k8s_client.configuration.host = 'https://localhost:{}'.format(port)
    return DynamicClient(k8s_client)


import re
import os

import requests
from bs4 import BeautifulSoup
from sqlalchemy import create_engine, Column, Integer, String
from sqlalchemy.ext.declarative import declarative_base

basedir = os.path.abspath(os.path.dirname(__file__))
SQLALCHEMY_DATABASE_URI = 'sqlite:///' + os.path.join(basedir, 'app.db')

engine = create_engine(SQLALCHEMY_DATABASE_URI, convert_unicode=True, echo=False)
Base = declarative_base()
Base.metadata.reflect(engine)

from sqlalchemy.orm import relationship

class Item(Base):
    __table__ = Base.metadata.tables['items']

countries = ['Australia', 'Canada', 'China', 'Ireland',
                  'New Zealand', 'United States', 'United Kingdom']
country_base_url = 'http://www.numbeo.com/cost-of-living/country_result.jsp?country='

def get_city_urls():
    # example: {city:{country:country, url:url}}
    cities = {}
    for country in countries:
        country_url = country_base_url + country.replace(' ', '+')
        r = requests.get(country_url)
        soup = BeautifulSoup(r.text)
        full_data_urls(country, cities, soup)
        partial_data_urls(country, cities, soup)
    return cities

def full_data_urls(country, cities, soup):
        tds = soup.findAll('td', { 'class' : 'cityOrCountryInIndicesTable' })
        soup = BeautifulSoup(str(tds))
        anchors = soup.findAll('a')
        for a in anchors:
            cities[a.contents[0]] = {'country':country, 'url':a['href']}

def partial_data_urls(country, cities, soup):
        base_url = 'http://www.numbeo.com/cost-of-living/'
        first_sibling = soup.find(
                text = re.compile(
                "Cost of Living in  this country for some cities:"
                ))
        anchors = first_sibling.findAllNext('a',
                attrs={"href":re.compile("^city_result")})
        for a in anchors:
            cities[a.contents[0]] = {'country' : country,
                                    'url' : base_url + a['href']}

def get_cities_price_data(cities, db_session):
    for city in cities:
        data = cities[city]
        url = data['url']
        r = requests.get(url)
        soup = BeautifulSoup(r.text)
        last_updated = soup.find(text = re.compile("Last update"))
        trs = soup.findAll('tr', { 'class' : ['tr_standard', 'tr_highlighted'] })
        soup = BeautifulSoup(str(trs))
        tds = soup.findAll('td')
        for i in range(len(tds)):
            if tds[i].has_attr('class') and 'priceValue' in tds[i].attrs['class']:
                name = tds[i-1].contents[0]
                price = tds[i].contents[0]
                country = data['country']
                item = Item(name=name, price=price, city=city,
                            country=country, url=url,last_updated=last_updated)
                db_session.add(item)
        db_session.commit()

def delete_database(db_session):
    for item in db_session.query(Item).all():
        db_session.delete(item)
        print(item)
    db_session.commit()

if __name__ == '__main__':
    from sqlalchemy.orm import scoped_session, sessionmaker, Query
    db_session = scoped_session(sessionmaker(bind=engine))
    delete_database(db_session)
    get_cities_price_data(get_city_urls(), db_session)
    for item in db_session.query(Item).all():
        print((item.name, item.price, item.city))


from graph import graph, RDF, OWL, FOAF, VIVO, STUDYBOT
from data import universities
from util import to_snake_case

from rdflib import URIRef, Literal

def convert():
    for i, row in universities.iterrows():
        name = row['Name']
        sameAs = row['Same As']
        uni_resource = STUDYBOT[to_snake_case(name)]
        same_as_resource = URIRef(sameAs)

        graph.add((
            uni_resource, RDF.type, VIVO.University
        ))
        graph.add((
            uni_resource, FOAF.name, Literal(name, lang='en')
        ))
        graph.add((
            uni_resource, OWL.sameAs, same_as_resource
        ))

if __name__ == "__main__":
    convert()

    print(
        graph.serialize(format='turtle').decode('utf-8')
    )


"""
Create by Mike-Wen at 2019/7/3 21:03
"""
import json

from app.models.base import Base, db
from sqlalchemy import ForeignKey, orm  # 导入外键

# 公司注册信息表

class CompanyRegisitInfo(Base):
    __tablename__ = 'CompanyRegisitInfo'
    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
    company_charge_people = db.Column(db.String(255))                       # 公司负责人
    company_tel = db.Column(db.String(255))                                 # 联系方式
    company_addtax = db.Column(db.String(255))                              # 地方留存增值税
    company_address = db.Column(db.String(255))                             # 注册地
    company_incometax = db.Column(db.String(255))                           # 地方留存所得税
    company_local_incentives = db.Column(db.String(255))                    # 地方激励比例
    company_local_advantage = db.Column(db.String(255))                     # 地方优势
    company_special_note = db.Column(db.String(255))                        # 地方特别说明
    grade = db.Column(db.Integer)                                           # 优先级
# 公司注册经营范围表

class CompanyRegisitBusinessScope(Base):
    __tablename__ = 'CompanyRegisitBusinessScope'
    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
    company_regisit_id = db.Column(db.String(255), ForeignKey("CompanyRegisitInfo.main_id"))
    company_regisit_name = db.Column(db.String(255))
    company_business_scope = db.Column(db.String(255))
    status = db.Column(db.Integer, default=1)  # 0下架， 1正常
# 公司注册服务内容总表

class CompanyRegisitServiceAllContent(Base):
    __tablename__ = 'CompanyRegisitServiceAllContent'
    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
    company_all_service_content = db.Column(db.String(255))
    grade = db.Column(db.Integer, default=0)
# 公司注册信息总服务内容关联表

class CompanyInfoAndAllContent(Base):
    __tablename__ = 'CompanyInfoAndAllContent'
    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
    company_regisit_id = db.Column(db.String(255), ForeignKey("CompanyRegisitInfo.main_id"))
    company_allcontent_id = db.Column(db.String(255), ForeignKey("CompanyRegisitServiceAllContent.main_id"))

# 公司注册服务种类总表
class CompanyRegisitKind(Base):
    __tablename__ = 'CompanyRegisitKind'
    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
    company_all_kind = db.Column(db.String(255))
    grade = db.Column(db.Integer, default=0)

# 公司注册信息种类关联表
class CompanyInfoAndAllKind(Base):
    __tablename__ = 'CompanyInfoAndAllKind'
    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
    company_regisit_id = db.Column(db.String(255), ForeignKey("CompanyRegisitInfo.main_id"))
    company_kind_id = db.Column(db.String(255), ForeignKey("CompanyRegisitKind.main_id"))

# 公司注册服务内容子表

class CompanyRegisitServiceSonContent(Base):
    __tablename__ = 'CompanyRegisitServiceSonContent'
    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
    company_regisit_id = db.Column(db.String(255), ForeignKey("CompanyRegisitInfo.main_id"))
    company_son_service_content = db.Column(db.String(255))

# 公司注册法人到场环节总表

class CompanyRegisitAllLegalPerson(Base):
    __tablename__ = 'CompanyRegisitAllLegalPerson'
    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
    company_all_legal_name = db.Column(db.String(255))
    grade = db.Column(db.Integer, default=0)

# 公司注册信息总服务内容关联表

class CompanyInfoAndAllLegal(Base):
    __tablename__ = 'CompanyInfoAndAllLegal'
    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
    company_regisit_id = db.Column(db.String(255), ForeignKey("CompanyRegisitInfo.main_id"))
    company_alllegal_id = db.Column(db.String(255), ForeignKey("CompanyRegisitAllLegalPerson.main_id"))

# 公司注册法人到场环节子表

class CompanyRegisitSonLegalPerson(Base):
    __tablename__ = 'CompanyRegisitSonLegalPerson'
    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
    company_regisit_id = db.Column(db.String(255), ForeignKey("CompanyRegisitInfo.main_id"))
    company_son_legal_name = db.Column(db.String(255))

# 公司注册政府文件表

class CompanyRegisitGovenmentFile(Base):
    __tablename__ = 'CompanyRegisitGovenmentFile'
    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
    company_regisit_id = db.Column(db.String(255), ForeignKey("CompanyRegisitInfo.main_id"))
    company_govenment_file_name = db.Column(db.String(255))
    company_govenment_file_link = db.Column(db.String(255))

# 公司注册营业执照文件表

class CompanyRegisitLicenseFile(Base):
    __tablename__ = 'CompanyRegisitLicenseFile'
    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
    company_business_id = db.Column(db.String(255), ForeignKey("CompanyRegisitBusinessScope.main_id"))
    company_license_file_name = db.Column(db.String(255))
    company_license_file_link = db.Column(db.String(255))

class CompanyRegisitFileZip(Base):
    __tablename__ = 'CompanyRegisitFileZip'
    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
    company_regisit_id = db.Column(db.String(255), ForeignKey("CompanyRegisitInfo.main_id"))
    company_regisit_file_name = db.Column(db.String(255))
    company_regisit_file_link = db.Column(db.String(255))

class CompanyProviderInfo(Base):
    __tablename__ = 'CompanyProviderInfo'  # 数据库名称
    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
    company_regisit_id = db.Column(db.String(255), ForeignKey("CompanyRegisitInfo.main_id"))
    company_serviceprovider_address = db.Column(db.String(255))  # 服务商地址
    company_serviceprovider_name = db.Column(db.String(255))  # 服务商名称
    company_serviceprovider_person = db.Column(db.String(255))  # 服务商联系人
    company_serviceprovider_phone = db.Column(db.String(255))  # 服务商联系电话
    company_serviceprovider_beizhu=db.Column(db.Text) #服务商备注
    status = db.Column(db.Integer, default=1)
    # company_serviceprovider_sign = db.Column(db.String(255))  # 服务商是否签名

class CompanyProviderCustomers(Base):
    __tablename__ = 'CompanyProviderCustomers'  # 数据库名称
    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
    provider_regisit_id = db.Column(db.String(255), ForeignKey("CompanyProviderInfo.main_id"))  # 服务商id
    company_servicecustomers_id = db.Column(db.String(255))       # 合同号
    company_servicecustomers_people = db.Column(db.String(255))   # 服务商客户
    company_serviceprovider_feedback = db.Column(db.String(255))  # 反馈

class CompanyProviderContent(Base):
    __tablename__ = 'CompanyProviderContent'  # 数据库名称
    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
    provider_regisit_id = db.Column(db.String(255), ForeignKey("CompanyProviderInfo.main_id"))  # 服务商id
    provider_regisit_info = db.Column(db.String(255))  # 服务商可注册内容

# 服务商服务内容总表

class CompanyProviderAllContent(Base):
    __tablename__ = 'CompanyRegisitProviderAllContent'
    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
    provider_all_service_content = db.Column(db.String(255))            # 服务商服务内容
    grade = db.Column(db.Integer, default=0)
# 服务商总服务内容关联表

class CompanyProviderAndAllContent(Base):
    __tablename__ = 'CompanyProviderAndAllContent'
    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
    provider_regisit_id = db.Column(db.String(255), ForeignKey("CompanyProviderInfo.main_id"))       # 服务商Id
    provider_allcontent_id = db.Column(db.String(255), ForeignKey("CompanyRegisitProviderAllContent.main_id")) # 服务商服务内容

class CompanyProviderSonContent(Base):
    __tablename__ = 'CompanyProviderSonContent'
    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
    provider_regisit_id = db.Column(db.String(255), ForeignKey("CompanyProviderInfo.main_id"))        # 服务商Id
    provider_son_service_content = db.Column(db.String(255))                                                    # 服务商自定义内容


# -*- coding: utf-8 -*-

import time
import logging
import threading
from datetime import datetime
from itertools import repeat

from sqlalchemy.sql import not_

from nailgun.notifier import notifier
from nailgun.db import orm
from nailgun.settings import settings
from nailgun.api.models import Node
from nailgun.logger import logger

class KeepAliveThread(threading.Thread):

    def __init__(self, interval=None, timeout=None):
        super(KeepAliveThread, self).__init__()
        self.stoprequest = threading.Event()
        self.interval = interval or settings.KEEPALIVE['interval']
        self.timeout = timeout or settings.KEEPALIVE['timeout']
        self.db = orm()
        self.reset_nodes_timestamp()

    def reset_nodes_timestamp(self):
        for node_db in self.db.query(Node).all():
            node_db.timestamp = datetime.now()
            self.db.add(node_db)
        self.db.commit()

    def join(self, timeout=None):
        self.stoprequest.set()
        super(KeepAliveThread, self).join(timeout)

    def sleep(self, interval=None):
        map(
            lambda i: not self.stoprequest.isSet() and time.sleep(i),
            repeat(1, interval or self.interval)
        )

    def run(self):
        while not self.stoprequest.isSet():
            self.db.expire_all()
            for node_db in self.db.query(Node).filter(
                # nodes may become unresponsive while provisioning
                not_(Node.status == 'provisioning')
            ):
                timedelta = (datetime.now() - node_db.timestamp).seconds
                if timedelta > self.timeout:
                    logger.warning(
                        u"Node '{0}' seems to be offline "
                        "for {1} seconds...".format(
                            node_db.name,
                            timedelta
                        )
                    )
                    if node_db.online:
                        node_db.online = False
                        self.db.add(node_db)
                        self.db.commit()
                        notifier.notify(
                            "error",
                            u"Node '{0}' has gone away".format(
                                node_db.name or node_db.mac
                            ),
                            node_id=node_db.id
                        )
            self.sleep()


"""
This module contains wrapper code to the DEPRECATED paid thumbnail API v2.
Please use Image.get_thumbnail_url instead, as it uses the newer, complementary thumbnail service.
"""
import logging
try:
    from urllib.parse import urlencode
except ImportError:
    from urllib import urlencode

from piplapis.data import Image
from piplapis.data.utils import to_utf8

logger = logging.getLogger(__name__)
logger.warning("The thumbnail API in piplapis.thumbnail is depracated.")

BASE_URL = 'http://api.pipl.com/thumbnail/v2/?'
# HTTPS is also supported:
#BASE_URL = 'https://api.pipl.com/thumbnail/v2/?'

MAX_PIXELS = 500

# Default API key value, you can set your key globally in this variable instead
# of passing it in each call to generate_thumbnail_url().
# >>> import piplapis.thumbnail
# >>> piplapis.thumbnail.default_api_key = '<your_key>'
default_api_key = None

def generate_thumbnail_url(image_url, height, width, favicon_domain=None,
                           zoom_face=True, api_key=None):
    """Take an image URL and generate a thumbnail URL for that image.

    Args:

    image_url -- unicode (or utf8 encoded str), URL of the image you want to
                 thumbnail.
    height -- int, requested thumbnail height in pixels, maximum 500.
    width -- int, requested thumbnail width in pixels, maximum 500.
    favicon_domain -- unicode (or utf8 encoded str), optional, the domain of
                      the website where the image came from, the favicon will
                      be added to the corner of the thumbnail, recommended for
                      copyright reasones.
                      IMPORTANT: Don't assume that the domain of the website is
                      the domain from `image_url`, it's possible that
                      domain1.com hosts its images on domain2.com.
    zoom_face -- bool, indicates whether you want the thumbnail to zoom on the
                 face in the image (in case there is a face) or not.
    api_key -- str, a valid API key (use "samplekey" for experimenting).
               Note that you can set a default API key
               (piplapis.thumbnail.default_api_key = '<your_key>') instead of
               passing your key in each call.

    ValueError is raised in case of illegal parameters.

    Example (thumbnail URL from an image URL):

    >>> from piplapis.thumbnail import generate_thumbnail_url
    >>> image_url = 'http://a7.twimg.com/a/ab76f.jpg'
    >>> generate_thumbnail_url(image_url, 100, 100,
                               favicon_domain='twitter.com',
                               api_key='samplekey')
    'http://api.pipl.com/thumbnail/v2/?key=samplekey&
    favicon_domain=twitter.com&height=100&width=100&zoom_face=True&
    image_url=http%3A%2F%2Fa7.twimg.com%2Fa%2Fab76f.jpg'

    Example (thumbnail URL from a record that came in the response of our
    Search API):

    >>> from piplapis.thumbnail import generate_thumbnail_url
    >>> generate_thumbnail_url(record.images[0].url, 100, 100,
                               favicon_domain=record.source.domain,
                               api_key='samplekey')
    'http://api.pipl.com/thumbnail/v2/?key=samplekey&
    favicon_domain=twitter.com&height=100&width=100&zoom_face=True&
    image_url=http%3A%2F%2Fa7.twimg.com%2Fa%2Fab76f.jpg'

    """
    if not (api_key or default_api_key):
        raise ValueError('A valid API key is required')
    if not Image(url=image_url).is_valid_url:
        raise ValueError('image_url is not a valid URL')
    if not (0 < height <= MAX_PIXELS and 0 < width <= MAX_PIXELS):
        raise ValueError('height/width must be between 0 and %d' % MAX_PIXELS)
    query = {
        'key': to_utf8(api_key or default_api_key),
        'image_url': urllib.unquote(to_utf8(image_url)),
        'height': height,
        'width': width,
        'favicon_domain': to_utf8(favicon_domain or ''),
        'zoom_face': zoom_face,
    }
    return BASE_URL + urlencode(query)


from api.api import store, SchemaUtils
import logging.config

log = logging.getLogger(__name__)

class BasicAction(object):

    @staticmethod
    def push(request):
        request_type = SchemaUtils.get_type(request)
        print(request_type)
        savable_data = SchemaUtils.get_data(request, request_type)
        if not store.index_exists(request_type):
            store.create_index(request_type)
        store.insert(request_type, SchemaUtils.get_id(savable_data), savable_data)
        return True

    @staticmethod
    def search(request):
        return store.multi_search(SchemaUtils.get_type(request), request.get('text'), 1, 10, request.get('fields'))


"""
Módulo relacionado à preparação de comandos para execução no servidor de acesso
"""
import os
import sys

from jinja2 import Environment, FileSystemLoader, exceptions

class Comando:
    __TEMPLATES_DIR = '../includes/ps_templates'
    __TEMPLATES_EXTENSAO = '.j2'

    def __init__(self, comando, descricao=None, **kwargs):
        try:
            self.comando = comando
            self.descricao = descricao if descricao else self.comando
            self.args = kwargs
            j2_env = Environment(
                loader=FileSystemLoader(os.path.join(
                    os.path.dirname(__file__), Comando.__TEMPLATES_DIR)),
                trim_blocks=True)
            self.template = j2_env.get_template(
                self.comando + Comando.__TEMPLATES_EXTENSAO)
        except exceptions.TemplateNotFound as ex:
            print(
                f"Template '{ex}' não encontrado para o comando '{self.comando}'.")
            sys.exit(1)

    def imprimir(self):
        print('\n' + self.template.render(self.args) + '\n')

    def executar(self, servidor_acesso):
        conteudo_script = self.template.render(self.args)
        return servidor_acesso.executar_script(
            self.comando, conteudo_script)


import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F

class CNN_Sent(nn.Module):
    def __init__(self, voc, embedding_size, kernel_num, pretrained_weights, dropout=0.5):
        super(CNN_Sent, self).__init__()

        self.voc = voc
        self.embedding_size = embedding_size
        self.Ci = 1 # input channel
        self.Co = kernel_num    # 100个filter

        self.embed = nn.Embedding(voc.num_words, embedding_size, padding_idx=0)
        self.embed.weight.data.copy_(torch.from_numpy(pretrained_weights))

        self.conv1 = nn.Conv2d(self.Ci, self.Co, kernel_size=(3, embedding_size))
        self.conv2 = nn.Conv2d(self.Ci, self.Co, kernel_size=(4, embedding_size))
        self.conv3 = nn.Conv2d(self.Ci, self.Co, kernel_size=(5, embedding_size))

        self.dropout = nn.Dropout(dropout)
        self.fc = nn.Linear(self.Co*3, 2)

    def conv_and_pool(self, x, conv):
        x = F.relu(conv(x)).squeeze(3)  # (Batch, Co, conv_len) 将最后一维挤压掉
        x = F.max_pool1d(x, x.size(2)).squeeze(2)   # (Batch, Co)
        return x

    def forward(self, sent):
        x_embed = self.embed(sent)  # (Batch, seq_len, embed_size)
        x_embed = x_embed.unsqueeze(1)  # (Batch, Ci, seq_len, embed_size)

        x1 = self.conv_and_pool(x_embed, self.conv1)
        x2 = self.conv_and_pool(x_embed, self.conv2)
        x3 = self.conv_and_pool(x_embed, self.conv3)

        x = torch.cat((x1, x2, x3), dim=1)  # (Batch, Co*3)
        x = self.dropout(x)
        logit = self.fc(x)  # (Batch, Class_num)
        return logit


from django.conf.urls import url
from .views import PostListAPIView,PostCreateAPIView,DetailAPIView,PostUpdateAPIView, PostDeleteAPIView

urlpatterns = [
    url(r'^$',PostListAPIView.as_view(),name='list' ),
    url(r'^create/',PostCreateAPIView.as_view(),name="create"),
    url(r'^(?P<pk>\d+)/$',DetailAPIView.as_view() , name='detail'),
    url(r'^(?P<pk>\d+)/edit/$',PostUpdateAPIView.as_view() , name='update'),
    url(r'^(?P<pk>\d+)/delete/$',PostDeleteAPIView.as_view() , name='delete'),
    ]


from die import Die
import pygal

# create a two D6 dice
die_1 = Die(6)
die_2 = Die(6)
die_3 = Die(6)

# make some rolls
results = []
for roll_num in range(1000):
    result = die_1.roll() + die_2.roll() + die_3.roll()
    results.append(result)

# analyze the results
frequencies = []
max_result = die_1.num_sides + die_2.num_sides + die_3.num_sides
for value in range(3, max_result+1):
    frequency = results.count(value)
    frequencies.append(frequency)

# visualize the results
hist = pygal.Bar()

hist.title = 'Results of rolling three D6 1000 times'
hist.x_labels = []
for x in range(3, max_result+1):
    hist.x_labels.append(str(x))
hist.x_title = 'Result'
hist.y_title = 'Frequency of Result'

hist.add('D6 + D6 + D6', frequencies)
hist.render_to_file('die_visual.svg')

# "" """ ''


import os
import sys

import matplotlib.pyplot as plt
import numpy as np

from .csitools import get_CSI
from .filters import bandpass, hampel, running_mean
from .matlab import db
from .reader import get_reader

DEFAULT_PATH = "./data/intel/misc/log.all_csi.6.7.6.dat"
# DEFAULT_PATH = "./data/pi/walk_1597159475.pcap"

class BatchGraph:

    def __init__(self, path=DEFAULT_PATH):
        self.reader = get_reader(path)

    def prepostfilter(self):

        reader = self.reader

        csi_trace = reader.csi_trace
        finalEntry, no_frames, _ = get_CSI(csi_trace)

        finalEntry = finalEntry[15]

        hampelData = hampel(finalEntry, 10)
        smoothedData = running_mean(hampelData.copy(), 10)

        y = finalEntry
        y2 = hampelData
        y3 = smoothedData

        x = list([x["timestamp"] for x in csi_trace])

        if sum(x) == 0:
            x = np.arange(0, no_frames, 1)

        plt.plot(x, y, label="Raw")
        plt.plot(x, y2, label="Hampel")
        plt.plot(x, y3, "r", label="Hampel + Running Mean")

        plt.xlabel("Time (s)")
        plt.ylabel("Amplitude (dBm)")
        plt.legend(loc="upper right")

        plt.show()

    def plotAllSubcarriers(self):

        reader = self.reader

        scaled_csi = reader.csi_trace
        finalEntry, no_frames, _ = get_CSI(scaled_csi)

        for x in finalEntry:
            plt.plot(np.arange(no_frames)/20, x)

        plt.xlabel("Time (s)")
        plt.ylabel("Amplitude (dBm)")
        plt.legend(loc="upper right")

        plt.show()

    def heatmap(self):
        reader = self.reader

        csi_trace = reader.csi_trace
        finalEntry, no_frames, no_subcarriers = get_CSI(csi_trace, metric="amplitude")

        x_label = "Time (s)"
        x = list([x["timestamp"] for x in csi_trace])

        if sum(x) == 0:
            #Some files have invalid timestamp_low values which means we can't plot based on timestamps.
            #Instead we'll just plot by frame count.

            xlim = no_frames

            x_label = "Frame No."
        else:
            xlim = max(x)

        limits = [0, xlim, 1, no_subcarriers]
        # limits = [0, xlim, -64, 63]

        #TODO Add options for filtering.
        # for x in range(no_subcarriers):
        #     hampelData = hampel(finalEntry[x], 5, 3)
        #     runningMeanData = running_mean(hampelData, 20)
        #     # smoothedData = dynamic_detrend(runningMeanData, 5, 3, 1.2, 10)
        #     # doubleHampel = hampel(smoothedData, 10, 3)

        #     finalEntry[x] = bandpass(9, 1, 2, Fs, runningMeanData)
        #     # for i in range(0, 140):
        #     #     finalEntry[x][i] = 0

        _, ax = plt.subplots()
        im = ax.imshow(finalEntry, cmap="jet", extent=limits, aspect="auto")

        cbar = ax.figure.colorbar(im, ax=ax)
        cbar.ax.set_ylabel("Amplitude (dBm)")

        plt.xlabel(x_label)
        plt.ylabel("Subcarrier Index")

        plt.title(reader.filename.split("/")[-1])

        plt.show()

if __name__ == "__main__":
    bg = BatchGraph()
    bg.heatmap()


import namegenerator
import names
import numpy as np

class PreferenceDataset(object):
    """Preference dataset abstract class."""

    def __init__(self, objects, users, fields, ratings=None,
                 n_ratings=10):
        self.objects = objects
        self.users = users
        self.fields = fields
        if ratings is not None:
            self.ratings = ratings
        else:
            self._generate_many(n_ratings=n_ratings)

    def _generate_many(self, n_ratings=10):
        self.ratings = [self._generate_one() for _ in range(n_ratings)]

    def _generate_one(self):
        r = self._generate_one_impl()
        for x in ['o1', 'o2', 'ratings', 'user']:
            assert x in r, "Missing field %s" % x
        assert r['o1'] in self.objects, "Object not found %s" % r['o1']
        assert r['o2'] in self.objects, "Object not found %s" % r['o2']
        assert r['user'] in self.users, "Object not found %s" % r['user']
        for f in self.fields:
            assert f in r['ratings'], "Field not found %s" % f
            assert 0 <= r['ratings'][f] <= 100, "Value must be in [0, 100] %s %s" \
                                                % (f, str(r['ratings'][f]))
        return r

    def _generate_one_impl(self):
        raise NotImplementedError("This class is abstract")

    def __repr__(self):
        return "<PrefDataset %s~objects[%d] %s~fields[%d] %s~users[%d] %s~ratings[%d]" % \
               (np.random.choice(self.objects), len(self.objects),
                np.random.choice(self.fields), len(self.fields),
                np.random.choice(self.users), len(self.users),
                np.random.choice(self.ratings), len(self.ratings))

class ToyRandomDataset(PreferenceDataset):
    """Random dataset."""

    def __init__(self, n_objects=10, n_users=10):
        # list of objects to consider
        objects = [namegenerator.gen() for _ in range(n_objects)]

        # fields to consider (features)
        fields = [
            "unicorniness",
            "magic_skills",
            "number_of_trump_appearances"]

        # some users
        users = [names.get_full_name() for _ in range(n_users)]

        super(ToyRandomDataset, self).__init__(objects=objects,
                                               fields=fields,
                                               users=users)

    def _generate_one_impl(self):
        """Sample one rating."""
        o1, o2 = None, None
        while o1 == o2:
            o1 = np.random.choice(self.objects)
            o2 = np.random.choice(self.objects)

        def get_rating():
            return np.random.uniform(low=0, high=100)

        user = np.random.choice(self.users)
        ratings = {f: get_rating() for f in self.fields}
        weights = {f: np.random.rand() + 1 for f in self.fields}
        return {'o1': o1, 'o2': o2, 'ratings': ratings,
                'weights': weights,
                'user': user}

class ToyHardcodedDataset(PreferenceDataset):
    """Random dataset."""

    def __init__(self):
        users = ['sergei', 'le']
        objects = ['trump_video', 'science4all_video', 'bbc_video']
        fields = ['accuracy', 'pedagogy']

        self.scores_dict = \
            {'sergei':
                {
                    'accuracy':
                        {('trump_video', 'science4all_video'): 90,
                         ('trump_video', 'bbc_video'): 85,
                         ('science4all_video', 'bbc_video'): 0
                         },
                    'pedagogy':
                        {('trump_video', 'science4all_video'): 100,
                         ('trump_video', 'bbc_video'): 71,
                         ('science4all_video', 'bbc_video'): 1
                         },
                },
                'le':
                    {
                        'accuracy':
                            {('trump_video', 'science4all_video'): 99,
                             ('trump_video', 'bbc_video'): 80,
                             ('science4all_video', 'bbc_video'): 10
                             },
                        'pedagogy':
                            {('trump_video', 'science4all_video'): 100,
                             ('trump_video', 'bbc_video'): 70,
                             ('science4all_video', 'bbc_video'): 0
                             },
                }
             }

        super(ToyHardcodedDataset, self).__init__(objects=objects,
                                                  fields=fields,
                                                  users=users)

    def _generate_one_impl(self):
        """Sample one rating."""
        o1, o2 = None, None
        user = np.random.choice(self.users)
        while o1 == o2:
            o1 = np.random.choice(self.objects)
            o2 = np.random.choice(self.objects)

        def get_rating(f):
            scores = self.scores_dict[user]

            if (o1, o2) in scores[f]:
                return scores[f][(o1, o2)]
            elif (o2, o1) in scores[f]:
                return 100 - scores[f][(o2, o1)]
            else:
                raise Exception("score not found")

        ratings = {f: get_rating(f) for f in self.fields}
        return {'o1': o1, 'o2': o2, 'ratings': ratings,
                'user': user}


"""Functional wrapper around the pygmo, nlopt and scipy libraries."""
import numpy as np
from joblib import delayed
from joblib import Parallel

from estimagic.config import DEFAULT_DATABASE_NAME
from estimagic.dashboard.run_dashboard import run_dashboard_in_separate_process
from estimagic.decorators import negative_gradient
from estimagic.logging.update_database import update_scalar_field
from estimagic.optimization.broadcast_arguments import broadcast_arguments
from estimagic.optimization.check_arguments import check_arguments
from estimagic.optimization.pounders import minimize_pounders_np
from estimagic.optimization.pygmo import minimize_pygmo_np
from estimagic.optimization.reparametrize import reparametrize_from_internal
from estimagic.optimization.scipy import minimize_scipy_np
from estimagic.optimization.transform_problem import transform_problem

def maximize(
    criterion,
    params,
    algorithm,
    criterion_kwargs=None,
    constraints=None,
    general_options=None,
    algo_options=None,
    gradient=None,
    gradient_kwargs=None,
    gradient_options=None,
    logging=DEFAULT_DATABASE_NAME,
    log_options=None,
    dashboard=False,
    dash_options=None,
):
    """Maximize criterion using algorithm subject to constraints and bounds.
    Each argument except for general_options can also be replaced by a list of
    arguments in which case several optimizations are run in parallel. For this, either
    all arguments must be lists of the same length, or some arguments can be provided
    as single arguments in which case they are automatically broadcasted.
    Args:
        criterion (callable or list of callables):
            Python callable that takes a pandas DataFrame with parameters as the first
            argument. Supported outputs are:
                - scalar floating point
                - np.ndarray: contributions for the tao Pounders algorithm.
                - tuple of a scalar floating point and a pd.DataFrame:
                    In this case the first output is the criterion value.
                    The second output are the comparison_plot_data.
        params (pd.DataFrame or list of pd.DataFrames):
            See :ref:`params`.
        algorithm (str or list of strings): Specifies the optimization algorithm.
            See :ref:`list_of_algorithms`.
        criterion_kwargs (dict or list of dicts): Additional keyword arguments for
            criterion.
        constraints (list or list of lists): List with constraint dictionaries.
            See :ref:`constraints`.
        general_options (dict): Additional configurations for the optimization.
            Keys can include:
                - keep_dashboard_alive (bool): Do not terminate the dashboard process
                    after the optimization(s) finish(es).
        algo_options (dict or list of dicts): Algorithm specific configurations for the
            optimization.
        gradient (callable): Gradient of the criterion function. Takes params as first
            argument and returns the gradient as numpy array or pandas Series.
        gradient_kwargs (dict): Additional keyword arguments for the gradient.
        gradient_options (dict): Options for the gradient function.
        logging (str or pathlib.Path or list): Path(s) to (an) sqlite3 file(s) which
            typically has the file extension ``.db``. If the file does not exist,
            it will be created. See :ref:`logging` for details.
        log_options (dict or list of dict): Keyword arguments to influence the logging.
            See :ref:`logging` for details.
        dashboard (bool): Whether to create and show a dashboard, default is False.
            See :ref:`dashboard` for details.
        dash_options (dict or list of dict, optional): Options passed to the dashboard.
            Supported keys are:
                - port (int): port where to display the dashboard.
                - no_browser (bool): whether to display the dashboard in a browser.
                - rollover (int): how many iterations to keep in the convergence plots.
    Returns:
        results (tuple or list of tuples): Each tuple consists of the harmonized result
        info dictionary and the params DataFrame with the minimizing parameter values
        of the untransformed problem as specified of the user.
    """
    # Set a flag for a maximization problem.
    general_options = {} if general_options is None else general_options
    general_options["_maximization"] = True

    if isinstance(gradient, list):
        gradient = [negative_gradient(grad) for grad in gradient]
    else:
        gradient = negative_gradient(gradient)

    results = minimize(
        criterion=criterion,
        params=params,
        algorithm=algorithm,
        criterion_kwargs=criterion_kwargs,
        constraints=constraints,
        general_options=general_options,
        algo_options=algo_options,
        gradient=gradient,
        gradient_kwargs=gradient_kwargs,
        gradient_options=gradient_options,
        logging=logging,
        log_options=log_options,
        dashboard=dashboard,
        dash_options=dash_options,
    )

    # Change the fitness value. ``results`` is either a tuple of results and params or a
    # list of tuples.
    if not isinstance(results, list):
        results = [results]

    results = [_undo_sign_switch(res) for res in results]

    results = results[0] if len(results) == 1 else results

    return results

def _undo_sign_switch(res):
    info, params = res
    info = info.copy()
    info["fitness"] = -info["fitness"]
    if "jacobian" in info and info["jacobian"] is not None:
        info["jacobian"] = (-np.array(info["jacobian"])).tolist()
    if "hessian" in info and info["hessian"] is not None:
        info["hessian"] = (-np.array(info["hessian"])).tolist()
    return info, params

def minimize(
    criterion,
    params,
    algorithm,
    criterion_kwargs=None,
    constraints=None,
    general_options=None,
    algo_options=None,
    gradient=None,
    gradient_kwargs=None,
    gradient_options=None,
    logging=DEFAULT_DATABASE_NAME,
    log_options=None,
    dashboard=False,
    dash_options=None,
):
    """Minimize *criterion* using *algorithm* subject to *constraints* and bounds.
    Each argument except for ``general_options`` can also be replaced by a list of
    arguments in which case several optimizations are run in parallel. For this, either
    all arguments must be lists of the same length, or some arguments can be provided
    as single arguments in which case they are automatically broadcasted.
    Args:
        criterion (callable or list of callables):
            Python callable that takes a pandas DataFrame with parameters as the first
            argument. Supported outputs are:
                - scalar floating point
                - np.ndarray: contributions for the tao Pounders algorithm.
                - tuple of a scalar floating point and a pd.DataFrame:
                    In this case the first output is the criterion value.
                    The second output are the comparison_plot_data.
        params (pd.DataFrame or list of pd.DataFrames):
            See :ref:`params`.
        algorithm (str or list of strings): Specifies the optimization algorithm.
            See :ref:`list_of_algorithms`.
        criterion_kwargs (dict or list of dicts): Additional keyword arguments for
            criterion.
        constraints (list or list of lists): List with constraint dictionaries.
            See :ref:`constraints`.
        general_options (dict): Additional configurations for the optimization.
            Keys can include:
                - keep_dashboard_alive (bool): Do not terminate the dashboard process
                    after the optimization(s) finish(es).
        algo_options (dict or list of dicts): Algorithm specific configurations for the
            optimization.
        gradient (callable): Gradient of the criterion function. Takes params as first
            argument and returns the gradient as numpy array or pandas Series.
        gradient_kwargs (dict): Additional keyword arguments for the gradient.
        gradient_options (dict): Options for the gradient function.
        logging (str or pathlib.Path or list): Path(s) to (an) sqlite3 file(s) which
            typically has the file extension ``.db``. If the file does not exist,
            it will be created. See :ref:`logging` for details.
        log_options (dict or list of dict): Keyword arguments to influence the logging.
            See :ref:`logging` for details.
        dashboard (bool): Whether to create and show a dashboard, default is False.
            See :ref:`dashboard` for details.
        dash_options (dict or list of dict, optional): Options passed to the dashboard.
            Supported keys are:
                - port (int): port where to display the dashboard.
                - no_browser (bool): whether to display the dashboard in a browser.
                - rollover (int): how many iterations to keep in the convergence plots.
    Returns:
        results (tuple or list of tuples): Each tuple consists of the harmonized result
        info dictionary and the params DataFrame with the minimizing parameter values
        of the untransformed problem as specified of the user.
    """
    arguments = broadcast_arguments(
        criterion=criterion,
        params=params,
        algorithm=algorithm,
        criterion_kwargs=criterion_kwargs,
        constraints=constraints,
        general_options=general_options,
        algo_options=algo_options,
        gradient=gradient,
        gradient_kwargs=gradient_kwargs,
        gradient_options=gradient_options,
        logging=logging,
        log_options=log_options,
        dashboard=dashboard,
        dash_options=dash_options,
    )

    check_arguments(arguments)

    optim_arguments = []
    results_arguments = []
    database_paths_for_dashboard = []
    for single_arg in arguments:
        optim_kwargs, database_path, result_kwargs = transform_problem(**single_arg)
        optim_arguments.append(optim_kwargs)
        results_arguments.append(result_kwargs)
        if database_path is not None:
            database_paths_for_dashboard.append(database_path)

    if dashboard:
        dashboard_process = run_dashboard_in_separate_process(
            database_paths=database_paths_for_dashboard
        )

    if len(arguments) == 1:
        # Run only one optimization
        results = [_internal_minimize(**optim_arguments[0])]
    else:
        # Run multiple optimizations
        if "n_cores" not in optim_arguments[0]["general_options"]:
            raise ValueError(
                "n_cores need to be specified in general_options"
                + " if multiple optimizations should be run."
            )
        n_cores = optim_arguments[0]["general_options"]["n_cores"]

        results = Parallel(n_jobs=n_cores)(
            delayed(_internal_minimize)(**optim_kwargs)
            for optim_kwargs in optim_arguments
        )

    if dashboard and dashboard_process is not None:
        if not results_arguments[0]["keep_dashboard_alive"]:
            dashboard_process.terminate()

    results = _process_optimization_results(results, results_arguments)

    return results

def _internal_minimize(
    internal_criterion,
    internal_params,
    bounds,
    origin,
    algo_name,
    algo_options,
    internal_gradient,
    database,
    general_options,
):
    """Run one optimization of the transformed optimization problem.
    The transformed optimization problem is converted from the original problem
    which consists of the user supplied criterion, params DataFrame, criterion_kwargs,
    constraints and gradient (if supplied).
    In addition, the transformed optimization problem provides sophisticated logging
    tools if activated by the user.
    The transformed problem can be solved by almost any optimizer package:
        1. The only constraints are bounds on the parameters.
        2. The internal_criterion function takes an one dimensional np.array as input.
        3. The internal criterion function returns a scalar value
            (except for the case of the tao_pounders algorithm).
    Note that because of the reparametrizations done by estimagic to implement
    constraints on behalf of the user the internal params cannot be interpreted without
    reparametrizing it to the full params DataFrame.
    Args:
        internal_criterion (func): The transformed criterion function.
            It takes the internal_params numpy array as only argument, automatically
            enforcing constraints specified by the user. It calls the original
            criterion function after the necessary reparametrizations.
            If logging is activated it protocols every call automatically to the
            specified database.
        internal_params (np.array): One-dimenisonal array with the values of
            the free parameters.
        bounds (tuple): tuple of the length of internal_params. Every entry contains
            the lower and upper bound of the respective internal parameter.
        origin (str): Name of the package to which the algorithm belongs.
        algo_name (str): Name of the algorithm.
        algo_options (dict): Algorithm specific configurations.
        internal_gradient (func): The internal gradient
        database (sqlalchemy.MetaData or False). The engine that connects to the
            database can be accessed via ``database.bind``. This is only used to record
            the start and end of the optimization
        general_options (dict): Only used to pass the start_criterion_value in case
            the tao pounders algorithm is used.
    Returns:
        results (tuple): Tuple of the harmonized result info dictionary and the params
            DataFrame with the minimizing parameter values of the untransformed problem
            as specified of the user.
    """
    if database:
        update_scalar_field(database, "optimization_status", "running")

    if origin in ["nlopt", "pygmo"]:
        results = minimize_pygmo_np(
            internal_criterion,
            internal_params,
            bounds,
            origin,
            algo_name,
            algo_options,
            internal_gradient,
        )
    elif origin == "scipy":
        results = minimize_scipy_np(
            internal_criterion,
            internal_params,
            bounds=bounds,
            algo_name=algo_name,
            algo_options=algo_options,
            gradient=internal_gradient,
        )
    elif origin == "tao":
        crit_val = general_options["_start_criterion_value"]
        len_criterion_value = 1 if np.isscalar(crit_val) else len(crit_val)
        results = minimize_pounders_np(
            internal_criterion,
            internal_params,
            bounds,
            n_errors=len_criterion_value,
            **algo_options,
        )
    else:
        raise NotImplementedError("Invalid algorithm requested.")

    if database:
        update_scalar_field(database, "optimization_status", results["status"])

    return results

def _process_optimization_results(results, results_arguments):
    """Expand the solutions back to the original problems.
    Args:
        results (list):
            list of dictionaries with the harmonized results objects.
        results_arguments (list):
            each element is a dictionary supplying the start params DataFrame
            and the constraints to the original problem.
            The keys are "params", "constraints" and "keep_dashboard_alive".
    Returns:
        results (tuple): Tuple of the harmonized result info dictionary and the params
            DataFrame with the minimizing parameter values of the untransformed problem
            as specified of the user.
    """
    new_results = []
    for res, args in zip(results, results_arguments):
        res["x"] = list(res["x"])
        start_params = args["params"]
        params = reparametrize_from_internal(
            internal=np.array(res["x"]),
            fixed_values=start_params["_internal_fixed_value"].to_numpy(),
            pre_replacements=start_params["_pre_replacements"].to_numpy(dtype="int"),
            processed_constraints=args["constraints"],
            post_replacements=start_params["_post_replacements"].to_numpy(dtype="int"),
            processed_params=start_params,
        )
        new_results.append((res, params))

    if len(new_results) == 1:
        new_results = new_results[0]
    return new_results