# In this file, we made a sentiment analysis for general users.
# First, we manually labeled 1000 tweets into positive, neutral, and negative
# Then, the labeled tweets were fed into BERTweet model for prediction of unlabeled tweets
# Analysis was based on the whole dataset (manually labeled and predicted)

# import libraries
import pandas as pd
import preprocessor as p
from sklearn.model_selection import train_test_split
import tensorflow as tf
from transformers import AutoTokenizer
from transformers import TFAutoModelForSequenceClassification
import numpy as np
from sklearn.metrics import classification_report

# read file
labeled = pd.read_csv('your_file') # manually labeled tweets,
# four columns: period (before, during, after Olympics), athletes (white or minority), tweets, label (positive, neutral, negative)

# a summary of labeled data
labeled.groupby(['period', 'athletes', 'label']).count()

# preprocess tweets
labeled['tweets'] = labeled['tweets'].apply(lambda x: p.clean(x))

# train, validation, test split
X = list(labeled.tweets.values)  # the texts --> X
y = list(labeled.label.values)   # the labels we want to predict --> Y
labels = [0, 1, 2]

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1)
X_test, X_val, y_test, y_val = train_test_split(X_test, y_test, test_size=0.5, random_state=1)

# tokenize X
tokenizer = AutoTokenizer.from_pretrained('vinai/bertweet-base', normalization=True)

train_encodings = tokenizer(X_train, truncation=True, padding=True, max_length=128) # convert input strings to BERT encodings
test_encodings = tokenizer(X_test, truncation=True, padding=True,  max_length=128)
val_encodings = tokenizer(X_val, truncation=True, padding=True, max_length=128)

train_dataset = tf.data.Dataset.from_tensor_slices((dict(train_encodings), y_train)).shuffle(1000).batch(16) # convert the encodings to Tensorflow objects
val_dataset = tf.data.Dataset.from_tensor_slices((dict(val_encodings), y_val)).batch(64)
test_dataset = tf.data.Dataset.from_tensor_slices((dict(test_encodings), y_test)).batch(64)

# define and compile model (BERTweet)
model = TFAutoModelForSequenceClassification.from_pretrained('vinai/bertweet-base', num_labels=len(labels))
model.roberta.return_dict = False

callbacks = [tf.keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0,
                                              patience=3, verbose=0, mode='min',
                                              baseline=None, restore_best_weights=True)]
optimizer = tf.keras.optimizers.Adam(learning_rate=5e-5)
loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)

model.compile(optimizer=optimizer, loss=loss)

# model training
model.fit(train_dataset, epochs=10, callbacks=callbacks,
          validation_data=val_dataset, batch_size=16)

# model evluation
logits = model.predict(test_dataset)
y_preds = np.argmax(logits[0], axis=1)
print(classification_report(y_test, y_preds))

# predict unlabeled tweets
# import file
unlabeled = pd.read_csv('your_file')

# precrossing (tokenize, deal with na, and clean tweets)
tokenizer = AutoTokenizer.from_pretrained('vinai/bertweet-base', normalization=True)
unlabeled['tweets'] = unlabeled['tweets'].fillna('').apply(str)
unlabeled['tweets'] = unlabeled['tweets'].apply(lambda x: p.clean(x))

# prepare for prediction set
unlabeled_tweets = list(unlabeled.tweets.values)
encodings = tokenizer(unlabeled_tweets, truncation=True, padding=True, max_length=128)
examples_encodings = tf.data.Dataset.from_tensor_slices((dict(encodings))).batch(64)

# make prediction
pred_unlabeled = model.predict(examples_encodings)
y_preds = np.argmax(pred_unlabeled[0], axis=1)
unlabeled['label'] = y_preds

# make a summary
unlabeled.groupby(['athletes', 'label']).count()


from datetime import datetime

from django import forms

from apps.banking.models import Account, Category, Change, Depot

# depot
class DepotForm(forms.ModelForm):
    class Meta:
        model = Depot
        fields = ("name",)

    def __init__(self, user, *args, **kwargs):
        super(DepotForm, self).__init__(*args, **kwargs)
        self.instance.user = user

class DepotSelectForm(forms.Form):
    depot = forms.ModelChoiceField(widget=forms.Select, queryset=None)

    class Meta:
        fields = ("depot",)

    def __init__(self, user, *args, **kwargs):
        super(DepotSelectForm, self).__init__(*args, **kwargs)
        self.fields["depot"].queryset = user.banking_depots.all()

class DepotActiveForm(forms.ModelForm):
    class Meta:
        model = Depot
        fields = ("is_active",)

# account
class AccountForm(forms.ModelForm):
    class Meta:
        model = Account
        fields = ("name",)

    def __init__(self, depot, *args, **kwargs):
        super(AccountForm, self).__init__(*args, **kwargs)
        self.instance.depot = depot

class AccountSelectForm(forms.Form):
    account = forms.ModelChoiceField(widget=forms.Select, queryset=None)

    class Meta:
        fields = ("account",)

    def __init__(self, depot, *args, **kwargs):
        super(AccountSelectForm, self).__init__(*args, **kwargs)
        self.fields["account"].queryset = depot.accounts.all()

# category
class CategoryForm(forms.ModelForm):
    class Meta:
        model = Category
        fields = (
            "name",
            "description",
        )

    def __init__(self, depot, *args, **kwargs):
        super(CategoryForm, self).__init__(*args, **kwargs)
        self.instance.depot = depot

class CategorySelectForm(forms.Form):
    category = forms.ModelChoiceField(widget=forms.Select, queryset=None)

    class Meta:
        fields = ("category",)

    def __init__(self, depot, *args, **kwargs):
        super(CategorySelectForm, self).__init__(*args, **kwargs)
        self.fields["category"].queryset = depot.categories.all()

# change
class ChangeField(forms.DecimalField):
    def __init__(self, *args, **kwargs):
        attrs = {
            "pattern": r"-?\d+(.|,)?\d{0,2}",
            "title": "A number with 2 decimal places.",
        }
        super().__init__(widget=forms.TextInput(attrs=attrs), *args, **kwargs)

    def to_python(self, value):
        if isinstance(value, str):
            value = value.replace(",", ".")
        return super().to_python(value)

class ChangeForm(forms.ModelForm):
    date = forms.DateTimeField(
        widget=forms.DateTimeInput(
            attrs={"type": "datetime-local"}, format="%Y-%m-%dT%H:%M"
        ),
        input_formats=["%Y-%m-%dT%H:%M"],
        label="Date",
    )
    change = ChangeField()

    class Meta:
        model = Change
        fields = (
            "account",
            "date",
            "change",
            "category",
            "description",
        )

    def __init__(self, depot, *args, **kwargs):
        super(ChangeForm, self).__init__(*args, **kwargs)
        self.fields["account"].queryset = depot.accounts.all()
        self.fields["category"].queryset = depot.categories.all()
        self.fields["date"].initial = datetime.now()
        self.fields["description"].widget.attrs.update({"class": "small"})


#!/usr/bin/env python
# -*- coding: utf-8  -*-
# Read in a map, smooth it, and write it out
#
# History:
# v0.1  Mike Peel   10-Jan-2016   Initial version.
# v0.2  Mike Peel   8-Mar-2016    Start to generalise to cover other numbers of maps
# v0.3  Mike Peel   6-Sep-2016    Switch to using pyfits; use alms and window functions to smooth; add unit conversion functionality
# v0.4  Mike Peel   23-Sep-2016   Carry fits headers through to the output file. Make fwhm_arcmin optional so that the function can be used solely to ud_grade maps.
# v0.5  Mike Peel   23-Sep-2016   Adding calc_variance_windowfunction (port of Paddy Leahy's IDL code) to properly smooth variance maps.
# v0.6  Mike Peel   26-Sep-2016   Support Nobs maps, and conversion to variance maps, plus debugging/tidying.
# v0.7  Mike Peel   27-Jan-2017   Bug fix - ud_grading the variance maps should include a factor of (nside_orig/nside_new)^2
# V0.8  Adam Barr   Various       Various tweaks
# v0.9  Mike Peel   17 Sep 2017   Tweaks, add rescale parameter
# v0.9a Mike Peel   21 Sep 2017   More bug fixes
# v0.9b Mike Peel   22 Sep 2017   Add nosmooth and outputmaps parameters
# v0.9c Mike Peel   29 Sep 2017   More bug fixes
# v0.9d Mike Peel   01 Oct 2017   Input/output dir improvements. Fix bug in beam window functions.
# v1.0  Mike Peel   06 Oct 2017   Version 1.0. Add option to append maps (e.g. MC output)
# v1.1  Mike Peel   09 Oct 2017   Add CMB subtraction
# v1.2  Mike Peel   04 Jul 2019   Multiple options for window function tapering, min/max map values
# v1.3  Mike Peel   28 Aug 2019   Gaussian taper, options for normalising wf and unseen vs. 0 in map
# v1.4  Mike Peel   24 Jul 2020   Add an option to not smooth the variance maps (but save them in the output anyway)
# v1.4a Mike Peel   27 Jul 2020   Tweak to only check for NESTED when not using usehealpixfits
# v1.4b Mike Peel   07 Oct 2020   Tweak to use 'usehealpixfits' for 'subtractmap'
# v1.4c Mike Peel   22 Mar 2020   Add option for precomputed window function smoothing
# v1.5  Mike Peel   05 Apr 2021   Fix polarisation smoothing to be done simulatenously
# v1.5a Mike Peel   09 Apr 2021   Tweak to avoid healpix maps without ordering (assume ring)
# v1.5b Mike Peel   21 Apr 2021   Tweak to handle Q/U maps being different area than I!
# v1.5c Mike Peel   27 May 2021   Check length before padding if needed
#
# Requirements:
# Numpy, healpy, matplotlib

import numpy as np
import healpy as hp
import scipy as sp
import math as m

# Work-around for no X display
import matplotlib as mpl
mpl.use('Agg')

import astropy.io.fits as fits
import matplotlib.pyplot as plt
import os.path
from scipy import special
from scipy import optimize
from astrocode.spectra import *

def gaussfit(x, param):
	return hp.gauss_beam(np.radians(param/60.0),300)

def smoothmap(indir, outdir, inputfile, outputfile, fwhm_arcmin=-1, nside_out=0,maxnummaps=-1, frequency=10.0, units_in='',units_out='', windowfunction = [],windowfunction_pol = [],nobs_out=False,variance_out=True, sigma_0 = -1, sigma_0_unit='', rescale=1.0, nosmooth=[], outputmaps=[],appendmap='',appendmapname='',appendmapunit='',subtractmap='',subtractmap_units='',usehealpixfits=False,taper=False,lmin_taper=350,lmax_taper=600, cap_one=False, cap_oneall=False,minmapvalue=0,maxmapvalue=0,minmaxmaps=[0],taper_gauss=False,taper_gauss_sigma=0.0,normalise=True,useunseen=False,smoothvariance=False,use_precomputed_wf=False,do_pol_combined=False):
	ver = "1.5"

	if (os.path.isfile(outdir+outputfile)):
		print("You already have a file with the output name " + outdir+outputfile + "! Not going to overwrite it. Move it, or set a new output filename, and try again!")
		# exit()
		return

	# Check to see if we have a sigma_0 value to use when converting from Nobs maps and back.
	no_sigma_0 = False
	if (sigma_0 == -1):
		no_sigma_0 = True

	# Read in the fits map, and put it into the format Healpix expects
	# if usehealpixfits:
	# 	maps, newheader = hp.read_map(indir+inputfile,field=None,h=True)
	# 	nmaps = len(maps)
	# 	print newheader
	# else:
	inputfits = fits.open(indir+inputfile)
	print(inputfits.info())
	cols = inputfits[1].columns
	col_names = cols.names
	nmaps = len(cols)
	maps = []
	if usehealpixfits:
		maps = hp.read_map(indir+inputfile,field=None)
	else:
		for i in range(0,nmaps):
			maps.append(inputfits[1].data.field(i))
			print(len(maps[i]))
		print(len(maps[0]))
		# Check to see whether we have nested data, and switch to ring if that is the case.
		try:
			if (inputfits[1].header['ORDERING'] == 'NESTED'):
				maps = hp.reorder(maps,n2r=True)
		except:
			null = 0
	newheader = inputfits[1].header.copy(strip=False)
	inputfits.close()

	if do_pol_combined and nmaps < 3:
		print('do_pol_combined is True, need polarisation maps (only have ' + str(nmaps) + ' maps for ' + inputfile + ').')
		return 0

	# Do some cleanup of the new header to avoid issues later
	try:
		print(newheader['TUNIT1'])
	except:
		for i in range(0,nmaps):
			newheader['TUNIT'+str(i+1)] = 'None'

	# Crop to just have the maps we want to output
	nmaps_orig = nmaps
	if maxnummaps != -1:
		nmaps = maxnummaps
	if outputmaps == []:
		outputmaps = range(0,nmaps)
	maps = [maps[i] for i in outputmaps]
	noutputmaps = len(outputmaps)
	# print newheader
	for i in range(0,nmaps_orig):
		if i < noutputmaps:
			newheader['TTYPE'+str(i+1)] = newheader['TTYPE'+str(outputmaps[i]+1)]
			newheader['TFORM'+str(i+1)] = newheader['TFORM'+str(outputmaps[i]+1)]
			newheader['TUNIT'+str(i+1)] = newheader['TUNIT'+str(outputmaps[i]+1)]
		else:
			del newheader['TTYPE'+str(i+1)]
			del newheader['TFORM'+str(i+1)]
			del newheader['TUNIT'+str(i+1)]
	newheader['TFIELDS'] = noutputmaps
	newheader['NAXIS1'] = noutputmaps*4
	nmaps = noutputmaps
	print(newheader)

	# Calculate the unit conversion factor
	const = get_spectrum_constants()

	pix_area = hp.nside2pixarea(nside_out)

	nside = hp.get_nside(maps)
	if use_precomputed_wf:
		print('Using precomputed window function')
		conv_windowfunction = windowfunction
		if 3*nside - len(conv_windowfunction) > 0:
			conv_windowfunction = np.pad(conv_windowfunction, (0, 3*nside - len(conv_windowfunction)), 'constant')

	elif (fwhm_arcmin != -1):
		# if do_pol_combined:
		# 	conv_windowfunction = hp.gauss_beam(np.radians(fwhm_arcmin/60.0),3*nside,pol=True)
		# else:
		conv_windowfunction = hp.gauss_beam(np.radians(fwhm_arcmin/60.0),3*nside)
		if (windowfunction != []):
			window_len = len(conv_windowfunction)
			beam_len = len(windowfunction)

			if (beam_len > window_len):
				windowfunction  = windowfunction[0:len(conv_windowfunction)]
			else:
				# We want to pad the window function rather than crop the convolution
				# conv_windowfunction = conv_windowfunction[0:len(windowfunction)]
				windowfunction = np.pad(windowfunction, (0, window_len - beam_len), 'constant')

			conv_windowfunction[windowfunction!=0] /= windowfunction[windowfunction!=0]
			conv_windowfunction[windowfunction==0] = 0.0

		# Normalise window function
		if normalise:
			normval = conv_windowfunction[0]
			conv_windowfunction /= normval

		conv_windowfunction_before = conv_windowfunction.copy()
		# The below is only applied to the intensity window function at the moment.
		# If needed, apply a taper
		if taper:
			conv_windowfunction[lmin_taper:lmax_taper] = conv_windowfunction[lmin_taper:lmax_taper] * np.cos((np.pi/2.0)*((np.arange(lmin_taper,lmax_taper)-lmin_taper)/(lmax_taper-lmin_taper)))
			conv_windowfunction[lmax_taper:] = 0.0
		if cap_one:
			conv_windowfunction[conv_windowfunction >= 1.0] = 1.0
		if cap_oneall:
			trip = 0
			for l in range(0,len(conv_windowfunction)):
				if trip == 1:
					conv_windowfunction[l] = 1.0
				elif conv_windowfunction[l] >= 1.0:
					trip = 1
		if taper_gauss:
			trip = 0
			val = 0
			# If we haven't been given a FWHM, estimate it from the data at l<300
			if taper_gauss_sigma == 0:
				beam1 = hp.gauss_beam(np.radians(fwhm_arcmin/60.0),3*nside)
				param_est, cov_x = optimize.curve_fit(gaussfit, range(0,299), windowfunction[0:301], 60.0)
				print(param_est[0])
				taper_gauss_sigma = param_est[0]
			# beam2 = hp.gauss_beam(np.radians(taper_gauss_sigma/60.0),3*nside)
			# plt.plot(windowfunction[0:301])
			# plt.plot(beam2)
			# plt.savefig(outdir+'temp.pdf')
			# exit()
			sigma_final = np.radians(fwhm_arcmin/60.0)/np.sqrt(8*np.log(2))
			sigma_current = np.radians(taper_gauss_sigma/60.0)/np.sqrt(8*np.log(2))
			for l in range(1,len(conv_windowfunction)):
				if trip == 1:
					conv_windowfunction[l] = val * np.exp(-0.5*(sigma_final**2-sigma_current**2)*l*(l+1))
					# conv_windowfunction[l] = val * (beam1[l]/beam2[l])
				elif (conv_windowfunction[l]-conv_windowfunction[l-1]) > 0.0:
					print(l)
					trip = 1
					val = conv_windowfunction[l-1]/np.exp(-0.5*(sigma_final**2-sigma_current**2)*(l-1)*((l-1)+1))
					conv_windowfunction[l] = val * np.exp(-0.5*(sigma_final**2-sigma_current**2)*l*(l+1))
					# val = conv_windowfunction[l-1]/(beam1[l-1]/beam2[l-1])
					# conv_windowfunction[l] = val * (beam1[l]/beam2[l])

		plt.xscale('linear')
		plt.yscale('log')
		plt.plot(windowfunction,label='Window function')
		plt.plot(hp.gauss_beam(np.radians(fwhm_arcmin/60.0),3*nside),label='Gaussian')
		plt.legend()
		plt.savefig(outdir+'wf_orig_'+outputfile+'.pdf')
		plt.clf()

		plt.xscale('linear')
		plt.yscale('linear')
		plt.ylim(0.0,1.0)
		plt.plot(conv_windowfunction,label='Window function')
		plt.plot(conv_windowfunction_before,label='Window function before')
		plt.plot(hp.gauss_beam(np.radians(fwhm_arcmin/60.0),len(windowfunction)-1)/windowfunction,label='Comparison')
		plt.legend()
		plt.savefig(outdir+'wf_lin_'+outputfile+'.pdf')
		plt.yscale('log')
		plt.ylim(1e-8,1e4)
		plt.savefig(outdir+'wf_log_'+outputfile+'.pdf')
		plt.clf()
		# return

		# Check whether we'll need to smooth variances too.
		if smoothvariance != False:
			test = False
			for i in range(0,nmaps):
				if ('cov' in newheader['TTYPE'+str(i+1)]) or ('N_OBS' in newheader['TTYPE'+str(i+1)]):
					test = True
			if test:
				print('Covariance maps detected. Calculating variance window function (this may take a short while)')
				conv_windowfunction_variance = conv_windowfunction.copy()
				# conv_windowfunction_variance = calc_variance_windowfunction(conv_windowfunction)
				# conv_windowfunction_variance /= conv_windowfunction_variance[0]
				print(conv_windowfunction_variance[0])
				print('Done! Onwards...')

				plt.xscale('log')
				plt.yscale('log')
				plt.plot(conv_windowfunction,label='Window function')
				plt.plot(conv_windowfunction_variance,label='Variance window function')
				plt.legend()
				plt.savefig(outdir+'wf_'+outputfile+'.png')
				plt.xscale('log')
				plt.yscale('linear')
				plt.plot(conv_windowfunction,label='Window function')
				plt.plot(conv_windowfunction_variance,label='Variance window function')
				plt.legend()
				plt.savefig(outdir+'wf_lin_'+outputfile+'.png')

	# Do the smoothing
	print("Smoothing the maps")
	smoothed_map = maps
	for i in range(0,nmaps):
		if do_pol_combined and (i == 1 or i == 2):
			continue
		print('map ' + str(i))
		print(len(maps[i]))
		if i == 0 and do_pol_combined:
			print('Doing combined polarisation')
			map_before = maps[i][:].copy()
			map_before_Q = maps[i+1][:].copy()
			map_before_U = maps[i+2][:].copy()
			print(np.shape(map_before))
			for j in range(0,3):
				if useunseen == True:
					maps[i+j][~np.isfinite(maps[i+j][:])] = hp.UNSEEN
					# See if we want to cut based on min/max map values
					if minmapvalue != maxmapvalue:
						if i in minmaxmaps:
							maps[i+j][maps[i+j][:] < minmapvalue] = hp.UNSEEN
							maps[i+j][maps[i+j][:] > maxmapvalue] = hp.UNSEEN
				else:
					maps[i+j][maps[i+j][:] == hp.UNSEEN] = 0.0
					maps[i+j][~np.isfinite(maps[i+j][:])] = 0.0
					# See if we want to cut based on min/max map values
					if minmapvalue != maxmapvalue:
						if i in minmaxmaps:
							maps[i+j][maps[i+j][:] < minmapvalue] = 0.0
							maps[i+j][maps[i+j][:] > maxmapvalue] = 0.0

			# Check that we actually want to do smoothing, as opposed to udgrading. Also check to see if this is in the list of maps to not smooth
			if fwhm_arcmin != -1 and (i not in nosmooth):
				# Calculate the alm's, multiply them by the window function, and convert back to the map
				print('Smoothing...')
				print(conv_windowfunction)
				alms = hp.map2alm(maps[i:i+3],pol=True)
				alms = hp.sphtfunc.smoothalm(alms, beam_window=conv_windowfunction,pol=True)
				# alms[0] = hp.almxfl(alms[0], conv_windowfunction)
				# alms[1] = hp.almxfl(alms[1], conv_windowfunction_pol)
				# alms[2] = hp.almxfl(alms[2], conv_windowfunction_pol)
				newmap = hp.alm2map(alms, nside,verbose=False,pol=True)
				smoothed_map[i] = newmap[0]
				smoothed_map[i+1] = newmap[1]
				smoothed_map[i+2] = newmap[2]
				print(np.sum(smoothed_map[i]))
				print(np.median(smoothed_map[i]))
				smoothed_map[i][map_before[:] == hp.UNSEEN] = hp.UNSEEN
				smoothed_map[i+1][map_before_Q[:] == hp.UNSEEN] = hp.UNSEEN
				smoothed_map[i+2][map_before_U[:] == hp.UNSEEN] = hp.UNSEEN

		else:
			map_before = maps[i][:].copy()
			if useunseen == True:
				maps[i][~np.isfinite(maps[i][:])] = hp.UNSEEN
				# See if we want to cut based on min/max map values
				if minmapvalue != maxmapvalue:
					if i in minmaxmaps:
						maps[i][maps[i][:] < minmapvalue] = hp.UNSEEN
						maps[i][maps[i][:] > maxmapvalue] = hp.UNSEEN
			else:
				maps[i][maps[i][:] == hp.UNSEEN] = 0.0
				maps[i][~np.isfinite(maps[i][:])] = 0.0
				# See if we want to cut based on min/max map values
				if minmapvalue != maxmapvalue:
					if i in minmaxmaps:
						maps[i][maps[i][:] < minmapvalue] = 0.0
						maps[i][maps[i][:] > maxmapvalue] = 0.0

			# Check that we actually want to do smoothing, as opposed to udgrading. Also check to see if this is in the list of maps to not smooth
			if fwhm_arcmin != -1 and (i not in nosmooth):
				if 'N_OBS' in newheader['TTYPE'+str(i+1)]:
					print('Column '+str(i)+' is an N_OBS map ('+newheader['TUNIT'+str(i+1)]+') - converting to variance map.')
					print(np.sum(maps[i]))
					print(np.median(maps[i]))
					maps[i] = conv_nobs_variance_map(maps[i], sigma_0)
					print(np.sum(maps[i]))
					print(np.median(maps[i]))
					if (nobs_out == False and no_sigma_0 == False):
						# We don't want to convert back later.
						print('test')
						newheader['TTYPE'+str(i+1)] = 'II_cov'
						newheader['TUNIT'+str(i+1)] = '('+sigma_0_unit+')^2'

				# Calculate the alm's, multiply them by the window function, and convert back to the map
				if ('cov' in newheader['TTYPE'+str(i+1)]) or ('N_OBS' in newheader['TTYPE'+str(i+1)]):
					if smoothvariance != False:
						print('Column '+str(i)+' is a covariance matrix ('+newheader['TUNIT'+str(i+1)]+') - smoothing appropriately.')
						alms = hp.map2alm(maps[i])
						alms = hp.almxfl(alms, conv_windowfunction_variance)
						newmap = hp.alm2map(alms, nside,verbose=False)
					else:
						newmap = maps[i].copy()
				else:
					print('Smoothing...')
					print(conv_windowfunction)
					alms = hp.map2alm(maps[i])
					alms = hp.almxfl(alms, conv_windowfunction)
					newmap = hp.alm2map(alms, nside,verbose=False)
				smoothed_map[i] = newmap
				print(np.sum(smoothed_map[i]))
				print(np.median(smoothed_map[i]))
				smoothed_map[i][map_before[:] == hp.UNSEEN] = hp.UNSEEN

				if ('N_OBS' in newheader['TTYPE'+str(i+1)]) and (nobs_out or no_sigma_0):
					print('You\'ve either asked for an N_OBS map to be returned, or not set sigma_0, so you will get an N_OBS map returned in your data!')
					print(np.sum(smoothed_map[i]))
					smoothed_map[i] = conv_nobs_variance_map(smoothed_map[i], sigma_0)
					print(np.sum(smoothed_map[i]))
					newheader['TTYPE'+str(i+1)] = 'N_OBS'
	maps = 0
	newmap = 0

	# Do the ud_grading
	print("ud_grading the maps (if needed)")
	nobs_sum = 0
	if (nside_out == 0):
		nside_out = nside
	else:
		for i in range (0,nmaps):
			if 'N_OBS' in newheader['TTYPE'+str(i+1)]:
				nobs_sum = np.sum(smoothed_map[i])

			# Check to see which type of map we have, and adjust the factor of (nside/nside_out)^power appropriately
			power = 0.0
			if ('cov' in newheader['TTYPE'+str(i+1)]):
				power = 2.0
			elif ('N_OBS' in newheader['TTYPE'+str(i+1)]) or ('Hits' in newheader['TTYPE'+str(i+1)]):
				power = -2.0
			print(power)
			smoothed_map[i] = hp.ud_grade(smoothed_map[i], nside_out=nside_out, power=power)

			if 'N_OBS' in newheader['TTYPE'+str(i+1)]:
				null = 0
			else:
				# If we don't have an N_OBS map, then we might want to convert the units.
				if (units_out != ''):
					if (units_in == ''):
						unit = newheader['TUNIT'+str(i+1)]
						unit = unit.strip()
					else:
						# Assume the user is right to have specified different input units from what is in the file.
						unit = units_in

					newheader['TUNIT'+str(i+1)] = units_out
					power = 1.0
					if ('^2' in unit):
						power = 2.0
						unit = unit.replace(")^2",'').replace('(','')
						unit = unit.replace("^2",'')
						newheader['TUNIT'+str(i+1)] = '('+units_out+")^2"
					print(unit + " " + str(power))
					conversion = convertunits(const, unit, units_out, frequency, pix_area)
					print(conversion)
					smoothed_map[i] = smoothed_map[i] * conversion**power

	# All done - now just need to write it to disk.
	print("Writing maps to disk: " + outdir+outputfile)
	cols = []
	for i in range(0,nmaps):
		if ('cov' in newheader['TTYPE'+str(i+1)]):
			smoothed_map[i] = smoothed_map[i] * rescale**2
		else:
			smoothed_map[i] = smoothed_map[i] * rescale

	# If we want to subtract another map (e.g., a CMB map) then we need to read it in, check nside and units, and then subtract.
	if subtractmap != '':
		print('Subtracting CMB map ' + subtractmap)
		if usehealpixfits:
			sub_maps = hp.read_map(indir+subtractmap)
		else:
			sub_inputfits = fits.open(indir+subtractmap)
			sub_cols = sub_inputfits[1].columns
			sub_col_names = sub_cols.names
			sub_nmaps = len(sub_cols)
			sub_nmaps_orig = sub_nmaps
			sub_maps = []
			for i in range(0,sub_nmaps):
				sub_maps.append(sub_inputfits[1].data.field(i))
			# Check to see whether we have nested data, and switch to ring if that is the case.
			if (sub_inputfits[1].header['ORDERING'] == 'NESTED'):
				sub_maps = hp.reorder(sub_maps,n2r=True)
			# We want a maximum of one map to subtract (could be extended in the future)
			sub_maps = sub_maps[0]
			sub_inputfits.close()
		# We want to use the same output Nside - we'll assume they have the same resolution.
		sub_maps = hp.ud_grade(sub_maps, nside_out=nside_out)
		# Calculate a rescaling factor if needed
		conversion = 1.0
		if subtractmap_units != '':
			conversion = convertunits(const, subtractmap_units, units_out, frequency, pix_area)
		# ... and do the subtraction
		smoothed_map[0] = smoothed_map[0] - sub_maps * conversion

	for i in range(0,nmaps):
		cols.append(fits.Column(name=col_names[i], format='E', array=smoothed_map[i]))

	if appendmap != '':
		addmap = hp.read_map(appendmap)
		cols.append(fits.Column(name=appendmapname, format='E', array=addmap))
		newheader['TTYPE'+str(nmaps+1)] = appendmapname
		newheader['TFORM'+str(nmaps+1)] = 'E'
		newheader['TTYPE'+str(nmaps+1)] = appendmapname
		newheader['TFIELDS'] = newheader['TFIELDS']+1
		newheader['NAXIS1'] = newheader['TFIELDS']*4

	cols = fits.ColDefs(cols)
	bin_hdu = fits.BinTableHDU.from_columns(cols)
	# bin_hdu = fits.new_table(cols)
	# print newheader
	bin_hdu.header = newheader
	# print bin_hdu.header
	bin_hdu.header['ORDERING']='RING'
	bin_hdu.header['POLCONV']='COSMO'
	bin_hdu.header['PIXTYPE']='HEALPIX'
	bin_hdu.header['NSIDE']=nside_out
	bin_hdu.header['COMMENT']="Smoothed using Mike Peel's smoothmap.py version "+ver +" modified by Adam Barr"
	print(bin_hdu.header)

	bin_hdu.writeto(outdir+outputfile)

	return

def conv_nobs_variance_map(inputmap, sigma_0):
	newmap = sigma_0**2 / inputmap
	return newmap

# Code to replicate IDL's INT_TABULATED function
# From http://stackoverflow.com/questions/14345001/idls-int-tabulate-scipy-equivalent
def int_tabulated(x, f, p=5) :
    def newton_cotes(x, f) :
        if x.shape[0] < 2 :
            return 0
        rn = (x.shape[0] - 1) * (x - x[0]) / (x[-1] - x[0])
        weights = sp.integrate.newton_cotes(rn)[0]
        return (x[-1] - x[0]) / (x.shape[0] - 1) * np.dot(weights, f)
    ret = 0
    for idx in xrange(0, x.shape[0], p - 1) :
        ret += newton_cotes(x[idx:idx + p], f[idx:idx + p])
    return ret

def calc_variance_windowfunction(conv_windowfunction):
	# Calculate the window function for variance maps.
	# Based on IDL code by Paddy Leahy, 'write_cvbl_planck3.pro'

	const = get_spectrum_constants()
	nbl = len(conv_windowfunction)
	ll = np.arange(0,nbl,1)

	# Choose scale to match size of beam. First find half-power point
	lhalf = [ n for n,i in enumerate(conv_windowfunction) if i<0.5 ][0]

	# Calculate beam out to about 40 * half-power radius, roughly (note that
	# this gives 100 points out to the half-power point).
	numelements = 4000
	rad = np.arange(0,numelements,1)*10.0*const['pi']/(float(lhalf)*float(numelements))

	x = np.cos(rad)
	sinrad = np.sin(rad)

	lgndr = np.zeros((numelements,nbl))
	for i in range(0,nbl):
		lgndr[:,i] = special.lpmv(0, i, x)

	# Generate radial profile of convolving beam:
	conva = np.zeros(numelements)
	for j in range(0,numelements):
		conva[j] = np.sum((ll+0.5)*conv_windowfunction*lgndr[j,:])

	conva = conva / (2.0*const['pi'])
	print('Peak of convolving beam is ' + str(conva[0]) + " (check: " + str(np.max(conva)) + ")")

	# Square convolving beam and convert back to window function
	mult = sinrad*conva**2
	cvbl = np.zeros(nbl)
	print(nbl)
	for l in range(0,nbl):
		cvbl[l] = int_tabulated(rad,mult*lgndr[:,l])

	# Put in 2pi normalization factor:
	cvbl = 2.0*const['pi']*cvbl

	print('Max in the window function is ' + str(cvbl[0]) + " (check: " + str(np.max(cvbl)) + ")")

	return cvbl

# Read in a Planck (HFI or LFI) beam
def get_beam(FITSfile,hdu=0):
	# fits.info(FITSfile) # print list of extensions found in FITSfile
	data, header = fits.getdata(FITSfile, hdu, header=True) # read extension #10 (data and header)
	# data, header = fits.getdata(FITSfile, 'ABC', header=True) # read extension having EXTNAME='ABC' (data and header)
	# print(header) # print header
	# print(data.names) # print column names
	# pylab.plot( data.field(0).flatten() ) # plot 1st column of binary table
	newdata = np.zeros(len(data))
	for i in range(0,len(data)):
		newdata[i] = data[i][0]
	return newdata

# EOF


import pandas as pd

basic_mr = pd.read_csv('basic_trend_following.csv')
vol_mr = pd.read_csv('volatility_adjusted_trend_following.csv')

import matplotlib.pyplot as plt

basic_mr['BasicTrendFollowingPnl'].plot(x='Date', color='b', lw=1., legend=True)
vol_mr['VolatilityAdjustedTrendFollowingPnl'].plot(x='Date', color='g', lw=1., legend=True)
plt.show()


from test_suit.test_utils import RetryableError, NonRetryableError, ResetAndRetry
from time import sleep
from multiprocessing.dummy import Pool as ThreadPool
import json

class TestPressure:
    def __init__(self, driver, nameSalt):
        self.driver = driver
        self.topics = []
        self.topicNum = 200
        self.partitionNum = 12
        self.recordNum = 10000
        self.curTest = 0
        self.threadCount = 10
        self.fileName = "travis_pressure_string_json"
        self.connectorName = self.fileName + nameSalt
        for i in range(self.topicNum):
            self.topics.append(self.fileName + str(i) + nameSalt)

        for t in range(self.topicNum):
            self.driver.createTopics(self.topics[t], self.partitionNum, 1)

        sleep(5)

    def getConfigFileName(self):
        return self.fileName + ".json"

    def send(self):
        threadPool = ThreadPool(self.threadCount)

        args = []
        for t in range(self.topicNum):
            for p in range(self.partitionNum):
                args.append((t, p))
        threadPool.starmap(self.sendHelper, args)
        threadPool.close()
        threadPool.join()

    def sendHelper(self, t, p):
        value = []
        for e in range(self.recordNum):
            value.append(json.dumps(
                {'numbernumbernumbernumbernumbernumbernumbernumbernumbernumbernumbernumber': str(e)}
            ).encode('utf-8'))
        self.driver.sendBytesData(self.topics[t], value, [], p)
        # self.threadPool.starmap(self.driver.sendBytesData, [(self.topics[t], value, [], p)])

    def verify(self, round):
        for t in range(self.curTest, self.topicNum):
            res = self.driver.snowflake_conn.cursor().execute(
                "SELECT count(*) FROM {}".format(self.topics[t])).fetchone()[0]
            if res != self.partitionNum * self.recordNum * (round + 1):
                print("Round", round, "Result", res, "topicNum", t)
                raise RetryableError()

            if self.curTest <= t:
                self.curTest = t + 1
                raise ResetAndRetry()

        # after success, reset curTest for next round
        self.curTest = 0

    def clean(self):
        threadPool = ThreadPool(self.threadCount)
        args = []
        for t in range(self.topicNum):
            args.append((self.connectorName, self.topics[t], self.partitionNum))
        threadPool.starmap(self.driver.cleanTableStagePipe, args)
        threadPool.close()
        threadPool.join()


# Name: Marisa-Dodd
# Prog Purpose: This program creates a sales receipt for Brownsville Bagels
# Price for one bagel: $2.25
# Price for cream cheese: $1.25
# Sales tax rate: 6%

import datetime

# Define global variables
SALES_TAX_RATE = 0.06
PR_BAGEL = 2.25
PR_CREAMCH = 1.25
num_bagels = 0
num_creamch = 0
cost_bagels = 0
cost_creamch = 0
subtotal = 0
sales_tax = 0
total = 0

# Define program functions
def main():
    get_user_data()
    perform_calculations()
    display_results()

def get_user_data():
    global num_bagels, num_creamch
    num_bagels = int(input("Number of bagels: "))
    num_creamch = int(input("Number of cream cheese: "))

def perform_calculations():
    global cost_bagels, cost_creamch, subtotal, sales_tax, total
    cost_bagels = num_bagels * PR_BAGEL
    cost_creamch = num_creamch * PR_CREAMCH
    subtotal = cost_bagels + cost_creamch
    sales_tax = subtotal * SALES_TAX_RATE
    total = subtotal + sales_tax

def display_results():
    print('--------------------------------')
    print('BROWNSVILLE BAGEL COMPANY')
    print('Fresh-baked bagels every day!')
    print('--------------------------------')
    print(f'Bagels       $ {cost_bagels:8,.2f}')
    print(f'Cream Cheese $ {cost_creamch:.2f}')
    print('--------------------------------')
    print(f'Subtotal     $ {subtotal:.2f}')
    print(f'Sales Tax    $ {sales_tax:.2f}')
    print(f'Total        $ {total:.2f}')
    print('--------------------------------')
    print(datetime.datetime.now())

# Call the main program to execute
main()


from django.db import transaction
from rest_framework import serializers

from tests_builder import models

class TagSerializer(serializers.ModelSerializer):
    class Meta:
        model = models.Tag
        fields = '__all__'

class AnswerSerializer(serializers.ModelSerializer):
    is_correct = serializers.BooleanField()

    class Meta:
        model = models.Answer
        fields = ('id', 'text_answer', 'is_correct')

class AnswerCreateSerializer(serializers.ModelSerializer):
    is_correct = serializers.BooleanField()

    class Meta:
        model = models.Answer
        fields = ('id', 'text_answer', 'is_correct', 'question')

class QuestionSerializer(serializers.ModelSerializer):
    answers = AnswerSerializer(many=True)
    # tags = serializers.SlugRelatedField(required=False, many=True, slug_field='name', queryset=models.Tag.objects.all())

    class Meta:
        model = models.Question
        fields = '__all__'

    @transaction.atomic
    def create(self, validated_data):
        answers_data = validated_data.pop('answers')

        question = models.Question.objects.create(**validated_data)

        for answer_data in answers_data:
            models.Answer.objects.create(question=question, **answer_data)

        # try:
        #     tags_data = validated_data.pop('tags')
        #     for tag_data in tags_data:
        #         question.tags.add(tag_data)
        # except:
        #     pass

        return question

    @transaction.atomic
    def update(self, instance, validated_data):
        instance.text_question = validated_data.get('text_question', instance.text_question)
        # tags_data = validated_data.pop('tags')
        #
        # for tag_data in tags_data:
        #     instance.tags.add(tag_data)

        instance.save()

        return instance

class FixedTestQuestionSerializer(serializers.ModelSerializer):
    question = QuestionSerializer()

    class Meta:
        model = models.FixedTestQuestion
        fields = '__all__'

class CreateFixedTestQuestionSerializer(serializers.ModelSerializer):
    class Meta:
        model = models.FixedTestQuestion
        fields = ('position', 'question')

class FixedTestSerializer(serializers.ModelSerializer):
    questions = CreateFixedTestQuestionSerializer(source='test_to_question', many=True)
    created_by = serializers.HiddenField(default=serializers.CurrentUserDefault())

    class Meta:
        model = models.FixedTest
        fields = '__all__'
        read_only_fields = ('created',)

    @transaction.atomic
    def create(self, validated_data):
        questions_data = validated_data.pop('test_to_question')

        fixed_test = models.FixedTest.objects.create(**validated_data)

        positions = {question_data['position'] for question_data in questions_data}
        if len(positions) != len(questions_data):
            raise serializers.ValidationError('Questions positions are supposed to be unique in one test')

        models.FixedTestQuestion.objects.bulk_create([
            models.FixedTestQuestion(
                test=fixed_test, question=data['question'], position=data['position']
            ) for data in questions_data
        ])

        return fixed_test

    @transaction.atomic
    def update(self, instance, validated_data):
        instance.name = validated_data.get('name', instance.name)
        instance.section = validated_data.get('section', instance.section)
        instance.max_number_of_tries = validated_data.get('max_number_of_tries', instance.max_number_of_tries)
        instance.time_limit_for_try = validated_data.get('time_limit_for_try', instance.time_limit_for_try)
        instance.min_percent_for_success = validated_data.get('min_percent_for_success', instance.min_percent_for_success)
        instance.start_date = validated_data.get('start_date', instance.start_date)
        instance.due_date = validated_data.get('due_date', instance.due_date)
        instance.save()

        return instance

class RandomTestSerializer(serializers.ModelSerializer):
    # tags = serializers.SlugRelatedField(many=True, slug_field='name', queryset=models.Tag.objects.all())
    created_by = serializers.HiddenField(default=serializers.CurrentUserDefault())

    class Meta:
        model = models.RandomTest
        fields = '__all__'

    @transaction.atomic
    def create(self, validated_data):
        tags_data = validated_data.pop('tags')

        random_test = models.RandomTest.objects.create(**validated_data)

        for tag_data in tags_data:
            random_test.tags.add(tag_data)

        return random_test


import math
from typing import List, Optional, Tuple

import numpy as np
import torch
import torch.nn.functional as F

from espnet2.gan_tts.hifigan import HiFiGANGenerator
from espnet2.gan_tts.utils import get_random_segments
from espnet2.gan_tts.vits.duration_predictor import StochasticDurationPredictor
from espnet2.gan_tts.vits.posterior_encoder import PosteriorEncoder
from espnet2.gan_tts.vits.residual_coupling import ResidualAffineCouplingBlock
from espnet2.gan_tts.vits.text_encoder import TextEncoder
from espnet.nets.pytorch_backend.nets_utils import make_non_pad_mask

class VITSGenerator(torch.nn.Module):
    """Generator module in VITS.

    This is a module of VITS described in `Conditional Variational Autoencoder
    with Adversarial Learning for End-to-End Text-to-Speech`_.

    As text encoder, we use conformer architecture instead of the relative positional
    Transformer, which contains additional convolution layers.

    .. _`Conditional Variational Autoencoder with Adversarial Learning for End-to-End
        Text-to-Speech`: https://arxiv.org/abs/2006.04558

    """

    def __init__(
        self,
        vocabs: int,
        aux_channels: int = 513,
        hidden_channels: int = 192,
        spks: Optional[int] = None,
        langs: Optional[int] = None,
        spk_embed_dim: Optional[int] = None,
        global_channels: int = -1,
        segment_size: int = 32,
        text_encoder_attention_heads: int = 2,
        text_encoder_ffn_expand: int = 4,
        text_encoder_blocks: int = 6,
        text_encoder_positionwise_layer_type: str = "conv1d",
        text_encoder_positionwise_conv_kernel_size: int = 1,
        text_encoder_positional_encoding_layer_type: str = "rel_pos",
        text_encoder_self_attention_layer_type: str = "rel_selfattn",
        text_encoder_activation_type: str = "swish",
        text_encoder_normalize_before: bool = True,
        text_encoder_dropout_rate: float = 0.1,
        text_encoder_positional_dropout_rate: float = 0.0,
        text_encoder_attention_dropout_rate: float = 0.0,
        text_encoder_conformer_kernel_size: int = 7,
        use_macaron_style_in_text_encoder: bool = True,
        use_conformer_conv_in_text_encoder: bool = True,
        decoder_kernel_size: int = 7,
        decoder_channels: int = 512,
        decoder_upsample_scales: List[int] = [8, 8, 2, 2],
        decoder_upsample_kernel_sizes: List[int] = [16, 16, 4, 4],
        decoder_resblock_kernel_sizes: List[int] = [3, 7, 11],
        decoder_resblock_dilations: List[List[int]] = [[1, 3, 5], [1, 3, 5], [1, 3, 5]],
        use_weight_norm_in_decoder: bool = True,
        posterior_encoder_kernel_size: int = 5,
        posterior_encoder_layers: int = 16,
        posterior_encoder_stacks: int = 1,
        posterior_encoder_base_dilation: int = 1,
        posterior_encoder_dropout_rate: float = 0.0,
        use_weight_norm_in_posterior_encoder: bool = True,
        flow_flows: int = 4,
        flow_kernel_size: int = 5,
        flow_base_dilation: int = 1,
        flow_layers: int = 4,
        flow_dropout_rate: float = 0.0,
        use_weight_norm_in_flow: bool = True,
        use_only_mean_in_flow: bool = True,
        stochastic_duration_predictor_kernel_size: int = 3,
        stochastic_duration_predictor_dropout_rate: float = 0.5,
        stochastic_duration_predictor_flows: int = 4,
        stochastic_duration_predictor_dds_conv_layers: int = 3,
    ):
        """Initialize VITS generator module.

        Args:
            vocabs (int): Input vocabulary size.
            aux_channels (int): Number of acoustic feature channels.
            hidden_channels (int): Number of hidden channels.
            spks (Optional[int]): Number of speakers. If set to > 1, assume that the
                sids will be provided as the input and use sid embedding layer.
            langs (Optional[int]): Number of languages. If set to > 1, assume that the
                lids will be provided as the input and use sid embedding layer.
            spk_embed_dim (Optional[int]): Speaker embedding dimension. If set to > 0,
                assume that spembs will be provided as the input.
            global_channels (int): Number of global conditioning channels.
            segment_size (int): Segment size for decoder.
            text_encoder_attention_heads (int): Number of heads in conformer block
                of text encoder.
            text_encoder_ffn_expand (int): Expansion ratio of FFN in conformer block
                of text encoder.
            text_encoder_blocks (int): Number of conformer blocks in text encoder.
            text_encoder_positionwise_layer_type (str): Position-wise layer type in
                conformer block of text encoder.
            text_encoder_positionwise_conv_kernel_size (int): Position-wise convolution
                kernel size in conformer block of text encoder. Only used when the
                above layer type is conv1d or conv1d-linear.
            text_encoder_positional_encoding_layer_type (str): Positional encoding layer
                type in conformer block of text encoder.
            text_encoder_self_attention_layer_type (str): Self-attention layer type in
                conformer block of text encoder.
            text_encoder_activation_type (str): Activation function type in conformer
                block of text encoder.
            text_encoder_normalize_before (bool): Whether to apply layer norm before
                self-attention in conformer block of text encoder.
            text_encoder_dropout_rate (float): Dropout rate in conformer block of
                text encoder.
            text_encoder_positional_dropout_rate (float): Dropout rate for positional
                encoding in conformer block of text encoder.
            text_encoder_attention_dropout_rate (float): Dropout rate for attention in
                conformer block of text encoder.
            text_encoder_conformer_kernel_size (int): Conformer conv kernel size. It
                will be used when only use_conformer_conv_in_text_encoder = True.
            use_macaron_style_in_text_encoder (bool): Whether to use macaron style FFN
                in conformer block of text encoder.
            use_conformer_conv_in_text_encoder (bool): Whether to use covolution in
                conformer block of text encoder.
            decoder_kernel_size (int): Decoder kernel size.
            decoder_channels (int): Number of decoder initial channels.
            decoder_upsample_scales (List[int]): List of upsampling scales in decoder.
            decoder_upsample_kernel_sizes (List[int]): List of kernel size for
                upsampling layers in decoder.
            decoder_resblock_kernel_sizes (List[int]): List of kernel size for resblocks
                in decoder.
            decoder_resblock_dilations (List[List[int]]): List of list of dilations for
                resblocks in decoder.
            use_weight_norm_in_decoder (bool): Whether to apply weight normalization in
                decoder.
            posterior_encoder_kernel_size (int): Posterior encoder kernel size.
            posterior_encoder_layers (int): Number of layers of posterior encoder.
            posterior_encoder_stacks (int): Number of stacks of posterior encoder.
            posterior_encoder_base_dilation (int): Base dilation of posterior encoder.
            posterior_encoder_dropout_rate (float): Dropout rate for posterior encoder.
            use_weight_norm_in_posterior_encoder (bool): Whether to apply weight
                normalization in posterior encoder.
            flow_flows (int): Number of flows in flow.
            flow_kernel_size (int): Kernel size in flow.
            flow_base_dilation (int): Base dilation in flow.
            flow_layers (int): Number of layers in flow.
            flow_dropout_rate (float): Dropout rate in flow
            use_weight_norm_in_flow (bool): Whether to apply weight normalization in
                flow.
            use_only_mean_in_flow (bool): Whether to use only mean in flow.
            stochastic_duration_predictor_kernel_size (int): Kernel size in stochastic
                duration predictor.
            stochastic_duration_predictor_dropout_rate (float): Dropout rate in
                stochastic duration predictor.
            stochastic_duration_predictor_flows (int): Number of flows in stochastic
                duration predictor.
            stochastic_duration_predictor_dds_conv_layers (int): Number of DDS conv
                layers in stochastic duration predictor.

        """
        super().__init__()
        self.segment_size = segment_size
        self.text_encoder = TextEncoder(
            vocabs=vocabs,
            attention_dim=hidden_channels,
            attention_heads=text_encoder_attention_heads,
            linear_units=hidden_channels * text_encoder_ffn_expand,
            blocks=text_encoder_blocks,
            positionwise_layer_type=text_encoder_positionwise_layer_type,
            positionwise_conv_kernel_size=text_encoder_positionwise_conv_kernel_size,
            positional_encoding_layer_type=text_encoder_positional_encoding_layer_type,
            self_attention_layer_type=text_encoder_self_attention_layer_type,
            activation_type=text_encoder_activation_type,
            normalize_before=text_encoder_normalize_before,
            dropout_rate=text_encoder_dropout_rate,
            positional_dropout_rate=text_encoder_positional_dropout_rate,
            attention_dropout_rate=text_encoder_attention_dropout_rate,
            conformer_kernel_size=text_encoder_conformer_kernel_size,
            use_macaron_style=use_macaron_style_in_text_encoder,
            use_conformer_conv=use_conformer_conv_in_text_encoder,
        )
        self.decoder = HiFiGANGenerator(
            in_channels=hidden_channels,
            out_channels=1,
            channels=decoder_channels,
            global_channels=global_channels,
            kernel_size=decoder_kernel_size,
            upsample_scales=decoder_upsample_scales,
            upsample_kernel_sizes=decoder_upsample_kernel_sizes,
            resblock_kernel_sizes=decoder_resblock_kernel_sizes,
            resblock_dilations=decoder_resblock_dilations,
            use_weight_norm=use_weight_norm_in_decoder,
        )
        self.posterior_encoder = PosteriorEncoder(
            in_channels=aux_channels,
            out_channels=hidden_channels,
            hidden_channels=hidden_channels,
            kernel_size=posterior_encoder_kernel_size,
            layers=posterior_encoder_layers,
            stacks=posterior_encoder_stacks,
            base_dilation=posterior_encoder_base_dilation,
            global_channels=global_channels,
            dropout_rate=posterior_encoder_dropout_rate,
            use_weight_norm=use_weight_norm_in_posterior_encoder,
        )
        self.flow = ResidualAffineCouplingBlock(
            in_channels=hidden_channels,
            hidden_channels=hidden_channels,
            flows=flow_flows,
            kernel_size=flow_kernel_size,
            base_dilation=flow_base_dilation,
            layers=flow_layers,
            global_channels=global_channels,
            dropout_rate=flow_dropout_rate,
            use_weight_norm=use_weight_norm_in_flow,
            use_only_mean=use_only_mean_in_flow,
        )
        # TODO(kan-bayashi): Add deterministic version as an option
        self.duration_predictor = StochasticDurationPredictor(
            channels=hidden_channels,
            kernel_size=stochastic_duration_predictor_kernel_size,
            dropout_rate=stochastic_duration_predictor_dropout_rate,
            flows=stochastic_duration_predictor_flows,
            dds_conv_layers=stochastic_duration_predictor_dds_conv_layers,
            global_channels=global_channels,
        )

        self.upsample_factor = int(np.prod(decoder_upsample_scales))
        self.spks = None
        if spks is not None and spks > 1:
            assert global_channels > 0
            self.spks = spks
            self.global_emb = torch.nn.Embedding(spks, global_channels)
        self.spk_embed_dim = None
        if spk_embed_dim is not None and spk_embed_dim > 0:
            assert global_channels > 0
            self.spk_embed_dim = spk_embed_dim
            self.spemb_proj = torch.nn.Linear(spk_embed_dim, global_channels)
        self.langs = None
        if langs is not None and langs > 1:
            assert global_channels > 0
            self.langs = langs
            self.lang_emb = torch.nn.Embedding(langs, global_channels)

        # delayed import
        from espnet2.gan_tts.vits.monotonic_align import maximum_path

        self.maximum_path = maximum_path

    def forward(
        self,
        text: torch.Tensor,
        text_lengths: torch.Tensor,
        feats: torch.Tensor,
        feats_lengths: torch.Tensor,
        sids: Optional[torch.Tensor] = None,
        spembs: Optional[torch.Tensor] = None,
        lids: Optional[torch.Tensor] = None,
    ) -> Tuple[
        torch.Tensor,
        torch.Tensor,
        torch.Tensor,
        torch.Tensor,
        torch.Tensor,
        torch.Tensor,
        Tuple[
            torch.Tensor,
            torch.Tensor,
            torch.Tensor,
            torch.Tensor,
            torch.Tensor,
            torch.Tensor,
        ],
    ]:
        """Calculate forward propagation.

        Args:
            text (Tensor): Text index tensor (B, T_text).
            text_lengths (Tensor): Text length tensor (B,).
            feats (Tensor): Feature tensor (B, aux_channels, T_feats).
            feats_lengths (Tensor): Feature length tensor (B,).
            sids (Optional[Tensor]): Speaker index tensor (B,) or (B, 1).
            spembs (Optional[Tensor]): Speaker embedding tensor (B, spk_embed_dim).
            lids (Optional[Tensor]): Language index tensor (B,) or (B, 1).

        Returns:
            Tensor: Waveform tensor (B, 1, segment_size * upsample_factor).
            Tensor: Duration negative log-likelihood (NLL) tensor (B,).
            Tensor: Monotonic attention weight tensor (B, 1, T_feats, T_text).
            Tensor: Segments start index tensor (B,).
            Tensor: Text mask tensor (B, 1, T_text).
            Tensor: Feature mask tensor (B, 1, T_feats).
            tuple[Tensor, Tensor, Tensor, Tensor, Tensor, Tensor]:
                - Tensor: Posterior encoder hidden representation (B, H, T_feats).
                - Tensor: Flow hidden representation (B, H, T_feats).
                - Tensor: Expanded text encoder projected mean (B, H, T_feats).
                - Tensor: Expanded text encoder projected scale (B, H, T_feats).
                - Tensor: Posterior encoder projected mean (B, H, T_feats).
                - Tensor: Posterior encoder projected scale (B, H, T_feats).

        """
        # forward text encoder
        x, m_p, logs_p, x_mask = self.text_encoder(text, text_lengths)

        # calculate global conditioning
        g = None
        if self.spks is not None:
            # speaker one-hot vector embedding: (B, global_channels, 1)
            g = self.global_emb(sids.view(-1)).unsqueeze(-1)
        if self.spk_embed_dim is not None:
            # pretreined speaker embedding, e.g., X-vector (B, global_channels, 1)
            g_ = self.spemb_proj(F.normalize(spembs)).unsqueeze(-1)
            if g is None:
                g = g_
            else:
                g = g + g_
        if self.langs is not None:
            # language one-hot vector embedding: (B, global_channels, 1)
            g_ = self.lang_emb(lids.view(-1)).unsqueeze(-1)
            if g is None:
                g = g_
            else:
                g = g + g_

        # forward posterior encoder
        z, m_q, logs_q, y_mask = self.posterior_encoder(feats, feats_lengths, g=g)

        # forward flow
        z_p = self.flow(z, y_mask, g=g)  # (B, H, T_feats)

        # monotonic alignment search
        with torch.no_grad():
            # negative cross-entropy
            s_p_sq_r = torch.exp(-2 * logs_p)  # (B, H, T_text)
            # (B, 1, T_text)
            neg_x_ent_1 = torch.sum(
                -0.5 * math.log(2 * math.pi) - logs_p,
                [1],
                keepdim=True,
            )
            # (B, T_feats, H) x (B, H, T_text) = (B, T_feats, T_text)
            neg_x_ent_2 = torch.matmul(
                -0.5 * (z_p**2).transpose(1, 2),
                s_p_sq_r,
            )
            # (B, T_feats, H) x (B, H, T_text) = (B, T_feats, T_text)
            neg_x_ent_3 = torch.matmul(
                z_p.transpose(1, 2),
                (m_p * s_p_sq_r),
            )
            # (B, 1, T_text)
            neg_x_ent_4 = torch.sum(
                -0.5 * (m_p**2) * s_p_sq_r,
                [1],
                keepdim=True,
            )
            # (B, T_feats, T_text)
            neg_x_ent = neg_x_ent_1 + neg_x_ent_2 + neg_x_ent_3 + neg_x_ent_4
            # (B, 1, T_feats, T_text)
            attn_mask = torch.unsqueeze(x_mask, 2) * torch.unsqueeze(y_mask, -1)
            # monotonic attention weight: (B, 1, T_feats, T_text)
            attn = (
                self.maximum_path(
                    neg_x_ent,
                    attn_mask.squeeze(1),
                )
                .unsqueeze(1)
                .detach()
            )

        # forward duration predictor
        w = attn.sum(2)  # (B, 1, T_text)
        dur_nll = self.duration_predictor(x, x_mask, w=w, g=g)
        dur_nll = dur_nll / torch.sum(x_mask)

        # expand the length to match with the feature sequence
        # (B, T_feats, T_text) x (B, T_text, H) -> (B, H, T_feats)
        m_p = torch.matmul(attn.squeeze(1), m_p.transpose(1, 2)).transpose(1, 2)
        # (B, T_feats, T_text) x (B, T_text, H) -> (B, H, T_feats)
        logs_p = torch.matmul(attn.squeeze(1), logs_p.transpose(1, 2)).transpose(1, 2)

        # get random segments
        z_segments, z_start_idxs = get_random_segments(
            z,
            feats_lengths,
            self.segment_size,
        )

        # forward decoder with random segments
        wav = self.decoder(z_segments, g=g)

        return (
            wav,
            dur_nll,
            attn,
            z_start_idxs,
            x_mask,
            y_mask,
            (z, z_p, m_p, logs_p, m_q, logs_q),
        )

    def inference(
        self,
        text: torch.Tensor,
        text_lengths: torch.Tensor,
        feats: Optional[torch.Tensor] = None,
        feats_lengths: Optional[torch.Tensor] = None,
        sids: Optional[torch.Tensor] = None,
        spembs: Optional[torch.Tensor] = None,
        lids: Optional[torch.Tensor] = None,
        dur: Optional[torch.Tensor] = None,
        noise_scale: float = 0.667,
        noise_scale_dur: float = 0.8,
        alpha: float = 1.0,
        max_len: Optional[int] = None,
        use_teacher_forcing: bool = False,
    ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
        """Run inference.

        Args:
            text (Tensor): Input text index tensor (B, T_text,).
            text_lengths (Tensor): Text length tensor (B,).
            feats (Tensor): Feature tensor (B, aux_channels, T_feats,).
            feats_lengths (Tensor): Feature length tensor (B,).
            sids (Optional[Tensor]): Speaker index tensor (B,) or (B, 1).
            spembs (Optional[Tensor]): Speaker embedding tensor (B, spk_embed_dim).
            lids (Optional[Tensor]): Language index tensor (B,) or (B, 1).
            dur (Optional[Tensor]): Ground-truth duration (B, T_text,). If provided,
                skip the prediction of durations (i.e., teacher forcing).
            noise_scale (float): Noise scale parameter for flow.
            noise_scale_dur (float): Noise scale parameter for duration predictor.
            alpha (float): Alpha parameter to control the speed of generated speech.
            max_len (Optional[int]): Maximum length of acoustic feature sequence.
            use_teacher_forcing (bool): Whether to use teacher forcing.

        Returns:
            Tensor: Generated waveform tensor (B, T_wav).
            Tensor: Monotonic attention weight tensor (B, T_feats, T_text).
            Tensor: Duration tensor (B, T_text).

        """
        # encoder
        x, m_p, logs_p, x_mask = self.text_encoder(text, text_lengths)
        g = None
        if self.spks is not None:
            # (B, global_channels, 1)
            g = self.global_emb(sids.view(-1)).unsqueeze(-1)
        if self.spk_embed_dim is not None:
            # (B, global_channels, 1)
            g_ = self.spemb_proj(F.normalize(spembs.unsqueeze(0))).unsqueeze(-1)
            if g is None:
                g = g_
            else:
                g = g + g_
        if self.langs is not None:
            # (B, global_channels, 1)
            g_ = self.lang_emb(lids.view(-1)).unsqueeze(-1)
            if g is None:
                g = g_
            else:
                g = g + g_

        if use_teacher_forcing:
            # forward posterior encoder
            z, m_q, logs_q, y_mask = self.posterior_encoder(feats, feats_lengths, g=g)

            # forward flow
            z_p = self.flow(z, y_mask, g=g)  # (B, H, T_feats)

            # monotonic alignment search
            s_p_sq_r = torch.exp(-2 * logs_p)  # (B, H, T_text)
            # (B, 1, T_text)
            neg_x_ent_1 = torch.sum(
                -0.5 * math.log(2 * math.pi) - logs_p,
                [1],
                keepdim=True,
            )
            # (B, T_feats, H) x (B, H, T_text) = (B, T_feats, T_text)
            neg_x_ent_2 = torch.matmul(
                -0.5 * (z_p**2).transpose(1, 2),
                s_p_sq_r,
            )
            # (B, T_feats, H) x (B, H, T_text) = (B, T_feats, T_text)
            neg_x_ent_3 = torch.matmul(
                z_p.transpose(1, 2),
                (m_p * s_p_sq_r),
            )
            # (B, 1, T_text)
            neg_x_ent_4 = torch.sum(
                -0.5 * (m_p**2) * s_p_sq_r,
                [1],
                keepdim=True,
            )
            # (B, T_feats, T_text)
            neg_x_ent = neg_x_ent_1 + neg_x_ent_2 + neg_x_ent_3 + neg_x_ent_4
            # (B, 1, T_feats, T_text)
            attn_mask = torch.unsqueeze(x_mask, 2) * torch.unsqueeze(y_mask, -1)
            # monotonic attention weight: (B, 1, T_feats, T_text)
            attn = self.maximum_path(
                neg_x_ent,
                attn_mask.squeeze(1),
            ).unsqueeze(1)
            dur = attn.sum(2)  # (B, 1, T_text)

            # forward decoder with random segments
            wav = self.decoder(z * y_mask, g=g)
        else:
            # duration
            if dur is None:
                logw = self.duration_predictor(
                    x,
                    x_mask,
                    g=g,
                    inverse=True,
                    noise_scale=noise_scale_dur,
                )
                w = torch.exp(logw) * x_mask * alpha
                dur = torch.ceil(w)
            y_lengths = torch.clamp_min(torch.sum(dur, [1, 2]), 1).long()
            y_mask = make_non_pad_mask(y_lengths).unsqueeze(1).to(text.device)
            attn_mask = torch.unsqueeze(x_mask, 2) * torch.unsqueeze(y_mask, -1)
            attn = self._generate_path(dur, attn_mask)

            # expand the length to match with the feature sequence
            # (B, T_feats, T_text) x (B, T_text, H) -> (B, H, T_feats)
            m_p = torch.matmul(
                attn.squeeze(1),
                m_p.transpose(1, 2),
            ).transpose(1, 2)
            # (B, T_feats, T_text) x (B, T_text, H) -> (B, H, T_feats)
            logs_p = torch.matmul(
                attn.squeeze(1),
                logs_p.transpose(1, 2),
            ).transpose(1, 2)

            # decoder
            z_p = m_p + torch.randn_like(m_p) * torch.exp(logs_p) * noise_scale
            z = self.flow(z_p, y_mask, g=g, inverse=True)
            wav = self.decoder((z * y_mask)[:, :, :max_len], g=g)

        return wav.squeeze(1), attn.squeeze(1), dur.squeeze(1)

    def _generate_path(self, dur: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
        """Generate path a.k.a. monotonic attention.

        Args:
            dur (Tensor): Duration tensor (B, 1, T_text).
            mask (Tensor): Attention mask tensor (B, 1, T_feats, T_text).

        Returns:
            Tensor: Path tensor (B, 1, T_feats, T_text).

        """
        b, _, t_y, t_x = mask.shape
        cum_dur = torch.cumsum(dur, -1)
        cum_dur_flat = cum_dur.view(b * t_x)
        path = torch.arange(t_y, dtype=dur.dtype, device=dur.device)
        path = path.unsqueeze(0) < cum_dur_flat.unsqueeze(1)
        path = path.view(b, t_x, t_y).to(dtype=mask.dtype)
        # path will be like (t_x = 3, t_y = 5):
        # [[[1., 1., 0., 0., 0.],      [[[1., 1., 0., 0., 0.],
        #   [1., 1., 1., 1., 0.],  -->   [0., 0., 1., 1., 0.],
        #   [1., 1., 1., 1., 1.]]]       [0., 0., 0., 0., 1.]]]
        path = path - F.pad(path, [0, 0, 1, 0, 0, 0])[:, :-1]
        return path.unsqueeze(1).transpose(2, 3) * mask


import numpy as np
import pydot
import argparse
import seaborn as sns
import pandas as pd
import matplotlib.pyplot as plt

class DrawStateDiag:
    def __init__(self, transmat, state_probs=None, isotype_mapping=None, color_encoding=None, cmap_name="rocket_r") -> None:

        if isotype_mapping is None:
            isotype_mapping = {i: i for i in range(transmat.shape[0])}

        self.isotype_mapping = isotype_mapping
        self.state_probs = state_probs
        self.cmap = sns.color_palette(cmap_name, as_cmap=True)
        self.states = [i for i in range(transmat.shape[0])]
        self.transmat = transmat

        if color_encoding is None:
            self.color_encoding =  {
                0 : "#f0f0f0",
                1 : "#FFEDA0",
                2 : "#FD8D3C",
                3 : "#E31A1C",
                4 : "#800026",
                5 : "#74C476",
                6 : "#6A51A3",
                7 : "darkgoldenrod",
                8 : "mediumseagreen",
                9 : "thistle1"

            }
            # self.color_encoding =  {
            #     0 : "antiquewhite",
            #     1 : "turquoise1",
            #     2 : "darkcyan",
            #     3 : "cornflowerblue",
            #     4 : "darksalmon",
            #     5 : "mediumseagreen",
            #     6 : "orangered",
            #     7 : "orchid",
            #     8 : "darkgoldenrod",
            #     9 : "thistle1"
            # }

        self.graph = pydot.Dot("state_diagram", graph_type="digraph", bgcolor="white")
        added_nodes = []
        for s in self.states:
            lab =isotype_mapping[s]
            if state_probs is not None:
                if state_probs[s] >= 0.00:

                    added_nodes.append(s)
                    lab +=  "\n" + str(round(state_probs[s],3))

            col = self.color_encoding[s]
            self.graph.add_node(pydot.Node(isotype_mapping[s], shape="circle", color=col, style='filled',label=lab))

        for s in self.states:
            # if s not in added_nodes:
            #     continue
            for t in self.states:
                if self.transmat[s,t] > 0.01:# and t in added_nodes:
                    # if t not in added_nodes and s == t:
                    #     continue

                    lab = str(round(self.transmat[s,t],3))
                    new_edge = pydot.Edge(dst=isotype_mapping[t], src=isotype_mapping[s], color="black", label=lab)

                    self.graph.add_edge(new_edge)

    def heatmap(self,fname):
        plt.figure()
        labs= [self.isotype_mapping[s] for s in self.states]
        df = pd.DataFrame(self.transmat, index=labs, columns=labs)
        fig =sns.heatmap(df, annot=True, fmt=".02f", cmap=self.cmap)

        fig.set(xlabel="Isotype", ylabel="Isotype")
        plt.savefig(fname)

    def state_heatmap(self, fname):
        labs= [self.isotype_mapping[s] for s in self.states]
        df = pd.DataFrame(self.state_probs.reshape(1,-1), columns=labs)
        fig =sns.heatmap(df, annot=True, fmt=".02f", cmap=self.cmap, square=True, yticklabels=False, vmin=0, vmax=1)

        fig.set(xlabel="Isotype")
        plt.savefig(fname)

    def save(self, fname):
        self.graph.write_png(fname)

    def save_pdf(self, fname):

        self.graph.write_pdf(fname)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()

    parser.add_argument("-t", "--transmat", required=True, type=str,
        help="filename of tree file in parent edge list form")
    parser.add_argument("-s", "--state_probs", required=False, type=str,
        help="filename of tree file in parent edge list form")

    parser.add_argument("-e", "--encoding", required=False, type=str,
        help="filename of input transition matrix")

    parser.add_argument("-o", "--outfile", type=str)
    parser.add_argument("--pdf", type=str)
    parser.add_argument("--heatmap", type=str)
    parser.add_argument("--statemap", type=str)
    args= parser.parse_args()

    # args = parser.parse_args([
    #     "-t", "/scratch/projects/tribal/benchmark_pipeline/sim_data/transmats/transmat1.txt",
    #     "-e", "/scratch/projects/tribal/benchmark_pipeline/sim_encoding.txt",
    #     "--heatmap", "src/test/heatmap.png"

    # ])

    tmat = np.loadtxt(args.transmat)
    if args.state_probs is not None:
        state_probs = np.loadtxt(args.state_probs)
    else:
        state_probs = None
    if args.encoding is not None:

        with open(args.encoding, 'r+') as file:
                    isotype_encoding = {}
                    counter = 0
                    for line in file:
                        isotype_encoding[counter] = line.strip()
                        counter += 1
    else:
        isotype_encoding = None

    ds = DrawStateDiag(tmat, state_probs, isotype_encoding)
    if args.pdf is not None:
        ds.save_pdf(args.pdf)

    if args.outfile is not None:
        ds.save(args.outfile)

    if args.heatmap is not None:
        ds.heatmap(args.heatmap)

    if args.statemap is not None:
        ds.state_heatmap(args.statemap)


from collections import deque
import sys
input = sys.stdin.readline
delta = ((0, 1), (1, 0), (0, -1), (-1, 0))

def find(q):
    tmp = deque()
    while q:
        r, c = q.popleft()
        if r == end[0] and c == end[1]:
            return 1, None
        for dr, dc in delta:
            nr, nc = dr+r, dc+c
            if 0 <= nr < R and 0 <= nc < C and not visited[nr][nc]:
                if lst[nr][nc] == 'X':
                    tmp.append((nr, nc))
                else:
                    q.append((nr, nc))
                visited[nr][nc] = 1
    return 0, tmp

def melt_ice(water):
    next_water = deque()
    while water:
        r, c = water.popleft()
        for dr, dc in delta:
            nr, nc = dr+r, dc+c
            if 0 <= nr < R and 0 <= nc < C and lst[nr][nc] == 'X':
                next_water.append((nr, nc))
                lst[nr][nc] = '.'
    return next_water

R, C = map(int, input().split())
start, end = [], []
lst = []
water = deque()
for i in range(R):
    input_ = list(input().rstrip())
    for j in range(C):
        if input_[j] == '.' or input_[j] == 'L':
            water.append((i, j))
        if input_[j] == 'L':
            if not start:
                start = [i, j]
            else:
                end = [i, j]
            input_[j] = '.'
    lst.append(input_)

time = 0
visited = [[0]*C for _ in range(R)]
q = deque()
q.append((start[0], start[1]))
visited[start[0]][start[1]] = 1
while 1:
    res, q = find(q)
    if res:
        break
    water = melt_ice(water)
    time += 1

print(time)


from subprocess import run, Popen, DEVNULL,PIPE
from time import sleep
from json import loads
from http.server import BaseHTTPRequestHandler, HTTPServer
import os

try:
    from requests import get
except:
    run("pip install requests")
    from requests import get

class  httpHandler(BaseHTTPRequestHandler):
    html = 'html ancora non letto'
    def do_GET(self):
        self.send_response(200)
        self.send_header('Content-type','text/html')
        self.end_headers()
        message = self.html
        self.wfile.write(bytes(message, "utf8"))

def getconfig():
    r = open("./config.json","r")
    c = loads(r.read())
    r.close()
    try:
        a = c['ngrok_token']
        a = c['pagekite_url']
        a = c['pagekite_token']
    except:
        raise Exception('config.json formattato male')
    return c

def htmltostr():
    a = open("./page.html","r")
    s = a.read()
    a.close()
    return s

def httpserver():
    server_address = ('127.0.0.1', 8080)
    handelr = httpHandler
    handelr.html = htmltostr().replace("//tcp ip qua","var a = '"+getngrokip()[6:]+"'")
    httpd = HTTPServer(server_address, handelr)
    print("http server avviato | http localhost:8080")
    httpd.serve_forever()

def ngrok():
    run("ngrok.exe config add-authtoken "+config['ngrok_token'],stdout=DEVNULL,stderr=DEVNULL)
    run("taskkill /F /IM  ngrok.exe /T",stdout=DEVNULL,stderr=DEVNULL)
    Popen(["cd",os.getcwd(),"&&","ngrok.exe","tcp", "25565"], shell = True,stdout=DEVNULL,stderr=DEVNULL)
    i = 0
    while i<10:
        sleep(1)
        r = get("http://127.0.0.1:4040/api/tunnels")
        a = loads(r.text)
        if(len(a["tunnels"])>0):
            break
        i+=1
    print("ngrok avviato | tcp localhost:25565 -> "+a["tunnels"][0]["public_url"])
    return(a["tunnels"][0]["public_url"])

def kite():
    p = Popen('python '+os.getcwd()+'\pagekite.py --defaults --service_on=http,https:{0}:localhost:8080:{1}'.format(config['pagekite_url'],config['pagekite_token']),stdout=DEVNULL,stderr=PIPE)
    a = p.stderr.read(2)
    if(a==b"\r\n"):
        raise Exception("pagekite non avviato, url o token forniti non validi")
    print("pagekite avviato | http localhost:8080 -> "+config["pagekite_url"])
    return

def getngrokip():
    try:
        r = get("http://127.0.0.1:4040/api/tunnels")
    except:
        raise Exception("Ngrok non avviato, token errato")
    return loads(r.text)["tunnels"][0]["public_url"]

config = getconfig()
ngrok()
kite()
httpserver()


import os
import sys
import json

from viva.utils.config import viva_setup
spark = viva_setup()

from timeit import default_timer as now
from typing import Callable, NamedTuple, Tuple, List, Any, Dict, Type

from pyspark.sql import Window
import pyspark.sql.dataframe as ppd
from pyspark.sql.functions import row_number, col

from viva.sparkmodels import IngestVideo
from viva.nodes.data_nodes import WalkRows
from viva.plans.tasti_plan import Img2VecPlan
from viva.plans.ingest_opt_plan import Plan as IngestPlan

import numpy as np
import pandas as pd
from sklearn.cluster import KMeans
from sklearn.metrics.pairwise import cosine_similarity

#def gen_input_df(frame_limit: int) -> ppd.DataFrame:
def gen_input_df(fraction_to_sample: float) -> ppd.DataFrame:
    videos = ['data/']
    data = WalkRows(videos, ['mp4']).custom_op(None)
    df_i = spark.createDataFrame(data, IngestVideo)

    for node in IngestPlan:
        df_i = node.apply_op(df_i)
        df_i = node.apply_filters(df_i)

    w = Window.partitionBy().orderBy(col("id"))
    df_i = df_i.withColumn("rn",row_number().over(w)).filter(col("rn") % int(1/fraction_to_sample) == 0)    .drop(*["rn"])

    #return df_i.limit(frame_limit)
    return df_i

def gen_indexes(df: ppd.DataFrame, vector_size: int, k_value: int) -> Dict:
    # Any plan will do
    plan = Img2VecPlan.all_plans[0]

    for i,node in enumerate(plan):
        df = node.apply_op(df)

        if i >= 10:
            break

    df = df.drop('uri', 'id', 'width', 'height', 'framebytes')
    col_to_select = ['img2vec'] + [col(f'{n}.label').alias(n) for n in df.schema.names if n != 'img2vec']
    df = df.select(*col_to_select)
    df_vec = df.collect()

    vec_np = np.array([np.frombuffer(bytearray(b.img2vec), dtype=np.float32).reshape(vector_size,) for b in df_vec])

    kmeans = KMeans(init='k-means++', n_clusters=k_value, n_init=10)
    kmeans.fit(vec_np)
    preds = kmeans.predict(vec_np)
    centers = kmeans.cluster_centers_

    # For each cluster, find the furthest point (FPF) and save its index
    # {Key: cluster_index, Value: FPF_index}
    fpf_map = {}
    for i,p in enumerate(preds):
        next_center = centers[p]
        next_vec = vec_np[i,:]
        sim = cosine_similarity(next_center.reshape((1, -1)), next_vec.reshape((1, -1)))[0][0]

        if p not in fpf_map:
            fpf_map[p] = i
        else:
            if fpf_map[p] > sim:
                fpf_map[p] = i

    # For each prediction (center), determine the furthest point from it, and assign that the label (i.e., FPF)
    rep_map = {} # {Key: model, Value: [(cluster_center, label(s)), ...]}
    for n in df.schema.names:
        if n != 'img2vec':
            for k,v in fpf_map.items():
                cluster_center = centers[k]
                label = df_vec[v][n]
                if n not in rep_map:
                    rep_map[n] = []
                rep_map[n].append((cluster_center, label))

    for k,v in rep_map.items():
        print(k)
        for vv in v:
            print('\t', vv[1])

    return rep_map

def run_tasti(output_name: str, fraction_to_sample: float, vector_size: int, k_value: int) -> None:
    df = gen_input_df(fraction_to_sample)
    generated_indexes = gen_indexes(df, vector_size, k_value)

    pandas_df = pd.DataFrame(generated_indexes)
    pd.set_option('display.max_colwidth', None)
    pandas_df.to_pickle(output_name)

if __name__ == '__main__':
    fname = 'long_tasti_index.bin'
    fraction_to_sample = 0.9
    vector_size = 512 # ResNet-18
    k_value = 50
    default_output_name = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../data/', fname)
    run_tasti(output_name=default_output_name, fraction_to_sample=fraction_to_sample, vector_size=vector_size, k_value=k_value)


# -*- coding: utf-8 -*-
"""
Created on Wed May  9 09:35:10 2018

Administrator
"""
import cv2
import numpy as np
a=cv2.imread(r"../image/lenacolor.png")
rows,cols,chn=a.shape
b = np.zeros((rows,cols),dtype=a.dtype)
g = np.zeros((rows,cols),dtype=a.dtype)
r = cv2.split(a) [2]
m = cv2.merge([b,g,r])
cv2.imshow("merge",m)
cv2.waitKey()
cv2.destroyAllWindows()