import pathlib as pl
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import sklearn.metrics
import statsmodels.api as sm
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import Ridge, LinearRegression
from sklearn.decomposition import PCA
from sklearn.pipeline import Pipeline
from sklearn.model_selection import KFold, GridSearchCV
from sklearn.base import BaseEstimator, TransformerMixin

""" ##############################################################
Preprocessing
###############################################################"""
path_to_data = pl.Path(__file__).parent.joinpath('prostate_poprawione.csv')
df = pd.read_csv(path_to_data)
df = df.drop(['id'], axis=1)
df_train = df[df['train'] == 'T'].drop(['train'], axis=1)
X, y = df_train.values[:, :-1], df_train.values[:, -1]
scaler = StandardScaler()
scaler.fit(df_train)
df_train_scaled = scaler.transform(df_train)
X_scaled, y_scaled = df_train_scaled[:, :-1], df_train_scaled[:, -1]

""" ##############################################################
Zheng Log method implementation (for PCR_ord)
###############################################################"""
def compute_sort_keys_from_tvalues(X):
    if_passed = False
    while if_passed is False:
        try:
            X_pca = PCA(n_components=X.shape[1]).fit_transform(X)
        except np.linalg.LinAlgError:
            pass
        else:
            if_passed = True

    tvalues = sm.OLS(y_scaled, sm.add_constant(X_pca)).fit().tvalues[1:]
    tvalues_square = tvalues * tvalues
    return np.argsort(-tvalues_square)

SORT_KEYS = compute_sort_keys_from_tvalues(X_scaled)

class ZhengLohTransformer(BaseEstimator, TransformerMixin):
    def __init__(self, n_components=None):
        self.n_components = n_components

    def fit(self, X, y=None):
        return self

    def transform(self, X):
        global SORT_KEYS
        return X[:, SORT_KEYS][:, 0:self.n_components]

""" ##############################################################
Cross validation
###############################################################"""
def perform_cv(model, param_name, param_range, X=X_scaled):
    cv = KFold(n_splits=10)
    scorer = sklearn.metrics.make_scorer(sklearn.metrics.mean_squared_error)
    grid = GridSearchCV(model, scoring=scorer, param_grid={param_name: param_range}, cv=cv)
    grid.fit(X, y_scaled)
    results = pd.DataFrame.from_dict(grid.cv_results_)[['param_' + param_name, 'mean_test_score', 'std_test_score']]
    results['std_test_score'] = results['std_test_score'] / np.sqrt(results.shape[0])
    return results

results_RR = perform_cv(
    Ridge(), 'alpha', [0.1, 4.3, 11.2, 22.8, 36.6, 74.4, 151.1, 388.8, 1000.0]
)
results_PCR = perform_cv(
    Pipeline(steps=[('pca', PCA()), ('linreg', LinearRegression())]), 'pca__n_components', [1, 2, 3, 4, 5, 6, 7, 8]
)
results_PCRord = perform_cv(
    Pipeline(steps=[('zlt', ZhengLohTransformer()), ('linreg', LinearRegression())]), 'zlt__n_components',
    [1, 2, 3, 4, 5, 6, 7, 8], X=PCA(n_components=8).fit_transform(X_scaled)
)

""" ##############################################################
Converting  "alpha" from RR to the "degrees of freedom".
###############################################################"""
def calculate_degrees_freedom(X, alpha):
    """ function which converts  "alpha" from RR to "degrees of freedom"""
    inner_factor = np.linalg.inv(X.T.__matmul__(X) + alpha * np.identity(X.shape[1]))
    hat_matrix = (X.__matmul__(inner_factor)).__matmul__(X.T)
    return np.trace(hat_matrix)

results_RR['param_alpha'] = results_RR['param_alpha'].apply(lambda x: calculate_degrees_freedom(X_scaled, x))
results_RR = results_RR.rename(columns={'param_alpha': 'param_df'})
results_RR = results_RR.sort_values(by=['param_df'])
results_RR = results_RR.reset_index(drop=True)

""" ##############################################################
Plotting
###############################################################"""
def plot_results(ax, results, param_name, title):
    """ function which creates plot for single model"""
    ax.plot(results[param_name], results['mean_test_score'])
    ax.scatter(results[param_name], results['mean_test_score'])
    ax.errorbar(results[param_name], results['mean_test_score'], results['std_test_score'])
    one_error_above_min = results['mean_test_score'].min() + results['std_test_score'][results['mean_test_score'].argmin()]
    one_error_below_min = results['mean_test_score'].min() - results['std_test_score'][results['mean_test_score'].argmin()]
    ax.hlines(one_error_above_min, 0, 8, linestyles='dashed')
    ax.hlines(one_error_below_min, 0, 8, linestyles='dashed')
    ax.set_xticks(np.linspace(0, 8, 9))
    ax.set_yticks(np.linspace(0, 1.6, 9))
    ax.set_xlabel(param_name)
    ax.set_ylabel('mean squared error')
    ax.set_title(title)

fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2)
plot_results(ax1, results_RR, 'param_df', 'ridge')
plot_results(ax2, results_PCR, 'param_pca__n_components', 'pcr')
plot_results(ax3, results_PCRord, 'param_zlt__n_components', 'pcr_ord')


#!/usr/bin/python3

from evdev import uinput, ecodes as e
from time import sleep
from random import random
import sys

def scaleTime(time):
	return time * (random() * 2)

try:
	logName = sys.argv[1]
except IndexError:
	print("Please pass in a log file to use.")
	sys.exit(1)

log = []
with open(logName) as f:
	for line in f.read().split('\n'):
		if line == '': continue
		time, key, event = line.split(' ')
		log.append({"time": float(time), "key": int(key), "event": int(event)})

sleep(5)

for event in log:
	sleep(scaleTime(event["time"]))
	with uinput.UInput() as ui:
		print(event)
		ui.write(e.EV_KEY, event["key"], event["event"])
		ui.syn()


import re
import time
from collections import defaultdict

def main():
    guards = defaultdict(list)

    current_guard = None
    sleep_start = None
    for action, value in get_input():
        if action == 'guard':
            current_guard = value
        if action == 'sleep':
            sleep_start = value
        if action == 'wake':
            guards[current_guard].append([sleep_start, value])

    minutes = max([[count_minutes(times), guard] for guard, times in guards.items()])
    print(minutes)
    return minutes[0][1] * minutes[1]

def count_minutes(times):
    minutes = [0] * 60

    for start, end in times:
        for i in range(end - start):
            minutes[start + i] += 1
    minute, count = max(enumerate(minutes), key=lambda x: x[1])
    return count, minute

def get_input():
    re_guard = re.compile(r' #(\d+)')
    re_minute = re.compile(r':(\d+)')

    f = open('input/day_4.1.txt')
    log = sorted(f.readlines())

    for row in log:
        if 'Guard' in row:
            yield 'guard', int(re_guard.search(row).groups()[0])
        if 'falls asleep' in row:
            yield 'sleep', int(re_minute.search(row).groups()[0])
        if 'wakes up' in row:
            yield 'wake', int(re_minute.search(row).groups()[0])

if __name__ == '__main__':
    start = time.monotonic()
    print(main())
    time_spent = time.monotonic() - start
    print(f'took: {time_spent:.3f} s')


import fnmatch
import argparse
import os
import mc2xml
import visa2xml
import logging
from matching_db import *

__version_info__ = ('2018','09','27')
__version__ = '-'.join(__version_info__)

def _parse_arguments():
    """
    argparse parser

    :return: parsed arguments
    """

    parser = argparse.ArgumentParser()
    parser.add_argument("filenames", action='append', nargs='+', help="Enter filenames")
    parser.add_argument('-V', '--version', action='version', version='%(prog)s ' + __version__)
    parser.add_argument('-u', dest='unmask', help='Unmask PAN', action='store_false')
    parser.add_argument('-c', dest='comment', help='Comment to add in result XML file')
    parser.add_argument('-l', dest='logging', action='store_true', help='Enable full logging mode')
    parser.add_argument('-v', dest='visa', action='store_true', help='parse as VISA')
    parser.add_argument('-m', dest='mc', action='store_true', help='parse as MasterCard')
    parser.add_argument('-t', dest='test', choices=['prime_acq_tst', 'prime_acq_openstack', 'greenplum'],
                        help='Check ARN by Database')
    parser.add_argument('-o', dest='oracle', default="C:\\app\\instantclient_12_2",
                        help='Oracle instant client datapath')

    return parser

def open_in_mps(filename, unmask, mes_comment, visa, mc):
    """
    Jacket will try to parse file in all parsers
    :param filename: import filename
    :param unmask: bool value unmasking PAN
    :param mes_comment: comment to add to file
    :param visa: parse file only as VISA
    :param mc: parse file only as MC

    """

    mc2xml.comment = mes_comment
    mc2xml.unmask = unmask

    visa2xml.unmask = unmask
    visa2xml.comment = mes_comment

    if visa:
        visa2xml.exctract_visa(filename)
    elif mc:
        mc2xml.extract_mc(filename)
    else:
        try:
            mc2xml.extract_mc(filename)
        except KeyError:
            visa2xml.exctract_visa(filename)

def isASCII(data):
    try:
        data[:200].decode('ASCII')
    except UnicodeDecodeError as exc:
        return False
    else:
        return True

def _main_mps2xml(args):
    """
    mc2xml main processing
    :param args: argparse arguments
    :return: exit code
    """

    if hasattr(args, 'unmask'):
        unmask = args.unmask
    else:
        unmask = True

    mc2xml.unmask = unmask
    visa2xml.unmask = unmask

    if hasattr(args, 'comment'):
        comment = args.comment
        mc2xml.comment = comment
        visa2xml.comment = comment

    if hasattr(args, 'visa'):
        is_visa = True

    if hasattr(args, 'mc'):
        is_visa = False

    if hasattr(args, 'test'):
        test = args.test

    if hasattr(args, 'oracle'):
        oracle = args.oracle

    if hasattr(args, 'logging'):
        if args.logging:
            logging.basicConfig(filename=os.path.join("mps2xml.log"), level=logging.DEBUG,
                                format='%(asctime)s - %(message)s')

    # exit if input file does not exist
    if not hasattr(args, 'filenames'):
        print("Input file not found")
        exit(8)

    filenames = args.filenames[0]

    for file in filenames:
        # Search files by mask
        if '*' in file:
            dir = os.path.dirname(os.path.abspath(file))
            onlyfiles = [f for f in os.listdir(dir) if os.path.isfile(os.path.join(dir, f))]
            for cfile in onlyfiles:
                if fnmatch.fnmatch(cfile, file):
                    print("Processed file {}".format(cfile))
                    process_file(file, is_visa, test, oracle)

        # Processed current file
        else:
            if os.path.isfile(file):
                print("Processed file {}".format(file))
                try:
                    process_file(file, is_visa, test, oracle)
                except Exception as exc:
                    print (str(exc))
            else:
                print("File {} not found!".format(file))

def process_file(filename, is_visa, testDB, oracle):
    # Get input data
    # Read input file
    with open(filename, 'rb') as infile:
        input_file = infile.read()
    if isASCII(input_file) or is_visa or input_file[:2] == '90':
        #context = [line for line in input_file.decode('ascii').split('\n') if line]
        try:
            return_xml = visa2xml.process_visa(filename)
            is_visa = True
        except Exception:
            return_xml = mc2xml.process_mc(input_file)
            is_visa = False
    else:
        try:
            return_xml = mc2xml.process_mc(input_file)
            is_visa = False
        except KeyError:
            context = [line for line in input_file.decode('ascii').split('\n') if line]
            return_xml = visa2xml.process_visa(context)
            is_visa = True

    if testDB:
        rough_string = ElementTree.tostring(return_xml, 'utf-8')
        if testDB == 'greenplum':
            return_xml = checkXMLbyDWH(rough_string, is_visa)
        else:
            return_xml = checkXMLbyOracle(testDB, rough_string, oracle, is_visa)

    # Make XML pretty
    xmlstr = prettify(return_xml)

    # Write to file
    output_filename = filename + ".xml"
    try:
        with open(output_filename, "w") as f:
            f.write(xmlstr)
        print("Successfully write to file: {}".format(output_filename))
    except Exception as exc:
        print("Cannot write to XML file: {} {}".format(output_filename, exc))

if __name__ == '__main__':
    _main_mps2xml(_parse_arguments().parse_args())


from pynput.keyboard import Key, Listener
import time
from threading import Thread

class Key_listener():

    def __init__(self):
        self.queue = [2,1,1,1,1,1,1,1]
        self.itime = time.time()
        self.scanned = False

        start_thread = Thread(target=self.start)
        start_thread.start()

    def on_press(self,key):
        try:
            if key.char in '0123456789':
                # print('key char',key.char)
                self.queue.append(time.time() - self.itime)
                self.queue.pop(0)
                # print(self.queue)
                self.itime = time.time()
            if self.queue[1] < .005:
                print("scan detected")
                self.queue = [2,1,1,1,1,1,1,1]
                self.scanned = True
        except:
            pass
    def start(self):
        with Listener(on_press=self.on_press) as listener:
            listener.join()

    def reset(self):
        self.scanned = False

def main():
    kl = Key_listener()
    # kl.start()
    while True:

        print(kl.scanned)

        if kl.scanned:
            kl.reset()
        time.sleep(1)

if __name__ == '__main__':
    main()


import logging

from django.contrib.auth.backends import RemoteUserBackend
from django.contrib.auth.models import Group

from eyewearstore.customauth.models import AccessToken

logger = logging.getLogger(__name__)

class AccessTokenBackend(RemoteUserBackend):
    """
    Authenticates against auth token
    """
    create_unknown_user = True

    def authenticate(self, access_token=None, **kwargs):
        if not access_token:
            return

        token = AccessToken.find_valid(value=access_token)
        if not token:
            logger.warn('unknown token %s', access_token)
            return

        logger.info("found token %s", token)

        # token is valid once
        email = token.email
        token.expire()

        return super(AccessTokenBackend, self).authenticate(remote_user=email)

    def configure_user(self, user):
        # manager by defaults
        group = Group.objects.get(name='Manager')
        group.user_set.add(user)

        # has /admin access
        user.is_staff = True
        # save email
        user.email = user.username
        user.save()

        return user


from __future__ import division, print_function
import os
import sys
import glob

import healpy as hp
import numpy as np
import matplotlib.pyplot as plt
from scipy import interpolate
import copy

import Tools as tl
import ReadMC as rmc

import qubic
from qubic import gal2equ, equ2gal
from qubic import Xpol
from qubic import apodize_mask

thespec = ['TT', 'EE', 'BB', 'TE', 'EB', 'TB']
stokes = ['I', 'Q', 'U']

#Coordinates of the zone observed in the sky
center = equ2gal(0., -57.)

#============= Get the simulation files ==================
#Simulation repository
rep_simu = '/home/louisemousset/QUBIC/Qubic_work/SpectroImagerie/SimuJCResults/Duration20'

#Number of subbands used during the simulation
nsubvals = np.array([1,2,3,4])

#Archetypes of the files .fits you want to work on
arch_conv, arch_recon = [], []
for isub in range(len(nsubvals)):
	arch_conv.append('mpiQ_Nodes_2_Ptg_40000_Noutmax_6_Tol_1e-4_*_nf{}_maps_convolved.fits'.format(nsubvals[isub]))
	arch_recon.append('mpiQ_Nodes_2_Ptg_40000_Noutmax_6_Tol_1e-4_*_nf{}_maps_recon.fits'.format(nsubvals[isub]))

#Get all maps
allmaps_conv, seenmap_conv = rmc.get_all_maps(rep_simu, arch_conv, nsubvals)
allmaps_recon, seenmap_recon = rmc.get_all_maps(rep_simu, arch_recon, nsubvals)

#Number of pixels and nside
npix = len(seenmap_recon)
ns = int(np.sqrt(npix/12))

# Angle associated to each pixel
ang = tl.pix2ang(ns, center, seenmap_recon)
print(ang.shape)
plt.plot(np.sort(ang))
plt.show()

#================= Noise Evolution as a function of the subband number=======================
#To do that, you need many realisations

allmeanmat = rmc.get_rms_covar(nsubvals, seenmap_recon, allmaps_recon)[1]
rmsmap_cov = rmc.get_rms_covarmean(nsubvals, seenmap_recon, allmaps_recon, allmeanmat)[1]
mean_rms_cov = np.sqrt(np.mean(rmsmap_cov**2, axis=2))

plt.plot(nsubvals, np.sqrt(nsubvals), 'k', label='Optimal $\sqrt{N}$', lw=2)
for i in range(3):
    plt.plot(nsubvals, mean_rms_cov[:,i] / mean_rms_cov[0,i] * np.sqrt(nsubvals), label=stokes[i], lw=2, ls='--')
plt.xlabel('Number of sub-frequencies')
plt.ylabel('Relative maps RMS')
plt.legend()

#===================== Residus estimation ===================

#Two ways
residus = []
for j in range(len(allmaps_conv)):
	residus.append(allmaps_recon[j] - allmaps_conv[j])

residus = []
for j in range(len(allmaps_conv)):
	residus.append(allmaps_recon[j] - np.mean(allmaps_recon[j], axis=0))

#Histogram of the residus
plt.clf()
for i in range(3):
	plt.subplot(1, 3, i+1)
	plt.hist(np.ravel(residus[0][:,3,:,i]), range=[-20,20], bins=100)
	plt.title(stokes[i])

#================ Look at the maps =================
isub = 0
real = 0
freq = 0

maps_conv = np.zeros((12*ns**2, 3))
maps_conv[seenmap_recon, :] = allmaps_conv[isub][real,freq,:,:]

maps_recon = np.zeros((12*ns**2, 3))
maps_recon[seenmap_conv, :] = allmaps_recon[isub][real,freq,:,:]

maps_residus = np.zeros((12*ns**2, 3))
maps_residus[seenmap_conv, :] = residus[isub][real,freq,:,:]

#hp.mollview(maps_conv[:,1], title='maps_conv')

plt.figure('maps')
for i in range(3):
	if i==0:
		min=None
		max=None
	else:
		min=None
		max=None
	hp.gnomview(maps_conv[:,i], rot=center, reso=9, sub=(3,3,i+1), title='conv '+stokes[i], min=min, max=max)
	hp.gnomview(maps_recon[:,i], rot=center, reso=9, sub=(3,3,3+i+1), title='recon '+stokes[i], min=min, max=max)
	hp.gnomview(maps_residus[:,i], rot=center, reso=9, sub=(3,3,6+i+1), title='residus '+stokes[i], min=min, max=max)
plt.show()

#==================== Make fake reconstructed maps ======================

######## Old version : Bruit estime pour une seule sous-bande, considere comme la reference

#Mean and std on each pixel, on moyenne sur les realisations
pixmean = np.mean(residus[0][:,0,:,:], axis=0)
pixstd = np.std(residus[0][:,0,:,:], axis=0)

#On reconstruit une carte complete
mapsmean = np.zeros((12*ns**2, 3))
mapsstd = np.zeros((12*ns**2, 3))
mapsmean[seenmap_recon, :] = pixmean
mapsstd[seenmap_recon, :] = pixstd

hp.mollview(mapsstd[:,1], title='std')
hp.mollview(mapsmean[:,1], title='mean')

plt.figure('Mean and Std for 1 subband for I Q U maps')
for i in range(3):
	hp.gnomview(mapsmean[:,i], rot=center, reso=12, sub=(2,3,i+4), title='mean'+stokes[i])
	hp.gnomview(mapsstd[:,i], rot=center, reso=12, sub=(2,3,i+1), title='std'+stokes[i])

#Fake map using the noise distribution calculated above
#Correct by a factor sqrt(N)
fake_mapsrecon = []
for j in range((len(nsubvals))):
	print('For nsub = {}'.format(nsubvals[i]))
	sh = allmaps_recon[j].shape
	newnoise = np.zeros(sh)
	for k in range(sh[0]):
		for l in range(sh[1]):
			noise = np.random.randn(len(pixmean),3) * pixstd * np.sqrt(j+1) + pixmean
			newnoise[k,l,:,:] = noise
	fake_mapsrecon.append(allmaps_conv[j] + newnoise)

#Correct by the real evolution, not exactly sqrt(N)
fake_mapsrecon = []
for isub in range((len(nsubvals))):
	print('For nsub = {}'.format(nsubvals[i]))
	sh = allmaps_recon[isub].shape
	correction = mean_rms_cov[isub,:] / mean_rms_cov[0,:] * np.sqrt(isub+1)
	newnoise = np.zeros(sh)
	for k in range(sh[0]):
		for l in range(sh[1]):
			noise = np.random.randn(sh[2], sh[3]) * pixstd * correction + pixmean
			newnoise[k,l,:,:] = noise
			if (isub,k,l) == (2,0,0):
				print(correction, np.sqrt(isub+1))
				print(noise.shape, newnoise.shape)
	fake_mapsrecon.append(allmaps_conv[isub] + newnoise)

######## New version : Correlations du bruit entre I Q U et les sous-bandes prises en compte
#On plot le std des residus en fonction de l'angle pour les differentes cartes
#Pour se debarrasser de cette dependence, on fabrique des residus_correct en divisant par std_profile
residus_correct = []
plt.clf()
for isub in range(len(nsubvals)):
	sh = residus[isub].shape
	nreals = sh[0]
	nsub = sh[1]
	residu_new = np.zeros(sh)
	bin_centers, std_bin, allstd_profile = tl.myprofile(ang, residus[isub], nbins=15)
	for l in range(nsub):
		for i in range(3):
			plt.subplot(4, 3, 3*isub+i+1)
			p = plt.plot(bin_centers, std_bin[:,l,i], 'o', label='subband'+str(l))
			plt.plot(ang, allstd_profile[3*l+i], ',', color=p[0].get_color())
			#plot(ang, pixstd[:,i], ',', color=p[0].get_color())
			if isub == 0: plt.title(stokes[i])
			plt.xlabel('angle (deg)')
			if i == 0 and isub == 3:
				plt.legend()

			#Residus corriges
			residu_new[:,l,:,i] = residus[isub][:,l,:,i] / allstd_profile[3*l+i]

			if i == 0: print(len(allstd_profile), allstd_profile[3*l+i].shape)
	residus_correct.append(residu_new)

#On verifie que les residus_correct ont un std autour de 1 qui n'evolue plus avec l'angle
np.std(residus_correct[3][:,0,:,:], axis=0)

plt.figure()
for isub in range(len(nsubvals)):
	sh = residus_correct[isub].shape
	nreals = sh[0]
	nsub = sh[1]

	bin_centers, std_bin, allstd_profile = tl.myprofile(ang, residus_correct[isub], nbins=15)
	for l in range(nsub):
		for i in range(3):
			plt.subplot(4, 3, 3*isub +1 +i)
			p = plt.plot(bin_centers, std_bin[:,l,i], 'o', label='subband'+str(l+1))
			print(np.round(np.mean(std_bin[:, l, i], axis=0),5))
			if isub == 0: plt.title(stokes[i])
			plt.xlabel('angle (deg)')
			if i == 0 and isub == 3:
				plt.legend()

#Covariance matrices and mean of the residus_correct
allmean, allcov = tl.covariance_IQU_subbands(residus_correct)
allmean, allcov = tl.covariance_IQU_subbands(residus)

#test de la fonction covariance_IQU_subbands
toto = []
for isub in range(len(nsubvals)):
	toto.append(np.random.randn(10, isub+1, 100014, 3))
m, c = tl.covariance_IQU_subbands(toto)

#Plot of the correlation or covariance matrices
plt.clf()
for isub in range(len(nsubvals)):
	plt.subplot(1, len(nsubvals), isub + 1)
	plt.imshow(tl.cov2corr(allcov[isub]), interpolation='nearest', vmin=-1, vmax=1)
	#plt.imshow(allcov[isub], interpolation='nearest')
	if isub == 0: plt.colorbar()
	plt.title(str(isub))

#Fake map using the noise distribution of the covariance matrix
#On remultiplie par le std_profile pour avoir la dependance du std en fonction de l'angle

residus_fake = []
for isub in range((len(nsubvals))):
	print('For nsub = {}'.format(nsubvals[isub]))
	sh = allmaps_recon[isub].shape
	nreals = sh[0]
	nsub = sh[1]
	npixok = sh[2]

	allstd_profile = tl.myprofile(ang, residus[isub], nbins=15)[2]

	noise4D = np.zeros(sh)

	for k in range(nreals):
		noise = np.random.multivariate_normal(allmean[isub], allcov[isub], size=npixok) #bruit pour la realisation k
		if k == 0: print(noise.shape)
		for l in range(nsub):
			noise4D[k, l, :, :] = noise[:, 3*l:3*l+3]
			for i in range(3):
				noise4D[k, l, :, i] *= allstd_profile[3*l+i]
	residus_fake.append(noise4D)

############# TEST #################

#Histogramme des residus_fake
plt.clf()
for i in range(3):
	plt.subplot(1, 3, i+1)
	plt.hist(np.ravel(residus_fake[0][:,0,:,i]), range=[-2,2], bins=100)
	plt.title(stokes[i])

#Matrice de cov des residus_fake corriges
residus_correct_fake = []
plt.clf()
for isub in range(len(nsubvals)):
	sh = residus_fake[isub].shape
	nreals = sh[0]
	nsub = sh[1]

	bin_centers, std_bin, allstd_profile = tl.myprofile(ang, residus_fake[isub], nbins=15)

	residu_new = np.zeros(sh)
	for l in range(nsub):
		for i in range(3):
			#Std profile en fonction de l'angle
			plt.subplot(4, 3, 3*isub+i+1)
			p = plt.plot(bin_centers, std_bin[:,l,i], 'o', label='subband'+str(l))
			plt.plot(ang, allstd_profile[3*l+i], ',', color=p[0].get_color())
			#plt.plot(ang, pixstd[:,i], ',', color=p[0].get_color())
			if isub == 0: plt.title(stokes[i])
			plt.xlabel('angle (deg)')
			if i == 0 and isub == 3:
				plt.legend()

			#Residus corriges
			residu_new[:,l,:,i] = residus_fake[isub][:,l,:,i] / allstd_profile[3*l+i]
	residus_correct_fake.append(residu_new)

allmean_fake, allcov_fake = tl.covariance_IQU_subbands(residus_correct_fake)

#Comparaison entre les matrices de covariance des residus_correct et des residus_correct_fake
alldiff = []
for isub in range(len(nsubvals)):
	alldiff.append(allcov_fake[isub] - allcov[isub])

#Plot of the correlation matrices
plt.clf()
for isub in range(len(nsubvals)):
	plt.subplot(1, len(nsubvals), isub + 1)
	#imshow(tl.cov2corr(alldiff[isub]), interpolation='nearest', vmin=-0.1, vmax=0.1)
	plt.imshow(alldiff[isub], interpolation='nearest')
	if isub == 0: plt.colorbar()
	plt.title(str(isub))
########################## FIN DU TEST ################################

#Fausse carte
fake_mapsrecon = []
for isub in range(len(nsubvals)):
	#avec les map conv
	fake_mapsrecon.append(allmaps_conv[isub] + residus_fake[isub])
	#que du bruit
	#fake_mapsrecon.append(residus_fake[isub])
	#avec les map recon
	#fake_mapsrecon.append(np.mean(allmaps_recon[isub], axis=0) + residus_fake[isub])

####### Look at one fake map
mapsfake = np.zeros((12*ns**2, 3))
mapsfake[seenmap_conv, :] = fake_mapsrecon[0][0,0,:,:]

hp.mollview(mapsfake[:,0], title='Fake map recon')

plt.figure('Fake map recon')
plt.clf()
for i in range(3):
	hp.gnomview(mapsfake[:,i], rot=center, reso=12, sub=(1,3,i+1), title=stokes[i])

#======================= Apply Xpoll on fake maps ============================
lmin = 20
lmax = 2 * ns
delta_ell = 20

#Xpoll needs a mask
mymask = apodize_mask(seenmap_conv, 5)
xpol = Xpol(mymask, lmin, lmax, delta_ell)
ell_binned = xpol.ell_binned
nbins = len(ell_binned)
print('nbins = {}'.format(nbins))

mcls, mcls_in = [], []
scls, scls_in = [], []

#Input, ce qu'on doit trouver
mapsconv = np.zeros((12*ns**2, 3))

#Output, ce qu'on trouve
mapsfake = np.zeros((12*ns**2, 3))

for isub in range(len(nsubvals)):
	sh = allmaps_conv[isub].shape
	nreals = sh[0]
	nsub = sh[1]
	cells = np.zeros((6, nbins, nsub, nreals))
	cells_in = np.zeros((6, nbins, nsub, nreals))
	print(cells.shape)
	for real in range(nreals):
		for n in range(nsub):
			for i in range(3):
				mapsconv[seenmap_conv, i] = allmaps_conv[isub][real,n,:,i] * mymask[seenmap_conv]

				mapsfake[seenmap_conv, i] = fake_mapsrecon[isub][real,n,:,i]* mymask[seenmap_recon]

			cells_in[:, :, n , real] = xpol.get_spectra(mapsconv)[1]

			cells[:, :, n, real] = xpol.get_spectra(mapsfake)[1]

	mcls.append(np.mean(cells, axis = 3))
	mcls_in.append(np.mean(cells_in, axis = 3))
	scls.append(np.std(cells, axis = 3))
	scls_in.append(np.std(cells_in, axis = 3))


#!/usr/bin/env python

import cv2
import sys
import copy
import rospy
from std_msgs.msg import String
import depthai
import platform
import signal
import subprocess
import json
import datetime

from calibration.srv import Capture
import time
import numpy as np
import os
from pathlib import Path
import shutil
import consts.resource_paths
from cv_bridge import CvBridge
from sensor_msgs.msg import Image
import shutil
from depthai_helpers.calibration_utils import *

from depthai_helpers.pygame_checkbox import Checkbox, pygame_render_text
import pygame
from pygame.locals import *

from depthai_helpers import utils
os.environ['SDL_VIDEO_WINDOW_POS'] = '100,50'

on_embedded = platform.machine().startswith('arm') or platform.machine().startswith('aarch64')

def find_chessboard(frame):
    chessboard_flags = cv2.CALIB_CB_ADAPTIVE_THRESH + cv2.CALIB_CB_FAST_CHECK + cv2.CALIB_CB_NORMALIZE_IMAGE
    small_frame = cv2.resize(frame, (0, 0), fx=0.3, fy=0.3)
    return cv2.findChessboardCorners(small_frame, (9, 6), chessboard_flags)[0] and \
           cv2.findChessboardCorners(frame, (9, 6), chessboard_flags)[0]

white  = [255, 255, 255]
orange = [143, 122, 4]
red    = [230, 9, 9]
green  = [4, 143, 7]
black  = [0, 0, 0]
pygame.init()

class depthai_calibration_node:
    def __init__(self, depthai_args):
        self.package_path = depthai_args['package_path']
        self.args = depthai_args
        self.bridge = CvBridge()
        self.is_service_active = False
        self.config = {
            'streams':
                ['left', 'right', 'meta_d2h', 'color'] if not on_embedded else
                [{'name': 'left', "max_fps": 30.0}, {'name': 'right', "max_fps": 30.0}, {'name': 'meta_d2h', "max_fps": 30.0}, {'name': 'color', "max_fps": 30.0}],
            'depth':
                {
                    'calibration_file': consts.resource_paths.calib_fpath,
                    'padding_factor': 0.3
                },
            'ai':
                {
                    'blob_file': consts.resource_paths.blob_fpath,
                    'blob_file_config': consts.resource_paths.blob_config_fpath,
                    'shaves' : 7,
                    'cmx_slices' : 7,
                    'NN_engines' : 1,
                },
            'board_config':
                {
                    'swap_left_and_right_cameras': self.args['swap_lr'],
                    'left_fov_deg':  self.args['field_of_view'],
                    'left_to_right_distance_cm': self.args['baseline'],
                    'override_eeprom': False,
                    'stereo_center_crop': True,
                },
            'camera':
                {
                    'mono':
                    {
                        # 1280x720, 1280x800, 640x400 (binning enabled)
                        'resolution_h': 800,
                        'fps': 30.0,
                    },
                },
        }

        if arg['board']:
            board_path = Path(arg['board'])
            if not board_path.exists():
                board_path = Path(consts.resource_paths.boards_dir_path) / Path(arg['board'].upper()).with_suffix('.json')
                print(board_path)
                if not board_path.exists():
                    raise ValueError('Board config not found: {}'.format(board_path))
            with open(board_path) as fp:
                self.board_config = json.load(fp)
        utils.merge(self.board_config, self.config)

        self.aruco_dictionary = cv2.aruco.Dictionary_get(cv2.aruco.DICT_4X4_1000)
        # self.charuco_board = aruco.CharucoBoard_create(
        #                                     22,16,
        #                                     self.args['square_size_cm'],
        #                                     self.args['marker_size_cm'],
        #                                     self.aruco_dictionary)
        self.start_device()

        self.disp = pygame.display
        self.screen = self.disp.set_mode((800, 600))
        self.screen.fill(white)
        self.disp.set_caption("Calibration - Device check ")
        title = "Device Status"
        pygame_render_text(self.screen, title, (350,20), orange, 50)
        self.auto_checkbox_names = ["USB3", "Left camera connected", "Right camera connected",
                                    "Left Stream", "Right Stream"]
        if self.args['enable_IMU_test']:
            self.auto_checkbox_names.append("IMU connected")

        y = 110
        x = 200
        self.start_disp = False
        font = pygame.font.Font(None, 20)
        self.auto_checkbox_dict = {}
        for i in range(len(self.auto_checkbox_names)):
            w, h = font.size(self.auto_checkbox_names[i])
            x_axis = x - w
            y_axis = y +  (40*i)
            font_surf = font.render(self.auto_checkbox_names[i], True, green)
            self.screen.blit(font_surf, (x_axis,y_axis))
            self.auto_checkbox_dict[self.auto_checkbox_names[i]] = Checkbox(self.screen, x + 10, y_axis-5, outline_color=green,
                                                        check_color=green, check=False)

        # text = 'call rosservice of device_status_handler to update the device status'
        for i in range(len(self.auto_checkbox_names)):
            self.auto_checkbox_dict[self.auto_checkbox_names[i]].render_checkbox()
        pygame.draw.rect(self.screen, red, no_button)
        pygame_render_text(self.screen, 'Exit', (500, 505))
        self.no_active = False
        self.click = False
        # self.disp.update()
        ## creating services and publishers at the end to avoid calls before initialization
        self.capture_srv = rospy.Service(self.args["capture_service_name"], Capture, self.capture_servive_handler)
        self.calib_srv = rospy.Service(self.args["calibration_service_name"], Capture, self.calibration_servive_handler)
        self.dev_status_srv = rospy.Service("device_status", Capture, self.device_status_handler)
        self.image_pub_left = rospy.Publisher("left",Image, queue_size=10)
        self.image_pub_right = rospy.Publisher("right",Image, queue_size=10)

    def capture_exit(self):
        is_clicked = False
        for event in pygame.event.get():
            # self.disp.update()
            if event.type == pygame.MOUSEMOTION:
                x, y = event.pos
                px, py, w, h = no_button
                if px < x < px + w and py < y < py + h:
                    self.no_active = True
                    pygame.draw.rect(self.screen, orange, no_button)
                    pygame_render_text(self.screen, 'Exit', (500, 505))
                else:
                    self.no_active = False
                    pygame.draw.rect(self.screen, red, no_button)
                    pygame_render_text(self.screen, 'Exit', (500, 505))

            if event.type == pygame.MOUSEBUTTONDOWN and self.no_active:
                print("setting click")
                self.click = True
            if event.type == pygame.MOUSEBUTTONUP:
                if self.no_active and self.click:
                    print("No clicked")
                    is_clicked = True
                    # is_device_ready = False
                    # if arm_control is not None:
                        # arm_control.go_to_next_joint_state(joints_goal_dict['sleep'])
                    break
        # pygame.event.pump()
        return is_clicked

    def start_device(self):
        self.device = depthai.Device('', False)
        # self.device = depthai.Device('/home/nuc/Desktop/depthai/.fw_cache/depthai-6fc8c54e33b8aa6d16bf70ac5193d10090dcd0d8.cmd', '')
        self.pipeline = self.device.create_pipeline(self.config)
        self.mx_id = self.device.get_mx_id()

    def publisher(self):
        while not rospy.is_shutdown():
            if self.capture_exit():
                print("signaling...")
                rospy.signal_shutdown("Finished calibration")
            if self.start_disp:
                self.disp.update()
            # else:
            #     pygameX.event.pump()
            # print("updating dis-----")
            if not self.is_service_active:
                # pygame.draw.rect(self.screen, red, no_button)
                # pygame_render_text(self.screen, 'Exit', (500, 505))
                # print("SERVICE NOT ACTIVE")
                if not hasattr(self, "pipeline"):
                    self.start_device()
                    # print("restarting device---->")
                _, data_list = self.pipeline.get_available_nnet_and_data_packets(True)
                for packet in data_list:
                    # print("found packets:")
                    # print(packet.stream_name)
                    if packet.stream_name == "left":
                        recent_left = packet.getData()
                        # print(recent_left.shape)
                        self.image_pub_left.publish(self.bridge.cv2_to_imgmsg(recent_left, "passthrough"))
                    elif packet.stream_name == "right":
                        recent_right = packet.getData()
                        self.image_pub_right.publish(self.bridge.cv2_to_imgmsg(recent_right, "passthrough"))
                    elif packet.stream_name == "meta_d2h":
                        str_ = packet.getDataAsStr()

    def parse_frame(self, frame, stream_name, file_name):
        file_name += '.png'
        # filename = image_filename(stream_name, self.current_polygon, self.images_captured)
        # print(self.package_path + "/dataset/{}/{}".format(stream_name, file_name))
        ds_path = self.package_path + "/dataset/{}".format(stream_name)
        # print(ds_path)
        if not os.path.exists(ds_path):
            os.makedirs(ds_path)

        cv2.imwrite(self.package_path + "/dataset/{}/{}".format(stream_name, file_name), frame)
        print("py: Saved image as: " + str(file_name))
        return True

    def is_markers_found(self, frame):
        # print(frame.shape)
        # gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        marker_corners, _, _ = cv2.aruco.detectMarkers(frame, self.aruco_dictionary)
        return not (len(marker_corners) == 0)

    def capture_servive_handler(self, req):
        print("Capture image Service Started")
        recent_left = None
        recent_right = None
        finished = False
        self.is_service_active = True
        # now = rospy.get_rostime()
        # pygame.draw.rect(self.screen, white, no_button)
        while not finished:
            _, data_list = self.pipeline.get_available_nnet_and_data_packets()
            # print(len(data_list))

            for packet in data_list:
                # print(packet.stream_name)
                # print("packet time")
                # print(packet.getMetadata().getTimestamp())
                # print("ros time")
                # print(now.secs)
                if packet.stream_name == "left":
                    recent_left = packet.getData()
                elif packet.stream_name == "right":
                    recent_right = packet.getData()

            if recent_left is not None and recent_right is not None:
                finished = True

        # print("looping")
        # is_board_found_l = find_chessboard(recent_left)
        # is_board_found_r = find_chessboard(recent_right)
        # is_board_found_l = True
        # is_board_found_r = True
        is_board_found_l = self.is_markers_found(recent_left)
        is_board_found_r = self.is_markers_found(recent_right)
        if is_board_found_l and is_board_found_r:
            print("Found------------------------->")
            self.parse_frame(recent_left, "left", req.name)
            self.parse_frame(recent_right, "right", req.name)
        else:
            print("Not found--------------------->")
            self.is_service_active = False
            self.parse_frame(recent_left, "left_not", req.name)
            self.parse_frame(recent_right, "right_not", req.name)
            return (False, "Calibration board not found")
        # elif is_board_found_l and not is_board_found_r: ## TODO: Add errors after srv is built
        print("Service ending")
        self.is_service_active = False
        return (True, "No Error")

    def device_status_handler(self, req):
        self.is_service_active = True
        self.start_disp = True
        # pygame.draw.rect(self.screen, white, no_button)
        while not self.device.is_device_changed():
            text = "Waiting for device change"
            pygame_render_text(self.screen, text, (250,400), orange, 40)

        # to remove waiting for device change
        fill_color =  pygame.Rect(200, 400, 450, 55)
        pygame.draw.rect(self.screen, white, fill_color)

        # to remove previous date and stuff
        fill_color =  pygame.Rect(400, 70, 500, 150)
        pygame.draw.rect(self.screen, white, fill_color)
        now_time = datetime.datetime.now()
        text = "date/time : " + now_time.strftime("%m-%d-%Y %H:%M:%S")
        pygame_render_text(self.screen, text, (400,80), black, 30)
        text = "device Mx_id : " + self.device.get_mx_id()
        pygame_render_text(self.screen, text, (400,120), black, 30)
        rospy.sleep(3)
        fill_color_2 =  pygame.Rect(50, 520, 400, 80)
        pygame.draw.rect(self.screen, white, fill_color_2)
        while self.device.is_device_changed():
            # print(self.device.is_device_changed())
            # if self.capture_exit():
            #     print("signaling...")
            #     rospy.signal_shutdown("Finished calibration")
            is_usb3 = False
            left_mipi = False
            right_mipi = False
            if self.args['enable_IMU_test']:
                is_IMU_connected = False
            else:
                is_IMU_connected = True
            is_usb3 = self.device.is_usb3()
            left_status = self.device.is_left_connected()
            right_status = self.device.is_right_connected()

            imu_times = 0
            if self.capture_exit():
                rospy.signal_shutdown("Finished calibration")

            # else
            if left_status and right_status:
                # mipi check using 20 iterations
                # ["USB3", "Left camera connected", "Right camera connected", "left Stream", "right Stream"]
                for _ in range(90):
                    _, data_list = self.pipeline.get_available_nnet_and_data_packets(True)
                    # print(len(data_list))
                    for packet in data_list:
                        # print("found packets:")
                        # print(packet.stream_name)
                        if packet.stream_name == "left":
                            recent_left = packet.getData()
                            left_mipi = True
                            self.image_pub_left.publish(self.bridge.cv2_to_imgmsg(recent_left, "passthrough"))
                        elif packet.stream_name == "right":
                            recent_right = packet.getData()
                            right_mipi = True
                            self.image_pub_right.publish(self.bridge.cv2_to_imgmsg(recent_right, "passthrough"))
                        elif packet.stream_name == "meta_d2h":
                            str_ = packet.getDataAsStr()
                            if not self.args['enable_IMU_test']:
                                continue
                            dict_ = json.loads(str_)
                            if 'imu' in dict_:
                                if imu_times >= 5:
                                    is_IMU_connected = True
                                fill_color =  pygame.Rect(50, 500, 400, 100)
                                pygame.draw.rect(self.screen, white, fill_color)
                                selected_clr = red

                                if dict_['imu']['status'] == 'IMU init FAILED':
                                    selected_clr = red
                                else:
                                    imu_times += 1
                                    selected_clr = green
                                    text = 'IMU acc x: {:7.4f}  y:{:7.4f}  z:{:7.4f}'.format(dict_['imu']['accel']['x'], dict_['imu']['accel']['y'], dict_['imu']['accel']['z'])
                                    pygame_render_text(self.screen, text, (50, 545), font_size=25)
                                    text = 'IMU acc-raw x: {:7.4f}  y:{:7.4f}  z:{:7.4f}'.format(dict_['imu']['accelRaw']['x'], dict_['imu']['accelRaw']['y'], dict_['imu']['accelRaw']['z'])
                                    pygame_render_text(self.screen, text, (50, 570), font_size=25)

                                text = 'IMU status: ' + dict_['imu']['status']
                                pygame_render_text(self.screen, text, (50, 500), font_size=30, color=selected_clr)

                            # print(dict_)
                            # if 'logs' in dict_:
                            #     for log in dict_['logs']:
                            #         if log != 'IMU init FAILED':
                            #             text = 'IMU status: ' + log
                            #             is_IMU_connected = True
                            #             pygame_render_text(self.screen, text, (50, 500), font_size=25, color=green)
                            #         else:
                            #             text = 'IMU status: ' + log
                            #             pygame_render_text(self.screen, text, (50, 500), font_size=25, color=red)
                                # print('meta_d2h LOG------------------>:', log)
                    if left_mipi and right_mipi and is_IMU_connected:
                        if is_usb3:
                            self.device.reset_device_changed()
                        # for i in range(len(self.auto_checkbox_names)):
                        #     self.auto_checkbox_dict[self.auto_checkbox_names[i]].render_checkbox()
                        # print("device reste")
                        # print(self.device.is_device_changed())
                        break

            is_usb3 = self.device.is_usb3()
            left_status = self.device.is_left_connected()
            right_status = self.device.is_right_connected()

            if not is_usb3:
                self.auto_checkbox_dict["USB3"].uncheck()
            else:
                self.auto_checkbox_dict["USB3"].check()

            if not left_status:
                self.auto_checkbox_dict["Left camera connected"].uncheck()
            else:
                self.auto_checkbox_dict["Left camera connected"].check()

            if not left_mipi:
                self.auto_checkbox_dict["Left Stream"].uncheck()
            else:
                self.auto_checkbox_dict["Left Stream"].check()

            if not right_status:
                self.auto_checkbox_dict["Right camera connected"].uncheck()
            else:
                self.auto_checkbox_dict["Right camera connected"].check()

            if not right_mipi:
                # img = np.zeros((720, 1280, 3), np.uint8)
                # img[:] = (0, 0, 255)
                # cv2.putText(img, "right mipi",  (100, 300), cv2.FONT_HERSHEY_DUPLEX, 10.0, (0,0,0), 20)
                # cv2.putText(img, "Failed ", (140, 600), cv2.FONT_HERSHEY_SIMPLEX, 10.0, (0,0,0), 20)
                # cv2.imshow('right mipi camera failed', img)
                self.auto_checkbox_dict["Right Stream"].uncheck()
            else:
                self.auto_checkbox_dict["Right Stream"].check()

            if self.args['enable_IMU_test']:
                if is_IMU_connected:
                    self.auto_checkbox_dict["IMU connected"].check()
                else:
                    self.auto_checkbox_dict["IMU connected"].uncheck()

            for i in range(len(self.auto_checkbox_names)):
                self.auto_checkbox_dict[self.auto_checkbox_names[i]].render_checkbox()

        self.is_service_active = False
        return (True, self.device.get_mx_id())

    def calibration_servive_handler(self, req):
        self.is_service_active = True
        print("calibration Service Started")
        # pygame.draw.rect(self.screen, white, no_button)

        mx_serial_id = self.device.get_mx_id()
        calib_dest_path = os.path.join(arg['calib_path'], arg["board"] + '_' + mx_serial_id + '.calib')

        flags = [self.config['board_config']['stereo_center_crop']]
        cal_data = StereoCalibration()
        avg_epipolar_error, calib_data = cal_data.calibrate(
                            self.package_path + "/dataset",
                            self.args['square_size_cm'],
                            calib_dest_path,
                            flags,
                            req.name,
                            self.args['marker_size_cm'])

        text = "Avg Epipolar error : " + format(avg_epipolar_error, '.6f')
        pygame_render_text(self.screen, text, (400,160), green, 30)
        if avg_epipolar_error > 0.5:
            text = "Failed due to high calibration error"
            pygame_render_text(self.screen, text, (400,200), red, 30)
            return (False, text)
        # self.rundepthai()

        dev_config = {
            'board': {},
            '_board': {}
        }
        dev_config["board"]["clear-eeprom"] = False
        dev_config["board"]["store-to-eeprom"] = True
        dev_config["board"]["override-eeprom"] = False
        dev_config["board"]["swap-left-and-right-cameras"] = self.board_config['board_config']['swap_left_and_right_cameras']
        dev_config["board"]["left_fov_deg"] = self.board_config['board_config']['left_fov_deg']
        dev_config["board"]["rgb_fov_deg"] = self.board_config['board_config']['rgb_fov_deg']
        dev_config["board"]["left_to_right_distance_m"] = self.board_config['board_config']['left_to_right_distance_cm'] / 100
        dev_config["board"]["left_to_rgb_distance_m"] = self.board_config['board_config']['left_to_rgb_distance_cm'] / 100
        dev_config["board"]["name"] = self.board_config['board_config']['name']
        dev_config["board"]["stereo_center_crop"] = True
        dev_config["board"]["revision"] = self.board_config['board_config']['revision']
        dev_config["_board"]['calib_data'] = list(calib_data)
        dev_config["_board"]['mesh_right'] = [0.0]
        dev_config["_board"]['mesh_left'] =  [0.0]

        self.device.write_eeprom_data(dev_config)

        # calib_src_path = os.path.join(arg['depthai_path'], "resources/depthai.calib")
        # shutil.copy(calib_src_path, calib_dest_path)
        print("finished writing to EEPROM with Epipolar error of")

        print(avg_epipolar_error)
        print('Validating...')
        is_write_succesful = False
        run_thread = True
        while run_thread:
            _, data_packets = self.pipeline.get_available_nnet_and_data_packets(blocking=True)
            for packet in data_packets:
                if packet.stream_name == 'meta_d2h':
                    str_ = packet.getDataAsStr()
                    dict_ = json.loads(str_)
                    if 'logs' in dict_:
                        for log in dict_['logs']:
                            print(log)
                            if 'EEPROM' in log:
                                if 'write OK' in log:
                                    text = "EEPROM Write succesfull"
                                    pygame_render_text(self.screen, text, (400,200), green, 30)
                                    is_write_succesful = True
                                    run_thread = False
                                elif 'FAILED' in log:
                                    text = "EEPROM write Failed"
                                    pygame_render_text(self.screen, text, (400,200), red, 30)
                                    is_write_succesful = False
                                    run_thread = False
        self.is_service_active = False
        if is_write_succesful:
            return (True, "EEPROM written succesfully")
        else:
            return (False, "EEPROM write Failed!!")

    def rundepthai(self):
        test_cmd = "python3 " + self.args['depthai_path'] + "/depthai_demo.py -e -brd " + self.args['brd']
        print(test_cmd)
        # test_cmd = """python3 depthai.py -brd bw1098obc -e""" ## TODO(sachin): Parameterize the board name
        self.p = subprocess.Popen(test_cmd, shell=True, preexec_fn=os.setsid)
        time.sleep(4)
        os.killpg(os.getpgid(self.p.pid), signal.SIGKILL)

        time.sleep(1)

no_button =  pygame.Rect(490, 500, 80, 45)

if __name__ == "__main__":

    rospy.init_node('depthai_calibration', anonymous=True)
    arg = {}
    arg["swap_lr"] = rospy.get_param('~swap_lr')
    arg["field_of_view"] = rospy.get_param('~field_of_view')
    arg["baseline"] = rospy.get_param('~baseline')
    arg["package_path"] = rospy.get_param('~package_path')
    arg["square_size_cm"] = rospy.get_param('~square_size_cm')
    arg["marker_size_cm"] = rospy.get_param('~marker_size_cm')

    # arg["depthai_path"] = rospy.get_param('~depthai_path') ## Add  capture_checkerboard to launch file
    arg["board"] = rospy.get_param('~brd') ## Add  capture_checkerboard to launch file
    arg["capture_service_name"] = rospy.get_param('~capture_service_name') ## Add  capture_checkerboard to launch file
    arg["calibration_service_name"] = rospy.get_param('~calibration_service_name') ## Add  capture_checkerboard to launch file
    arg["depthai_path"] = rospy.get_param('~depthai_path') ## Path of depthai repo
    arg["enable_IMU_test"] = rospy.get_param('~enable_IMU_test')
    arg["calib_path"] = str(Path.home()) + rospy.get_param('~calib_path') ## local path to store calib files with using mx device id.
    print("Hone------------------------>")
    print(type(arg["enable_IMU_test"]))
    if not os.path.exists(arg['calib_path']):
        os.makedirs(arg['calib_path'])

    if arg['board']:
        board_path = Path(arg['board'])
        if not board_path.exists():
            board_path = Path(consts.resource_paths.boards_dir_path) / Path(arg['board'].upper()).with_suffix('.json')
            print(board_path)
            if not board_path.exists():
                raise ValueError('Board config not found: {}'.format(board_path))
        with open(board_path) as fp:
            board_config = json.load(fp)
    # assert os.path.exists(arg['depthai_path']), (arg['depthai_path'] +" Doesn't exist. \
    #     Please add the correct path using depthai_path:=[path] while executing launchfile")

    depthai_dev = depthai_calibration_node(arg)
    depthai_dev.publisher()
    rospy.spin()


import numpy as np
import matplotlib.pyplot as plt
import mysql.connector
import time
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import linear_kernel

# Get data (titles, overviews, genres and production companies) from database
def get_data_from_DB(source):
    title_list = []
    overview_list = []
    genres_list = []
    production_companies_list = []
    cnx = mysql.connector.connect(user='root', database='recomvee')
    cursor = cnx.cursor()

    if source is not None :
        query = ("SELECT title, overview, genres, production_companies FROM movie WHERE data_source = '" + source +"'")
    else :
        query = ("SELECT title, overview, genres, production_companies FROM movie")
    cursor.execute(query)

    for (title, overview, genres, production_companies) in cursor :
        title_list.append(format(title))
        overview_list.append(format(overview))
        genres_list.append(format(genres))
        production_companies_list.append(format(production_companies))

    cursor.close()
    cnx.close()
    return title_list, overview_list, genres_list, production_companies_list

# Create the Item-Item matrix
def createItemItemMatrix(length, titles_output_file, matrix_output_file,source):
    title_list, overview_list, genres_list, production_companies_list  = get_data_from_DB(source)

    criteria = []
    criteria.append(overview_list[:length])
    criteria.append(title_list[:length])
    criteria.append(genres_list[:length])
    criteria.append(production_companies_list[:length])

    cosine_sim = np.zeros((length,length))

    for k in range(len(criteria)) :

        tfidf_vectorizer = TfidfVectorizer()
        tfidf_matrix = tfidf_vectorizer.fit_transform(criteria[k])
        cosine_sim += linear_kernel(tfidf_matrix, tfidf_matrix)

    cosine_sim /= len(criteria)

    np.save(titles_output_file, title_list[:length])
    np.save(matrix_output_file, cosine_sim)

# Get recommendation's execution time graphics
def exec_time_graph():
    exec_time_matrix_creation = []
    for j in range(5):
        length = (j+1)*5000
        print("--- Exec Time Step %s ---" % (j+1))

        print("Matrix calculation ...")
        start_time = time.time()
        createItemItemMatrix(length, "titles" + str(j) + ".npy","matrix" + str(j) + ".npy")
        exec_time = time.time() - start_time
        exec_time_matrix_creation.append(exec_time)
        print("Done")

    x = [5000,10000,15000,20000,25000]
    plt.plot(x,exec_time_matrix_creation)
    plt.xlabel('Amount of movie')
    plt.ylabel('Execution Time (seconds)')
    plt.title("Evolution of the matrix's calculation time compared to the amount of movie")
    plt.show()

source = 'kaggle'

if source == 'kaggle' :
    titles_output_file = "titles_kaggle.npy"
    matrix_output_file = "matrix_kaggle.npy"
elif source == 'allocine' :
    titles_output_file = "titles_allocine.npy"
    matrix_output_file = "matrix_allocine.npy"

createItemItemMatrix(1000, titles_output_file, matrix_output_file, source)


import logging
from pyAitu import executor, Bot, Dispatcher
from pyAitu.models import Message, Options, Form, Header, FormClosed, LabeledText, Indent

API_TOKEN = 'YOUR_API_TOKEN'

bot = Bot(token=API_TOKEN)
dp = Dispatcher(bot)

logging.basicConfig(level=logging.INFO)

@dp.message_handler()
async def send_ui(message: Message):
    header = Header(
        _type="toolbar",
        title="Title",
        options=Options(
            closeable=True
        )
    )
    text = LabeledText(
        content_id="testid",
        label="Some short title for label",
        title="Label text. It has fixed decoration and formatting, cannot change it with options",
        options=Options(
            indent_inner=Indent(
                right=5,
                top=5,
                bottom=10
            )
        )
    )
    form = Form(_id="lol", header=header, content=text, options=Options(fullscreen=True))
    await bot.send_form(message.chat.id, form=form)

@dp.form_closed_handler()
async def get_form_closed(fc: FormClosed):
    await bot.send_message(fc.chat.id, "form closed")

if __name__ == '__main__':
    executor.start_polling(dp)


from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import linear_kernel
import numpy as np
import pandas as pd

class Similarity:
    def __init__(self, verbose=True):
        self.vectorizer = TfidfVectorizer(stop_words='english', max_df=0.05)
        self.tfidf = None

    def process(self, df: pd.DataFrame):
        original_columns = list(df.columns)
        df['similarities'] = self.fit(df['description'])
        docs_to_merge = self.to_merge(df)
        df = self.merge(df, docs_to_merge)
        df = df[original_columns]
        return df

    def fit(self, descriptions):
        self.tfidf = self.vectorizer.fit_transform(descriptions)
        similarities = []
        rows = len(descriptions)
        i = 0
        for row in self.tfidf:
            similarities.append(self.cosine_similarity(row))
            if i % 100 == 0:
                print(str(i/rows*100)+'%')
            i +=1
        return similarities

    def cosine_similarity(self, array):
        cosine_similarities = linear_kernel(array, self.tfidf).flatten()
        return [i for i in range(len(cosine_similarities)) if cosine_similarities[i] > 0.95]

    def to_merge(self, df: pd.DataFrame):
        only_similarities = df['similarities'].tolist()
        rows_to_merge = []
        for item in only_similarities:
            if len(item) > 1:
                if len(rows_to_merge) == 0:
                    rows_to_merge.append(set(item))
                else:
                    s_item = set(item)
                    matched = False
                    temp_rows_to_merge = []
                    for row in rows_to_merge:
                        if len(s_item.intersection(row)) > 0:
                            s_item.update(row)
                            temp_rows_to_merge.append(s_item)
                            matched = True
                        else:
                            temp_rows_to_merge.append(row)
                    if not matched:
                        temp_rows_to_merge.append(s_item)
                    rows_to_merge = temp_rows_to_merge

        return rows_to_merge

    def merge(self, df_original: pd.DataFrame, docs_to_merge: list):
        df = df_original.copy()
        for subset in docs_to_merge:
            try:
                sublist = list(subset)
                current_rows = df.iloc[sublist, :]
                merged = self.merge_doc_sets(current_rows)
                df = df.drop(sublist)
                df = df.append(merged)
            except Exception as e:
                print(subset)
                print(e)
        return df

    @staticmethod
    def merge_doc_sets(rows: pd.DataFrame):
        rows.sort_values(by='site')
        rows = rows.fillna(method='bfill')
        new_row = rows.iloc[1].copy()
        return new_row


from collections import OrderedDict

import torch
import torch.nn as nn
import torch.nn.functional as F

class MDNet(nn.Module):
    def __init__(self, model_path=None, K=1):
        super(MDNet, self).__init__()
        self.K = K
        self.layers = nn.Sequential(OrderedDict([
            ('conv1', nn.Sequential(nn.Conv2d(3, 96, kernel_size=7, stride=2),
                                    nn.ReLU(inplace=True),
                                    nn.LocalResponseNorm(2),
                                    nn.MaxPool2d(kernel_size=3, stride=2))),
            ('conv2', nn.Sequential(nn.Conv2d(96, 256, kernel_size=5, stride=2),
                                    nn.ReLU(inplace=True),
                                    nn.LocalResponseNorm(2),
                                    nn.MaxPool2d(kernel_size=3, stride=2))),
            ('conv3', nn.Sequential(nn.Conv2d(256, 512, kernel_size=3, stride=1),
                                    nn.ReLU(inplace=True))),
            ('fc4',   nn.Sequential(nn.Linear(512 * 3 * 3, 512),
                                    nn.ReLU(inplace=True))),
            ('fc5',   nn.Sequential(nn.Dropout(0.5),
                                    nn.Linear(512, 512),
                                    nn.ReLU(inplace=True)))]))

        self.branches = nn.ModuleList([nn.Sequential(nn.Dropout(0.5),
                                                     nn.Linear(512, 2)) for _ in range(K)])

        for m in self.layers.modules():
            if isinstance(m, nn.Linear):
                nn.init.normal_(m.weight, 0, 0.01)
                nn.init.constant_(m.bias, 0.1)
        for m in self.branches.modules():
            if isinstance(m, nn.Linear):
                nn.init.normal_(m.weight, 0, 0.01)
                nn.init.constant_(m.bias, 0)

        self.layers.load_state_dict(torch.load(model_path)['shared_layers'])
        self.params = OrderedDict()

        def _append(m, n):
            for c in m.children():
                for k, p in c._parameters.items():
                    if p is not None:
                        name = n + ('_bn_' if isinstance(c, nn.BatchNorm2d) else '_') + k
                        if name not in self.params:
                            self.params[name] = p

        for name, module in self.layers.named_children():
            _append(module, name)
        for k, module in enumerate(self.branches):
            _append(module, 'fc6_{:d}'.format(k))

    def forward(self, x, k=0, in_layer='conv1', out_layer='fc6'):
        # forward model from in_layer to out_layer
        run = False
        for name, module in self.layers.named_children():
            if name == in_layer:
                run = True
            if run:
                x = module(x)
                if name == 'conv3':
                    x = x.view(x.size(0), -1)
                if name == out_layer:
                    return x

        x = self.branches[k](x)
        if out_layer == 'fc6':
            return x
        elif out_layer == 'fc6_softmax':
            return F.softmax(x, dim=1)

class BCELoss(nn.Module):
    def forward(self, pos_score, neg_score, average=True):
        pos_loss = -F.log_softmax(pos_score, dim=1)[:, 1]
        neg_loss = -F.log_softmax(neg_score, dim=1)[:, 0]

        loss = pos_loss.sum() + neg_loss.sum()
        if average:
            loss /= (pos_loss.size(0) + neg_loss.size(0))
        return loss


import random

import discord
import httpx
from discord import app_commands
from discord.ext import commands

class Dice(commands.Cog):
    def __init__(self, bot: commands.Bot):
        self.bot = bot

    @app_commands.command(name="dice", description="Valorantのマップをランダムに返します。")
    async def d(self, interaction: discord.Interaction):
        # interactionは3秒以内にレスポンスしないといけないとエラーになるのでこの処理を入れる。
        await interaction.response.defer()

        url = "https://valorant-api.com/v1/maps"
        async with httpx.AsyncClient() as client:
            res = await client.get(url)
        json = res.json()

        data = json["data"]

        data_len = len(data)
        data_list = list(range(data_len))

        for range_idx in range(data_len):
            if data[range_idx]["uuid"] == "ee613ee9-28b7-4beb-9666-08db13bb2244":  # 射撃場のuuid
                break

        data_list.remove(range_idx)
        num = random.choice(data_list)

        displayName = data[num]["displayName"]
        listViewIcon = data[num]["listViewIcon"]
        # displayIcon = data[num]['displayIcon']

        embed = discord.Embed()
        embed.title = displayName
        embed.set_image(url=listViewIcon)
        # embed.set_thumbnail(url=displayIcon)
        embed.color = discord.Color.dark_blue()
        await interaction.followup.send(embed=embed)

async def setup(bot: commands.Bot):
    await bot.add_cog(Dice(bot), guilds=[discord.Object(id=731366036649279518)])


from user.user import User
from db import dbskiut_con
from globals import send_skiutc_mail
import pymysql
from datetime import datetime, timedelta

class Shotgun:
    """
    Shotgun handler class
    - Handle shotgun api calls
    - Build the shotgun
    - Check for users

    A shotgunner shotguns by its login

    we need to store in db :
    login
    timestamp

    reset auto increment :
    ALTER TABLE `shotgun-etu_2020` DROP `idshotgun`;
    ALTER TABLE `shotgun-etu_2020` AUTO_INCREMENT = 1;
    ALTER TABLE `shotgun-etu_2020` ADD `idshotgun` int UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST;
    """
    def __init__(self, login=""):
        self.login = login
        self.shotgun = False
        if len(login) < 9:
            self._add_to_shotgun()

    @staticmethod
    def _organize_table():
        con = dbskiut_con()
        con.begin()
        with con:
            cur = con.cursor()
            sql = "SELECT * FROM `shotgun-etu_2020` WHERE idshotgun < 293"
            try:
                cur.execute(sql)
            except pymysql.err.InternalError as error:
                code, message = error.args
                print (">>>>>>>>>>>>>", code, message)
            except pymysql.err.IntegrityError as error:
                code, message = error.args
                print (">>>>>>>>>>>>>", code, message)
            shotgun_list = cur.fetchall()
            for shotgun in shotgun_list:
                user = User.build_user_from_login(shotgun["login"])
                if not user.is_valid() or not user.is_cotisant():
                    sql = "DELETE FROM `shotgun-etu_2020` WHERE login=%s"
                    try:
                        cur.execute(sql,shotgun["login"])
                    except pymysql.err.InternalError as error:
                        code, message = error.args
                        print(">>>>>>>>>>>>>", code, message)
                    except pymysql.err.IntegrityError as error:
                        code, message = error.args
                        print(">>>>>>>>>>>>>", code, message)
                    cur.commit()
            cur.close()

    def _add_to_shotgun(self):
        """
        Function to add row in shotgun used for the shotgun
        Need to check after for cotisant and valid !
        :return: True if added, false else
        """
        con = dbskiut_con()
        con.begin()
        with con:
            cur = con.cursor()
            sql = "INSERT INTO `shotgun-etu_2020` (login) VALUES (%s)"
            try:
                cur.execute(sql, self.login)
                con.commit()
            except pymysql.InternalError as error:
                code, message = error.args
                print (">>>>>>>>>>>>>", code, message)
                self.shotgun = False
            except pymysql.err.IntegrityError as error:
                code, message = error.args
                print (">>>>>>>>>>>>>", code, message)
                self.shotgun = False
            finally:
                if cur.lastrowid:
                    self.shotgun = True
                else:
                    self.shotgun = False
                cur.close()

    def shotgun_valid(self):
        return self.shotgun

    @staticmethod
    def get_shotgun():
        """
        :return: Array of object returning shotgun
        """
        con = dbskiut_con()
        con.begin()
        with con:
            cur = con.cursor()
            sql = "SELECT * FROM `shotgun-etu_2020` WHERE idshotgun < 293"
            try:
                cur.execute(sql)
            except pymysql.err.InternalError as error:
                code, message = error.args
                print (">>>>>>>>>>>>>", code, message)
            except pymysql.err.IntegrityError as error:
                code, message = error.args
                print (">>>>>>>>>>>>>", code, message)
            finally:
                cur.close()
            shotgun = cur.fetchall()
            #array of all shotgun in object
            return shotgun

    @staticmethod
    def check_time():
        valid = False
        validity_date = datetime(2019, 10, 14, 7, 0, 0, 0)
        if datetime.now() - validity_date > timedelta(0):
            valid = True
        return valid

    @staticmethod
    def send_result_mail():
        """
        Send mail to every user in shotgun table
        - shotgun réussi
        - shotgun F
        :return:
        """
        con = dbskiut_con()
        con.begin()
        with con:
            cur = con.cursor()
            sql = "SELECT * FROM `shotgun-etu_2020`"
            try:
                cur.execute(sql)
            except pymysql.err.InternalError as error:
                code, message = error.args
                print(">>>>>>>>>>>>>", code, message)
            except pymysql.err.IntegrityError as error:
                code, message = error.args
                print(">>>>>>>>>>>>>", code, message)
            finally:
                cur.close()
        all_shotgun = cur.fetchall()
        for user, place in enumerate(all_shotgun):
            login = user["login"]
            mail = login + "@etu.utc.fr"
            #check si l'utilisateur a reussi le shotgun
            msg = ""
            if place < 293:
                msg = """Bonsoir,

                    Voici votre résultat à Shotgun Ski\'UTC 2020 pour le semestre de A19 : A, MENTION

                    Vous aurez l\'honneur d\'être parmi nous lors de ce voyage de folie !

                    Vous allez recevoir sous peu un autre mail avec plus d\'informations.

                    Ski\'UTC 2020 qui vous aime <3

                    ---------------

                    Ceci est un mail automatique, Merci de ne pas y répondre.
                    """
            else:
                msg = """Bonsoir,

                      Voici votre résultat à Shotgun Ski\'UTC 2020 pour le semestre de A19 : FX, INSUFFISANT

                      Ne vous inquiétez pas.. C\'est pas si difficile de se lever à 1h pour un shotgun physique !

                      Vous allez recevoir bientôt un autre mail avec plus d\'informations.

                      Ski\'UTC 2020 qui vous aime quand même <3

                       ---------------

                       Ceci est un mail automatique, Merci de ne pas y répondre.
                    """

            send_skiutc_mail(mail, "SKI'UTC 20 - Votre resultat", msg)


import sys
from collections import OrderedDict

import os
import torch
from torch import nn
from torch.autograd import Variable
from torchvision import transforms

class MarrNet(nn.Module):
    def __init__(self, cuda=False):
        super(MarrNet, self).__init__()
        from marrnet.models.step1 import step1
        from marrnet.models.step2 import step2
        self.step1 = step1
        self.step2 = step2
        script_dir = os.path.join(os.path.dirname(__file__), 'models')
        self.step1.load_state_dict(torch.load(os.path.join(script_dir, 'step1.pth')))
        self.step2.load_state_dict(torch.load(os.path.join(script_dir, 'step2.pth')))
        self.silhouette_threshold = 30
        if cuda is not False:
            self.cuda(cuda if cuda is not True else None)

    def forward(self, input):
        step1_out = self.step1(input)
        masked_normal_depth = self.mask(step1_out)
        step2_out = self.step2(masked_normal_depth)
        assert list(step2_out.size())[-3:] == [128, 128, 128]
        return step2_out

    def mask(self, input):
        normal = input[0]
        depth = input[1]
        silhouette = input[2][:, 0, :, :]
        silhouette_mask = torch.le(silhouette, self.silhouette_threshold)

        # TODO: solve this within PyTorch. Work around in-place limitation
        masked_normal = normal.data.numpy()
        masked_depth = depth.data.numpy()
        silhouette_mask = silhouette_mask.data.numpy().astype(bool)
        for i in range(3):
            masked_normal[:, i, :, :][silhouette_mask] = 100
        masked_depth[:, 0, :, :][silhouette_mask] = 0
        return torch.cat((Variable(torch.FloatTensor(masked_normal)), Variable(torch.FloatTensor(masked_depth))), 1)

def preprocess(normalize_mean=[0.485, 0.456, 0.406], normalize_std=[0.229, 0.224, 0.225], scale_dim=256):
    return transforms.Compose([
        transforms.Scale(scale_dim),
        transforms.ToTensor(),
        transforms.Normalize(mean=normalize_mean, std=normalize_std),
        lambda img: img.unsqueeze(0)
    ])

def summary(model, x, file=sys.stdout):
    """
    adapted from https://github.com/pytorch/pytorch/pull/3043
    Use input x directly instead of creating from input_size
    """

    def register_hook(module):
        def hook(module, input, output):
            if module._modules:  # only want base layers
                return
            class_name = str(module.__class__).split('.')[-1].split("'")[0]
            module_idx = len(summary)
            m_key = '%s-%i' % (class_name, module_idx + 1)
            summary[m_key] = OrderedDict()
            summary[m_key]['input_shape'] = list(input[0].size())
            summary[m_key]['input_shape'][0] = None
            if output.__class__.__name__ == 'tuple':
                summary[m_key]['output_shape'] = list(output[0].size())
            else:
                summary[m_key]['output_shape'] = list(output.size())
            summary[m_key]['output_shape'][0] = None

            params = 0
            # iterate through parameters and count num params
            for name, p in module._parameters.items():
                params += torch.numel(p.data) if p is not None else 0
                summary[m_key]['trainable'] = p.requires_grad if p is not None else False

            summary[m_key]['nb_params'] = params

        if not isinstance(module, torch.nn.Sequential) and \
                not isinstance(module, torch.nn.ModuleList) and \
                not (module == model):
            hooks.append(module.register_forward_hook(hook))

    # create properties
    summary = OrderedDict()
    hooks = []
    # register hook
    model.apply(register_hook)
    # make a forward pass
    model(x)
    # remove these hooks
    for h in hooks:
        h.remove()

    # print out neatly
    def get_names(module, name, acc):
        if not module._modules:
            acc.append(name)
        else:
            for key in module._modules.keys():
                p_name = key if name == "" else name + "." + key
                get_names(module._modules[key], p_name, acc)

    names = []
    get_names(model, "", names)

    col_width = 25  # should be >= 12
    summary_width = 61

    def crop(s):
        return s[:col_width] if len(s) > col_width else s

    print('_' * summary_width, file=file)
    print('{0: <{3}} {1: <{3}} {2: <{3}}'.format(
        'Layer (type)', 'Output Shape', 'Param #', col_width), file=file)
    print('=' * summary_width, file=file)
    total_params = 0
    trainable_params = 0
    for (i, l_type), l_name in zip(enumerate(summary), names):
        d = summary[l_type]
        total_params += d['nb_params']
        if 'trainable' in d and d['trainable']:
            trainable_params += d['nb_params']
        print('{0: <{3}} {1: <{3}} {2: <{3}}'.format(
            crop(l_name + ' (' + l_type[:-2] + ')'), crop(str(d['output_shape'])),
            crop(str(d['nb_params'])), col_width), file=file)
        if i < len(summary) - 1:
            print('_' * summary_width, file=file)
    print('=' * summary_width, file=file)
    print('Total params: ' + str(total_params), file=file)
    print('Trainable params: ' + str(trainable_params), file=file)
    print('Non-trainable params: ' + str((total_params - trainable_params)), file=file)
    print('_' * summary_width, file=file)


import tweepy
from config import TwitterAuthenticate
from tweepy import API
from tweepy import Cursor
import json
import re
import pandas as pd
import numpy as np
from textblob import TextBlob
from pymongo import  MongoClient

class TwitterClient():
    """
    Class retrieves tweets from a particular user and saves them to a json file
    """
    def __init__(self, twitter_user=None):
        self.auth = TwitterAuthenticate().authenticate_twitter_app()
        self.twitter_client = API(self.auth)
        # self.fetched_tweets_filename = fetched_tweets_filename
        self.twitter_user = twitter_user

    def get_twitter_client_api(self):
        return self.twitter_client

    def get_user_timeline_tweets(self, num_tweets):
        tweets = []

        for tweet in Cursor(self.twitter_client.user_timeline, id=self.twitter_user).items(num_tweets):

            tweets.append(tweet)
        return tweets

class TweetAnalyzer():
    '''
    Create pd data frame
    '''
    def tweets_to_data_frame(self, tweets):
        df = pd.DataFrame(data=[tweet.created_at for tweet in tweets], columns=['date'])

        df['user'] = np.array([tweet.user.name for tweet in tweets])
        df['text'] = np.array([tweet.text for tweet in tweets])
        df['char_count'] = np.array([len(tweet.text) for tweet in tweets])
        df['retweets'] = np.array([tweet.retweet_count for tweet in tweets])
        return df

    def clean_text(self, text):
        return ' '.join(re.sub(r"(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)", " ", text).split())

    def analyze_sentiment(self, text):
        analysis = TextBlob(self.clean_text(text))
        return analysis.sentiment.polarity

if __name__ == "__main__":
    user = "realDonaldTrump"
    tweet_analyzer = TweetAnalyzer()
    twitter_client = TwitterClient()
    api = twitter_client.get_twitter_client_api()

    # desired_width = 320
    # pd.set_option('display.width', desired_width)
    # np.set_printoptions(linewidth=desired_width)
    # pd.set_option('display.max_columns', 10)

    tweets = []
    for status in tweepy.Cursor(api.user_timeline, screen_name=user, count=200, exclude_replies=True).items():
        tweets.append(status)

    df = tweet_analyzer.tweets_to_data_frame(tweets)
    df['sentiment'] = np.array([tweet_analyzer.analyze_sentiment(text) for text in df['text']])

    client = MongoClient('localhost', 27017)
    db = client.twitter_data
    trump_tweets = db.trump_tweets
    trump_tweets.drop()
    df_dict = df.to_dict("records")
    for i, row in enumerate(df_dict):
        row['_id'] = i
        trump_tweets.insert_one(row)


from abra.abra_pb2 import User, SnapshotWrapper
import click
from abra.common import *
from abra.errors import UnsupportedMessageQueueError
from flask import Flask, request
import json
import os
import random
from termcolor import cprint
from pathlib import Path
from abra.mq.mq_handlers import RabbitMQHandler

DEBUG = 0
_WORK_DIR = "state_of_minds"
app = Flask(__name__)
_publish = None
dbg_num = 0

# FIXME add errors everywhere
@click.group()
def main():
    pass

@main.command('run-server')
@click.option("--host", "-h", default=APP_SERVER_HOST)
@click.option("--port", "-p", default=APP_SERVER_PORT)
@click.argument('publish')
def run_server_cli(host, port, publish):
    run_server(host, port, publish)

def run_server(host=APP_SERVER_HOST, port=APP_SERVER_PORT, publish=None):
    global _publish
    _publish = publish

    cprint('Creating your Server!', 'white', 'on_blue', attrs=['bold'])
    app.run(host, port)

@app.route('/msg', methods=['POST'])
def new_msg():
    snapshot_wrapper = SnapshotWrapper()
    msg = request.data
    snapshot_wrapper.ParseFromString(msg)
    user, snapshot, msg_num = snapshot_wrapper.user, snapshot_wrapper.snapshot, snapshot_wrapper.msg_num
    unique_path = Path(VOLUME_PATH) / f'{_WORK_DIR}/{user.user_id}_{msg_num}_{random.random()}'
    f_path = save_snapshot(snapshot, unique_path)
    while f_path is None:
        unique_path = Path(VOLUME_PATH) / f'{_WORK_DIR}/{user.user_id}_{msg_num}_{random.random()}'
        f_path = save_snapshot(snapshot, unique_path)

    snap_path_json = json.dumps({
        USER_ID_KEY: user.user_id,
        USERNAME_KEY: user.username,
        USER_SNAPSHOTS_URL_KEY: f_path,
        SNAPSHOT_ID_KEY: create_snapshot_id(user, snapshot.datetime)
    })

    if callable(_publish):
        _publish(snap_path_json)
        return SUCCESS

    # For debug:
    # with open("dbg.json", "w") as of:
    #     json.dump(snap_path_json, of)

    publish_msg(snap_path_json)
    return snap_path_json

@app.route('/user', methods=['POST'])
def register_new_user():
    user = User()
    user_data = request.data
    user.ParseFromString(user_data)

    if callable(_publish):
        _publish(user)
        return SUCCESS

    json_user = jsonify_user(user)
    publish_user(json_user)
    return json_user

def publish_msg(msg):
    mq_handler = None
    try:
        mq_handler = RabbitMQHandler(_publish)
    except UnsupportedMessageQueueError as e:
        print(e)
        exit(FAIL)
    mq_handler.publish_to_parsers(msg)

def publish_user(user):
    mq_handler = None
    try:
        mq_handler = RabbitMQHandler(_publish)
    except UnsupportedMessageQueueError as e:
        print(e)
        exit(FAIL)
    mq_handler.publish_to_parsed_data_exchange(msg=user, topic="user")

def save_snapshot(snapshot, path):
    if os.path.isdir(path):
        return None
    try:
        path = Path(path)
        Path(path).mkdir(parents=True, exist_ok=True)
        f = path / SNAPSHOT_JSON_FILE_NAME
        f.write_bytes(snapshot.SerializeToString())
        return str(f.absolute())
    except OSError as err:
        print(f"Error: {err}")
        return None
    except ValueError as err:
        print(f"Error: {err}")
        return None

def jsonify_user(user):
    json_user = {
        TOPIC_KEY: USER_TOPIC,
        USER_ID_KEY: user.user_id,
        USERNAME_KEY: user.username,
        BIRTHDAY_KEY: user.birthday,
        GENDER_KEY: user.gender
    }
    return json.dumps(json_user)

def create_snapshot_id(user_id, date):
    st = str(user_id)+str(date)
    hashed = int(hash(st)) + random.randint(1, 1e+18)
    return abs(hashed & 0xffffffff)

if __name__ == '__main__':
    main()


import logging
import os
import pickle
import jellyfish

# db_file = "honyme_db.pkl"
# noiser = HomophoneNoise(db_file)
import logger
from noise.noise_base import NoiseGenerator

class NoiseFromDict(NoiseGenerator):
    # database is created with words that have jaro_winkler threshold >= 0.5 ...
    def __init__(self, db_file_list, threshold=0.8, cnt_error_samples=5, max_N=1):  # >= 0.8
        self.logger = logging.getLogger(logger.BasicLogger.logger_name)

        NoiseGenerator.__init__(self)
        self.threshold = threshold
        if not 0.0 <= self.threshold <= 1.0:
            self.logger.error("jaro-winkler-threshold must be in [0,1], but: {} was given".format(self.threshold))
            raise ValueError

        self.cnt_error_samples = cnt_error_samples
        self.max_N = max_N

        # Parse file with list of the dictionaries
        hash_dicts, dicts = self.parse_file_list(db_file_list=db_file_list)

        # Load dictionaries to memory !!
        self.hash_dictionaries = []
        self.dictionaries = []

        self.load_dicits_to_memory(hash_dicts, dicts)

    def get_name(self):
        return "dictionary"

    def load_dicits_to_memory(self, hash_dicts, dicts):
        for hash_dict in hash_dicts:
            path, type_, from_M, to_N = hash_dict

            with open(path, 'rb') as handle:
                self.logger.info("Loading hashed dict: {}".format(path))
                _dict = pickle.load(handle)
                self.hash_dictionaries.append((_dict, type_, from_M, to_N))

        for dict in dicts:
            path, m, n = dict
            with open(path, 'rb') as handle:
                self.logger.info("Loading regular dict: {}".format(path))
                _dict = pickle.load(handle)
                self.dictionaries.append((_dict, m, n))

    def parse_file_list(self, db_file_list):

        hash_dicts = []
        dicts = []
        if not os.path.isfile(db_file_list):
            self.logger.error("Source list not found: {}".format(db_file_list))
            raise FileNotFoundError

        # load paths to dictionary files (and skip comments)
        with open(db_file_list) as f:
            files_list = [line.strip().split() for line in f]

        # remove comments and empty lines
        files_list[:] = [line for line in files_list if not (len(line) == 0 or line[0].startswith("#"))]

        for i, file_info in enumerate(files_list):
            if len(file_info) < 3 or len(file_info) > 4:
                self.logger.error("list with dictionary files: {} BAD SYNTAX ON LINE: {}".format(db_file_list, i + 1))
                raise SyntaxError

            path = file_info[0]
            if not os.path.isfile(path):
                self.logger.error("Path to dictionary file not found: {}".format(path))
                raise FileNotFoundError

            type_ = file_info[1].upper()
            if type_ not in ["DICTIONARY", "SOUNDEX", "METAPHONE", "MRA", "NYSIIS"]:
                self.logger.error(
                    "Unkown dictionary type: {} found in list with dictionary files: {}".format(type_, db_file_list))

            if type_ == "DICTIONARY":
                if len(file_info) != 4:
                    self.logger.error(
                        "list with dictionary files: {} BAD SYNTAX ON LINE: {}".format(db_file_list, i + 1))
                    raise SyntaxError

                from_M = float(file_info[2])
                to_N = float(file_info[3])
                dicts.append((path, from_M, to_N))

            else:
                if len(file_info) != 4:
                    self.logger.error(
                        "list with dictionary files: {} BAD SYNTAX ON LINE: {}".format(db_file_list, i + 1))
                    raise SyntaxError
                from_M = int(file_info[2])
                to_N = int(file_info[3])
                hash_dicts.append((path, type_, from_M, to_N))

        if len(hash_dicts) == 0 and len(dicts) == 0:
            self.logger.error("No dictionaries were found in dictionary list: {}".format(db_file_list))
            raise ValueError

        return hash_dicts, dicts

    def load_data(self, name):
        with open(name, 'rb') as f:
            return pickle.load(f)

    def generate_pronunciation(self, words_alpha_pos):
        pass

    def recursive_merge(self, word_list, curr_depth, word_part, tmp):
        # word_list = []
        # curr_depth = 0
        # word_part  = (None,  0.0)
        # tmp =  [ [('Yolk', 1.0), ('and', 1.0)] ]
        if curr_depth == len(tmp):
            return [word_part]  # we must return list ...

        curr_level = tmp[curr_depth]  # [ (word, prob) ,  (word, prob) ... ]

        for i in range(len(curr_level)):
            wrd, score = word_part  # (None, 0.0)
            if wrd is None:
                wrd = ""
            else:
                wrd += " "
            wrd += curr_level[i][0]  # (word, prob)
            score += (curr_level[i][1] / len(tmp))  # average score
            new_word = (wrd, score)
            final_wrd = self.recursive_merge(word_list, curr_depth + 1, new_word, tmp)
            word_list.extend(final_wrd)  # extend otherwise ...

        return word_list

    def add_parsed_result(self, node_id, to_N, synthetised_word, score):
        if node_id in self.parsed_results:
            if to_N in self.parsed_results[node_id]:
                self.parsed_results[node_id][to_N].append((synthetised_word, score))
            else:
                self.parsed_results[node_id][to_N] = [(synthetised_word, score)]
        else:
            self.parsed_results[node_id] = {to_N: [(synthetised_word, score)]}

    def generate_noise(self, list_sampling_nodes):
        """
            generate P2G using the provided dictionary
            :param list_sampling_nodes:
            :return: results: { node_id : {to_n : [(error, score)] } }
        """
        self.parsed_results = {}

        for node in list_sampling_nodes:
            for j in range(self.max_N):
                N = j + 1
                M = node.from_M_variant
                list_words = []
                word = node.word
                # ......................................
                self.list_words_from_dicts(word, list_words, M=M, N=N)  # dict (M:N)
                self.list_words_from_hashes(word, list_words, N=N)     # hash(w1...wM) -> N (M:N)
                # ......................................
                noise_result = self.get_noise_results(orig_word=word, list_of_similar=list_words)

                if len(noise_result) == 0:
                    noise_result = self.fallback_M_all(word=word, M=M)
                    #if len(noise_result) == 0 and M > 1:
                    #    noise_result = self.fallback_split(word=word, N=N)
                # print("'{}':\t\t {}".format(word, noise_result))
                for result in noise_result:
                    synth_word, prob_score = result
                    self.add_parsed_result(node.sampling_id, N, synth_word, prob_score)

        # parsed results may be empty dictionary ...
        return self.parsed_results

    def get_hash(self, word, hash_type):
        word = word.replace(" ", "")
        if hash_type == "SOUNDEX":
            hash = jellyfish.soundex(word)
        elif hash_type == "NYSIIS":
            hash = jellyfish.nysiis(word)
        elif hash_type == "MRA":
            hash = jellyfish.match_rating_codex(word)
        elif hash_type == "METAPHONE":
            hash = jellyfish.metaphone(word)
        else:
            raise NotImplementedError("approach '{}' not implemented".format(hash_type))
        return hash

    def fallback_split(self, word, N):
        # print("fallback split")
        final_results = []
        split = word.split()
        if len(split) > N:
            split = split[:N]  # [w1,..wn]

        # add noise to individual words ... # w1: [w1_h1, w1_h2 ...]  ... wn : [wn_h1, wn_h2 ... ]
        tmp = []
        for w in split:
            ttmp = []
            self.list_words_from_dicts(w, ttmp, M=1, N=1)
            partial_result = self.get_noise_results(orig_word=w, list_of_similar=ttmp)
            tmp.append(partial_result)  # append not extend !!

        # and merge it back together ... all options [w1_h1 wm_h1 ...  ]  [w1_h1 wm_h2 ...  ] ....
        if len(tmp) > 0:
            results = self.recursive_merge([], 0, (None, 0.0), tmp)
            final_results.extend(results)
            # print("from split_merge [{}]:  {}".format(word, results))

        return final_results

    def fallback_M_all(self, word, M):
        # A. try try_similar_from_dicts M -> [..]
        lst_similar = []
        self.list_words_from_dicts(word, lst_similar, M=M, N=None)  # N=None ... to arbitrary N ...
        return self.get_noise_results(orig_word=word, list_of_similar=lst_similar)

    def list_words_from_dicts(self, word, lst_similar, M, N):
        for i, dic in enumerate(self.dictionaries):
            _dict, M__, N__ = dic
            if ( N is None and M == M__) or (M == M__ and N == N__):
                sim = self.similar_list_from_dict(word, i)
                if len(sim) > 0:
                    lst_similar.extend(sim)

    def similar_list_from_dict(self, word, dict_id):
        word = word.lower()
        if word in self.dictionaries[dict_id][0]:  # word exist # (dict,m,n)
            if len(self.dictionaries[dict_id][0]) > 0:  # safety
                return self.dictionaries[dict_id][0]
        return []

    def list_words_from_hashes(self, word, lst_similar, N):
        for i, tuple_ in enumerate(self.hash_dictionaries):
            if tuple_[3] == N:
                sim = self.similar_list_from_hash_dict(word, i)
                if len(sim) > 0:
                    lst_similar.extend(sim)

    def similar_list_from_hash_dict(self, word, hash_id):
        # _dict, type_, from_M, to_N
        hash_type = self.hash_dictionaries[hash_id][1]
        hashed = self.get_hash(word, hash_type)
        if hashed in self.hash_dictionaries[hash_id][0]:
            # for M:M we need at least 2 items in list ...
            if self.hash_dictionaries[hash_id][2] == self.hash_dictionaries[hash_id][3]:
                if len(self.hash_dictionaries[hash_id][0][hashed]) > 1:
                    return self.hash_dictionaries[hash_id][0][hashed]
            # for M:N 1 item is enough...
            else:
                if len(self.hash_dictionaries[hash_id][0][hashed]) >= 1:
                    return self.hash_dictionaries[hash_id][0][hashed]
        return []

    def get_noise_results(self, orig_word, list_of_similar):
        similar = []
        added_words = set({})
        for w2 in list_of_similar:
            if orig_word == w2:
                continue

            score = jellyfish.jaro_winkler_similarity(orig_word, w2)
            if score >= self.threshold and w2 not in added_words:  # and score must be higher than threshold
                similar.append((w2, score))  # word, pronunciation, score
                added_words.add(w2)

        # few or zero results ..
        if len(similar) < self.cnt_error_samples:
            return similar

        # else return top scoring
        # similar = sorted(similar, key=lambda entry: entry[1], reverse=True)  # by score from greatest
        return similar[:self.cnt_error_samples]