#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Mar 21 15:29:25 2020

@author: sergio.lordano
"""
import numpy
import numpy as np
from matplotlib import pyplot as plt
from scipy.interpolate import interp1d
from scipy.optimize import minimize
from optlnls.math import derivate

def crop_height_error_matrix(matrix, L, W, height_offset=False):
    """
    This function receives as input a 2D mirror height error matrix and crop it\
    to new length and width within the matrix range. Input and Output format is a 2D matrix in which\
    the first row is the longitudinal coordinates, the first column is sagittal coordinates,\
    elements [1:,1:] are the corresponding height errors and element [0,0] is unused.\
    Length, width and matrix must be in same units.\n
    :matrix: 2D matrix in the above mentioned format
    :L: total mirror length in which the matrix must be cropped
    :W: total mirror width in which the matrix must be cropped
    :height_offset: if True, a height offset will be added in all points so that the median value of central line is zero.
    """
    import numpy as np
    idx_min_L = np.abs(matrix[0,:]-(-L/2.0)).argmin() # search for the coordinate values which are the closest from desired L and W.
    idx_max_L = np.abs(matrix[0,:]-(L/2.0)).argmin()
    idx_min_W = np.abs(matrix[:,0]-(-W/2.0)).argmin()
    idx_max_W = np.abs(matrix[:,0]-(W/2.0)).argmin()
#    print(idx_min_L, idx_max_L, idx_min_W, idx_max_W)

    height2D_part = matrix[idx_min_W:idx_max_W+1,idx_min_L:idx_max_L+1]
    height2D_fmt = np.zeros((len(height2D_part[:,0])+1,len(height2D_part[0,:])+1))
    height2D_fmt[0,:][1:] = matrix[0,:][idx_min_L:idx_max_L+1]
    height2D_fmt[:,0][1:] = matrix[:,0][idx_min_W:idx_max_W+1]
    if(height_offset):
        vert_displacement = (max(height2D_part[int(len(height2D_part)/2.0)+1,:])+min(height2D_part[int(len(height2D_part)/2.0)+1,:]))/2.0
    else:
        vert_displacement = 0
    height2D_fmt[1:,1:] = height2D_part - vert_displacement

    return height2D_fmt

def write_shadow_height_error(matrix, file_name, unit_factor, delimiter=' '):
    """
    Creates a mirror height error file in shadow/dabam format from a 2D matrix in which\
    the first row is the longitudinal coordinates, the first column is sagittal coordinates,\
    elements [1:,1:] are the corresponding height errors and element [0,0] is unused.\n
    :matrix: 2D matrix in the above mentioned format
    :file_name: file name containing extension, e.g. 'mirror_error.dat'
    :unit_factor: unit factor from matrix units, that is, 1e+3 to write file in [mm] if matrix is in [m]
    :delimiter: space as standard for shadow format, or alternatively tab.
    """
    nl = len(matrix[:,0])-1
    nc = len(matrix[0,:])-1
    with open(file_name, 'w') as ofile:
        ofile.write('{0}'.format(nl)+ delimiter +'{0}\n'.format(nc))
        for line in range(nl+1):
            for column in range(nc+1):
                if not(line==0 and column==0):
                    ofile.write('{0:.3e}'.format(matrix[line, column]*unit_factor)+delimiter)
            ofile.write('\n')
    ofile.closed

def from_shadow_to_matrix(file_path, unit_factor):
    """
    Imports a mirror height error file in shadow/dabam format and returns a 2D matrix (in meters) in which\
    the first row is the longitudinal coordinates, the first column is sagittal coordinates,\
    elements [1:,1:] are the corresponding height errors and element [0,0] is unused.\n
    :file_path: path to file including file name.
    :unit_factor: unit factor from the file units to meter, that is, 1e-3 if file is in [mm] for instance.

    """
    import numpy as np
    data = [] # array to store all elements in 1D list
    with open(file_path, 'r') as datafile: # reads and append every element sequentially until readline() fails.
        while True:
            read_line = datafile.readline()
            if not read_line: break
            for element in read_line.split():
                data.append(element)
    datafile.close()

    nl, nc = int(data[0]), int(data[1])
    matrix_data = np.zeros((nl+1, nc+1)) # allocate matrix
    matrix_data[0,:][1:] = np.array(data[2:nc+2], dtype='float')*unit_factor # [m] Longitudinal coordinates
    for line in range(nl): # associate elements in data array to the matrix lines
        matrix_data[line+1,:] = np.array(data[1+(nc+1)*(line+1):1+(nc+1)*(line+2)], dtype='float')*unit_factor # [m]
    return matrix_data

def SRW_figure_error(file_name, unit_factor, angle_in, angle_out, orientation_x_or_y, crop=False, height_offset=False, L=5e3, W=5e3):
    """
    Returns an instance of srwlib.SRWLOptT() which simulates a mirror height error in SRW module.\
    To run this function, it is necessary to have srwlib able to import.\n
    :file_name: filename of file in shadow/dabam format
    :unit_factor: unit factor from the file units to meter, that is, 1e-3 if file is in [mm] for instance.
    :angle_in: incidence angle [rad] relative to mirror surface
    :angle_out: reflection angle [rad] relative to mirror surface
    :orientation_x_or_y: 'x' for horizontal or 'y' for vertical deflection
    :crop: (optional) if True, crops the matrix to new length L and width W and optionally offset data to make median value equal zero
    :height_offset: if True, a height offset will be added in all points so that the median value of central line is zero.
    :L: total mirror length in which the matrix must be cropped
    :W: total mirror width in which the matrix must be cropped
    """
    from srwlib import srwl_opt_setup_surf_height_2d

    height2D = from_shadow_to_matrix(file_name, unit_factor)
    if(crop):
        height2D_cropped = crop_height_error_matrix(height2D, L, W, height_offset)
        print('Actual L x W: {0:.3f} m x {1:.3f} m'.format(height2D_cropped[0,-1]-height2D_cropped[0,1], height2D_cropped[-1,0]-height2D_cropped[1,0]))
        return srwl_opt_setup_surf_height_2d(height2D_cropped, orientation_x_or_y, angle_in, angle_out)
    else:
        return srwl_opt_setup_surf_height_2d(height2D, orientation_x_or_y, angle_in, angle_out)

def analyze_height_error(filelist, unit_factor, plotting=True, workingFolder=''):

    import os
    from optlnls.math import derivate_keeping_size, psd
    from optlnls.plot import set_ticks_size

    # === CREATES SUBFOLDERS === #
    if(workingFolder != ''):
        os.chdir(workingFolder)

    if not (os.path.exists('2D')): os.mkdir('2D')
    if not (os.path.exists('meridional')): os.mkdir('meridional')
    if not (os.path.exists('sagittal')): os.mkdir('sagittal')
    if not (os.path.exists('PSD')): os.mkdir('PSD')
    if not (os.path.exists('statistics')): os.mkdir('statistics')

    for slopefile in filelist:

        # *** Finds adequate dimensions and writes new Shadow input file *** #
        filename = slopefile.split('/')[-1]
        slope2d = from_shadow_to_matrix(slopefile, unit_factor)
        new_mtx = slope2d #mypkg.select_part_surface(slope2d, L, W)
        nW = len(new_mtx[1:,0])
        nL = len(new_mtx[0,1:])
        realL = new_mtx[0,-1]-new_mtx[0,1]
        realW = new_mtx[-1,0]-new_mtx[1,0]
        X = new_mtx[0, 1:]

        if(0):
            write_shadow_height_error(new_mtx, filename.split('.')[0]+'_ctr.dat', unit_factor, delimiter=' ')

        #==================================================#
        # MERIDIONAL
        #==================================================#
        mer_cut = new_mtx[int(np.ceil((len(new_mtx)-1)/2)+1),1:]
        mer_cut_slope = derivate_keeping_size(X, mer_cut)
        PV_mer_cut = (np.max(mer_cut)-np.min(mer_cut))*1e9
        std_height = np.std(mer_cut)*1e9
        std_slope = np.std(mer_cut_slope)*1e6

        PSD, frequencies = psd(X, mer_cut)

        mer_errors = []
        for i in [-2, -1, 0, +1, +2]:
            idx = int(np.ceil((nW-1)/2.0)+1 + ((nW-1)/5.0)*i)
            mer_errors.append(calc_errors(new_mtx[0,1:], new_mtx[idx,1:], new_mtx[idx,0]))
        mer_errors = np.array(mer_errors)

        length_scan = []
        for i in [realL/8.0, realL/4.0, realL/2.0]:
            idx_i, idx_f = np.abs(X+i).argmin(), np.abs(X-i).argmin()
            length_scan.append(calc_errors(X[idx_i:idx_f+1], new_mtx[int(np.ceil((nW-1)/2.0)+1),1:][idx_i:idx_f+1], X[idx_f]-X[idx_i]))
        length_scan = np.array(length_scan)

        #==================================================#
        # SAGITTAL
        #==================================================#
        sag_cut = new_mtx[1:,int(np.ceil((len(new_mtx[0,:])-1)/2)+1)]
        sag_cut_slope = derivate_keeping_size(new_mtx[1:,0], sag_cut)
        PV_sag_cut = (np.max(sag_cut)-np.min(sag_cut))*1e9
        std_height_sag = np.std(sag_cut)*1e9
        std_slope_sag = np.std(sag_cut_slope)*1e6

        sag_errors = []
        for i in [-2, -1, 0, +1, +2]:
            idx = int(np.ceil((nL-1)/2.0)+1 + ((nL-1)/5.0)*i)
            sag_errors.append(calc_errors(new_mtx[1:,0], new_mtx[1:,idx], new_mtx[0,idx]))
        sag_errors = np.array(sag_errors)

        # === WRITES STATISTICS === #
        if(1):
            write_stats(filename, length_scan, mer_errors, sag_errors, new_mtx)

        #==================================================#
        # PLOTS MERIDIONAL
        #==================================================#
        fig_size = 5.5
        fsize = 14

        plt.figure(figsize=(2.5*fig_size,fig_size))

        plt.subplot(121)
        plt.plot(new_mtx[0][1:]*1e3, mer_cut*1e9, '.-', markersize=2.0)
        plt.title(filename+'\nL = {0} mm ; W = {1} mm'.format(realL*1e3, realW*1e3), fontsize=fsize)
        plt.xlabel('Mirror Length [mm]', fontsize=fsize)
        plt.ylabel('Height Error [nm]', fontsize=fsize)
        plt.grid(True)
        plt.text(0.95,0.95,"RMS = {0:.2f} nm".format(std_height)+'\n'+'PV = {0:.2f} nm'.format(PV_mer_cut), verticalalignment="top", horizontalalignment="right", transform=plt.gca().transAxes)
        plt.ylim(plt.ylim()[0], plt.ylim()[1]*1.15)
        set_ticks_size(fsize)

        plt.subplot(122)
        plt.plot(new_mtx[0][1:]*1e3, mer_cut_slope*1e6)
        plt.title(filename+'\nL = {0} mm ; W = {1} mm'.format(realL*1e3, realW*1e3), fontsize=fsize)
        plt.xlabel('Mirror Length [mm]', fontsize=fsize)
        plt.ylabel(r'Slope Error [$\mu rad$]', fontsize=fsize)
    #    plt.ylim(-2.0,2.0)
        plt.grid(True)
        plt.text(0.95,0.95,"RMS = {0:.3f} ".format(std_slope)+r'$\mu rad$', verticalalignment="top", horizontalalignment="right", transform=plt.gca().transAxes)
        set_ticks_size(fsize)
        #plt.show()

        plt.savefig(os.path.join(os.getcwd(), 'meridional', filename.split('.')[0]+'_mer.png'), dpi=300)

        #==================================================#
        # PLOTS SAGITTAL
        #==================================================#
        plt.figure(figsize=(2.5*fig_size,fig_size))

        plt.subplot(121)
        plt.plot(new_mtx[1:,0]*1e3, sag_cut*1e9, '.-', markersize=2.0)
        plt.title(filename+'\nL = {0} mm ; W = {1} mm'.format(realL*1e3, realW*1e3), fontsize=fsize)
        plt.xlabel('Mirror Width [mm]', fontsize=fsize)
        plt.ylabel('Height Error [nm]', fontsize=fsize)
        plt.grid(True)
        plt.text(0.95,0.95,"RMS = {0:.2f} nm".format(std_height_sag)+'\n'+'PV = {0:.2f} nm'.format(PV_sag_cut), verticalalignment="top", horizontalalignment="right", transform=plt.gca().transAxes)
        plt.ylim(plt.ylim()[0], plt.ylim()[1]*1.15)
        set_ticks_size(fsize)

        plt.subplot(122)
        plt.plot(new_mtx[1:,0]*1e3, sag_cut_slope*1e6)
        plt.title(filename+'\nL = {0} mm ; W = {1} mm'.format(realL*1e3, realW*1e3), fontsize=fsize)
        plt.xlabel('Mirror Width [mm]', fontsize=fsize)
        plt.ylabel(r'Slope Error [$\mu rad$]', fontsize=fsize)
    #    plt.ylim(-2.0,2.0)
        plt.grid(True)
        plt.text(0.95,0.95,"RMS = {0:.3f} ".format(std_slope_sag)+r'$\mu rad$', verticalalignment="top", horizontalalignment="right", transform=plt.gca().transAxes)
        set_ticks_size(fsize)
        #plt.show()

        plt.savefig(os.path.join(os.getcwd(), 'sagittal', filename.split('.')[0]+'_sag.png'), dpi=300)

        #==================================================#
        # PLOTS 2D SURFACE
        #==================================================#
        plt.figure(figsize=(12,3.0))
        #fig = plt.gcf()
        ax = plt.gca()
        #cbound = np.max([np.abs(np.floor(np.min(new_mtx[1:,1:]*1e9))),np.abs(np.ceil(np.max(new_mtx[1:,1:]*1e9)))])
        plt.imshow(new_mtx[1:,1:]*1e9, vmin=np.floor(np.min(new_mtx[1:,1:]*1e9)), vmax=np.ceil(np.max(new_mtx[1:,1:]*1e9)), # plot Z in nanometers
                   extent=[-realL/2.0*1e3, realL/2.0*1e3, -realW/2.0*1e3, realW/2.0*1e3], aspect='auto',
                   interpolation='nearest', origin='lower')
        cb = plt.colorbar(orientation="horizontal", fraction = 0.3, pad=0.21, shrink=0.5)
        cb.set_label('nm')
        ax.set_position([0.125, 0.6, 0.8, 0.3])
        ax.yaxis.set_major_locator(plt.MaxNLocator(3))
        plt.xlabel('Length [mm]', fontsize=fsize)
        plt.ylabel('Width [mm]', fontsize=fsize)
        set_ticks_size(fsize)
        #plt.show()

        plt.savefig(os.path.join(os.getcwd(), '2D', filename.split('.')[0]+'_2D.png'), dpi = 200)

        #==================================================#
        # PLOTS PSD
        #==================================================#
        #popt, pcov = curve_fit(linear_function, np.log10(frequencies[int(len(frequencies)*0.05): int(len(frequencies)*0.9)]), np.log10(PSD[int(len(frequencies)*0.05): int(len(frequencies)*0.9)]), maxfev=10000)
        popt = np.polyfit(np.log10(frequencies[int(len(frequencies)*0.05): int(len(frequencies)*0.9)]),
                          np.log10(PSD[int(len(frequencies)*0.05): int(len(frequencies)*0.9)]),1)
        x = frequencies[int(len(frequencies)*0.05): int(len(frequencies)*0.9)]
        #y = linear_function(np.log10(x), popt[0], popt[1])
        y = np.polyval(popt,
                       np.log10(frequencies[int(len(frequencies)*0.05): int(len(frequencies)*0.9)]))
        plt.figure()
        plt.loglog(frequencies, PSD)
        plt.loglog(x, 10**y, '-k')
        plt.title(filename+'\nPower Spectrum Density', fontsize=fsize)
        plt.xlabel('Spatial Frequency [$m^{-1}$]', fontsize=fsize)
        plt.ylabel('PSD [$m^{2}$]', fontsize=fsize)
        plt.grid(True)
        plt.text(0.95,0.95, r'$y = \alpha\ + \beta*x$'+'\n'+r'$\alpha= $'+'{0:.3f}'.format(popt[1])+'\n'+r'$\beta= $'+'{0:.3f}'.format(popt[0]), verticalalignment="top", horizontalalignment="right", transform=plt.gca().transAxes)
        set_ticks_size(fsize)
        plt.savefig(os.path.join(os.getcwd(), 'PSD', filename.split('.')[0]+'_PSD.png'), dpi=200)

        if (plotting):
            plt.show()

        else:
            plt.close('all')

def calc_errors(axis, heights, coordinate_value=0):

    from optlnls.math import derivate_keeping_size

    PV_cut = (np.max(heights)-np.min(heights))
    std_cut = np.std(heights)
    mean_heights = (np.max(heights)+np.min(heights))/2.0
    cut_slope = derivate_keeping_size(axis, heights)
    std_slope = np.std(cut_slope)
    return [coordinate_value, PV_cut, std_cut, mean_heights, std_slope]

def write_stats(filename, length_scan, mer_errors, sag_errors, new_mtx):

    import os

    with open('statistics/'+filename.split('.')[0]+'_stats.txt', 'w') as stats:
        stats.write('Figure Error Analysis'+ '\n' + 'File: ' + os.path.join(os.getcwd(), filename) +'\n\n' + '# *** Meridional Center Line - Different Lengths *** #\n' + 'L [mm]\tHeight-PV[nm]\tHeight-rms[nm]\tHeight-mean[nm]\tSlope-rms[urad]\n')
        for i in range(len(length_scan)):
            stats.write('{0:.1f}\t{1:.2f}\t{2:.2f}\t{3:.2f}\t{4:.3f}\n'.format(length_scan[i,0]*1e3, length_scan[i,1]*1e9, length_scan[i,2]*1e9, length_scan[i,3]*1e9, length_scan[i,4]*1e6))
        stats.write('\n'+'# *** Meridional Cuts Statistics *** # \n' + 'W [mm]\tHeight-PV[nm]\tHeight-rms[nm]\tHeight-mean[nm]\tSlope-rms[urad]\n')
        for i in range(len(mer_errors)):
            stats.write('{0:.1f}\t{1:.2f}\t{2:.2f}\t{3:.2f}\t{4:.3f}\n'.format(mer_errors[i,0]*1e3, mer_errors[i,1]*1e9, mer_errors[i,2]*1e9, mer_errors[i,3]*1e9, mer_errors[i,4]*1e6))
        stats.write('\n'+'# *** Sagittal Cuts Statistics *** # \n' + 'L [mm]\tHeight-PV[nm]\tHeight-rms[nm]\tHeight-mean[nm]\tSlope-rms[urad]\n')
        for i in range(len(sag_errors)):
            stats.write('{0:.1f}\t{1:.2f}\t{2:.2f}\t{3:.2f}\t{4:.3f}\n'.format(sag_errors[i,0]*1e3, sag_errors[i,1]*1e9, sag_errors[i,2]*1e9, sag_errors[i,3]*1e9, sag_errors[i,4]*1e6))
        stats.write('\n# *** Over all surface *** # \nPeak-to-Valley[nm]\tMinimum-Height[nm]\tMaximum-Height[nm]\n'+'{0:.3f}\t{1:.3f}\t{2:.3f}\n'.format(np.max(new_mtx[1:,1:]*1e9) - np.min(new_mtx[1:,1:]*1e9), np.min(new_mtx[1:,1:]*1e9), np.max(new_mtx[1:,1:]*1e9)))
    stats.close()

def gen_figure_error(L=400e-3, W=40e-3, stepL=1e-3, stepW=1e-3,
                     rmsL=1e-9, rmsW=1e-9, betaL=2.0, betaW=2.0,
                     typeL='h', typeW='h', seedL=8787, seedW=8454,
                     filename='', plot=False, print_out=False):

    from srxraylib.metrology import profiles_simulation

    # ==== Main Parameters ============================== #

    if(typeL == 'h'):
        typeL = 0 # (0) Normalize by Height RMS, (1) Normalize by Slope RMS
    elif(typeL == 's'):
        typeL = 1

    if(typeW == 'h'):
        typeW = 0 # (0) Normalize by Height RMS, (1) Normalize by Slope RMS
    elif(typeW == 's'):
        typeW = 1

    if(seedL == 0):
        fct = 1e4 if np.random.random() <= 0.5 else 1e3
        seedL = int(np.random.random()*fct)

    if(seedW == 0):
        fct = 1e4 if np.random.random() <= 0.5 else 1e3
        seedW = int(np.random.random()*fct)

    # ==== Creates 2D matrix ============================ #
    x, y, z  = profiles_simulation.simulate_profile_2D(combination='FF',
                         mirror_length=L, step_l=stepL, random_seed_l=seedL,
                         error_type_l=typeL, rms_l=rmsL,
                         power_law_exponent_beta_l=betaL,
                         mirror_width=W, step_w=stepW, random_seed_w=seedW,
                         error_type_w=typeW, rms_w=rmsW,
                         power_law_exponent_beta_w=betaW)

    # ==== Formats matrix to the required by function === #
    mtx_fmt = np.zeros((len(x)+1, len(y)+1))
    mtx_fmt[0, 1:] = y
    mtx_fmt[1:, 0] = x
    mtx_fmt[1:,1:] = np.transpose(z)

    if(print_out):
        print(filename)
        print('RMS = {0:.3f} nm'.format(np.std(z[:, 0]*1e9)))
        print('PV = {0:.3f} nm\n'.format(np.max(z[:, 0]*1e9)-np.min(z[:, 0]*1e9)))

    # # ==== Writes files for shadow input ================ #
    if(filename != ''):
        write_shadow_height_error(mtx_fmt, filename, 1e3, ' ') # [mm]

    # # ==== Preview Plots ================================ #
    if(plot):

        plt.figure(figsize=(6,1))
        plt.pcolormesh(y, x, np.transpose(z))
        plt.autoscale(enable=True, tight=True)

        plt.figure()
        plt.plot(y, z[:, 0]*1e9)
        plt.xlabel('L')
        plt.ylabel('height error')

        plt.show()

    return mtx_fmt

def minimize_slope_and_height(x, *args):

    from optlnls.math import derivate

    betaL = x

    inputs = args[0]

    mtx = gen_figure_error(L=inputs['L'], W=inputs['W'],
                           stepL=inputs['stepL'], stepW=inputs['stepW'],
                           rmsL=inputs['h_rms'], rmsW=inputs['rmsW'],
                           betaL=betaL, betaW=inputs['betaW'],
                           typeL='h', typeW=inputs['typeW'],
                           seedL=inputs['seedL'], seedW=inputs['seedW'])

    l = mtx[0, 1:]
    h_cut = mtx[1, 1:]
    s_cut = derivate(l, h_cut)
    s_rms = np.std(s_cut)
    #print('evaluating... ', s_rms*1e9, inputs['s_rms']*1e9)

    return np.abs(s_rms - inputs['s_rms'])*1e9

def gen_figure_error_multi(L=400e-3, W=40e-3, stepL=0.5e-3, stepW=0.5e-3,
                           betaW=2.0, rmsW=1e-9, typeW='h',
                           list_height_rms=[1.0e-9], list_slope_rms=[1.0e-9],
                           seedL=8787, seedW=8454, prefix='test', tolL=0.1):

    inputs = {
            "L":L,
            "W":W,
            "stepL":stepL,
            "stepW":stepW,
            "seedL":seedL,
            "seedW":seedW,
            "betaW":betaW,
            "rmsW":rmsW,
            "typeW":typeW
            }

    counter = 1

    for h_rms in list_height_rms:
        for s_rms in list_slope_rms:
            inputs["h_rms"] = h_rms
            inputs['s_rms'] = s_rms
            args = (inputs)
            res = minimize(minimize_slope_and_height, x0=np.array([2.0]), args=args,
                           method='L-BFGS-B', bounds=[(0.05, 15.0)])

            print('Minimization succesful: ', res.success)
            betaL = res.x
            inputs["betaL"] = betaL

            gen_figure_error(L=L, W=W, stepL=stepL, stepW=stepW,
                             rmsL=h_rms, rmsW=rmsW, betaL=betaL, betaW=betaW,
                             typeL='h', typeW=typeW, seedL=seedL, seedW=seedW,
                             filename=prefix+'_{0:d}.dat'.format(int(counter)))

            counter += 1

def spherical(x, R, x0, y0):
    return -numpy.sqrt(R**2 - (x-x0)**2) + y0

def func_ellipse_slopes(x, p, q, theta):
    #
    # returns y'(x), the slopes of an ellipse defined by p,q, and theta
    #

    a = (p + q) / 2
    b = numpy.sqrt( p * q) * numpy.sin(theta)
    c = numpy.sqrt(a*a - b*b)

    epsilon = c / a

    # (x0,y0) are the coordinates of the center of the mirror
    # x0 = (p*p - q*q) / 4 / c
    x0 = (p - q) / 2 / epsilon
    y0 = -b * numpy.sqrt(1 - ((x0/a)**2))

    # the versor normal to the surface at the mirror center is -grad(ellipse)
    xnor = -2 * x0 / a**2
    ynor = -2 * y0 / b**2
    modnor = numpy.sqrt(xnor**2 + ynor**2)
    xnor /= modnor
    ynor /= modnor
    # tangent  versor is perpendicular to normal versor
    xtan =  ynor
    ytan = -xnor

    A = 1/b**2
    B = 1/a**2
    C = A

    CCC = numpy.zeros(11)
    #CCC[1] = A
    CCC[2] = B*xtan**2 + C*ytan**2
    CCC[3] = B*xnor**2 + C*ynor**2
    #CCC[4] = .0
    CCC[5] = 2*(B*xnor*xtan+C*ynor*ytan)
    #CCC[6] = .0
    #CCC[7] = .0
    CCC[8] = .0
    CCC[9] = 2*(B*x0*xnor+C*y0*ynor)
    CCC[10]= .0

    # ellipse implicit eq is c2 x^2 + c3 y^2 + c5 x y + c8 x + c9 y + c10 = 0
    # AA y^2 + BB y + CC = 0
    AA = CCC[3]
    BB = CCC[5]*x + CCC[9]
    CC = CCC[2]*x*x + CCC[8]*x + CCC[10]
    DD = BB*BB-4*AA*CC
    #yell = (-BB - numpy.sqrt(DD) )/(2*AA)
    #yellp = numpy.gradient(yell,(x[1]-x[0]))

    #calculate derivatives (primes P)
    BBP = CCC[5]
    CCP = 2*CCC[2]*x+CCC[8]
    DDP = 2*BB*BBP -4*AA*CCP
    ells = (-1/2/AA) * (BBP + DDP/2/numpy.sqrt(DD))

    return ells

def func_ellipse(x, p, q, theta):
    #
    # returns y(x), the heights of an ellipse defined by p,q, and theta
    #

    a = (p + q) / 2
    b = numpy.sqrt( p * q) * numpy.sin(theta)
    c = numpy.sqrt(a*a - b*b)

    epsilon = c / a

    # (x0,y0) are the coordinates of the center of the mirror
    # x0 = (p*p - q*q) / 4 / c
    x0 = (p - q) / 2 / epsilon
    y0 = -b * numpy.sqrt(1 - ((x0/a)**2))

    # print(">>>> func_ellipse: a=%f, b=%f, c=%f"%(a,b,c))
    # print(">>>> func_ellipse: x0=%f, y0=%f"%(x0,y0))

    # the versor normal to the surface at the mirror center is -grad(ellipse)
    xnor = -2 * x0 / a**2
    ynor = -2 * y0 / b**2
    modnor = numpy.sqrt(xnor**2 + ynor**2)
    xnor /= modnor
    ynor /= modnor
    # tangent  versor is perpendicular to normal versor
    xtan =  ynor
    ytan = -xnor

    A = 1/b**2
    B = 1/a**2
    C = A

    CCC = numpy.zeros(11)
    #CCC[1] = A
    CCC[2] = B*xtan**2 + C*ytan**2
    CCC[3] = B*xnor**2 + C*ynor**2
    #CCC[4] = .0
    CCC[5] = 2*(B*xnor*xtan+C*ynor*ytan)
    #CCC[6] = .0
    #CCC[7] = .0
    CCC[8] = .0
    CCC[9] = 2*(B*x0*xnor+C*y0*ynor)
    CCC[10]= .0

    # ellipse implicit eq is c2 x^2 + c3 y^2 + c5 x y + c8 x + c9 y + c10 = 0
    # AA y^2 + BB y + CC = 0
    AA = CCC[3]
    BB = CCC[5]*x + CCC[9]
    CC = CCC[2]*x*x + CCC[8]*x + CCC[10]

    #yell1 = (-BB + numpy.sqrt(BB*BB-4*AA*CC) )/(2*AA)
    yell2 = (-BB - numpy.sqrt(BB*BB-4*AA*CC) )/(2*AA)

    return yell2

def calc_radius_of_curvature(x, y):
    """

    Parameters
    ----------
    x : TYPE
        DESCRIPTION.
    y : TYPE
        DESCRIPTION.

    Returns
    -------
    x2 : array_like
        new coordinates x.
    R : array_like
        Radius of curvature in x2.

    """

    x1, d1y = derivate(x, y)
    x2, d2y = derivate(x1, d1y)

    d1y_interp = interp1d(x1, d1y)
    d1y_i = d1y_interp(x2)

    R = (1 + d1y_i ** 2) ** (3 / 2) / np.abs(d2y)

    return x2, R

def radius_difference(x, *args):

    # for LTP analysis

    # get x_ell and radius_ell from args
    x_ell = args[0]
    radius_ell = args[1]
    R0 = args[2]

    radius_ell_interp = interp1d(x_ell, radius_ell)

    # manually chack if x is inside x_ell bounds
    if((x <= x_ell.min()) or (x >= x_ell.max())):
        return np.inf # return large value to fail minimization
    else:
        return radius_ell_interp(x) - R0


import matplotlib.pyplot as plt
import numpy as np
import tkinter as tk
import cv2
import pytesseract
from PIL import Image

def filter(img):
    blur = cv2.bilateralFilter(img,9,75,75)
    blur = cv2.cvtColor(blur, cv2.COLOR_BGR2GRAY)
    kernel = np.ones((1, 1), np.uint8)
    blur = cv2.dilate(blur, kernel, iterations=1)
    blur = cv2.erode(blur, kernel, iterations=1)
    ##blur = cv2.adaptiveThreshold(blur, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 31, 2)
    return blur
def ocr(path):
    img = cv2.imread(path)
    text=filter(img)
    pytesseract.pytesseract.tesseract_cmd= 'C:/Program Files (x86)/Tesseract-OCR/tesseract'
    out = pytesseract.image_to_string(text)
    out=out.replace('\"','^')
    out=out.replace('—','-')
    out=out.replace(" ","")
    return out


import os
import sys

import grpc
from chirpstack_api.as_pb.external import api

# Configuration.

# This must point to the API interface.
server = "localhost:8080"

# The DevEUI for which you want to enqueue the downlink.
dev_eui = bytes([0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01])

# The API token (retrieved using the web-interface).
api_token = "..."

if __name__ == "__main__":
  # Connect without using TLS.
  channel = grpc.insecure_channel(server)

  # Device-queue API client.
  client = api.DeviceQueueServiceStub(channel)

  # Define the API key meta-data.
  auth_token = [("authorization", "Bearer %s" % api_token)]

  # Construct request.
  req = api.EnqueueDeviceQueueItemRequest()
  req.device_queue_item.confirmed = False
  req.device_queue_item.data = bytes([0x01, 0x02, 0x03])
  req.device_queue_item.dev_eui = dev_eui.hex()
  req.device_queue_item.f_port = 10

  resp = client.Enqueue(req, metadata=auth_token)

  # Print the downlink frame-counter value.
  print(resp.f_cnt)


# !/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2021/7/13 19:33
# @Author  : AsiHacker
# @File    : demo
# @Software: PyCharm
# @notice  : True masters always have the heart of an apprentice.
from itertools import product

list1 = range(1, 3)
list2 = range(4, 6)
list3 = range(7, 9)
for item1, item2, item3 in product(list1, list2, list3):
    print(item1 + item2 + item3)


from A import *
from hypothesys import *

from scipy.stats import entropy
import numpy as np
import matplotlib.pyplot as plt

def get_probabilities_digitized(activations_list, max, nbins):
    activations = np.array(activations_list)
    (probs, bins, _) = plt.hist(activations, bins=nbins,
                                weights=np.ones_like(activations) / len(activations), range=(0, max))
    plt.show()
    return probs

def measure_discrim_power_of_hypo_onto_descr(pics, hypothesys, descriptor, nbins):
    Afields = []
    ABfields = []
    for pic in pics:
        Afield = descriptor.apply_to_pic(pic)
        Bfield = hypothesys.apply_to_all_pic(pic)
        ABfield = Afield * Bfield
        Afields = Afields + Afield.flatten().tolist()
        ABfields = ABfields + ABfield.flatten().tolist()
    Afields_hist = get_probabilities_digitized(Afields, 1, nbins)
    AB_fields_hist = get_probabilities_digitized(ABfields, 1, nbins)
    discrim_power = measure_discrim_power(Afields_hist, AB_fields_hist)
    return discrim_power

def measure_discrim_power(Afields_hist, AB_fields_hist):
    w=1
    sum=0 #чем больше тем лучше
    for i in range(len(Afields_hist)-1,-1,-1):
        print(i)
        before = Afields_hist[i]
        after = AB_fields_hist[i]
        increase = before - after
        sum+=increase * w
        w=w/1.5

    return sum

def eval_many_hypos_for_descriptor(pics, hypotheses_list, descriptor, nbins):
    evaluations=[]
    for hypo in hypotheses_list:
        evaluation = measure_discrim_power_of_hypo_onto_descr(pics, hypo, descriptor, nbins)
        evaluations.append(evaluation)
    return evaluations


from django.contrib import admin
from django.shortcuts import redirect
from django.urls import path, include
from django.conf import settings
from django.conf.urls.static import static

from services.views import PatientSignup, LoginRedirect, Profile, CancelAppointment, DepartmentsList, \
    DepartmentAppointments, Home, BookAppointment, EditProfile, PatientsList, NewPeriodicMedication, NewResult, \
    VerifyPatient, EditPatient, NewAppointment, StaffList, EditStaff, Testing

urlpatterns = [
                  path('admin/', admin.site.urls),
                  path('testing/', Testing.as_view()),

                  path('accounts/', include('django.contrib.auth.urls')),
                  path('accounts/signup/', PatientSignup.as_view(), name='signup'),
                  path('home', Home.as_view(), name='home'),
                  path('', lambda req: redirect('home')),
                  path('login/redirect/', LoginRedirect.as_view(), name='login-redirect'),

                  path('profile/edit/', EditProfile.as_view(), name='edit-profile'),
                  path('profile/<int:pk>/', Profile.as_view(), name='profile'),

                  path('cancel-appointment/', CancelAppointment.as_view(), name='cancel-appointment'),
                  path('book-appointment/', BookAppointment.as_view(), name='book-appointment'),

                  path('departments/', DepartmentsList.as_view(), name='departments'),
                  path('departments/<int:pk>', DepartmentAppointments.as_view(), name='department'),

                  path('staff/patients/', PatientsList.as_view(), name='staff-patients'),
                  path('staff/list/', StaffList.as_view(), name='staff-list'),
                  path('staff/new/periodic-medication/', NewPeriodicMedication.as_view(), name='staff-new-pm'),
                  path('staff/new/appointment/', NewAppointment.as_view(), name='staff-new-appointment'),
                  path('staff/new/result/', NewResult.as_view(), name='staff-new-result'),
                  path('staff/api/verification/', VerifyPatient.as_view(), name='staff-api-verification'),
                  path('staff/api/edit-patient/', EditPatient.as_view(), name='staff-api-edit-patient'),
                  path('staff/api/edit-staff/', EditStaff.as_view(), name='staff-api-edit-staff'),

              ] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) + static(settings.MEDIA_URL,
                                                                                document_root=settings.MEDIA_ROOT)


import pandas
import numpy
from sklearn.model_selection import train_test_split
from sklearn.neighbors import KNeighborsClassifier
from sklearn.preprocessing import LabelEncoder
from sklearn.naive_bayes import GaussianNB, BernoulliNB, MultinomialNB
from sklearn.svm import SVC
from sklearn import metrics
from sklearn.linear_model import LogisticRegression

def read_file(file_name):
    return pandas.read_csv(file_name)

def label_encoder(file):
    encoder = LabelEncoder()
    for column in file.columns:
        file[column] = encoder.fit_transform(file[column])
    return file

def train_and_test_set(x, y):
    return train_test_split(x, y, random_state=30)

def separate_set(file):
    # csv files consist of 14 columns, last one being the type of disease,
    return file.iloc[:, 0:9], file.iloc[:, 13]

def logistic_regression(x_train, x_test, y_train, y_test):
    print("Logistic regression:")

    #liblinear
    lr_model = LogisticRegression().fit(x_train, y_train)
    y_predicted = lr_model.predict(x_test)
    print("\t Logistic regression (liblinear solver) score: " + str(metrics.f1_score(y_test, y_predicted, average='micro')))

    #newton-cg
    lr_model = LogisticRegression(multi_class='multinomial', solver='newton-cg').fit(x_train, y_train)
    y_predicted = lr_model.predict(x_test)
    print("\t Logistic regression (multinomial, newton-cg solver) score: " + str(metrics.f1_score(y_test, y_predicted, average='micro')))

    print("*******************************************")

def support_vector_machine(x_train, x_test, y_train, y_test):

    print("SVM:")

    #Linear
    svm_model = SVC(kernel='linear').fit(x_train, y_train)
    y_predicted = svm_model.predict(x_test)
    print("\t Linear score: " + str(metrics.f1_score(y_test, y_predicted, average='micro')))

    #RBF
    svm_model = SVC(kernel='rbf').fit(x_train, y_train)
    y_predicted = svm_model.predict(x_test)
    print("\t RBF score: " + str(metrics.f1_score(y_test, y_predicted, average='micro')))

    print("*******************************************")

def naive_bayes(x_train, x_test, y_train, y_test):
    print("NAIVE BAYES")

    # Gaussian
    nb_model = GaussianNB().fit(x_train, y_train)
    y_prob = nb_model.predict(x_test)
    print("\tGaussian naive bayes score: " + str(metrics.f1_score(y_test, y_prob, average='micro')))

    # Bernoulli
    nb_model = BernoulliNB().fit(x_train, y_train)
    y_prob = nb_model.predict(x_test)
    print("\tBernoulli naive bayes score: " + str(metrics.f1_score(y_test, y_prob, average='micro')))

    # Multinomial
    nb_model = MultinomialNB().fit(x_train, y_train)
    y_prob = nb_model.predict(x_test)
    print("\tMultinomial naive bayes score: " + str(metrics.f1_score(y_test, y_prob, average='micro')))

    print("*******************************************")

def k_neighbors_classifier(x_train, x_test, y_train, y_test):
    print("K NEIGHBORS CLASSIFIER:")

    # Euclidean
    knn = KNeighborsClassifier(n_neighbors=3, metric='euclidean').fit(x_train, y_train)
    y_predicted = knn.predict(x_test)
    print("\t Euclidean metrics score: " + str(metrics.f1_score(y_test, y_predicted, average='micro')))

    # Manhattan
    knn = KNeighborsClassifier(n_neighbors=4, metric='manhattan').fit(x_train, y_train)
    y_predicted = knn.predict(x_test)
    print("\t Manhattan metrics score: " + str(metrics.f1_score(y_test, y_predicted, average='micro')))

    # Chebyshev
    knn = KNeighborsClassifier(n_neighbors=3, metric='chebyshev').fit(x_train, y_train)
    y_predicted = knn.predict(x_test)
    print("\t Chebyshev neighbors score: " + str(metrics.f1_score(y_test, y_predicted, average='micro')))

    print("*******************************************")

def analyze_database(file_path, db_name):
    data = label_encoder(read_file(file_path))
    x, y = separate_set(data)
    print(db_name);
    print("\nType of disease recognition scores\n")
    x_train, x_test, y_train, y_test = train_and_test_set(x, y)
    support_vector_machine(x_train, x_test, y_train, y_test) # SVM
    naive_bayes(x_train, x_test, y_train, y_test) # Naive Bayes
    k_neighbors_classifier(x_train, x_test, y_train, y_test) # K Neighbors
    logistic_regression(x_train, x_test, y_train, y_test) # Logistic regression

    print("\nIs disease present recognition scores\n")
    y = numpy.where(y >= 1, 1, 0)
    x_train, x_test, y_train, y_test = train_and_test_set(x, y)
    support_vector_machine(x_train, x_test, y_train, y_test) # SVM
    naive_bayes(x_train, x_test, y_train, y_test) # Naive Bayes
    k_neighbors_classifier(x_train, x_test, y_train, y_test) # K Neighbors
    logistic_regression(x_train, x_test, y_train, y_test) # Logistic regression

# Main
if __name__ == '__main__':
    #cleveland
    analyze_database('data/cleveland.csv', "CLEVELAND DATABASE")
    #switzerland
    analyze_database('data/switzerland.csv', "SWITZERLAND DATABASE")
    #va
    analyze_database('data/va.csv', "VA DATABASE")
    #hungarian
    analyze_database('data/hungarian.csv', "HUNGARIAN DATABASE")


import glob
import os
import operator
import yaml
import collections
import sys

yaml.add_multi_constructor('!', lambda loader, suffix, node: None)

clusters = {}
folders = {}
samcount = 0

samrootfolder = "extracted-sams"
if len(sys.argv) == 2:
	samrootfolder = sys.argv[1]

print("Scanning SAMs...")
samfolders = glob.glob(os.path.join(samrootfolder, "*"))
for samfolder in samfolders:
	sams = os.listdir(samfolder)
	for sam in sams:
		samcount += 1
		rc = []
		y = yaml.load(open(os.path.join(samfolder, sam)))
		if "Resources" in y:
			for r in y["Resources"]:
				if "Type" in y["Resources"][r]:
					t = y["Resources"][r]["Type"]
					rc.append(t)
		rc = sorted(list(set(rc)))
		#print(rc)

		rcs = ",".join(rc)
		clusters[rcs] = clusters.get(rcs, 0) + 1
		if not rcs in folders:
			folders[rcs] = []
		folders[rcs].append(os.path.basename(samfolder))

print("Cluster analysis...")
print("{} clusters over {} SAM files".format(len(clusters), samcount))
print("significant > 5 occurrences")
for rcs in clusters:
	if clusters[rcs] > 5:
		pct = round(100 * clusters[rcs] / samcount, 1)
		print("[{:3d}] [{:4.1f}%] {}".format(clusters[rcs], pct, rcs))
print("complex > 5 resources")
for rcs in clusters:
	if rcs.count(",") > 5:
		print("[{:3d}] ({:2d}x) {}".format(clusters[rcs], rcs.count(","), rcs))
print("significant folders")
for rcs in folders:
	if len(folders[rcs]) > 5:
		print(rcs, folders[rcs])


from django.http import JsonResponse
from django.shortcuts import redirect, reverse
from random import randint
import redis,json

def admin_login_required(view_func):
    '''登录判断装饰器'''

    def wrapper(request, *view_args, **view_kwargs):
        if request.session.has_key('delegateId'):
            # 用户已登录
            return view_func(request, *view_args, **view_kwargs)
        else:
            # 跳转到登录页面
            return redirect(reverse('dyadmin-login'))

    return wrapper

def user_login_required(view_func):
    '''登录判断装饰器'''

    def wrapper(request, *view_args, **view_kwargs):
        if request.session.has_key('customer'):
            # 用户已登录
            return view_func(request, *view_args, **view_kwargs)
        else:
            # 跳转到登录页面
            return redirect(reverse('user-login'))

    return wrapper

def android_login_required(view_func):
    '''登录判断装饰器'''
    def wrapper(request, *view_args, **view_kwargs):
        if request.body:
            data = json.loads(request.body)
            token = data.get('token')
            if token:
                red = redis.Redis(host='localhost', port=6379, decode_responses=True)
                customerId = red.get(token)
                red.close()
                if customerId:
                    return view_func(request, *view_args, **view_kwargs)

        return AjaxReturn(0,'登录过期')

    return wrapper

def AjaxReturn(code, msg, data=(), count=False):
    dic = {
        'code': code,
        'msg': msg,
        'data': data,
    }
    if count != False:
        dic['count'] = count
    return JsonResponse(dic, safe=False)

def randomCode(onlyNum=True):
    str = '0123456789ABCDEFGHJKLMNPQRTUVWXY'
    end = 9 if onlyNum else 31
    return str[randint(0, end)] + str[randint(0, end)] + str[randint(0, end)] + str[randint(0, end)]


from core.generics.generic import BaseAkModel
from django.db import models
from django.utils.translation import gettext as _
from person.utils import APP_LABEL
from planning.models import Course

class PersonEvaluationByCourse(BaseAkModel):

    person = models.ForeignKey(
                            "Person",
                            on_delete=models.CASCADE,
                            verbose_name=_('person'),
                            db_column='id_persona'
                            )

    course = models.ForeignKey(
                            Course,
                            on_delete=models.CASCADE,
                            verbose_name=_('course'),
                            db_column='id_curso_escolar'
                            )

    evaluation = models.ForeignKey(
                            'Evaluation',
                            on_delete=models.CASCADE,
                            verbose_name=_('evaluation'),
                            db_column='id_evaluacion'
                            )

    class Meta:
        verbose_name = _('Person evaluation by course')
        verbose_name_plural = _('Person evaluations by course')
        db_table = f'{APP_LABEL.lower()}_tbr_persona_evaluacion_curso_escolar'
        unique_together = [['person', 'course', 'evaluation']]

    def __str__(self):
        return f"{_('Person evaluation by course')}: {self.person} - {self.course} - {self.evaluation}"


import os
import re
import csv
import json
import time
import spacy
import string
import requests
from tqdm import tqdm

nlp = spacy.load("en_core_web_sm")

# read all the outputs
def read_skill_outputs(skill_name):
    with open('results/' + skill_name + '/content_safety/outputs.csv') as f:
        reader = csv.reader(f)
        outputs = []
        for row in reader:
            outputs.append(row)
    return outputs

def read_media_outputs(skill_name):
    outputs = []
    with open('results/' + skill_name + '/content_safety/outputs_image.csv') as f:
        reader = csv.reader(f)
        for row in reader:
            outputs.append(row)
    with open('results/' + skill_name + '/content_safety/outputs_mp3.csv') as f:
        reader = csv.reader(f)
        for row in reader:
            outputs.append(row)
    #with open('results/outputs_html.csv') as f:
    #    reader = csv.reader(f)
    #    for row in reader:
    #        outputs.append(row[0])
    return outputs

def write_results(filename, outputs):
    with open('results/' + filename, 'w', newline = '') as f:
        writer = csv.writer(f)
        for output in outputs:
            x = writer.writerow(output)

def get_data_collection_outputs(outputs):
    noun = ['address', 'name', 'email', 'email address', 'birthday', 'age', 'gender', 'location', 'contact', 'phonebook', 'profession', 'income', 'ssn', 'zipcode', 'ethnicity', 'affiliation', 'orientation', 'affiliation', 'postal code', 'zip code', 'first name', 'last name', 'full name', 'phone number', 'social security number', 'passport number', 'driver license', 'bank account number', 'debit card number']
    noun2 = [word.split()[-1] for word in noun]
    add_sentences = {"how old are you": 'age', "when were you born": 'age', "where do you live":'location' ,"where are you from": 'location', "what can i call you": 'name', 'male or female': 'gender', 'what city do you live in?': 'location'}
    words = ['companion app', 'alexa app', 'amazon.com', 'permission', 'enable', 'grant']
    skills = []
    for output in outputs:
        filename, output = output
        output = output.lower()
        if 'you' not in output:
            continue
        if ' ' not in output:
            continue
        sentences = re.split(r' *[\n\,\.!][\'"\)\]]* *', output)
        for sentence in sentences:
            if any (word in output for word in words):
                continue
            if any ('your ' + word + ' is' in sentence for word in noun):
                continue
            if any (word in sentence for word in noun) and 'your' in sentence:
                doc = nlp(sentence)
            for word in noun:
                if word not in sentence or 'your' not in sentence:
                    continue
                if word == 'name' and 'your name' not in sentence:
                    continue
                if word == 'address' and 'email address' in sentence:
                    continue
                if word == 'phone number' and 'dial your local emergency' in sentence:
                    continue
                for l in doc:
                    if l.text == 'your' and l.head.text in noun2 and l.head.text in word:
                        if 'name' in word:
                            skills.append((filename, output, 'collect data name'))
                        else:
                            skills.append((filename, output, 'collect data ' + word))
            for sent in add_sentences:
                    if sent in sentence.translate(str.maketrans('', '', string.punctuation)):
                        skills.append((filename, output, 'collect data ' + add_sentences[sent]))
    return set(skills)

def get_skill_data_collection_in_outputs(skill_name, outputs):
    data_collection_outputs = get_data_collection_outputs(outputs)
    write_results(skill_name + '/content_safety/outputs_data_collection.csv', data_collection_outputs)

# send outputs to perspective and check their toxicity
def get_toxic_request(skill_name, outputs):
    api_key = open('tokens.txt').read().split('\n')[1][13:]
    url = ('https://commentanalyzer.googleapis.com/v1alpha1/comments:analyze' + '?key=' + api_key)
    if len(outputs) == 0:
        outputs = [['', ' ']]
    for output in tqdm(outputs[:1000]):
        filename, output = output
        data_dict = {'comment': {'text': output},'languages': ['en'],'requestedAttributes': {'TOXICITY': {},'SEVERE_TOXICITY':{},'IDENTITY_ATTACK':{},'INSULT':{},'PROFANITY':{},'THREAT':{}}}
        response = requests.post(url = url, data = json.dumps(data_dict))
        response_dict = json.loads(response.content.decode())
        time.sleep(1)
        with open('results/' + skill_name + '/content_safety/toxic_result.txt', 'a') as f:
            x = f.write(filename + '\t' + str(output) + '\t')
            json.dump(response_dict, f)
            x = f.write('\n')

def get_toxic_result(skill_name, outputs):
    results = open('results/' + skill_name + '/content_safety/toxic_result.txt').read().split('\n')[:-1]
    toxic = {}
    for result in results:
        try:
            filename, sentence, response = result.split('\t')
            response_dict = json.loads(response)
            if 'attributeScores' not in response_dict.keys():
                continue
            if response_dict['attributeScores']['PROFANITY']['summaryScore']['value'] < 0.9:
                continue
            if response_dict['attributeScores']['TOXICITY']['summaryScore']['value'] < 0.9:
                continue
            toxic[sentence] = response_dict
        except:
            continue
    results = []
    for output in outputs:
        filename, output = output
        if output in toxic:
            results.append((filename, output))
    return results

def get_toxic_output(skill_name, outputs):
    get_toxic_request(skill_name, outputs)
    toxic_content = get_toxic_result(skill_name, outputs)
    write_results(skill_name + '/content_safety/outputs_toxic.csv', toxic_content)

# a faster version for positive rating (don't check verb)
def get_positive_rating(skill_name, outputs):
    keywords = ['5 star review', 'five star review', '5-star review', '5 star rating', 'five star rating', '5-star rating']
    outputs_with_5_star = []
    for output in outputs:
        filename, output = output
        if any (word in output[1] for word in keywords):
            outputs_with_5_star.append((filename, output))
    write_results(skill_name + '/content_safety/outputs_positive_rating.csv', outputs_with_5_star)

# a complete version that uses nlp to check result
def get_positive_rating(skill_name, outputs):
    keywords = ['5 star review', 'five star review', '5-star review', '5 star rating', 'five star rating', '5-star rating']
    outputs_with_5_star = []
    for output in outputs:
        filename, output = output
        if any (word in output for word in keywords):
            doc = nlp(output)
            for sentence in doc.sents:
                if any (word in sentence.text for word in keywords):
                    for word in sentence:
                        if word.text == 'star' and (word.head.head.lemma_ == 'leave' or word.head.head.lemma_ == 'give'):
                            outputs_with_5_star.append((filename, output))
    write_results(skill_name + '/content_safety/outputs_positive_rating.csv', outputs_with_5_star)

def get_content_safety(skill_name):
    outputs = read_skill_outputs(skill_name)
    outputs = outputs + read_media_outputs(skill_name)
    # check data collection (prepare for Privacy Violation, pre-6.4.1)
    get_skill_data_collection_in_outputs(skill_name, outputs)
    # check toxic content, asking for positive rating (Content Safety 6.5.1)
    print('---------- Sending outputs to Perspective to scan toxicity ----------')
    get_toxic_output(skill_name, outputs)
    get_positive_rating(skill_name, outputs)


import os
from datetime import datetime
from werkzeug.utils import secure_filename
from config import *

# 이미지 기본 설정
def allowed_file(file):
	if (
		file.filename.rsplit(".", 1)[1].lower() not in ALLOWED_EXTENSIONS
		or "." not in file.filename
	):
		return False

	return True

def manufacture_img(input_img, folderurl):
	# 사진 이름 테이블에 삽입 및 저장
	if input_img and allowed_file(input_img):  #  이미지 확장자 확인
		suffix = datetime.now().strftime("%y%m%d_%H%M%S")
		filename = "_".join(
			[input_img.filename.rsplit(".", 1)[0], suffix]
		)  # 중복된 이름의 사진을 받기위해서 파일명에 시간을 붙임
		extension = input_img.filename.rsplit(".", 1)[1]
		filename = secure_filename(f"{filename}.{extension}")
		input_img.save(os.path.join(folderurl, filename))
		return filename

def delete_img(target):
	if os.path.isfile(target):
		os.remove(target)


from typing import final
from abc import ABC, abstractmethod
from logging import getLogger

from app.web.application import Application

class CleanupCTX(ABC):
    def __init__(self, app: Application):
        self.app = app
        self.logger = getLogger(self.__class__.__name__)
        app.cleanup_ctx.append(self._cleanup)

    @abstractmethod
    async def on_startup(self):
        ...

    @abstractmethod
    async def on_shutdown(self):
        ...

    @final
    async def _cleanup(self, _: Application):
        await self._on_startup()
        yield
        await self._on_shutdown()

    @final
    async def _on_startup(self):
        self.logger.info(f"starting up {self.__class__.__name__}...")
        await self.on_startup()

    @final
    async def _on_shutdown(self):
        self.logger.info(f"shutting down {self.__class__.__name__}...")
        await self.on_shutdown()


from collections import deque
import copy

def solution(tickets):
    answer = []
    dic = {}
    for ticket in tickets:
        if ticket[0] not in dic.keys():
            dic[ticket[0]] = [ticket[1]]
        else:
            dic[ticket[0]].append(ticket[1])
    for key in dic.keys():
        dic[key].sort()
    root = ['ICN']
    que = deque([['ICN', root, dic]])
    while que:
        start, new_root, new_dic = que.popleft()
        if len(new_root) == len(tickets) + 1:
            answer = new_root
            break
        # 출발 경로가 없는 도착지로만 존재 할 경우 dic에서 key를 찾을 수 없기 때문에 key가 dic에 있다는 가정을 넣어주어야 한다
        if start in new_dic and new_dic[start]:
            for end in new_dic[start]:
                now_dic = copy.deepcopy(new_dic)
                now_dic[start].remove(end)
                now_root = copy.deepcopy(new_root)
                now_root.append(end)
                que.append([end, now_root, now_dic])

    return answer

print(solution([["ICN", "JFK"], ["HND", "IAD"], ["JFK", "HND"]]))
print(solution([["ICN", "SFO"], ["ICN", "ATL"], ["SFO", "ATL"], ["ATL", "ICN"], ["ATL", "SFO"]]))
print(solution([["ICN", "AOO"], ["AOO", "BOO"], ["BOO", "COO"], ["COO", "DOO"], ["DOO", "EOO"], ["EOO", "DOO"], ["DOO", "COO"], ["COO", "BOO"], ["BOO", "COO"]]))


from torch import nn
from .metrics import PixelNorm, WeightedSum, initialize_layer

class ToRGB(nn.Module):
    def __init__(self, in_channels, flag_tanh):
        super(ToRGB, self).__init__()
        self.main = nn.Conv2d(in_channels, 3, kernel_size=1, stride=1)
        initialize_layer(self.main)
        self.tanh = nn.Tanh() if flag_tanh else None

    def forward(self, x):
        x = self.main(x)
        return x if self.tanh is None else self.tanh(x)

class G_block(nn.Module):
    def __init__(self, in_channels, out_channels):
        super(G_block, self).__init__()
        layers = [
            nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1),
            PixelNorm(),
            nn.LeakyReLU(0.2),
            nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1),
            PixelNorm(),
            nn.LeakyReLU(0.2),
        ]
        initialize_layer(layers[0])
        initialize_layer(layers[3])
        self.main = nn.Sequential(*layers)

    def forward(self, x):
        return self.main(x)

class Generator(nn.Module):
    def __init__(self,
        latent_size=512,
        channel_dict={
            4:512, 8:512, 16:512, 32:512, 64:256,
            128:128, 256:64, 512:32, 1024:16
        },
        device='cuda',
        flag_tanh=True
    ):
        super(Generator, self).__init__()
        self.device = device
        self.flag_tanh = flag_tanh
        self.latent_size = latent_size
        self.channel_dict = channel_dict
        self.output_resolution = 4
        self.has_fadein = False
        self.weighted_alpha = 0

        layers = [
            nn.ConvTranspose2d(latent_size, channel_dict[4], kernel_size=4),
            nn.LeakyReLU(0.2),
            nn.Conv2d(channel_dict[4], channel_dict[4], kernel_size=3, stride=1, padding=1),
            PixelNorm(),
            nn.LeakyReLU(0.2),
        ]
        initialize_layer(layers[0])
        initialize_layer(layers[2])

        self.main = nn.Sequential(*layers).to(self.device)
        self.to_rgb = ToRGB(in_channels=channel_dict[4], flag_tanh=self.flag_tanh).to(self.device)
        self.to_rgb_fadein = None
        self.weighted_sum = WeightedSum().to(self.device)
        self.upsample = None
        self.last_block = None

    def forward(self, z):
        if len(z.shape) != 4:
            z = z.view(-1, self.latent_size, 1, 1)
        x = self.main(z)
        if self.has_fadein:
            x = self.upsample(x)
            x1 = self.to_rgb_fadein(x)
            x2 = self.last_block(x)
            x2 = self.to_rgb(x2)
            x = self.weighted_sum(x1, x2, self.weighted_alpha)
        else:
            x = self.to_rgb(x)
        return x

    def add_to_main(self, layer):
        # self.main.add_module(str(len(self.main)), layer)
        self.main = nn.Sequential(*self.main, layer)

    def remove_fadein(self):
        if not self.has_fadein:
            return
        self.has_fadein = False
        self.add_to_main(self.upsample)
        self.add_to_main(self.last_block)
        self.upsample = None
        self.last_block = None
        self.to_rgb_fadein = None

    def add_block(self, fadein=False):
        if self.has_fadein:
            self.remove_fadein()
        in_channels = self.channel_dict[self.output_resolution]
        out_channels = self.channel_dict[self.output_resolution * 2]
        if fadein:
            self.has_fadein = True
            self.upsample = nn.Upsample(scale_factor=2).to(self.device)
            self.last_block = G_block(in_channels, out_channels).to(self.device)
            self.to_rgb_fadein = self.to_rgb
            self.weighted_alpha = 0
        else:
            self.add_to_main(nn.Upsample(scale_factor=2).to(self.device))
            self.add_to_main(G_block(in_channels, out_channels).to(self.device))

        self.to_rgb = ToRGB(out_channels, flag_tanh=self.flag_tanh).to(self.device)
        self.output_resolution *= 2

    def set_weighted_alpha(self, alpha):
        self.weighted_alpha = alpha


from collections import deque

N, M = list(map(int, input().split()))

restrict = deque([])
test = deque([])

for i in range(N):
    length, speed = list(map(int, input().split()))
    restrict.append((length, speed))

for i in range(M):
    length, speed = list(map(int, input().split()))
    test.append((length, speed))

max_speed = 0
test_length = 0
restrict_length = 0

while (test or restrict):
    if test_length > restrict_length:
        r_length, r_speed = restrict.popleft()
        restrict_length += r_length
        max_speed = max(max_speed, t_speed - r_speed)

    elif test_length < restrict_length:
        t_length, t_speed = test.popleft()
        test_length += t_length
        max_speed = max(max_speed, t_speed - r_speed)

    else:
        r_length, r_speed = restrict.popleft()
        t_length, t_speed = test.popleft()
        test_length += t_length
        restrict_length += r_length
        max_speed = max(max_speed, t_speed - r_speed)

print(max_speed)


"""
The corresponding TorchPlus layers to PyTorch's `torch.nn`.

They look identical but perform various magic behind the scenes to enable "+"
sequences, layers without parentheses, etc.
"""

from inspect import isclass
import sys
import torch
from torch.nn.modules import Module

from . import extra
from .internal import Keyword

def wrap_layer_classes(from_module, to_module):
    """
    Create a corresponding wrapper in to_module for each from_module nn.Module.
    """
    for class_name in dir(from_module):
        if class_name == 'Module':
            continue
        klass = getattr(from_module, class_name)
        if not isclass(klass):
            continue
        if not issubclass(klass, Module):
            continue
        assert not hasattr(to_module, class_name)
        wrapped = Keyword(klass)
        setattr(to_module, class_name, wrapped)

this_module = sys.modules[__name__]
wrap_layer_classes(torch.nn.modules, this_module)
wrap_layer_classes(extra, this_module)


"""
File that consists of the Fips Command Functionality
"""
from syslog import LOG_ERR, syslog
from typing import List, Tuple
from enum import IntEnum
from summit_rcm.at_interface.commands.command import Command
from summit_rcm.services.fips_service import FipsService, FipsUnsupportedError
from summit_rcm.services.system_service import SystemService

class States(IntEnum):
    fips = 0
    fips_wifi = 1
    unset = 2

class FipsCommand(Command):
    """
    AT Command to get/set fips state
    """

    NAME: str = "Fips"
    SIGNATURE: str = "at+fips"
    VALID_NUM_PARAMS: List[int] = [1, 2]
    VALID_STATE_VAL: List[str] = ["fips", "fips_wifi", "unset"]

    @staticmethod
    async def execute(params: str) -> Tuple[bool, str]:
        (valid, params_dict) = FipsCommand.parse_params(params)
        if not valid:
            syslog(LOG_ERR, "Invalid Parameters")
            return (True, "ERROR")
        try:
            if params_dict["state"]:
                if await FipsService().get_fips_state() == "unsupported":
                    raise FipsUnsupportedError("Fips Unsupported")
                success = await FipsService().set_fips_state(params_dict["state"])
                if not success:
                    return (True, "ERROR")
                if params_dict["reboot"]:
                    await SystemService().set_power_state("reboot")
                return (True, "OK")
            else:
                fips_str = await FipsService().get_fips_state()
            return (True, f"+FIPS: {fips_str}\r\nOK")
        except Exception as exception:
            syslog(LOG_ERR, f"Error setting/getting FIPS state: {str(exception)}")
            return (True, "ERROR")

    @staticmethod
    def parse_params(params: str) -> Tuple[bool, dict]:
        valid = True
        params_dict = {}
        params_list = params.split(",")
        given_num_param = len(params_list)
        valid &= given_num_param in FipsCommand.VALID_NUM_PARAMS
        if not valid:
            return (False, {})
        try:
            params_dict["state"] = (
                States(int(params_list[0])).name if params_list[0] else ""
            )
            if params_dict["state"] and given_num_param < 2:
                raise ValueError
            params_dict["reboot"] = (
                bool(int(params_list[1])) if given_num_param > 1 and params_list[1] else False
            )
        except ValueError:
            valid = False
        return (valid, params_dict)

    @staticmethod
    def usage() -> str:
        return "AT+FIPS[=<state>[,<reboot>]]"

    @staticmethod
    def signature() -> str:
        return FipsCommand.SIGNATURE

    @staticmethod
    def name() -> str:
        return FipsCommand.NAME


import sys
import os
import re
from Bio import SeqIO

in_file = "./Brassica.bac.final.0.03.rep.fasta"
out_file ="./Brassica.bac.final.0.03.rep.rename.fasta"

output_handle = open(out_file, "w")

OldseqID = list()
seqInfo = list()
NewseqID = list()

for seq_record in SeqIO.parse(in_file, "fasta"):
    OldseqID.append(seq_record.id)
    seqInfo.append(seq_record.description)
    seqID_New = seq_record.description
    seqID_New = re.findall('Otu.{6}',seq_record.description)
    NewseqID.append(seqID_New)
    seq_record.id = seqID_New[0]

    SeqIO.write(seq_record, output_handle, "fasta")

output_handle.close()
renamed = len(NewseqID)

print("done")
print("renamed %i sequence records" %(renamed))


from io import StringIO
import sys

def sandbox(code: str, block_globals: bool=False,
            block_locals: bool=False) -> tuple:
    """Runs the code-string and captures any errors

    Args:
     code: executable string
     block_globals: if True don't use global namespace
     block_ locals: if True don't use local namespace
    Returns:
     output, stderr, and any exception code
    """
    old_stdout = sys.stdout
    old_stderr = sys.stderr
    redirected_output = sys.stdout = StringIO()
    redirected_error = sys.stderr = StringIO()

    namespace_globals = {} if block_globals else globals()
    namespace_locals = {} if block_locals else locals()
    output, error, exception = None, None, None

    try:
        exec(code, namespace_globals, namespace_locals)
    except:
        import traceback
        exception = traceback.format_exc()

    output = redirected_output.getvalue()
    error = redirected_error.getvalue()

    # reset outputs to the original values
    sys.stdout = old_stdout
    sys.stderr = old_stderr
    return output, error, exception


# Section 12-6
# 파이선 데이터베이스 연동(SQLITE)
# 테이블 데이터 수정 및 삭제

import sqlite3
# DB생성(파일)

conn = sqlite3.connect('C:/python_basic/resource/database.db')

# Cursor 연결
c = conn.cursor()

# 데이터 수정1
c.execute("UPDATE users SET username = ? WHERE id = ?", ('niceman', 2))

# 데이터 수정2 //딕셔너리 형태로
c.execute("UPDATE users SET username = :name WHERE id = :id", {"name" : 'goodman', "id": 5})

# 데이터 수정3
c.execute("UPDATE users SET username = '%s' WHERE id = '%s'"  % ('badboy', 3))

# 중간 데이터 확인1
for user in c.execute("SELECT * FROM users"):
    print(user)

# Row Delete1
c.execute("DELETE FROM users WHERE id =?", (2,))

# Row Delete2
c.execute("DELETE FROM users WHERE id =:id", {"id":5})

# Row Delete3
c.execute("DELETE FROM users WHERE id = '%s'" %(4,))

print()

# 중간 데이터 확인1
for user in c.execute("SELECT * FROM users"):
    print(user)

# 테이블 전체 데이터 삭제
print("users db deleted :", conn.execute("DELETE FROM users").rowcount, " rows")

# 커밋
conn.commit() # 반드시 commit을 해야 데이터 수정이 된다

# 접속 해제
conn.close()


import datetime
import logging
import re
import sys
import time
from collections import defaultdict
from functools import wraps
from typing import List, Optional, Union

import pandas as pd
import progressbar
import pytz

from cityair_api.settings import CHECKINFO_PARSE_PATTERN
from .exceptions import EmptyDataException

logger = logging.getLogger(__name__)

FATHER_PREFIXES = ["CA", "ROOFTOP", "LAB"]

def is_main_device(serial):
    for prefix in FATHER_PREFIXES:
        if serial.startswith(prefix):
            return True
    return False

RIGHT_PARAMS_NAMES = {
        'FlagPs220': '220',
        'RecvDate': 'receive_date', 'Ps220': '220',
        'GsmRssi': 'rssi',
        'SendDate': 'date', 'Temperature': 'T',
        'BatLow': 'is_bat_low',
        'Humidity': 'RH', 'Pressure': 'P',
        'ChildDevices': 'children',
        'DataDeliveryPeriodSec': 'delivery_period',
        'Description': 'description',
        'DeviceCheckInfos': 'check_infos',
        'DeviceFirstDate': 'first_packet_date',
        'DeviceIMEI': 'IMEI',
        'DeviceIMSI': 'IMSI',
        'DeviceLastDate': 'last_packet_date',
        'DeviceLastGeo': 'coordinates',
        'DeviceName': 'name',
        'FlagBatLow': 'is_bat_low', 'DeviceId': 'id',
        'IsOffline': 'is_offline',
        'SerialNumber': 'serial_number',
        'SoftwareVersion': 'software',
        'SourceType': 'type', 'TagIds': 'tags',
        'MoId': 'id',
        'Name': 'name',
        'PublishName': 'internal_name',
        'PublishNameRu': 'name_ru',
        'ManualDeviceLinks': 'devices_manual',
        'DeviceLink': 'devices_auto',
        'GmtOffset': 'gmt_offset',
        'DotItem': 'coordinates',
        'Latitude': 'latitude',
        'Longitude': 'longitude',
        'LocationId': 'location',
        'GeoInfo': 'coordinates',
        'Geo': 'coordinates', 'DataAqi': 'AQI',
        'GmtHour': 'gmt_hour_diff',
        'PublishOnMap': 'is_public',
        'NameRu': 'name_ru',
        'PacketId': 'packet_id',
        "Humidity_": "RH"
        }
USELESS_COLS = ['220', 'BatLow', 'receive_date', 'GeoInfo', 'Geo', 'Date',
                'SendDate', 'ResetMoData', 'description', 'coordinates',
                'rssi', 'FlagBatLowHasFailed', 'FlagPs220HasFailed',
                'IsNotSaveData', 'ParentDeviceId', 'SourceType', 'tags',
                'DataProviderId', 'IsDeleted', 'IsManualParamLinks',
                'IsStartInterval1H', 'ManualPacketParamLinks', 'Timestamp',
                'is_bat_low', 'BounceNorth', 'BounceSouth', 'BounceEast',
                'BounceWest', 'CountryId', 'BounceNorth', 'BounceSouth',
                'BounceEast', 'BounceWest', 'GmtHour', 'LocationUrl',
                'DistributionSummary', 'SortRank', 'PacketId', 'packet_id',
                "DoS", "DDW", 'Pcf', 'Scf']

MAIN_DEVICE_PARAMS = ['serial_number', 'name', 'software', 'stations',
                      'children', 'check_infos']
MAIN_STATION_PARAMS = ['id', 'name', 'name_ru', 'location', 'gmt_offset',
                       'devices', 'latitude', 'longitude']

def add_progress_bar(method):
    """
    Decorator to display progress bar

    """
    progress_scaler = 10 ** 6
    default_take_count = 500

    @wraps(method)
    def progressed(*args, **kwargs):
        start_date = to_date(kwargs.get('start_date'))
        if not start_date:
            return method(*args, **kwargs)
        finish_date = to_date(
                kwargs.get('finish_date', datetime.datetime.utcnow()))
        kwargs.update(take_count=kwargs.get('take_count', default_take_count))
        bar = progressbar.ProgressBar(max_value=(finish_date - start_date)
                                      .total_seconds() / progress_scaler,
                                      widgets=[
                                              f"{args[1]}",
                                              ': ',
                                              progressbar.Percentage(),
                                              '    ',
                                              progressbar.Timer(),
                                              progressbar.Bar(),
                                              progressbar.ETA()
                                              ],
                                      max_error=False
                                      )
        if kwargs.get('format', 'df') == 'df':
            res = pd.DataFrame()
        else:
            res = defaultdict(pd.DataFrame)
        while finish_date - start_date > datetime.timedelta(hours=1):
            try:
                data = method(*args, **kwargs)
                if isinstance(res, pd.DataFrame):
                    res = pd.concat([res, data], sort=False)
                    start_date = res.index[-1]
                else:
                    for serial, df in data.items():
                        res[serial] = pd.concat([res[serial], df], sort=False)
                    start_date = max([res[key].index[-1] for key in res])
            except EmptyDataException:
                start_date += datetime.timedelta(days=2)
            kwargs.update(
                    start_date=start_date + datetime.timedelta(seconds=30))
            fetched_seconds = (finish_date.replace(tzinfo=pytz.utc)
                               - start_date.replace(
                            tzinfo=pytz.utc)).total_seconds()
            bar.update(bar.max_value - (fetched_seconds / progress_scaler))
        size = len(res) if isinstance(res, pd.DataFrame) else max(
                map(len, res.values()))
        logger.info(f'finished acquiring {args[1]} data of size {size}')
        if size == 0:
            raise EmptyDataException(request_args=kwargs)
        return res

    return progressed

def unpack_cols(df, cols_to_unpack, right_params_names=RIGHT_PARAMS_NAMES):
    for col in cols_to_unpack:
        df = df.assign(**df[col].apply(pd.Series)).drop(col, 1)
    df.rename(right_params_names, axis=1, inplace=True)
    return df

def prep_dicts(dicts, newkeys, keys_to_drop, dropna=True):
    res = []
    for d in dicts:
        new_dict = {}
        for key, value in zip(d.keys(), d.values()):
            if key in keys_to_drop:
                continue
            if dropna and value is None:
                continue
            if key == 'is_offline':
                new_dict['is_online'] = not value
                continue
            new_key = newkeys.get(key, key)
            if 'date' in new_key:
                value = to_date(value)
            new_dict[new_key] = value
        res.append(new_dict)
    return res

def parse_checkinfo(msg: str) -> List[dict]:
    parsed_checkinfo = []
    parse_re = re.compile(CHECKINFO_PARSE_PATTERN)
    infos = parse_re.findall(msg.replace(", ", ","))
    if not infos:
        raise ValueError("checkinfo data should be in the format"
                         f" of \"{CHECKINFO_PARSE_PATTERN}\"")
    for info in infos:
        module, status, count, details = info.split(',')
        parsed_checkinfo.append(dict(
                module=module,
                has_error=False if status == 'ok' else True,
                errors_count=int(count),
                details=str(details).replace("\"", "")))
    return parsed_checkinfo

def to_date(date: Union[datetime.datetime, str, None],
            format: str = 'date') -> Optional[Union[str, datetime.datetime]]:
    """

    :param date: date (datetime or string) to be converted
    :param format: str, 'str' or 'date', default 'date'
        return type
    :return:
    """
    if not date:
        return None
    if isinstance(date, datetime.datetime):
        pass
    elif isinstance(date, str):
        date = pd.to_datetime(date, dayfirst=True)
    else:
        raise ValueError(f"date should be 'str' or 'datetime', "
                         f"got {type(date)} instead")

    if format == 'str':
        return date.isoformat()
    elif format == 'date':
        return date.replace(tzinfo=None)
    else:
        raise ValueError(f"format arg should be one of the 'str', 'date', "
                         f"got {format} instead")

def prep_df(df: pd.DataFrame, right_param_names: dict = RIGHT_PARAMS_NAMES,
            cols_to_drop: List[str] = USELESS_COLS,
            dicts_cols: List[str] = [], dropna: bool = True,
            index_col: str = None, cols_to_unpack: List[str] = []):
    res = df.copy()
    res.dropna(how='all', axis=0)
    res.rename(right_param_names, axis=1, inplace=True)

    for col in dicts_cols:
        res[col] = res[col].apply(prep_dicts,
                                  args=[right_param_names, cols_to_drop,
                                        dropna])
    try:
        res = unpack_cols(res, cols_to_unpack)
    except (KeyError, TypeError):
        pass
    if dropna:
        res.dropna(how='all', axis=1, inplace=True)
    for col in res:
        if 'date' in col.lower():
            res[col] = res[col].apply(to_date)
    try:
        res['is_online'] = ~ res['is_offline']
        res.drop('is_offline', axis=1, inplace=True)
    except KeyError:
        pass
    if index_col and index_col in res.columns:
        res.set_index(index_col, inplace=True)
    res.rename(right_param_names, axis=1, inplace=True)
    res = res.drop(cols_to_drop, axis=1, errors='ignore')
    res = res.sort_index()
    return res

def timeit(method):
    """
    Decorator to print time elapsed

    """

    @wraps(method)
    def timed(*args, **kwargs):
        ts = time.time()
        result = method(*args, **kwargs)
        te = time.time()

        logger.debug(
                f"{te - ts:.2f} seconds took to {method.__name__} of size"
                f"{sys.getsizeof(result) / 1000: .2f} KB\nargs were"
                f" {', '.join(map(str, args[1:]))}\nkwargs were: {kwargs}")
        return result

    return timed


import rssi
import serial
import re
import time
import os

devices = ['/dev/'+port for port in os.listdir('/dev/')
        if 'ACM' in port
        and int(port[-1]) % 2 == 0 ]

serial_connections = [
        serial.Serial(port, 115200, timeout=2)
        for port in devices]

addresses = [None]*len(serial_connections)

def reading_matcher(reading):
    '''
    Intended to match a pattern such as
    "Channel: 11 ; RSSI is: -117"

    return (int, int)
    '''
    pattern = re.compile(r'Channel: (\d+) ; RSSI is: (.?\d+)')
    match = pattern.findall(reading)
    if len(match) == 0:
        return None
    result = tuple(map(int,match[0]))
    return result

def close_connections():
    for conn in serial_connections:
        conn.close()
import sys

protocol = [ '', 'In: OPMOD:', 'In: SAM_INTV:', 'In: CHNUM:']
args = []
for arg_num, prompt in enumerate(protocol):
    arg = raw_input(" {} :: {} : ".format(arg_num, prompt))
    args.append(bytes(arg))

protocol_args = zip(protocol, args)
args_map = dict(protocol_args)
if args_map["In: OPMOD:"] not in ["RX0", "RX1", "TX0", "TX1"]:
    print("OPMOD -{}- invalid!".format(args_map["In: OPMOD:"]))
    close_connections()
    sys.exit(1)

if not args_map["In: SAM_INTV:"].isdigit():
    print("-{}- is not a number".format(args_map["In: SAM_INTV:"]))
    close_connections()
    sys.exit(1)

chan_num = args_map["In: CHNUM:"]
if len(chan_num) == 0:
    print("Channel number is required")
    close_connections()
    sys.exit(1)

chan_num = chan_num[1:]
if not chan_num.isdigit():
    print("{} is not a valid channel number".format(chan_num))
    close_connections()
    sys.exit(1)

if int(chan_num) not in range(11, 26+1):
    print("{} is not within the range 11 and 26".format(chan_num))
    close_connections()
    sys.exit(1)

readers = [rssi.Serial_Reader(connection, reading_matcher, protocol_args, address)
            for connection, address in zip(serial_connections, addresses)]

print ("Collected all parameters. Starting polling.")

for reader in readers:
    reader.start()

time.sleep(15)
# while True:
#     end = raw_input(
# '''Enter q to stop\n
# Enter c for the current total\n''')
#     if end == "c":
#         print(rssi.Rssi.commits)
#     if end == "q":
#         print("Quitting after {} recordings".format(rssi.Rssi.commits))
#         break

print ("{} recordings made".format(rssi.Signal_Repository.commits))
est_skipped= sum(reader.times_data_failed_to_match for reader in readers)
print ('{} transmissions skipped'.format(est_skipped))

for reader in readers:
    reader.join()


import pdfplumber as pdfplumber
import os
import time
inpath='D:\\LDA\\LDAdata\\pdfdata'
outpath='D:\\LDA\\LDAdata\\txtdata-plumber'

def pdf_to_txt_plumber(folder):
    files=os.listdir(folder)
    pdfFiles=[f for f in files if f.endswith('.pdf')]

    for pdfFile in pdfFiles:
        print(pdfFile)
        pdfPath = os.path.join(folder, pdfFile)
        wdPath=os.path.join(outpath,pdfFile)
        if wdPath[-4:] != '.txt':
            wdPath = wdPath + '.txt'
        pdf=pdfplumber.open(pdfPath)
        f=open(wdPath,'w+',encoding='utf-8')
        i=0;
        for page in pdf.pages:
            text=pdf.pages[i].extract_text()
            # print(text)
            i+=1
            f.write(text)
            f.write('\n')
        # print("------------------------------")

if __name__=='__main__':
    tickets = time.time()
    print(tickets)
    pdf_to_txt_plumber(inpath)
    ticket = time.time()
    print(ticket)


import json
import requests
import geopandas as gpd
import numpy as np
import shapely.speedups
import pandas as pa
import matplotlib.pyplot as plt

# my_gdf.to_sql(name="test", con=engine, if_exists='append', index=False,
#               dtype={'geom': Geometry(geometry_type='POLYGON', srid=4326)})
def create_heirarchy(geo_data_frame, layers):
    filter = geo_data_frame["IndexOrder"] == layers[0]
    child_filter = geo_data_frame["IndexOrder"] == layers[1]
    parent_order = geo_data_frame[filter]
    child_order = geo_data_frame[child_filter]
    count = 0
    print("total parent ", parent_order.size)
    for row in parent_order.iterrows():
        count = count + 1
        print(count)
        ids = get_all_intersected_polygons(row[1].geometry, child_order)
        geo_data_frame.loc[geo_data_frame.unique_id.isin(ids), "parent_id"] = int(row[1].unique_id)
    geo_data_frame.to_file('/home/gulve/data-analysis/feature-comparator/data/complte_irl.gpkg', layer='index',
                           driver='GPKG')
    return geo_data_frame

def call_hierarchy(geo_data_frame, lowest_order):
    filter = geo_data_frame["IndexOrder"] == lowest_order
    geo_data_frame.unique_id.astype(str)
    # result_data_frame = gpd.GeoDataFrame(columns=["order10", "order9", "order8"])
    apts = gpd.read_file(
        "C:\\Amol_Parande\\8_MIS\Abhijeet\\contains_Python\\2_Geopakage\\Geo_Data.gpkg", layer="Points")
    for row in geo_data_frame[filter].iterrows():
        list = [row[1].unique_id]
        construct_hierarchy(geo_data_frame, row[1].parent_id, list)
        query = get_name_hierarchy_from_ids(geo_data_frame, list)
        # res = compare_pelias_hierarchy(geo_data_frame, query, list, {})
        print(query, ",", compare_with_apt(geo_data_frame, list, apts), percentage_match(None))

def percentage_match(ll):
    if ll == None:
        return 0
    count = 0
    for name in ll:
        if ll[name]:
            count = count + 1
    return count / len(ll)

api = "https://api.geocode.earth/v1/autocomplete?focus.point.lat=18.545351289248973&focus.point.lon=73.88522386550905&api_key=ge-8f137223ed5b405d&text="

def compare_pelias_hierarchy(geo_data_frame, query, hierarchy_ids, response_model_attribute):
    #     name  neighbourhood  locality
    resp = requests.get(api + query.strip())
    if resp.status_code != 200:
        raise Exception('GET /tasks/ {}'.format(resp.status_code))
    res_json = resp.json()
    if len(res_json["features"]) <= 0:
        return
    name_matching = res_json["features"][0]["properties"]
    # if "name" in name_matching and "neighbourhood" in name_matching and "locality" in name_matching:
    return json_value_math(name_matching, hierarchy_ids, geo_data_frame)

def json_value_math(json_obj, hierarchy_ids, geo_data_frame):
    json_obj = json.loads(json.dumps(json_obj))
    arr = {}
    for id in hierarchy_ids:
        name = get_area_name(geo_data_frame, id)
        if "County" not in name:
            temp = False
            for value in json_obj.values():
                temp |= name == value
            arr[name] = temp
        else:
            arr[name] = True
    return arr

def create_convex_hull_and_compare(apts_within_area, area):
    name_poly = check_for_same_name_poly_area(apts_within_area, area)
    if name_poly > 50:
        return name_poly
    group = apts_within_area.groupby("LOCALITY_NAME")
    for single_group in group:
        poly = single_group[1]["geometry"].unary_union.convex_hull
        intersection = poly.intersection(area[1].geometry)
        area[1].within(poly[1].geometry)

def check_for_same_name_poly_area(apts_within_area, area):
    fltr = apts_within_area[apts_within_area["LOCALITY_NAME"] == area[1].name]
    poly_with_area_name = fltr[1]["geometry"].unary_union.convex_hull
    intersection = poly_with_area_name.intersection(area[1].geometry)
    return (intersection.area / area.area) * 100

def compare_with_apt(geo_data_frame, hierarchical_ids, apts):
    global apt_group_by
    data_frame = geo_data_frame[geo_data_frame.unique_id.isin(hierarchical_ids[:2])]
    # data_frame = data_frame[not data_frame["name"].str.contains("County"
    for area in data_frame.iterrows():
        area_arr = []
        for touple in area.iteritems():
            area_arr.append(touple)
        gpd.GeoDataFrame(area_arr)
        apts_within_area = apts[apts.within(area[1].geometry)]
        group_by = apts_within_area.groupby("LOCALITY_NAME").aggregate("LOCALITY_NAME").count()
        arr = []
        for touple in group_by.iteritems():
            arr.append(touple)
        apt_group_by = pa.DataFrame(arr, columns=["LOCALITY_NAME", "count"])
        apt_group_by = remove_parent(apt_group_by, area, geo_data_frame)
        apt_group_by["perc"] = apt_group_by['count'] / (apt_group_by.sum()[1] / 100)
        apt_group_by = apt_group_by.sort_values(by="perc", ascending=False)
        aa = apt_group_by[apt_group_by["LOCALITY_NAME"] == area[1]["name"].upper()]
        if len(aa.perc.array) <= 0:
            return False
    return True

def filter_county_column_name(data_frame):
    for key in data_frame.keys():
        if "county" in key.lower():
            return key

def remove_parent(apt_group_by, area, geo_data_frame):
    list = []
    construct_hierarchy(geo_data_frame, area[1].parent_id, list)
    for id in list:
        row = geo_data_frame[geo_data_frame["unique_id"] == id]["name"].values[0].upper()
        apt_group_by = apt_group_by.drop(apt_group_by[apt_group_by["LOCALITY_NAME"] == row].index)
    return apt_group_by

def get_all_name_matching_poly(data_frame, poly):
    poly_list = poly[poly.LOCALITY_NAME.isin([x.upper() for x in data_frame.name.values])]
    return poly_list

def get_intersecting_polygons(poly_list, lowest_geo_data_frame):
    lowest_poly = poly_list[poly_list["LOCALITY_NAME"] == lowest_geo_data_frame.name.values[0].upper()]
    pip_mask = lowest_poly[lowest_poly.within(poly_list['geometry'])]
    for poly in poly_list.iterrows():
        poly[1].geometry

def get_area_name(geo_data_frame, id):
    return geo_data_frame.loc[geo_data_frame.unique_id == id].name.values[0]

def get_name_hierarchy_from_ids(geo_data_frame, list):
    string = ""
    for id in list:
        string += (geo_data_frame.loc[geo_data_frame.unique_id == id].name.values[0] + " ")
    return string

def construct_hierarchy(geo_data_frame, parent_id, list):
    if parent_id == '':
        return list
    list.append(np.int64(parent_id))
    construct_hierarchy(geo_data_frame,
                        geo_data_frame[geo_data_frame["unique_id"] == np.int64(parent_id)].parent_id.values[0],
                        list)

def get_all_intersected_polygons(parent_poly, child_data_frame):
    return child_data_frame[child_data_frame.within(parent_poly)].unique_id.array

def create_df_from_tuple(group_by, columns):
    arr = []
    index = []
    for touple in group_by.iteritems():
        arr.append(touple)
        index.append(touple)
    return pa.DataFrame(arr, columns=columns, index=index)

# df = gpd.read_file("/home/gulve/data-analysis/Geogesion/dublin.gpkg", driver="GPKG",
#                    layer="TTOM-Core::IndexArea")
df = gpd.read_file("/home/gulve/data-analysis/feature-comparator/data/complte_irl.gpkg",
                   driver="GPKG",
                   layer="index")
# df = df[df["IndexOrder"].notnull()]
# df['unique_id'] = df.name.map(hash)
# df['parent_id'] = ''
# create_heirarchy(df, ["Order9", "Order10"])

# df.loc[df["IndexOrder"] == "Order8", "parent_id"] = ''
call_hierarchy(df, "Order10")

def absolute_value(val):
    a = np.round(val, 0)
    return a
# csv = pa.read_csv("/home/gulve/data-analysis/hierarchy_res.csv")
# csv = csv.astype({"apt": str})
# both_matches_hierarchy = csv[csv["apt"] == csv["pelias"]]
#
# need_to_check = csv[csv["pelias"] == "None"]
# need_to_check = need_to_check[need_to_check["apt"] == "False"]
# print()
# not_matches = csv[csv["pelias"] != "None"]
# not_matches = not_matches[not_matches["apt"] != not_matches["pelias"]]
# print(need_to_check)
#
# pelias_distinct_values = create_df_from_tuple(csv.pelias.value_counts(), ["is_matches", "count"])
# pelias_distinct_values.plot.pie(y='count', autopct=absolute_value)
# apt = create_df_from_tuple(csv.apt.value_counts(), ["is_matches", "count"])
# apt = apt.astype({'count': np.int64})
# apt.plot.pie(y="count", x="apt", autopct=absolute_value)
# plt.show()
# df = gpd.read_file("/home/gulve/data-analysis/Geogesion/tt.gpkg", driver="GPKG",
#                    layer="TTOM-Core::IndexArea")
# create_convex_hull(df)


import xlrd
import re
import time
import xlwt
import pymysql
import pymysql.cursors

wb = xlwt.Workbook()
ws = wb.add_sheet('A Test Sheet')
config = {
    'host': '127.0.0.1',
    'port': 3306,
    'user': 'root',
    'password': '',
    'db': 'test',
    'charset': 'utf8'
}
for i in range(1, 5716):  # 循环逐行打印
    try:
        connection = pymysql.connect(**config)
        with connection.cursor() as cursor:
            # 执行sql语句，进行查询
            sql = "select * from ref where id=%s"
            cursor.execute(sql, (i))
            result = cursor.fetchall()
        # 没有设置默认自动提交，需要主动提交，以保存所执行的语句
        connection.commit()
    finally:
        connection.close();
    ws.write(i,0,result[0][1])
    ws.write(i, 1, result[0][2])
    if result[0][3]>1:
        ws.write(i, 2, result[0][1])
    ws.write(i, 3, result[0][3])
    print(i)
print("over")
wb.save('Ref-P.xls')


import sys
import csv
from py2neo import Graph, Node, Relationship, NodeMatcher

def loadEmployees(g, file):
   with open(file, 'r') as f:
       rows = list(csv.reader(f))
   for row in rows:
       r = ",".join(row).split(":")
       n = Node("Employee", first=r[0], mid=r[1], last=r[2], ssn=r[3], dob=r[4], address=r[5], sex=r[6],
         salary=int(r[7]), superV=r[8], deptId=int(r[9]))
       g.create(n)

def loadDepatments(g, file):
   with open(file,'r') as f:
       rows = list(csv.reader(f))
   for row in rows:
       r = row[0].split(":")
       print(r)
       n = Node('Department',dept=r[0], deptId=r[1], ssn=r[2], date=r[3])
       g.create(n)

def loadDepatments2(g, file):
   with open(file,'r') as f:
       rows = list(csv.reader(f))
   matcher = NodeMatcher(g)
   for row in rows:
       r = row[0].split(":")
       d = matcher.match("Department", ssn=r[2]).first()
       e = matcher.match("Employee", ssn=r[2]).first()
       #print(d)
       #n = Node('Departments',dept=r[0], deptId=r[1], date=r[3])
       rel1 = Relationship(d,"managed_by",e)
       rel2 = Relationship(e,"manages",d)
       g.create(rel1)
       g.create(rel2)

def loadEmployees2(g, file):
    with open(file, 'r') as f:
        rows = list(csv.reader(f))
    matcher = NodeMatcher(g)
    for row in rows:
        r = ",".join(row).split(":")
        if r[8] == "null":
            d = matcher.match("Department", deptId=r[9]).first()
            e = matcher.match("Employee", ssn=r[3]).first()
            rel1 = Relationship(d,"employs",e)
            rel2 = Relationship(e,"works_for",d)
            g.create(rel1)
            g.create(rel2)
        elif r[8] != "null":
            d = matcher.match("Department", deptId=r[9]).first()
            boss = matcher.match("Employee", ssn=r[8]).first()
            e = matcher.match("Employee", ssn=r[3]).first()
            rel1 = Relationship(d,"employs",e)
            rel2 = Relationship(e,"works_for",d)
            rel3 = Relationship(e,"supervisee",boss)
            rel4 = Relationship(boss,"supervisor",e)
            g.create(rel1)
            g.create(rel2)
            g.create(rel3)
            g.create(rel4)

def loadDependents(g, file):
    with open(file,'r') as f:
       rows = list(csv.reader(f))
    matcher = NodeMatcher(g)
    for row in rows:
       r = row[0].split(":")
       print(r)
       d = Node('Dependent',ssn=r[0], name=r[1], sex=r[2], birthdate=r[3], relationship=r[4])
       e = matcher.match("Employee", ssn=r[0]).first()
       rel1 = Relationship(d,"dependents_of",e)
       rel2 = Relationship(e,"dependent",d)
       g.create(rel1)
       g.create(rel2)

def loadProject(g, file):
   with open(file, 'r') as f:
       rows = list(csv.reader(f))
   matcher = NodeMatcher(g)
   for row in rows:
       r = row[0].split(":")
       print(r)
       p = Node("Project", name=r[0], pnumber=r[1], location=r[2], controllingDept=r[3])
       d = matcher.match("Department", deptId=r[3]).first()
       e = matcher.match("Employee", deptId=r[3]).first()
       rel1 = Relationship(d,"controls",p)
       rel2 = Relationship(p,"controlled_by",d)
       g.create(rel1)
       g.create(rel2)

def loadDeptLocations(g, file):
   with open(file,'r') as f:
       rows = list(csv.reader(f))
   matcher = NodeMatcher(g)
   for row in rows:
       r = row[0].split(":")
       rlen = len(r)
       d = matcher.match('Department',deptId=r[0]).first()
       n = Node('Department_location', deptId=r[0], address=','.join(r[1:rlen]).split(','))
       rel = Relationship(d,"is_located",n)
       rel1 = Relationship(n,"located_at",d)
       g.create(rel)
       g.create(rel1)

def loadWorksOn(g, file):
    with open(file, 'r') as f:
        rows = list(csv.reader(f))
    matcher = NodeMatcher(g)
    for row in rows:
        r = row[0].split(":")
        w = Node("WorksOn", ssn=r[0], pnumber=r[1], hours=r[2])
        p = matcher.match("Project", pnumber=r[1]).first()
        e = matcher.match("Employee",ssn=r[0]).first()
        rel1 = Relationship(e,"empWorkHours",w)
        rel2 = Relationship(p,"projectWorkHours",w)
        g.create(rel1)
        g.create(rel2)

def loadWorksOn2(g, file):
    with open(file, 'r') as f:
        rows = list(csv.reader(f))
    matcher = NodeMatcher(g)
    for row in rows:
        r = row[0].split(":")
        e = matcher.match('Employee',ssn=r[0]).first()
        p = matcher.match("Project", pnumber=r[1]).first()
        rel1 = Relationship(p,"worker",e)
        rel2 = Relationship(e,"works_on",p)
        g.create(rel1)
        g.create(rel2)

def main():
    g = Graph(auth=('neo4j','Adventure@25'))
    g.delete_all()
    loadEmployees(g,"./"+sys.argv[1]+"/EMPLOYEES.dat")
    loadDepatments(g,"./"+sys.argv[1]+"/DEPARTMENTS.dat")
    loadDepatments2(g,"./"+sys.argv[1]+"/DEPARTMENTS.dat")
    loadEmployees2(g,"./"+sys.argv[1]+"/EMPLOYEES.dat")
    loadDependents(g,"./"+sys.argv[1]+"/DEPENDENTS.dat")
    loadDeptLocations(g,"./"+sys.argv[1]+"/DEPT_LOCATIONS.dat")
    loadProject(g,"./"+sys.argv[1]+"/PROJECTS.dat")
    loadWorksOn(g,"./"+sys.argv[1]+"/WORKS_ON.dat")
    loadWorksOn2(g,"./"+sys.argv[1]+"/WORKS_ON.dat")

main()