import numpy as np
from matplotlib import colors
import matplotlib.pyplot as plt
from matplotlib.ticker import LogLocator
from datetime import datetime, timedelta
import matplotlib.dates as dts
import pandas as pd
import os
import locale
import warnings
import yaml
import re
import sys
from dateutil.parser import parse
from tinydb import TinyDB, Query
from tinydb.operations import add
import time
import json

# Fixed diameter and mobility bins
dp_ion = np.array([7.949610066873187275e-01,9.181737924552214603e-01,
1.060513600503926179e+00,1.224959679823698799e+00,1.414958699738506631e+00,
1.634499249798819331e+00,1.888198514085806856e+00,2.181403433339687226e+00,
2.520308747865528165e+00,2.912095102815642989e+00,3.365090891236600878e+00,
3.888962384293289887e+00,4.494937535166431353e+00,5.196070414640996837e+00,
6.007554438162747701e+00,6.947095098447752193e+00,8.035355151375323857e+00,
9.296489193192451594e+00,1.075878902024538242e+01,1.245546773082500103e+01,
1.442561898219513949e+01,1.671539984850161886e+01,1.937950186998520152e+01,
2.248299804137784363e+01,2.610368545677439300e+01,3.033508982931992648e+01,
3.529036394466827886e+01,4.110740875515996606e+01])

dp_par = np.array([7.498942093324539870e-01,8.659643233600640144e-01,
9.999999999999980016e-01,1.154781984689456031e+00,1.333521432163321974e+00,
1.539926526059490097e+00,1.778279410038920094e+00,2.053525026457140079e+00,
2.371373705661659947e+00,2.738419634264360081e+00,3.162277660168379967e+00,
3.651741272548380213e+00,4.216965034285819591e+00,4.869675251658620141e+00,
5.623413251903479626e+00,6.493816315762099833e+00,7.498942093324560076e+00,
8.659643233600640144e+00,1.000000000000000000e+01,1.154781984689457985e+01,
1.333521432163323972e+01,1.539926526059490008e+01,1.778279410038922137e+01,
2.053525026457139901e+01,2.371373705661660125e+01,2.738419634264360170e+01,
3.162277660168379967e+01,3.651741272548380124e+01,4.216965034285819769e+01])

mob_ion = np.array([3.162277660168379937e-04,2.371373705661659990e-04,
1.778279410038920258e-04,1.333521432163320159e-04,1.000000000000000048e-04,
7.498942093324559917e-05,5.623413251903490022e-05,4.216965034285820205e-05,
3.162277660168380208e-05,2.371373705661660125e-05,1.778279410038919852e-05,
1.333521432163319990e-05,1.000000000000000082e-05,7.498942093324561442e-06,
5.623413251903490361e-06,4.216965034285830030e-06,3.162277660168380038e-06,
2.371373705661659871e-06,1.778279410038920148e-06,1.333521432163330027e-06,
1.000000000000000167e-06,7.498942093324570124e-07,5.623413251903499890e-07,
4.216965034285829924e-07,3.162277660168379721e-07,2.371373705661660136e-07,
1.778279410038920042e-07,1.333521432163329868e-07])*1e4

# Some other values calculated from the fixed bins
mob_ion_geomeans=np.array([2.73841963e-04, 2.05352503e-04, 1.53992653e-04, 1.15478198e-04,
       8.65964323e-05, 6.49381632e-05, 4.86967525e-05, 3.65174127e-05,
       2.73841963e-05, 2.05352503e-05, 1.53992653e-05, 1.15478198e-05,
       8.65964323e-06, 6.49381632e-06, 4.86967525e-06, 3.65174127e-06,
       2.73841963e-06, 2.05352503e-06, 1.53992653e-06, 1.15478198e-06,
       8.65964323e-07, 6.49381632e-07, 4.86967525e-07, 3.65174127e-07,
       2.73841963e-07, 2.05352503e-07, 1.53992653e-07])*1e4

dp_par_geomeans=np.array([ 0.80584219,  0.93057204,  1.07460783,  1.24093776,  1.43301257,
        1.6548171 ,  1.91095297,  2.20673407,  2.54829675,  2.94272718,
        3.39820833,  3.92418976,  4.53158364,  5.23299115,  6.0429639 ,
        6.97830585,  8.05842188,  9.30572041, 10.74607828, 12.40937761,
       14.3301257 , 16.548171  , 19.10952975, 22.06734069, 25.48296748,
       29.42727176, 33.98208329, 39.24189758])

dlogmob_ion=np.array([0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125,
       0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125,
       0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125,
       0.125])

dlogdp_ion=np.array([0.06257907, 0.06258521, 0.06259845, 0.06261376, 0.06263147,
       0.06265194, 0.06267563, 0.06270305, 0.06273478, 0.06277153,
       0.06281409, 0.06286343, 0.06292064, 0.06298703, 0.06306411,
       0.06315368, 0.06325786, 0.06337916, 0.06352054, 0.06368553,
       0.06387836, 0.06410408, 0.06436873, 0.06467961, 0.06504553,
       0.06547715, 0.06598741, 0.06626396])

dlogdp_par=np.array([0.0625, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625,
       0.0625, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625,
       0.0625, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625,
       0.0625, 0.0625, 0.0625, 0.0625, 0.0625])

# Names and naming formats encountered
filename_formats = [
['%Y-%m-%d.ions.nds','%Y-%m-%d.particles.nds','%Y-%m-%d.log'],
['%Y%m%d-block-ions.spectra','%Y%m%d-block-particles.spectra','%Y%m%d-block.records'],
['%Y%m%d-block-ions.spectra','%Y%m%d-block-particles.spectra','%Y%m%d-block.diagnostics']]

# Possible names for diagnostic parameters
possible_sampleflow_names = [
'pos_sampleflow.mean',
'neg_sampleflow.mean',
'pos_sampleflow',
'neg_sampleflow',
'sampleflow',
'Flowaer']

possible_temperature_names = [
'temperature.mean',
'temperature',
'temp']

possible_pressure_names = [
'baro.mean',
'baro']

# Define standard conditions
temp_ref = 273.15 # K, 0C
pres_ref = 101325.0 # Pa, 1atm

def make_config():
    """ Make a configuration file for processing NAIS data """

    # Collect data from the user
    print()
    print("Enter name of configuration file.")
    print("E.g. ./configs/campaign.yml")
    while True:
        config_file = input("> ")
        if len(config_file)>0:
            if not config_file.lower().endswith(".yml"):
                config_file = config_file + ".yml"
            break
        else:
            continue

    print()
    print("Give path(s) to raw data. If multiple paths give them as comma separated list.")
    print("E.g. /data/nais/2021,/data/nais/2022")
    while True:
        user_input = input("> ")
        if len(user_input)>0:
            load_path=user_input.split(",")
            break
        else:
            continue

    print()
    print("Path to where processed data is saved.")
    print("E.g. ./data/campaign/processed")
    while True:
        save_path = input("> ")
        if len(save_path)>0:
            break
        else:
            continue

    print()
    print("Path to where figures are saved. Leave empty if no figures.")
    print("E.g. ./data/campaign/figures")
    fig_path = input("> ")

    print()
    print("Start of measurement (YYYY-MM-DD)")
    while True:
        start_date = input("> ")
        try:
            start_dt = pd.to_datetime(start_date)
            break
        except:
            continue

    print()
    print("End of measurement (YYYY-MM-DD)")
    print("If empty processor assumes current day, use for continuous measurements.")
    while True:
        end_date = input("> ")
        if len(end_date)==0:
            break
        try:
            end_dt = pd.to_datetime(end_date)
            break
        except:
            continue

    print()
    print("Enter name of database file")
    print("E.g. ./logs/campaign.json")
    while True:
        database_file = input("> ")
        if len(database_file)>0:
            if not database_file.lower().endswith(".json"):
                database_file = database_file + ".json"
            break
        else:
            continue

    print()
    print("Measurement location")
    print("E.g. Helsinki, Kumpula")
    location = input("> ")

    print()
    print("Apply corrections to data? (True/False)")
    print("Requires a NAIS with temperature and pressure sensors.")
    while True:
        apply_corrections = input("> ")
        if ((apply_corrections=='True') or (apply_corrections=='False')):
            if apply_corrections=='True':
                apply_corrections=True
            else:
                apply_corrections=False
                sealevel_correction=False
                inlet_length = 0.0
            break
        else:
            continue

    if apply_corrections:
        print()
        print("Length of the inlet in meters")
        while True:
            inlet_length = input("> ")
            try:
                inlet_length = float(inlet_length)
                break
            except:
                continue

        print()
        print("Correct concentrations to sealevel conditions? (True/False)")
        while True:
            sealevel_correction = input("> ")
            if ((sealevel_correction=='True') or (sealevel_correction=='False')):
                if sealevel_correction=='True':
                    sealevel_correction=True
                else:
                    sealevel_correction=False
                break
            else:
                continue

    print()
    print("Configuration saved to: %s"%config_file)
    print()

    # Make a dictionary out of the user input
    config_info = {
        "data_folder": load_path,
        "processed_folder": save_path,
        "figure_folder": fig_path,
        "start_date": start_date,
        "end_date": end_date,
        "database_file": database_file,
        "location": location,
        "inlet_length": inlet_length,
        "apply_corrections":apply_corrections,
        "sealevel_correction": sealevel_correction
    }

    # Save the config file
    with open(config_file,"w") as cf:
        yaml.dump(config_info,cf)

# FUNCTIONS TO CALCULATE LOSSES IN THE INLET
def visc(temp):
    """ Calculate viscosity of air """

    nyy_ref=18.203e-6
    S=110.4
    temp_ref=293.15
    nyy=nyy_ref*((temp_ref+S)/(temp+S))*((temp/temp_ref)**(3./2.))
    return nyy

def rlambda(temp,press):
    """ Calculate mean-free path """

    kaasuv=8.3143
    dm=3.7e-10
    avoc=6.022e23
    return kaasuv*temp/(np.sqrt(2.)*avoc*press*np.pi*dm*dm)

def cunn(Dp,temp,press):
    """ Calculate Cunningham correction """

    lambd = rlambda(temp,press)
    return 1. + 2.*lambd/Dp * (1.165 + 0.483 * np.exp(-0.997*Dp/lambd))

def diffuus(dpp,temp,press):
    """ Calculate diffusion coefficient """

    K=1.38e-23
    return (K*temp*cunn(dpp,temp,press))/(3.*np.pi*visc(temp)*dpp)

def tubeloss(dpp,pflow,plength,temp,press):
    """ Laminar diffusion losses in circular straight conduit """

    DPP,TEMP = np.meshgrid(dpp,temp)
    DPP,PRESS = np.meshgrid(dpp,press)
    DPP,PFLOW = np.meshgrid(dpp,pflow)

    rmuu = np.pi*diffuus(DPP,TEMP,PRESS)*plength/PFLOW;
    pene = np.zeros(rmuu.shape)

    cond1=rmuu<0.02
    cond2=rmuu>=0.02
    pene[cond1]=1. - 2.56*rmuu[cond1]**(2./3.) + 1.2*rmuu[cond1]+0.177*rmuu[cond1]**(4./3.)
    pene[cond2]=1. - 2.56*rmuu[cond2]**(2./3.) + 1.2*rmuu[cond2]+0.177*rmuu[cond2]**(4./3.)

    return pene

# CONVERT TO MATLAB DATENUM
def datetime2datenum(dt):
    """ Convert from python datetime to matlab datenum """

    mdn = dt + timedelta(days = 366)
    frac = (dt-datetime(dt.year,dt.month,dt.day,0,0,0,tzinfo=dt.tzinfo)).seconds \
           / (24.0 * 60.0 * 60.0)
    return mdn.toordinal() + frac

# PLOTTING FUNCTION
def plot_sumfile(handle,v,clim=(10,100000)):
    """ Plot UHEL's sum-formatted aerosol number-size distribution """

    time = v[1:,0] # This is datenum
    dp = v[0,2:]
    data = v[1:,2:]
    data[data<=0]=1e-30 # No holes in plots
    mesh_dp, mesh_time = np.meshgrid(dp,time)
    pcolorplot = handle.pcolormesh(mesh_time,mesh_dp,data,
                                   norm=colors.LogNorm(),
                                   linewidth=0,rasterized=True,cmap='jet')
    handle.set_yscale('log')
    pcolorplot.set_clim(clim)
    pcolorplot.set_edgecolor('face')
    handle.autoscale(tight='true')
    #handle.set_xlim((np.floor(time[0]),np.floor(time[0])+1)) # makes 24-h axis

    handle.grid('on',which='both',linestyle='--',color='w',lw=0.5)
    handle.xaxis.set_major_locator(dts.HourLocator(interval=1))
    handle.xaxis.set_major_formatter(dts.DateFormatter('%H'))
    #handle.set_xticks(np.floor(time[0])+np.arange(0,25)/24.0)
    #handle.set_xticklabels(["%2.2i" % x for x in np.arange(0,25)])
    plt.setp(handle.get_xticklabels(), rotation=80)
    handle.set_ylabel('Dp, [m]')
    handle.set_xlabel('UTC'+'%+d'%v[0,0]+', [h]')
    cbar = plt.colorbar(pcolorplot, ax=handle,
                        ticks=LogLocator(subs=range(10)))
    cbar.set_label('dN/dlogDp, [cm-3]')
    return pcolorplot

# PARSE THROUGH THE RAW DATA FILES
def read_file(fn):
    """ Read the data files to a pandas dataframe """
    with open(fn) as f:

        header_found = False
        data_matrix=[]
        line = f.readline()
        while line:

             line = line.rstrip("\n")

             # Skip completely empty line
             if len(line)==0:
                 line = f.readline()
                 continue

             # Skip commented line
             if line[0]=='#':
                 line = f.readline()
                 continue

             # Extract header line
             if header_found==False:
                 delimiter = re.search('(.)opmode',line).group(1)
                 header = line.split(delimiter)
                 number_of_columns = len(header)
                 header_found = True
                 line = f.readline()
                 continue

             data_line = line.split(delimiter)

             # Add data line to data matrix only if correct number of columns
             # and not a header line if header is inserted mid file
             # when the NAIS is restarted
             if ((len(data_line)==number_of_columns) & ("opmode" not in data_line)):
                 data_matrix.append(data_line)

             line = f.readline()

    # Construct data frame
    # In the data convert anything that can be converted to float and the rest is coerced to NaNs
    df = pd.DataFrame(columns = header, data = data_matrix)
    df.iloc[:,3:] = df.iloc[:,3:].apply(pd.to_numeric, errors='coerce').astype(float)

    # Convert infinities to nan also in case any were present for some reason
    df.iloc[:,3:] = df.iloc[:,3:].replace([np.inf,-np.inf],np.nan)

    return df

# FUNCTIONS TO AVERAGE DATA INTO FIXED BINS
def average_mob(y,h):
    data = np.nan*np.ones((y.shape[0],len(mob_ion)))

    for i in range(0,len(mob_ion_geomeans)):
        if i==0:
            y_block = y[:,h>mob_ion_geomeans[i]]
        else:
            y_block = y[:,((h>mob_ion_geomeans[i]) & (h<=mob_ion_geomeans[i-1]))]

        data[:,i] = np.nanmean(y_block,axis=1)

    y_block = y[:,h<=mob_ion_geomeans[i]]
    data[:,i+1] = np.nanmean(y_block,axis=1)

    return data

def average_dp(y,h):
    data = np.nan*np.ones((y.shape[0],len(dp_par)))

    for i in range(0,len(dp_par_geomeans)):
        if i==0:
            y_block = y[:,h<dp_par_geomeans[i]]
        else:
            y_block = y[:,((h<dp_par_geomeans[i]) & (h>=dp_par_geomeans[i-1]))]

        data[:,i] = np.nanmean(y_block,axis=1)

    y_block = y[:,h>=dp_par_geomeans[i]]
    data[:,i+1] = np.nanmean(y_block,axis=1)

    return data

def find_diagnostic_names(diag_params):
    """
    Find the names of important diagnostic
    parameters in the diagnostic file

    """
    sampleflow_name=None
    sampleflow_names=None
    temperature_name=None
    pressure_name=None

    for temp_name in possible_temperature_names:
         if temp_name in diag_params:
             temperature_name = temp_name
             break

    for pres_name in possible_pressure_names:
        if pres_name in diag_params:
            pressure_name = pres_name
            break

    sf_name = []
    for flow_name in possible_sampleflow_names:
        if flow_name in diag_params:
            sf_name.append(flow_name)

    if len(sf_name)==2:
        sampleflow_names = sf_name
    if len(sf_name)==1:
        sampleflow_name = sf_name

    return temperature_name, pressure_name, sampleflow_names, sampleflow_name

def process_data(
    df,
    rec,
    mode,
    apply_corr,
    sealevel_corr,
    pipel):
    """ Create the number-size distribution files and apply corrections """

    try:
        df = df.set_index(df.columns[0])
        df.index = [parse(y) for y in df.index]
        df_columns = df.columns
        df_inverter_reso = int((len(df_columns)-2)/4)

        # get the number densities and interpolate out the nans
        neg_df = df.iloc[:,2:2+df_inverter_reso+1].astype(float)
        neg_df = neg_df.interpolate().values
        pos_df = df.iloc[:,2+2*df_inverter_reso:2+3*df_inverter_reso+1].astype(float)
        pos_df = pos_df.interpolate().values

        if mode=="ions":
            mob_ion_inv = np.array([float(re.findall(r"[-+]?\d*\.\d+|\d+",y)[0])
                                    for y in df_columns[2:2+df_inverter_reso+1]])
            neg_df = average_mob(neg_df,mob_ion_inv)
            pos_df = average_mob(pos_df,mob_ion_inv)

            # get the number size distributions
            neg_df = neg_df * dlogmob_ion / dlogdp_ion
            pos_df = pos_df * dlogmob_ion / dlogdp_ion

        if mode=="particles":
            dp_par_inv = 2.0*np.array([float(re.findall(r"[-+]?\d*\.\d+|\d+",y)[0])
                                       for y in df_columns[2:2+df_inverter_reso+1]])
            neg_df = average_dp(neg_df,dp_par_inv)
            pos_df = average_dp(pos_df,dp_par_inv)

        # If all data is NaNs then skip
        if (np.all(np.isnan(neg_df)) |
            np.all(np.isnan(pos_df)) ):
            return None

        if apply_corr:
            rec = rec.set_index('opmode')

            # If relevant diagnostic data to make corrections does not exist:
            # return nothing
            t_name,p_name,sf_names,sf_name = find_diagnostic_names(list(rec))
            if ((t_name is not None) & (p_name is not None) &
                ((sf_names is not None) | (sf_name is not None))):
                pass
            else:
                return None

            # Then extract the records that match the polarity
            if mode=="ions":
                df_rec = rec.loc['ions'].set_index(rec.columns[0])
            if mode=="particles":
                df_rec = rec.loc['particles'].set_index(rec.columns[0])

            df_rec.index = [parse(y) for y in df_rec.index]

            # Match records to data according to time
            df_rec = df_rec.reindex(index=df.index,method='nearest')

            # Temperature
            t_df = 273.15 + df_rec[t_name].astype(float)
            t_df = t_df.interpolate().values.flatten()

            # Pressure
            p_df = 100.0 * df_rec[p_name].astype(float)
            p_df = p_df.interpolate().values.flatten()

            # Sampleflow
            if sf_names is not None:
                flow_df = df_rec[sf_names].astype(float).sum(axis=1)
                flow_df = flow_df.interpolate().values.flatten()
            if sf_name is not None:
                flow_df = df_rec[sf_name].astype(float)
                flow_df = flow_df.interpolate().values.flatten()

            # If any of the relevant data are all NaNs return nothing
            if (np.all(np.isnan(neg_df)) |
                np.all(np.isnan(pos_df)) |
                np.all(np.isnan(flow_df)) |
                np.all(np.isnan(p_df)) |
                np.all(np.isnan(t_df))):
                return None

            # Test if the sampleflow is in cm3/s (old models) or l/min and possibly convert to l/min
            if flow_df[0]>100:
                flow_df = (flow_df/1000.0) * 60.0
            else:
                pass

            # Correct the number concentrations to standard conditions (optional)
            if (sealevel_corr):
                stp_corr_df = (pres_ref*t_df)/(temp_ref*p_df)
                neg_df = (stp_corr_df*neg_df.T).T
                pos_df = (stp_corr_df*pos_df.T).T

            # Diffusion loss correction
            if mode=="ions":
                throughput_df = tubeloss(dp_ion*1e-9,flow_df*1.667e-5,pipel,t_df,p_df)
            if mode=="particles":
                throughput_df = tubeloss(dp_par*1e-9,flow_df*1.667e-5,pipel,t_df,p_df)
            neg_df = neg_df / throughput_df
            pos_df = pos_df / throughput_df

            # Robert Wagner's calibration (only ions)
            if mode=="ions":
                roberts_corr = 0.713*dp_ion**0.120
                neg_df = neg_df / roberts_corr
                pos_df = pos_df / roberts_corr

        # CREATE FINAL DATA MATRICES

        # Integrate total number concentrations
        if mode=="ions":
            total_neg_df = np.nansum(neg_df*dlogdp_ion,axis=1)[np.newaxis].T
            total_pos_df = np.nansum(pos_df*dlogdp_ion,axis=1)[np.newaxis].T
        if mode=="particles":
            total_neg_df = np.nansum(neg_df*dlogdp_par,axis=1)[np.newaxis].T
            total_pos_df = np.nansum(pos_df*dlogdp_par,axis=1)[np.newaxis].T

        # Get the utc offset
        if df.index[0].utcoffset()==None:
            utc_offset = 0
        else:
            utc_offset = df.index[0].utcoffset().total_seconds()/(60.0*60.0)

        time_df = np.array([datetime2datenum(x) for x in df.index])[np.newaxis].T

        # Construct the headers
        if mode=="ions":
            df_header = np.insert(dp_ion*1e-9,0,(utc_offset,0))[np.newaxis]
        if mode=="particles":
            df_header = np.insert(dp_par*1e-9,0,(utc_offset,0))[np.newaxis]

        # Construct the sum-files
        negdf = np.concatenate((df_header,np.concatenate((time_df,total_neg_df,neg_df),axis=1)))
        posdf = np.concatenate((df_header,np.concatenate((time_df,total_pos_df,pos_df),axis=1)))

        return [negdf,posdf]

    except:
        return None

def nais_processor(config_file):
    """ Function that processes data from the NAIS

    The function creates specially formatted number-size distribution
    files from neutral cluster and air ion spectrometer (NAIS) data files
    and optionally corrects for losses, applies ion mode calibration
    and corrects concentrations to standard conditions.

    A measurement setup specific configuration file is needed in the
    processing. A configuration file can be created using make_config().

    Four types of files are created:

    NAISn[yyyymmdd]nds.sum
        ion number size distribution, negative polarity
    NAISp[yyyymmdd]nds.sum
        ion number size distribution, positive polarity
    NAISn[yyyymmdd]np.sum
        particle number size distribution, negative polarity
    NAISp[yyyymmdd]np.sum
        particle number-size distribution, positive polarity

    The created processed files have the following format
        [0,0]  = UTC offset in hours
        [1:,0] = time (MATLAB datenum)
        [0,2:] = geometric mean diameter of size-channel (m)
        [1:,1] = integrated total number concentration (cm-3)
        [1:,2:] = normalized number concentrations, dN/dlogDp (cm-3)

    Function arguments:
        Name of the configuration file (str)

    Example:
        nais_processor('/home/user/data/config.yml')

    """

    # Find out today
    today_dt = datetime.today()
    today = today_dt.strftime('%Y%m%d')

    # Check that the config file exists
    if os.path.isfile(config_file)==False:
        print('"%s" does not exist' % config_file)
        return
    else:
        # Try to parse the config file
        with open(config_file,'r') as stream:
            try:
                config = yaml.safe_load(stream)

                load_path = config['data_folder']
                save_path = config['processed_folder']
                start_date = config['start_date']
                database = config['database_file']
                location = config['location']
                end_date = config['end_date']
                if len(end_date)==0:
                    end_date = today
                pipelength = config['inlet_length']
                sealevel_correction = config['sealevel_correction']
                apply_corrections = config['apply_corrections']
            except:
                print("Something went wrong with parsing %s",config_file)
                return

    # Then check if you can initialize th database
    try:
      db = TinyDB(database)
      check = Query()
    except:
        print("Could not initialize database")
        return

    # Test if the configuration information is valid
    try:
        float(pipelength)
    except:
        print('"%s" must be a number' % pipelength)
        return

    # Test if start and end dates are valid
    try:
       start_dt = pd.to_datetime(start_date)
       end_dt = pd.to_datetime(end_date)
    except:
       print('bad start_date or end_date')
       return

    # Check if given data folders exist
    for x in load_path:
        if os.path.exists(x):
            continue
        else:
            print("At least one data folder does not exist")
            return

    # Check for save path and create folders if they do not exist.
    if os.path.exists(save_path):
        pass
    else:
        print("Save path does not exist")
        return

    print("Configuration file: %s" % config_file)

    start_date_str = start_dt.strftime("%Y%m%d")
    end_date_str = end_dt.strftime("%Y%m%d")

    # Convert load and save paths to absolute paths
    load_path = [os.path.abspath(x) + '/' for x in load_path]
    save_path = os.path.abspath(save_path) + '/'

    # Make a list of datetimes that the config file is interested in
    list_of_datetimes = pd.date_range(start=start_date_str, end=end_date_str)

    # list existing dates
    list_of_existing_dates = [x['timestamp'] for x in db.search(check.diagnostics.exists())]
    earliest_existing_date = np.min(pd.to_datetime(list_of_existing_dates))

    # Add unprocessed datafiles to the database
    for x in list_of_datetimes:
        if ((x.strftime('%Y%m%d') in list_of_existing_dates) |
            (x < earliest_existing_date)):
            continue
        else:
            files_found=False
            for z in load_path:
                for y in filename_formats:

                    if ( (os.path.exists(z+x.strftime(y[0])) | # ions
                         os.path.exists(z+x.strftime(y[1]))) & # particles
                         os.path.exists(z+x.strftime(y[2])) # diagnostics
                       ):

                        db.insert(
                            {'timestamp':x.strftime('%Y%m%d'),
                            'diagnostics':z+x.strftime(y[2])}
                            )

                        if os.path.exists(z+x.strftime(y[0])):
                            db.update(
                                {'ions':z+x.strftime(y[0])},
                                check.timestamp==x.strftime('%Y%m%d'))

                        if os.path.exists(z+x.strftime(y[1])):
                            db.update(
                                {'particles':z+x.strftime(y[1])},
                                check.timestamp==x.strftime('%Y%m%d'))

                        files_found=True
                        break

                if files_found:
                    break

    # From the database find the last day with processed data
    processed_days = db.search(
        check.processed_neg_ion_file.exists() |
        check.processed_pos_ion_file.exists() |
        check.processed_neg_particle_file.exists() |
        check.processed_pos_particle_file.exists())

    if len(processed_days)!=0:
        last_day=np.max([datetime.strptime(x['timestamp'],'%Y%m%d') for x in processed_days]).strftime('%Y%m%d')
    else:
        last_day=None

    # Iterate through data that can be processed
    # But is still not processed
    for x in iter(db.search(
        ((check.timestamp==last_day) &
         (check.timestamp>=start_date_str) &
         (check.timestamp<=end_date_str)) |
         (check.diagnostics.exists() &
          (check.ions.exists() |
          check.particles.exists()) &
          ~check.processed_neg_ion_file.exists() &
          ~check.processed_pos_ion_file.exists() &
          ~check.processed_neg_particle_file.exists() &
          ~check.processed_pos_particle_file.exists() &
          (check.timestamp>=start_date_str) &
          (check.timestamp<=end_date_str)))):

        print('processing %s' % x['timestamp'])

        # Read the diagnostics
        records = read_file(x['diagnostics'])

        ions_exist=db.search(
            check.ions.exists() &
            (check.timestamp==x['timestamp']))
        particles_exist=db.search(
            check.particles.exists() &
            (check.timestamp==x['timestamp']))

        # ions
        if ions_exist:

            ions = read_file(x['ions'])
            ion_datamatrices = process_data(
                 ions,
                 records,
                 "ions",
                 apply_corrections,
                 sealevel_correction,
                 pipelength)

            if ion_datamatrices is not None:

                # Save the sum matrices using the standard names
                np.savetxt(save_path+'NAISn'+x['timestamp']+'nds.sum',ion_datamatrices[0])
                np.savetxt(save_path+'NAISp'+x['timestamp']+'nds.sum',ion_datamatrices[1])

                # Update the database
                db.update(
                    {'processed_neg_ion_file': save_path+'NAISn'+x['timestamp']+'nds.sum',
                    'processed_pos_ion_file': save_path+'NAISp'+x['timestamp']+'nds.sum'},
                    check.timestamp==x['timestamp'])

        # particles
        if particles_exist:

            # Process particles
            particles = read_file(x['particles'])
            particle_datamatrices = process_data(
                particles,
                records,
                "particles",
                apply_corrections,
                sealevel_correction,
                pipelength)

            if particle_datamatrices is not None:

                # Save the sum matrices using the standard names
                np.savetxt(save_path+'NAISn'+x['timestamp']+'np.sum',particle_datamatrices[0])
                np.savetxt(save_path+'NAISp'+x['timestamp']+'np.sum',particle_datamatrices[1])

                # Update the database
                db.update(
                    {'processed_neg_particle_file': save_path+'NAISn'+x['timestamp']+'np.sum',
                    'processed_pos_particle_file': save_path+'NAISp'+x['timestamp']+'np.sum'},
                    check.timestamp==x['timestamp'])

    print("Done!")

def do_figs(config_file):
    """ Plot the processed NAIS data

    Function arguments:
        Name of the configuration file (str)

    Example:
        do_figs('/home/user/data/config.yml')

    """

    # Find out today
    today_dt = datetime.today()
    today = today_dt.strftime('%Y%m%d')

    # Check that the config file exists
    if os.path.isfile(config_file) == False:
        print('"%s" does not exist' % config_file)
        return
    else:
        # Try to parse the config file
        with open(config_file,'r') as stream:
            try:
                config = yaml.safe_load(stream)

                save_path = config['processed_folder']
                database = config['database_file']
                location = config['location']
                fig_path = config['figure_folder']
                if len(fig_path)==0:
                    fig_path = None
            except:
                print("Something went wrong with parsing %s",config_file)
                return

    # Check if you can initialize the database
    try:
      db = TinyDB(database)
      check = Query()
    except:
        print("Could not initialize database")
        return

    # Check if processed data path exists
    if not os.path.exists(save_path):
        print("Path to processed data is invalid")
        return

    # Check the fig path exists
    if fig_path is not None:
        if not os.path.exists(fig_path):
            print('Figure path does not exist')
            return
    else:
        print("figure path not given")
        return

    fig_path = os.path.abspath(fig_path) + '/'

    # Define some plotting styles
    plt.style.use('dark_background')

    fontsize = 14
    plt.rcParams.update({'font.size': fontsize,
                         'axes.titlesize': fontsize,
                         'axes.labelsize': fontsize,
                         'xtick.labelsize': fontsize,
                         'ytick.labelsize': fontsize,
                         'figure.titlesize': fontsize,
                         'legend.fontsize': fontsize})

    # From the database find the last day with processed data
    processed_days = db.search(
        check.processed_neg_ion_file.exists() |
        check.processed_pos_ion_file.exists() |
        check.processed_neg_particle_file.exists() |
        check.processed_pos_particle_file.exists())

    if len(processed_days)!=0:
        last_day=np.max([datetime.strptime(x['timestamp'],'%Y%m%d') for x in processed_days]).strftime('%Y%m%d')
    else:
        last_day=None

    # Iterate through data that can be plotted, but is not
    for x in iter(db.search(
        (
          (check.processed_neg_ion_file.exists() &
          check.processed_pos_ion_file.exists() &
          (check.timestamp==last_day)) |
          (check.processed_neg_particle_file.exists() &
          check.processed_pos_particle_file.exists() &
          (check.timestamp==last_day)) |
          (check.processed_neg_ion_file.exists() &
          check.processed_pos_ion_file.exists() &
          ~check.ion_figure.exists()) |
          (check.processed_neg_particle_file.exists() &
          check.processed_pos_particle_file.exists() &
          ~check.particle_figure.exists())
        )
      )
    ):

        print('plotting %s' % x['timestamp'])

        ions_exist=db.search(
            check.processed_neg_ion_file.exists() &
            check.processed_pos_ion_file.exists() &
            (check.timestamp==x['timestamp']))
        particles_exist=db.search(
            check.processed_neg_particle_file.exists() &
            check.processed_pos_particle_file.exists() &
            (check.timestamp==x['timestamp']))

        if ions_exist:
            negion=np.loadtxt(x["processed_neg_ion_file"])
            posion=np.loadtxt(x["processed_pos_ion_file"])
            fig,ax = plt.subplots(2,1,figsize=(7,7.5),dpi=100)
            ax = ax.flatten()
            plot_sumfile(ax[0],posion,clim=(10,10000))
            plot_sumfile(ax[1],negion,clim=(10,10000))
            ax[0].set_title('Positive ions')
            ax[1].set_title('Negative ions')
            plt.tight_layout(rect=[0, 0.0, 1, 0.96])
            fig.suptitle('NAIS' + ' ' + x['timestamp'] + '\n' + location, y=1.0)
            plt.savefig(fig_path+'NAIS_ions_'+ x['timestamp'] +'.png',dpi=100,bbox_inches='tight')
            db.update({'ion_figure': fig_path+'NAIS_ions_'+ x['timestamp'] +'.png'}, check.timestamp==x['timestamp'])
            plt.close()

        if particles_exist:
            negpar=np.loadtxt(x["processed_neg_particle_file"])
            pospar=np.loadtxt(x["processed_pos_particle_file"])
            fig,ax = plt.subplots(2,1,figsize=(7,7.5),dpi=100)
            ax = ax.flatten()
            plot_sumfile(ax[0],pospar,clim=(10,100000))
            plot_sumfile(ax[1],negpar,clim=(10,100000))
            ax[0].set_title('Particles (positive polarity)')
            ax[1].set_title('Particles (negative polarity)')
            plt.tight_layout(rect=[0, 0.0, 1, 0.96])
            fig.suptitle('NAIS' + ' ' + x['timestamp'] + '\n' + location, y=1.0)
            plt.savefig(fig_path+'NAIS_particles_'+ x['timestamp'] +'.png',dpi=100,bbox_inches='tight')
            db.update({'particle_figure': fig_path+'NAIS_particles_'+x['timestamp'] +'.png'}, check.timestamp==x['timestamp'])
            plt.close()

    print("Done!")


import os
import sqlite3
from sqlite3 import Error

############## Conxão com Banco de Dados ##############
dbfile = 'C:\\Users\\magal\\Desktop\\input\\mysite\\db.sqlite3'
con = sqlite3.connect(dbfile)
cur = con.cursor()

############## Variavel global ##############
idPessoal = ""

############## Criar tabela para dados Pessoais - PESSOAL ##############
def createTable():
    try:
        cur.execute('''CREATE TABLE PESSOAL (
                        ID_PESSOAL INTEGER PRIMARY KEY AUTOINCREMENT,
                        NAME       VARCHAR (10, 200),
                        TELEFONE   VARCHAR (10, 11),
                        EMAIL      VARCHAR (10, 200),
                        NASCIMENTO VARCHAR (10, 100)
                    )''')
        con.commit()
    except Error as er:
        print(er)

############## Inserir Dados Pessoais na Tabela ##############
def insertData():
    flag = "n"
    while flag!="s":
        try:
            nome = input("Digite seu nome: ")
            telefone = input("Digite seu telefone: ")
            email = input("Digite seu email: ")
            nasc = input("Digite sua data de nascimento: ")
            dadosPessoais = "INSERT INTO PESSOAL (NAME, TELEFONE, EMAIL, NASCIMENTO) VALUES('"+nome+"', '"+telefone+"', '"+email+"', '"+nasc+"')"
            os.system('cls')
            print("Cadastro efetuado com sucesso!")
            flag = input("Deseja parar de cadastrar dados? [s/n]")
            cur.execute(dadosPessoais)
            os.system('cls')
            con.commit()
        except Error as er:
            print(er)

############## Consultar perfis de Dados Pessoais na Tabela ##############
def viewData():
    try:
        allData = "SELECT * FROM PESSOAL"
        registros = cur.execute(allData)
        for r in registros:
            print(r)
        con.commit()
    except Error as er:
        print(er)

############## Atualizar Dados Pessoais na Tabela ##############
def updateData():
    try:
        idPessoal = input("Escolha qual perfil deseja editar: ")
        newName = input("Digite o novo nome: ")
        newTel = input("Digite o novo telefone: ")
        newMail = input("Digite o novo email: ")
        newBorn = input("Digite a nova dota de nascimento: ")

        att = "UPDATE PESSOAL SET NAME = '"+newName+"', TELEFONE = '"+newTel+"', EMAIL = '"+newMail+"', NASCIMENTO = '"+newBorn+"' WHERE ID_PESSOAL = "+idPessoal+""
        cur.execute(att)
        con.commit()
    except Error as er:
        print(er)

############## Deletar Dados Pessoais na Tabela ##############
def delData():
    try:
        idPessoal = input("Escolha qual perfil deseja excluir: ")
        deletarPessoal = "DELETE FROM PESSOAL WHERE ID_PESSOAL="+idPessoal+""
        deletado = "SELECT * FROM PESSOAL WHERE ID_PESSOAL="+idPessoal+""
        deleted = cur.execute(deletado)
        for d in deleted:
            print(" O perfil: " + str(d) + " foi deletado com sucesso!")
        cur.execute(deletarPessoal)
        con.commit()
        os.system('pause')
    except Error as er:
        print(er)

############## Executar funções para teste ##############
createTable()
#insertData()
#updateData()
#viewData()
#con.commit()
#con.close()


#!/usr/bin/env python3
import contextlib
import logging
import os
import sys

import boto3
import requests
import retrying
from botocore import exceptions

log = logging.getLogger(__name__)
logging.basicConfig(format='[%(levelname)s] %(message)s', level='INFO')

def is_rate_limit_error(exception):
    if exception in [exceptions.ClientError, exceptions.WaiterError]:
        if isinstance(exception, exceptions.ClientError):
            error_code = exception.response['Error']['Code']
        elif isinstance(exception, exceptions.WaiterError):
            error_code = exception.last_response['Error']['Code']
        if error_code in ['Throttling', 'RequestLimitExceeded']:
            print('AWS rate-limit encountered!')
            return True
    return False

@contextlib.contextmanager
def _remove_env_vars(*env_vars):
    environ = dict(os.environ)

    for env_var in env_vars:
        try:
            del os.environ[env_var]
        except KeyError:
            pass

    try:
        yield
    finally:
        os.environ.clear()
        os.environ.update(environ)

@retrying.retry(
    wait_exponential_multiplier=1000,
    wait_exponential_max=300 * 1000,
    stop_max_delay=1800 * 1000,
    retry_on_exception=is_rate_limit_error)
def delete_ec2_volume(name, timeout=600):
    """Delete an EC2 EBS volume by its "Name" tag

    Args:
        timeout: seconds to wait for volume to become available for deletion

    """
    def _force_detach_volume(volume):
        log.info("Force detaching all volume attachments.")
        for attachment in volume.attachments:
            try:
                log.info("Volume has attachment: {}".format(attachment))
                log.info("Detaching volume from instance: {}".format(attachment['InstanceId']))
                volume.detach_from_instance(
                    DryRun=False,
                    InstanceId=attachment['InstanceId'],
                    Device=attachment['Device'],
                    Force=True)
            except exceptions.ClientError as exc:
                log.exception("Failed to detach volume")
                # See the following link for the structure of the exception:
                # https://github.com/boto/botocore/blob/4d4c86b2bdd4b7a8e110e02abd4367f07137ca47/botocore/exceptions.py#L346
                err_message = exc.response['Error']['Message']
                err_code = exc.response['Error']['Code']
                # See the following link for details of the error message:
                # https://jira.mesosphere.com/browse/DCOS-37441?focusedCommentId=156163&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-156163
                available_msg = "is in the 'available' state"
                if err_code == 'IncorrectState' and available_msg in err_message:
                    log.info("Ignoring benign exception")
                    return
                raise

    @retrying.retry(wait_fixed=30 * 1000, stop_max_delay=timeout * 1000,
                    retry_on_exception=lambda exc: isinstance(exc, exceptions.ClientError))
    def _delete_volume(volume):
        log.info("Trying to delete volume...")
        _force_detach_volume(volume)
        try:
            log.info("Issuing volume.delete()")
            volume.delete()  # Raises ClientError (VolumeInUse) if the volume is still attached.
        except exceptions.ClientError:
            log.exception("volume.delete() failed.")
            raise

    def _get_current_aws_region():
        try:
            return requests.get('http://169.254.169.254/latest/meta-data/placement/availability-zone').text.strip()[:-1]
        except requests.RequestException as ex:
            print("Can't get AWS region from instance metadata: {}".format(ex))
            return None

    # Remove AWS environment variables to force boto to use IAM credentials.
    with _remove_env_vars('AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY'):
        volumes = list(boto3.session.Session(
            # We assume we're running these tests from a cluster node, so we
            # can assume the region for the instance on which we're running is
            # the same region in which any volumes were created.
            region_name=_get_current_aws_region(),
        ).resource('ec2').volumes.filter(Filters=[{'Name': 'tag:Name', 'Values': [name]}]))

    if len(volumes) == 0:
        raise Exception('no volumes found with name {}'.format(name))
    elif len(volumes) > 1:
        raise Exception('multiple volumes found with name {}'.format(name))
    volume = volumes[0]
    log.info("Found volume {}".format(volume))

    try:
        _delete_volume(volume)
    except retrying.RetryError as ex:
        raise Exception('Operation was not completed within {} seconds'.format(timeout)) from ex

if __name__ == '__main__':
    log.info("Deleting volume {}".format(sys.argv[1]))
    delete_ec2_volume(sys.argv[1])


from django.shortcuts import render_to_response
from django.views.generic import TemplateView
from django.http import HttpResponseRedirect
from django.contrib import auth
from django.template.context_processors import csrf
from django.shortcuts import render
from Registration.models import Voter,Candidate
from question.models import Reward
from vote.models import Votes,Election
from django.db.models import Sum,Max
from django.template import RequestContext
from django.template import Context
from django.db.models import Q

def home(request):
    q1={}
    try:
        x=request.session['user']
        print ("hello check",str(x))
    except Exception as e:
        print("check")
        dict={'invalid':True}
        q1.update(dict)
        q1.update(csrf(request))
        return render_to_response('Login1.html', q1)
        #return HttpResponseRedirect('/loginmodule/login/')
    p={'user1':request.session['user']}
    q1.update(p)
    print(p)
    q1.update(csrf(request))
    return render_to_response('home.html', q1)

def vote1(request):
    q1={}
    try:
        x=request.session['user']
        print ("hello check",str(x))
    except Exception as e:
        print("check")
        dict={'invalid':True}
        q1.update(dict)
        q1.update(csrf(request))
        return render_to_response('Login1.html', q1)
        #return HttpResponseRedirect('/loginmodule/login/')
    q1.update(csrf(request))
    area=Voter.objects.get(vunm=x)
    li=list(Candidate.objects.filter(area=area.area))
    #li=Candidate.objects.order_by('-reward').all()[:3]
    #lt=Candidate.objects.values('cunm').annotate(reward_count=Count('reward')).order_by('-reward_count')[:5]
    #ca=Candidate.objects.values_list('cunm')
    #can={'cand':li}
    q1['cc']=li
    #c.update(can)
    user=Voter.objects.get(vunm=x)
    if user.flag :
        dict={'voteOnce':True}
        q1.update(dict)
    q1.update(csrf(request))
    return render_to_response('castVote.html', q1)

def castVote(request):
    q1={}
    try:
        x1=request.session['user']
        print ("hello check",str(x1))
    except Exception as e:
        print("check")
        dict1={'invalid':True}
        q1.update(dict1)
        q1.update(csrf(request))
        return render_to_response('Login1.html', q1)
        #return HttpResponseRedirect('/loginmodule/login/')
    q1.update(csrf(request))

    p=request.GET.get("candidate",'')
    q=Candidate.objects.get(cunm=p)

    val=Voter.objects.get(vunm=x1)

    date =Election.objects.latest('Edate').Edate
    x=Votes.objects.get(vid=q.id,ElectionDate=date)
    x.vote=x.vote + 1
    x.save()
    val.flag=1
    val.save()
    return render_to_response('thnxVote.html', q1)

#def showResult(request):
#    q1={}
#    try:
#        x=request.session['user']
#        print ("hello check",str(x))
#    except Exception as e:
#        print("check")
#        dict={'invalid':True}
#        q1.update(dict)
#        q1.update(csrf(request))
#        return render_to_response('Login1.html', q1)
        #return HttpResponseRedirect('/loginmodule/login/')

#    total=Votes.objects.aggregate(Sum('vote'))
#    can=Votes.objects.all()
#    for i in can:
#        temp=can.vid
#        cand[i]=temp.candidate_name
#        result[i]=100*(can.vote/total)
#    q1.update(csrf(request))
#    return render_to_response('showResult.html', q1)

def result(request):
    q1={}
    try:
        x=request.session['user']
        print ("hello check",str(x))
    except Exception as e:
        print("check")
        dict={'invalid':True}
        q1.update(dict)
        q1.update(csrf(request))
        return render_to_response('Login1.html', q1)
        #return HttpResponseRedirect('/loginmodule/login/')

    date =Election.objects.latest('Edate').Edate
    total=Votes.objects.filter(ElectionDate=date).aggregate(Sum('vote'))
    #can=Votes.objects.order_by('-vid__reward').order_by('-vote').all()[:3]
    area=Voter.objects.get(vunm=x)
    cand=(Candidate.objects.filter(area=area.area))
    result=[]
    votes=[]
    for i in cand:
        v=Votes.objects.get(vid=i,ElectionDate=date)
        votes.append(v.vote)
        result.append(100*(v.vote/total['vote__sum']))
        #print(cand)
        #print(result)
    q1['name']=cand
    q1['per']=result
    q1['headcount']=votes

    voteper=[]
    vtotal=len(Voter.objects.all())
    voted=len(Voter.objects.filter(flag=1).all())
    voteper.append(round(100*(voted/vtotal)))
    voteper.append(round(100-100*(voted/vtotal)))
    q1['voteper']=voteper

    v=[]
    parties=Candidate.objects.values_list('party_name', flat=True).distinct()
    for q in parties:
        count=Votes.objects.filter(vid__party_name=q,ElectionDate=date).aggregate(Sum('vote'))
        v.append(round(100*(count['vote__sum']/total['vote__sum'])))

    q1['partyper']=v
    q1['party1']=parties
    j=[]
    area=Candidate.objects.values_list('area', flat=True).distinct()
    for q in area:
        vote=Votes.objects.filter(vid__area=q,ElectionDate=date).aggregate(Max('vote'))['vote__max']
        cand=Votes.objects.filter(vid__area=q,ElectionDate=date,vote=vote).all()
        print(cand)
        j.append(cand)
    q1['warea']=area
    q1['wvote']=j

    q1.update(csrf(request))
    return render_to_response('showResult.html', q1)

def logout(request):
    q1={}
    q1.update(csrf(request))
    del request.session['user']
    return render_to_response('Login1.html',q1)

def AboutUs(request):
    q1={}
    try:
        x=request.session['user']
        print ("hello check",str(x))
    except Exception as e:
        print("check")
        dict={'invalid':True}
        q1.update(dict)
        q1.update(csrf(request))
        return render_to_response('Login1.html', q1)

    state=list(Candidate.objects.values('area').distinct())
    q1['state']=state

    comp=request.GET.get('candidate1')
    comp2=request.GET.get('candidate2')
    p=request.GET.get('city')
    can=Candidate.objects.all()
    if(p):
        y=[]
        z=[]
        y1=[]
        z1=[]
        y2=[]
        z2=[]
        y3=[]
        z3=[]
        q=Candidate.objects.filter(area=p)
        print(q)
        for data in q:
            r=Reward.objects.filter(v_Id=data).all()
            s=Votes.objects.filter(vid=data).all()
            for rate in r:
                z.append(rate.reward)
                z1.append(rate.month)
            y.append(z)
            y1.append(z1)
            z=[]
            z1=[]
            for vote in s:
                z2.append(vote.ElectionDate.year)
                z3.append(vote.vote)
            y2.append(z2)
            y3.append(z3)
            z2=[]
            z3=[]
        q1['candid']=q
        q1['rate']=y
        q1['ratevote']=y3
        q1['ratemonth']=y1
        q1['voteyr']=y2
    else:
        y=[]
        z=[]
        y1=[]
        z1=[]
        y2=[]
        z2=[]
        y3=[]
        z3=[]

        q=Candidate.objects.all()[:3]
        for data in q:
            r=Reward.objects.filter(v_Id=data).all()
            s=Votes.objects.filter(vid=data).all()
            for rate in r:
                z.append(rate.reward)

                z1.append(rate.month.strftime("%b"))
            y.append(z)
            y1.append(z1)
            z=[]
            z1=[]
            for vote in s:
                z2.append(vote.ElectionDate.year)
                z3.append(vote.vote)
            y2.append(z2)
            y3.append(z3)
            z2=[]
            z3=[]
        q1['candid']=q
        q1['rate']=y
        q1['ratevote']=y3
        q1['ratemonth']=y1
        q1['voteyr']=y2

    if(comp and comp2):
        dict={'compare':True}
        q1.update(dict)
        y=[]
        z=[]
        y1=[]
        z1=[]
        y2=[]
        z2=[]
        y3=[]
        z3=[]
        qq=Candidate.objects.filter(Q(candidate_name=comp)|Q(candidate_name=comp2))
        for data in qq:
            r=Reward.objects.filter(v_Id=data).all()
            s=Votes.objects.filter(vid=data).all()
            for rate in r:
                z.append(rate.reward)

                z1.append(rate.month.strftime("%b"))
            y.append(z)
            y1.append(z1)
            z=[]
            z1=[]
            for vote in s:
                z2.append(vote.ElectionDate.year)
                z3.append(vote.vote)
            y2.append(z2)
            y3.append(z3)
            z2=[]
            z3=[]
        q1['candid1']=qq
        q1['rate1']=y
        q1['ratevote1']=y3
        q1['ratemonth1']=y1
        q1['voteyr1']=y2
    q1['candidlist']=can
    q1.update(csrf(request))
    return render_to_response('AboutUs.html', q1)


import numpy as np
import scipy.sparse as sp
from time import time
from Dataset import Dataset
import subprocess as sub
import pandas as pd
import sys,pdb

class ItemView_Dataset(Dataset):
    def __init__(self,args):
        Dataset.__init__(self,args)

        self.same_entity_list       = eval(args.same_entity)
        assert len(self.same_entity_list) == args.num_views, 'length of same_entity and num_views should be same.'

        self.item_view_matrix,self.num_row,self.num_col,self.adjacency_view_matrix = [],[],[],[]
        for view in range(args.num_views):
            num_row, num_col = self.get_row_column_count(self.embed_path + ".view_matrix" + str(view+1))
            self.num_row.append(num_row) # this is only for test purpose. self.num_item is used instead
            self.num_col.append(num_col)

            if self.same_entity_list[view] == -1: #-1 means entities are different
                self.item_view_matrix.append(self.load_rating_file_as_matrix_for_views(self.embed_path + ".view_matrix" + str(view+1),self.num_row[view],self.num_col[view]))
                self.adjacency_view_matrix.append(self.get_adjacency_matrix_sparse(self.item_view_matrix[view],self.item_view_matrix[view].T)) # Note num_items is used
            else: # 1 or other number means it is same entity both side
                self.item_view_matrix.append(self.load_rating_file_as_matrix_for_views(self.embed_path + ".view_matrix" + str(view+1), max(self.num_row[view],self.num_col[view]),max(self.num_row[view],self.num_col[view])))
                _A_obs = self.item_view_matrix[view] + self.item_view_matrix[view].T # Note num_items is used
                _A_obs[_A_obs > 1] = 1
                self.adjacency_view_matrix.append(_A_obs) # Note num_items is used

    def get_row_column_count(self, filename):
        num_users, num_items = 0, 0
        with open(filename, "r") as f:
            line = f.readline().strip()
            while line != None and line != "":
                arr         = line.split("\t")
                u, i        = int(arr[0]), int(arr[1])
                num_users   = max(num_users, u)
                num_items   = max(num_items, i)
                line = f.readline()
        return num_users+1, num_items+1

    def load_rating_file_as_matrix_for_views(self, filename, num_row, num_col):
        mat     = sp.dok_matrix((num_row,num_col), dtype=np.float32)
        with open(filename, "r") as f:
            line = f.readline()
            while line != None and line != "":
                arr = line.split("\t")
                user, item = (int(arr[0]), int(arr[1]))
                mat[user, item] = 1.0
                line = f.readline()
        return mat

    def get_adjacency_matrix_sparse(self,mat1,mat2): ## exactly same as param
        num_row,num_col = (mat1.shape[0] + mat2.shape[0], mat1.shape[1] + mat2.shape[1])
        mat = sp.lil_matrix((num_row,num_col),dtype=np.float32)
        assert num_row == num_col, 'In adj matrix conv. row and col should be equal.'
        mat[0:mat1.shape[0],mat1.shape[0]:] = mat1.astype(np.float32).tolil()
        mat[mat1.shape[0]:,0:mat1.shape[0]] = mat2.astype(np.float32).tolil()
        return mat.tocsr()

    def get_num_social(self,fname):
        df = pd.read_csv(fname,delimiter='\t',header=None)
        return df[1].max()+1

    def load_item_embed_as_mat(self, filename, zero_flag=False):
        # Construct matrix
        mat = np.zeros((self.num_item,self.item_attr_dim),dtype=np.float32)
        item_set = set()
        with open(filename, "r") as f:
            line = f.readline()
            while line != None and line != "":
                toks    = line.replace("\n","").split("::")
                itemid  = int(toks[0])
                embed   = np.array(toks[1].split(" ")).astype(np.float)
                mat[itemid] = embed
                line = f.readline()
                item_set.add(itemid)
            avg = np.mean(mat,axis=0)
            if zero_flag == False:
                for itemid in range(self.num_item):
                    if itemid not in item_set:
                        mat[itemid] = avg
        return mat


import logging
import os
import sys
import traceback
from functools import wraps
from typing import List
from typing import Union
from uuid import uuid4 as uuid

from flask import jsonify
from flask import redirect
from flask import render_template
from flask import request
import json
from flask import send_from_directory
from git.cmd import Git
from werkzeug.wrappers import Response

from dino import environ
from dino import utils
from dino import validation
from dino.admin.forms import SearchHistoryForm
from dino.admin.orm import acl_manager
from dino.admin.orm import blacklist_manager
from dino.admin.orm import broadcast_manager
from dino.admin.orm import channel_manager
from dino.admin.orm import room_manager
from dino.admin.orm import storage_manager
from dino.admin.orm import spam_manager
from dino.admin.orm import user_manager
from dino.config import ApiTargets
from dino.config import ConfigKeys
from dino.exceptions import ChannelNameExistsException
from dino.exceptions import NoSuchRoomException
from dino.exceptions import EmptyChannelNameException
from dino.exceptions import InvalidAclTypeException
from dino.exceptions import InvalidAclValueException
from dino.exceptions import NoSuchUserException
from dino.exceptions import UnknownBanTypeException
from dino.exceptions import ValidationException
from dino.web import app

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

__author__ = 'Oscar Eriksson <oscar.eriks@gmail.com>'

acl_config = environ.env.config.get(ConfigKeys.ACL)

home_dir = os.environ.get('DINO_HOME', default=None)
environment = os.environ.get('DINO_ENVIRONMENT', default=None)

if home_dir is None:
    home_dir = '.'
tag_name = Git(home_dir).describe()

def is_blank(s: str):
    return s is None or len(s.strip()) == 0

def api_response(code, data: Union[dict, List[dict]]=None, message: Union[dict, str]=None):
    if data is None:
        data = dict()
    if message is None:
        message = ''

    return jsonify({
        'status_code': code,
        'data': data,
        'message': message
    })

def internal_url_for(url):
    return app.config['ROOT_URL'] + url

def is_authorized():
    if not environ.env.config.get(ConfigKeys.OAUTH_ENABLED, default=False, domain=ConfigKeys.WEB):
        return True
    if 'token' not in request.cookies:
        return False

    logging.info(str(request.cookies))
    return environ.env.web_auth.check(request.cookies.get('token'))

def requires_auth(f):
    @wraps(f)
    def decorated(*args, **kwargs):
        state = is_authorized()

        if state is False:
            if request.path.startswith('/api'):
                return api_response(400, message="Invalid authentication.")
            return redirect(internal_url_for('/login'))

        if isinstance(state, Response):
            return state
        return f(*args, **kwargs)
    return decorated

@app.route('/login')
def login():
    root_url = environ.env.config.get(ConfigKeys.ROOT_URL, domain=ConfigKeys.WEB, default='/')
    callback_url = environ.env.config.get(ConfigKeys.CALLBACK_URL, domain=ConfigKeys.WEB, default=root_url)
    return environ.env.web_auth.auth.authorize(callback=callback_url)

@app.route('/logout')
def logout():
    request.cookies.pop('token', None)
    return redirect(internal_url_for('/login'))

@app.route('/login/callback')
def authorized():
    return environ.env.web_auth.authorized()

@app.route('/workaround', methods=['GET'])
@requires_auth
def workaround():
    floating_menu = str(environ.env.config.get(ConfigKeys.USE_FLOATING_MENU, domain=ConfigKeys.WEB))
    floating_menu = floating_menu.strip().lower() in {'yes', 'y', 'true'}
    logger.info('using floating menu? "%s"' % str(floating_menu))
    return render_template(
        'workaround.html',
        environment=environment,
        config={
            'ROOT_URL': environ.env.config.get(ConfigKeys.ROOT_URL, domain=ConfigKeys.WEB),
            'FLOATING_MENU': floating_menu
        },
        version=tag_name)

@app.route('/', methods=['GET'])
@requires_auth
def index():
    floating_menu = str(environ.env.config.get(ConfigKeys.USE_FLOATING_MENU, domain=ConfigKeys.WEB))
    floating_menu = floating_menu.strip().lower() in {'yes', 'y', 'true'}
    logger.info('using floating menu? "%s"' % str(floating_menu))
    return render_template(
        'index.html',
        environment=environment,
        config={
            'ROOT_URL': environ.env.config.get(ConfigKeys.ROOT_URL, domain=ConfigKeys.WEB),
            'FLOATING_MENU': floating_menu
        },
        version=tag_name)

@app.route('/api/acls', methods=['GET'])
def acl_list():
    acls = acl_manager.get_acls()
    result = {
        'channel': {},
        'room': {}
    }

    for action in acls['channel']:
        result['channel'][action] = acls['channel'][action]['acls']
    for action in acls['room']:
        result['room'][action] = acls['room'][action]['acls']

    return api_response(200, data=result)

@app.route('/api/acl/actions/<channel_or_room>', methods=['GET'])
def acl_list_actions(channel_or_room):
    return api_response(200, data=[action for action in acl_manager.get_acl_actions(channel_or_room)])

@app.route('/api/acl/validation/<acl_type>', methods=['GET'])
def acl_validation_for_type(acl_type):
    return api_response(200, data={'validation': acl_manager.get_validation_for_type(acl_type)})

@app.route('/api/acl/types/<channel_or_room>/<action>', methods=['GET'])
def acl_list_types_for_action(channel_or_room, action):
    return api_response(200, data=[
        action for action in
        acl_manager.get_acl_types_for_action(channel_or_room, action)
    ])

@app.route('/api/acl/validate/<acl_type>/<acl_value>')
def acl_validate_type_and_value(acl_type, acl_value):
    is_valid, message = validation.acl.is_acl_valid(acl_type, acl_value)
    if is_valid:
        return api_response(200)
    return api_response(400, message=message)

####################################
#             Channels             #
####################################
@app.route('/api/channels', methods=['GET'])
@requires_auth
def channels():
    """ Get all channels. """
    return api_response(200, channel_manager.get_channels())

@app.route('/api/channels', methods=['POST'])
@requires_auth
def create_channel():
    """ Create new channel """
    form = request.get_json()
    channel_name = form['name']
    channel_uuid = str(uuid())
    user_uuid = form['owner']

    message = {}
    if is_blank(channel_name):
        message['name'] = "Channel name can't be none."
    if is_blank(user_uuid):
        message['owner'] = "Owner can't be none."

    if len(message):
        return api_response(400, message=message)
    result = channel_manager.create_channel(channel_name, channel_uuid, user_uuid)

    if result is not None:
        return api_response(400, message=result)
    return api_response(200, {'sort': 1, 'name': channel_name, 'uuid': channel_uuid})

@app.route('/api/channels/<channel_uuid>', methods=['DELETE'])
@requires_auth
def delete_channel(channel_uuid: str):
    channel_manager.remove_channel(channel_uuid)
    return api_response(200)

@app.route('/api/channels/<channel_uuid>/name', methods=['PUT'])
@requires_auth
def update_channel_name(channel_uuid: str):
    form = request.get_json()
    name = form['name']

    try:
        channel_manager.rename(channel_uuid, name)
    except ChannelNameExistsException:
        return api_response(400, message='A channel with that name already exists')
    except EmptyChannelNameException:
        return api_response(400, message='Blank channel name is not allowed')

    return api_response(200)

@app.route('/api/channels/<channel_uuid>/order', methods=['PUT'])
@requires_auth
def update_channel_order(channel_uuid: str):
    form = request.get_json()
    order = form['order']
    channel_manager.update_sort(channel_uuid, order)
    return api_response(200)

@app.route('/api/channels/<channel_uuid>', methods=['GET'])
@requires_auth
def get_channel(channel_uuid: str):
    """ Get channel owners/admins/acls """
    acls = acl_manager.get_acls_channel(channel_uuid)
    acls_decoded = list()
    for acl in acls:
        acl['value'] = utils.b64d(acl['value'])
        acls_decoded.append(acl)

    return api_response(200, {
        'owners': channel_manager.get_owners(channel_uuid),
        'admins': channel_manager.get_admins(channel_uuid),
        'acls': acls_decoded,
    })

@app.route('/api/channels/<channel_uuid>/owners', methods=['POST'])
@requires_auth
def add_channel_owner(channel_uuid: str):
    form = request.get_json()
    user_uuid = form['owner']

    if is_blank(user_uuid):
        return api_response(400, message='Blank user id is not allowed')

    try:
        user = user_manager.get_user(user_uuid)
        user['name'] = utils.b64d(user['name'])
    except NoSuchUserException:
        return api_response(400, message='No Such User.')

    user_manager.add_channel_owner(channel_uuid, user_uuid)
    return api_response(200, user)

@app.route('/api/channels/<channel_uuid>/owners/<user_uuid>', methods=['DELETE'])
@requires_auth
def remove_channel_owner(channel_uuid: str, user_uuid: str):
    user_manager.remove_channel_owner(channel_uuid, user_uuid)
    return api_response(200)

@app.route('/api/channels/<channel_uuid>/admins', methods=['POST'])
@requires_auth
def add_channel_admin(channel_uuid: str):
    form = request.get_json()
    user_uuid = form['admin']

    if is_blank(user_uuid):
        return api_response(400, message='Blank user id is not allowed.')
    try:
        user = user_manager.get_user(user_uuid)
        user['name'] = utils.b64d(user['name'])
    except NoSuchUserException:
        return api_response(400, message='No Such User.')
    user_manager.add_channel_admin(channel_uuid, user_uuid)
    return api_response(200, user)

@app.route('/api/channels/<channel_uuid>/admins/<user_uuid>', methods=['DELETE'])
@requires_auth
def remove_channel_admin(channel_uuid: str, user_uuid: str):
    user_manager.remove_channel_admin(channel_uuid, user_uuid)
    return api_response(200)

@app.route('/api/channels/<channel_uuid>/acls', methods=['POST'])
@requires_auth
def create_channel_acl(channel_uuid: str):
    form = request.get_json()
    action = form['action']
    acl_type = form['type']
    acl_value = form['value']

    message = {}
    if is_blank(acl_type):
        message['type'] = 'Blank type is not allowed.'
    if is_blank(acl_value):
        message['value'] = 'Blank value is not allowed.'
    if len(message):
        return api_response(400, message=message)

    try:
        acl_manager.add_acl_channel(channel_uuid, action, acl_type, acl_value)
    except InvalidAclValueException:
        return api_response(400, message='Invalid ACL value %s' % acl_value)
    except InvalidAclTypeException:
        return api_response(400, message='Invalid ACL type %s' % acl_type)
    except ValidationException as e:
        return api_response(400, message='Invalid ACL: %s' % e.msg)
    except Exception as e:
        logger.exception(traceback.format_exc())
        return api_response(400, message='could not create acl for channel %s: %s' % (channel_uuid, str(e)))

    return api_response(200)

@app.route('/api/channels/<channel_uuid>/acls/<action>/<acl_type>', methods=['PUT'])
@requires_auth
def update_channel_acl(channel_uuid: str, action: str, acl_type: str):
    form = request.get_json()
    value = form['value']

    try:
        acl_manager.update_channel_acl(channel_uuid, action, acl_type, value)
    except InvalidAclValueException:
        return api_response(400, message='Invalid ACL value %s' % value)
    except InvalidAclTypeException:
        return api_response(400, message='Invalid ACL type %s' % acl_type)
    except ValidationException as e:
        return api_response(400, message='Invalid ACL: %s' % e.msg)
    except Exception as e:
        logger.exception(traceback.format_exc())
        return api_response(400, message='could not update acl for channel %s: %s' % (channel_uuid, str(e)))

    return api_response(200)

@app.route('/api/channels/<channel_uuid>/acls/<action>/<acl_type>', methods=['DELETE'])
@requires_auth
def delete_channel_acl(channel_uuid: str, action: str, acl_type: str):
    acl_manager.delete_acl_channel(channel_uuid, action, acl_type)
    return api_response(200)

####################################
#               Rooms              #
####################################
@app.route('/api/channels/<channel_uuid>/rooms', methods=['GET'])
@requires_auth
def rooms_for_channel(channel_uuid: str):
    """ Get rooms of channel """
    return api_response(200, room_manager.get_rooms(channel_uuid))

@app.route('/api/channels/<channel_uuid>/rooms', methods=['POST'])
@requires_auth
def create_room(channel_uuid: str):
    form = request.get_json()
    room_name = form['name']
    room_uuid = str(uuid())
    user_uuid = form['owner']

    message = {}
    if is_blank(room_name):
        message['name'] = 'Blank room name is not allowed.'
    if is_blank(user_uuid):
        message['owner'] = 'Blank owner is not allowed.'

    if len(message):
        return api_response(400, message=message)
    try:
        result = room_manager.create_room(room_name, room_uuid, channel_uuid, user_uuid)
        if result is not None:
            return api_response(400, message=result)

        return api_response(200, {
            'sort': 10,
            'name': room_name,
            'uuid': room_uuid,
            'is_admin': False,
            'is_default': False,
            'is_ephemeral': False,
        })
    except NoSuchUserException:
        return api_response(400, message={
            'owner': 'No such user',
        })

@app.route('/api/rooms/<room_uuid>/order', methods=['PUT'])
@requires_auth
def update_room_order(room_uuid: str):
    form = request.get_json()
    order = form['order']
    room_manager.update_sort(room_uuid, order)
    return api_response(200)

@app.route('/api/channels/<channel_uuid>/rooms/<room_uuid>/name', methods=['PUT'])
@requires_auth
def update_room_name(channel_uuid: str, room_uuid: str):
    form = request.get_json()
    name = form['name']

    result = room_manager.rename(channel_uuid, room_uuid, name)
    if result is not None:
        return api_response(400, message=result)
    return api_response(200)

@app.route('/api/channels/<channel_uuid>/rooms/<room_uuid>', methods=['GET'])
@requires_auth
def get_room(channel_uuid: str, room_uuid: str):
    acls = acl_manager.get_acls_room(room_uuid)
    acls_decoded = list()
    for acl in acls:
        acl['value'] = utils.b64d(acl['value'])
        acls_decoded.append(acl)

    return api_response(200, {
        'channel': {
            'uuid': channel_uuid,
            'name': channel_manager.name_for_uuid(channel_uuid)
        },
        'acls': acls_decoded,
        'owners': room_manager.get_owners(room_uuid),
        'moderators': room_manager.get_moderators(room_uuid)
    })

@app.route('/api/rooms/<room_uuid>/set-default', methods=['PUT'])
@requires_auth
def set_default_room(room_uuid: str):
    try:
        room_manager.set_default_room(room_uuid)
    except Exception as e:
        logger.error('Could not set room as default: %s' % str(e))
        return api_response(400, message='Could not set room as default: %s' % str(e))
    return api_response(200)

@app.route('/api/rooms/<room_uuid>/unset-default', methods=['PUT'])
@requires_auth
def unset_default_room(room_uuid: str):
    try:
        room_manager.unset_default_room(room_uuid)
    except Exception as e:
        logger.error('Could not set room as default: %s' % str(e))
        return api_response(400, message='Could not set room as default: %s' % str(e))
    return api_response(200)

@app.route('/api/rooms/<room_uuid>/set-ephemeral', methods=['PUT'])
@requires_auth
def set_ephemeral_room(room_uuid: str):
    """ Set as ephemeral room """
    try:
        room_manager.set_ephemeral_room(room_uuid)
    except Exception as e:
        logger.error('Could not set room as ephemeral: %s' % str(e))
        return api_response(400, message='Could not set room as ephemeral: %s' % str(e))
    return api_response(200)

@app.route('/api/rooms/<room_uuid>/unset-ephemeral', methods=['PUT'])
@requires_auth
def unset_ephemeral_room(room_uuid: str):
    try:
        room_manager.unset_ephemeral_room(room_uuid)
    except Exception as e:
        logger.error('Could not set room as ephemeral: %s' % str(e))
        return api_response(400, message='Could not set room as ephemeral: %s' % str(e))
    return api_response(200)

@app.route('/api/rooms/<room_uuid>/set-admin', methods=['PUT'])
@requires_auth
def set_admin_room(room_uuid: str):
    try:
        room_manager.set_admin_room(room_uuid)
    except Exception as e:
        logger.error('Could not set room as admin: %s' % str(e))
        return api_response(400, message='Could not set room as admin: %s' % str(e))
    return api_response(200)

@app.route('/api/rooms/<room_uuid>/unset-admin', methods=['PUT'])
@requires_auth
def unset_admin_room(room_uuid: str):
    try:
        room_manager.unset_admin_room(room_uuid)
    except Exception as e:
        logger.error('Could not set room as admin: %s' % str(e))
        return api_response(400, message='Could not set room as admin: %s' % str(e))
    return api_response(200)

@app.route('/api/channels/<channel_uuid>/rooms/<room_uuid>', methods=['DELETE'])
@requires_auth
def delete_room(channel_uuid: str, room_uuid: str):
    room_manager.remove_room(channel_uuid, room_uuid)
    return api_response(200)

@app.route('/api/rooms/<room_uuid>/owners', methods=['POST'])
@requires_auth
def add_room_owner(room_uuid: str):
    form = request.get_json()
    user_uuid = form['owner']

    if is_blank(user_uuid):
        return api_response(400, message='Blank user id is not allowed')

    try:
        user = user_manager.get_user(user_uuid)
        user['name'] = utils.b64d(user['name'])
    except NoSuchUserException:
        return api_response(400, message='No Such User.')

    user_manager.add_room_owner(room_uuid, user_uuid)
    return api_response(200, user)

@app.route('/api/rooms/<room_uuid>/owners/<user_uuid>', methods=['DELETE'])
@requires_auth
def delete_room_owner(room_uuid: str, user_uuid: str):
    user_manager.remove_room_owner(room_uuid, user_uuid)
    return api_response(200)

@app.route('/api/rooms/<room_uuid>/moderators', methods=['POST'])
@requires_auth
def add_room_moderator(room_uuid: str):
    form = request.get_json()
    user_uuid = form['moderator']

    if is_blank(user_uuid):
        return api_response(400, message='Blank user id is not allowed.')
    try:
        user = user_manager.get_user(user_uuid)
        user['name'] = utils.b64d(user['name'])
    except NoSuchUserException:
        return api_response(400, message='No Such User.')
    user_manager.add_room_moderator(room_uuid, user_uuid)
    return api_response(200, user)

@app.route('/api/rooms/<room_uuid>/moderators/<user_uuid>', methods=['DELETE'])
@requires_auth
def delete_room_moderator(room_uuid: str, user_uuid: str):
    user_manager.remove_room_moderator(room_uuid, user_uuid)
    return api_response(200)

@app.route('/api/rooms/<room_uuid>/acls', methods=['POST'])
@requires_auth
def create_room_acl(room_uuid: str):
    form = request.get_json()
    action = form['action']
    acl_type = form['type']
    acl_value = form['value']

    message = {}
    if is_blank(action):
        message['action'] = 'Blank action is not allowed.'
    if is_blank(acl_type):
        message['type'] = 'Blank type is not allowed.'
    if is_blank(acl_value):
        message['value'] = 'Blank value is not allowed.'
    if len(message):
        return api_response(400, message=message)

    try:
        acl_manager.add_acl_room(room_uuid, action, acl_type, acl_value)
    except InvalidAclValueException:
        return api_response(400, message='Invalid ACL value %s' % acl_value)
    except InvalidAclTypeException:
        return api_response(400, message='Invalid ACL type %s' % acl_type)
    except ValidationException as e:
        return api_response(400, message='Invalid ACL: %s' % e.msg)
    except Exception as e:
        logger.exception(traceback.format_exc())
        return api_response(400, message='could not create acl for room %s: %s' % (room_uuid, str(e)))

    return api_response(200)

@app.route('/api/channels/<channel_uuid>/rooms/<room_uuid>/acls/<action>/<acl_type>', methods=['PUT'])
@requires_auth
def update_room_acl(channel_uuid: str, room_uuid: str, action: str, acl_type: str):
    form = request.get_json()
    value = form['value']

    try:
        acl_manager.update_room_acl(channel_uuid, room_uuid, action, acl_type, value)
    except InvalidAclValueException:
        return api_response(400, message='Invalid ACL value %s' % value)
    except InvalidAclTypeException:
        return api_response(400, message='Invalid ACL type %s' % acl_type)
    except ValidationException as e:
        return api_response(400, message='Invalid ACL: %s' % e.msg)
    except Exception as e:
        logger.exception(traceback.format_exc())
        return api_response(400, message='could not update acl for room %s: %s' % (room_uuid, str(e)))

    return api_response(200)

@app.route('/api/rooms/<room_uuid>/acls/<action>/<acl_type>', methods=['DELETE'])
@requires_auth
def delete_room_acl(room_uuid: str, action: str, acl_type: str):
    acl_manager.delete_acl_room(room_uuid, action, acl_type)
    return api_response(200)

####################################
#               Users              #
####################################

@app.route('/api/rooms/<room_uuid>/users', methods=['GET'])
@requires_auth
def get_users_for_room(room_uuid: str):
    return api_response(200, user_manager.get_users_for_room(room_uuid))

@app.route('/api/users/<user_uuid>', methods=['GET'])
@requires_auth
def get_user(user_uuid: str):
    try:
        user = user_manager.get_user(user_uuid)
    except NoSuchUserException:
        return api_response(400, message='No Such User.')
    return api_response(200, user)

@app.route('/api/rooms/<room_uuid>/users/<user_uuid>/kick', methods=['POST'])
@requires_auth
def kick_user(room_uuid: str, user_uuid: str):
    try:
        user_manager.kick_user(room_uuid, user_uuid)
    except Exception as e:
        logger.error('Could not kick user %s' % str(e))
        return api_response(400, message='Could not kick user %s' % str(e))
    return api_response(200)

@app.route('/api/bans', methods=['GET'])
@requires_auth
def banned_users():
    bans = user_manager.get_banned_users()
    result = {'global': list(), 'channel': list(), 'room': list()}

    channel_bans = bans['channels']
    for channel_id in channel_bans:
        channel = {'name': utils.b64d(channel_bans[channel_id]['name']), 'uuid': channel_id}
        for user_id in channel_bans[channel_id]['users']:
            user = channel_bans[channel_id]['users'][user_id]
            user['uuid'] = user_id
            user['name'] = utils.b64d(user['name'])
            user['channel'] = channel
            result['channel'].append(user)

    room_bans = bans['rooms']
    for room_id in room_bans:
        room = {'name': utils.b64d(room_bans[room_id]['name']), 'uuid': room_id}
        for user_id in room_bans[room_id]['users']:
            user = room_bans[room_id]['users'][user_id]
            user['uuid'] = user_id
            user['name'] = utils.b64d(user['name'])
            user['room'] = room
            result['room'].append(user)

    global_bans = bans['global']
    for user_id in global_bans:
            user = global_bans[user_id]
            user['uuid'] = user_id
            user['name'] = utils.b64d(user['name'])
            result['global'].append(user)
    return api_response(200, result)

@app.route('/api/bans', methods=['POST'])
@requires_auth
def ban_user():
    form = request.get_json()
    target = form['target']
    target_uuid = form['target_uuid']
    user_uuid = form['user_uuid']
    duration = form['duration']

    try:
        user_manager.ban_user(user_uuid, target_uuid, duration, target)
    except ValidationException as e:
        return api_response(400, message='invalid duration: %s' % str(e))
    except UnknownBanTypeException as e:
        return api_response(400, message='could not ban user: %s' % str(e))
    except Exception as e:
        logger.exception(traceback.format_exc())
        return api_response(400, message=str(e))

    try:
        user = user_manager.get_user(user_uuid)
        user['name'] = utils.b64d(user['name'])
        user['duration'] = duration
    except NoSuchUserException:
        return api_response(400, message="No such user.")

    if target == 'channel':
        user['channel'] = {
            'uuid': target_uuid,
            'name': channel_manager.name_for_uuid(target_uuid)
        }
    elif target == 'room':
        user['room'] = {
            'uuid': target_uuid,
            'name': room_manager.name_for_uuid(target_uuid)
        }
    return api_response(200, user)

@app.route('/api/bans/<user_uuid>/delete', methods=['POST'])
@requires_auth
def remove_ban(user_uuid: str):
    form = request.get_json()
    target = form['target']
    target_uuid = form['target_uuid']
    user_manager.remove_ban(user_uuid, target_uuid, target)
    return api_response(200)

####################################
#           Super Users            #
####################################

@app.route('/api/super-users', methods=['GET'])
@requires_auth
def super_users():
    return api_response(200, user_manager.get_super_users())

@app.route('/api/super-users', methods=['POST'])
@requires_auth
def create_super_user():
    form = request.get_json()
    user_name = str(form['name']).strip()
    user_uuid = str(form['uuid']).strip()

    message = {}
    if is_blank(user_name):
        message['name'] = 'Blank user name is not allowed.'
    if is_blank(user_uuid):
        message['uuid'] = 'Blank user id is not allowed.'

    if len(message):
        return api_response(400, message=message)
    user_manager.create_super_user(user_name, user_uuid)
    return api_response(200)

@app.route('/api/super-users/<user_uuid>', methods=['POST'])
@requires_auth
def set_super_user(user_uuid: str):
    user_manager.set_super_user(user_uuid)
    return api_response(200)

@app.route('/api/super-users/<user_uuid>', methods=['DELETE'])
@requires_auth
def remove_super_user(user_uuid: str):
    user_manager.del_super_user(user_uuid)
    return api_response(200)

@app.route('/api/users/search/<query>', methods=['GET'])
@requires_auth
def search_user(query: str):
    return api_response(200, user_manager.search_for(query))

####################################
#             Spam                 #
####################################

@app.route('/api/spam', methods=['GET'])
@requires_auth
def latest_spam():
    try:
        msgs = spam_manager.get_latest_spam()
    except Exception as e:
        logger.error('Could not get messages: %s' % str(e))
        logger.exception(traceback.format_exc())
        return api_response(400, message='Could not get message: %s' % str(e))
    return api_response(200, msgs)

@app.route('/api/spam/<spam_id>', methods=['GET'])
@requires_auth
def get_one_spam(spam_id):
    try:
        msgs = spam_manager.get_spam(spam_id)
    except Exception as e:
        logger.error('Could not get messages: %s' % str(e))
        logger.exception(traceback.format_exc())
        return api_response(400, message='Could not get message: %s' % str(e))
    return api_response(200, msgs)

@app.route('/api/spam/search', methods=['POST'])
@requires_auth
def search_spam():
    form = request.get_json()

    if form is None:
        return api_response(400, message='no json data in request')

    user_uuid = form.get('user', None)
    room_uuid = form.get('room', None)
    from_time = form.get('from', None)
    to_time = form.get('to', None)

    user_name = get_user_name(user_uuid)
    room_name = get_room_name(room_uuid)

    try:
        msgs, real_from_time, real_to_time = spam_manager.find(room_uuid, user_uuid, from_time, to_time)
    except Exception as e:
        logger.error('Could not get messages: %s' % str(e))
        logger.exception(traceback.format_exc())
        return api_response(400, message='Could not get message: %s' % str(e))

    return api_response(200, {
        'message': msgs,
        'real_from_time': real_from_time,
        'real_to_time': real_to_time,
        'user_name': user_name,
        'room_name': room_name,
    })

@app.route('/api/spam/<spam_id>/correct', methods=['POST'])
@requires_auth
def set_spam_correct(spam_id):
    try:
        spam_manager.set_correct_or_not(spam_id, True)
    except Exception as e:
        logger.error('Could not get messages: %s' % str(e))
        logger.exception(traceback.format_exc())
        return api_response(400, message='Could not get message: %s' % str(e))

    return api_response(200)

@app.route('/api/spam/<spam_id>/incorrect', methods=['POST'])
@requires_auth
def set_spam_incorrect(spam_id):
    try:
        spam_manager.set_correct_or_not(spam_id, False)
    except Exception as e:
        logger.error('Could not get messages: %s' % str(e))
        logger.exception(traceback.format_exc())
        return api_response(400, message='Could not get message: %s' % str(e))

    return api_response(200)

@app.route('/api/spam/settings', methods=['POST'])
@requires_auth
def spam_set_settings():
    try:
        form = request.get_json()

        if form is None:
            return api_response(400, message='no json data in request')

        enabled = form.get('enabled', None)
        max_length = form.get('max_length', None)
        min_length = form.get('min_length', None)
        threshold = form.get('threshold', None)
        should_delete = form.get('should_delete', None)
        ignore_emoji = form.get('ignore_emoji', None)
        should_save = form.get('should_save', None)

        settings = spam_manager.set_settings(
            enabled, max_length, min_length,
            should_delete, should_save, threshold, ignore_emoji
        )
    except Exception as e:
        msg = 'Could not set settings: {}'.format(str(e))
        logger.error(msg)
        logger.exception(traceback.format_exc())
        environ.env.capture_exception(sys.exc_info())
        return api_response(400, message=msg)

    return api_response(200, settings)

@app.route('/api/spam/settings', methods=['GET'])
@requires_auth
def spam_get_settings():
    try:
        settings = {
            setting.replace('spam_', ''): val
            for setting, val in spam_manager.get_settings().items()
        }
    except Exception as e:
        msg = 'Could not fet settings: {}'.format(str(e))
        logger.error(msg)
        logger.exception(traceback.format_exc())
        environ.env.capture_exception(sys.exc_info())
        return api_response(400, message=msg)

    return api_response(200, settings)

@app.route('/api/spam/set/minlen/<min_length>', methods=['PUT'])
@requires_auth
def spam_set_min_length(min_length: int):
    return _spam_set_setting(spam_manager.set_min_length, min_length, 'set min length')

@app.route('/api/spam/set/maxlen/<max_length>', methods=['PUT'])
@requires_auth
def spam_set_max_length(max_length: int):
    return _spam_set_setting(spam_manager.set_max_length, max_length, 'set max length')

@app.route('/api/spam/disable/save', methods=['POST'])
@requires_auth
def spam_disable_save():
    return _enable_disable(spam_manager.disable_save, 'disable spam classifier')

@app.route('/api/spam/disable/delete', methods=['POST'])
@requires_auth
def spam_disable_delete():
    return _enable_disable(spam_manager.disable_delete, 'disable spam classifier')

@app.route('/api/spam/enable/save', methods=['POST'])
@requires_auth
def spam_enable_save():
    return _enable_disable(spam_manager.enable_save, 'enable spam classifier')

@app.route('/api/spam/enable/delete', methods=['POST'])
@requires_auth
def spam_enable_delete():
    return _enable_disable(spam_manager.enable_delete, 'enable spam classifier')

@app.route('/api/spam/enable', methods=['POST'])
@requires_auth
def enable_spam_classifier():
    return _enable_disable(spam_manager.enable, 'enable spam classifier')

@app.route('/api/spam/disable', methods=['POST'])
@requires_auth
def disable_spam_classifier():
    return _enable_disable(spam_manager.disable, 'disable spam classifier')

def _spam_set_setting(func, value, msg):
    try:
        func(value)
    except Exception as e:
        msg = 'Could not {}: {}'.format(msg, str(e))
        logger.error(msg)
        logger.exception(traceback.format_exc())
        environ.env.capture_exception(sys.exc_info())
        return api_response(400, message=msg)
    return api_response(200)

def _enable_disable(func, msg):
    try:
        func()
    except Exception as e:
        msg = 'Could not  {}: {}'.format(msg, str(e))
        logger.error(msg)
        logger.exception(traceback.format_exc())
        environ.env.capture_exception(sys.exc_info())
        return api_response(400, message=msg)
    return api_response(200)

####################################
#             History              #
####################################

@app.route('/history', methods=['GET', 'POST'])
def history_workaround():
    form = SearchHistoryForm(request.form)
    return render_template(
            'history.html',
            form=form,
            messages=list(),
            environment=environment,
            version=tag_name)

@app.route('/history/<message_id>/delete', methods=['PUT'])
def delete_message_workaround(message_id: str):
    storage_manager.delete_message(message_id)
    return jsonify({'status_code': 200})

@app.route('/history/<message_id>/undelete', methods=['PUT'])
def undelete_message_workaround(message_id: str):
    storage_manager.undelete_message(message_id)
    return jsonify({'status_code': 200})

@app.route('/history/search', methods=['POST'])
def search_history_workaround():
    form = SearchHistoryForm(request.form)
    user_id = form.user_id.data
    room_id = form.room_id.data
    from_time = form.from_time.data
    to_time = form.to_time.data
    msgs = list()

    try:
        msgs, real_from_time, real_to_time = storage_manager.find_history(room_id, user_id, from_time, to_time)
    except Exception as e:
        logger.error('Could not get messages: %s' % str(e))
        logger.exception(traceback.format_exc())
        return render_template(
                'history.html',
                form=form,
                messages=msgs
        )

    return render_template(
            'history.html',
            form=form,
            messages=msgs
    )

@app.route('/api/history/stream', methods=['POST'])
@requires_auth
def stream_history():
    form = request.get_json()
    user_uuid = form['user']
    room_uuid = form['room']
    from_time = form['from']
    to_time = form['to']

    try:
        msgs, real_from_time, real_to_time = storage_manager.find_history(room_uuid, user_uuid, from_time, to_time)
    except Exception as e:
        logger.error('Could not get messages: %s' % str(e))
        logger.exception(traceback.format_exc())
        return api_response(400, message='Could not get message: %s' % str(e))

    def generate_messages():
        batch = list()
        n_messages = len(msgs)
        n_batch = 0
        batch_size = 100

        user_name = get_user_name(user_uuid)
        room_name = get_room_name(room_uuid)

        for message in msgs:
            try:
                json_body = message['body']
                json_body = json.loads(json_body)
                json_body = json_body.get('text')
                message['body'] = json_body
            except Exception:
                pass  # ignore, use original

            batch.append(message)
            if len(batch) >= batch_size:
                yield api_response(200, {
                    'batch': n_batch,
                    'total_batches': int(n_messages / batch_size),
                    'message': batch,
                    'real_from_time': real_from_time,
                    'real_to_time': real_to_time,
                    'username': user_name,
                    'room': room_name,
                })
                n_batch += 1
                batch.clear()

    return Response(generate_messages(), mimetype='application/json')

def get_room_name(room_uuid: str) -> str:
    if room_uuid is not None and len(room_uuid.strip()) > 0:
        try:
            return utils.get_room_name(room_uuid)
        except NoSuchRoomException:
            pass
    return ''

def get_user_name(user_id: str) -> str:
    if user_id is not None and len(user_id.strip()) > 0:
        try:
            return utils.get_user_name_for(user_id)
        except NoSuchUserException:
            pass
    return ''

@app.route('/api/history', methods=['POST'])
@requires_auth
def search_history():
    form = request.get_json()
    user_uuid = form['user']
    room_uuid = form['room']
    from_time = form['from']
    to_time = form['to']

    user_name = get_user_name(user_uuid)
    room_name = get_room_name(room_uuid)

    try:
        msgs, real_from_time, real_to_time = storage_manager.find_history(room_uuid, user_uuid, from_time, to_time)
    except Exception as e:
        logger.error('Could not get messages: %s' % str(e))
        logger.exception(traceback.format_exc())
        return api_response(400, message='Could not get message: %s' % str(e))

    try:
        clean_msgs = list()
        for message in msgs:
            try:
                json_body = message['body']
                json_body = json.loads(json_body)
                json_body = json_body.get('text')
                message['body'] = json_body
            except Exception:
                pass  # ignore, use original
            clean_msgs.append(message)
    except Exception as e:
        logger.error('Could not clean messages, will use original: %s' % str(e))
        clean_msgs = msgs

    return api_response(200, {
        'message': clean_msgs,
        'real_from_time': real_from_time,
        'real_to_time': real_to_time,
        'username': user_name,
        'room': room_name,
    })

@app.route('/api/history/<message_id>', methods=['DELETE'])
@requires_auth
def delete_message(message_id: str):
    storage_manager.delete_message(message_id)
    return api_response(200)

@app.route('/api/history/<message_id>/undelete', methods=['PUT'])
@requires_auth
def undo_delete_message(message_id: str):
    storage_manager.undelete_message(message_id)
    return api_response(200)

####################################
#            Blacklist             #
####################################

@app.route('/api/blacklist', methods=['GET'])
@requires_auth
def blacklist():
    return api_response(200, blacklist_manager.get_black_list())

@app.route('/api/blacklist', methods=['POST'])
@requires_auth
def add_to_blacklist():
    form = request.get_json()
    words = form['words']
    try:
        blacklist_manager.add_words(words)
    except Exception as e:
        logger.error('Could not add word to blacklist: %s' % str(e))
        return api_response(400, message='Could not add word to blacklist: %s' % str(e))
    return api_response(200)

@app.route('/api/blacklist/<word_id>', methods=['DELETE'])
@requires_auth
def remove_from_blacklist(word_id: str):
    try:
        blacklist_manager.remove_word(word_id)
    except Exception as e:
        logger.error('Could not remove word from blacklist: %s' % str(e))
        return api_response(400, message='Could not remove word from blacklist: %s' % str(e))
    return api_response(200)

@app.route('/api/broadcast', methods=['POST'])
@requires_auth
def send_broadcast():
    form = request.get_json()
    verb = form['verb']
    content = form['content']

    message = {}
    if is_blank(verb):
        message['verb'] = 'Verb may not be empty.'
    if is_blank(content):
        message['content'] = 'Content may not be empty.'

    if len(message):
        return api_response(400, message=message)

    try:
        content = utils.b64e(content)
        broadcast_manager.send(content, verb)
    except Exception as e:
        logger.error('Could not send broadcast: %s' % str(e))
        logger.exception(traceback.format_exc())
        return api_response(400, message='Could not send broadcast')
    return api_response(200)

@app.route('/api/acls/<target>/actions/<api_action>/types', methods=['GET'])
@requires_auth
def acl_types_for_target_and_action(target: str, api_action: str):
    if target not in [ApiTargets.CHANNEL, ApiTargets.ROOM]:
        return api_response(400, message='unknown target type "%s"' % target)

    config = acl_config[target][api_action]
    acls = set(config['acls'])
    excludes = set()
    if 'exclude' in config:
        excludes = set(config['exclude'])

    output = list()
    for acl in acls:
        if acl in excludes:
            continue

        output.append({
            'acl_type': acl,
            'name': acl.capitalize()
        })
    return api_response(200, output)

@app.route('/static/<path:path>')
def send_static(path):
    return send_from_directory('admin/static/', path)

@app.route('/static/custom/<path:path>')
def send_custom(path):
    return send_from_directory('admin/static/custom/', path)

@app.route('/images/<path:path>')
def send_images(path):
    return send_from_directory('admin/static/vendor/images/', path)

@app.route('/staticv/<path:path>')
def send_staticv(path):
    return send_from_directory('admin/static/vendor/', path)

@app.route('/fonts/<path:path>')
def send_fonts(path):
    return send_from_directory('admin/static/vendor/fonts/', path)

@app.errorhandler(404)
def page_not_found(_):
    # your processing here
    return index()