# -*- coding: utf-8 -*-
"""
Created on Thu Dec 06 09:49:58 2018

@author: daniel
"""

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import math
import os
import csv
import os.path
import tarfile
import shutil
import collections

"""
Function that checks if there is no ocean/sea on the left side of the model domain, if there is it finds the highest positioned
point of the inland area and clips the model accordingly (keeping the domain to the right side of that point). This simulates the
position of the water divide.

top_lst = model.top_elev
ib_arr = model.ibound_arr
model_obj = model

"""
def trim_model_domain(model_obj, top_lst, ib_arr):

    #   first check if there really is a sea on the left side of the model domain, if there actually is one then find the index of the
    #   left coastline position. Trim the topo values in case there is also sea/ocean at the left side of the land (land looks like an island)

    left_side = top_lst[:25]
    neg_lst = [i for i in left_side if i < 0]
    if len(neg_lst) > 0:
        start = 0
        for f in range(100):
            cut_lst = top_lst[start : start + 10]
            if len([i for i in cut_lst if i < 0]) >= 5:
                start += 10
            else:
                break
        idx_neg = start#left_side.index([i for i in left_side if i < 0][-1])
        while top_lst[idx_neg] < -0.:
            idx_neg += 1
        new_st_idx = idx_neg

    else:
        idx_neg = 0
        while top_lst[idx_neg] < -0.:
            idx_neg += 1
        new_st_idx = 0

    left_cst_idx = idx_neg

    #   check that the landward boundary doesnt have any empty columns on the inland side of the model domain
    for z in range(ib_arr.shape[-1]):
        ibound_act_lay_idxs = [i for i, x in enumerate(model_obj.ibound_arr[:, 0, z].tolist()) if x == 1]
        if ibound_act_lay_idxs != []:
            break#pass
        else:
            new_st_idx += 1
            pass#break
    ib_arr = ib_arr[:, :, new_st_idx:]

    #   check if there is any column that contains only inactive cells - if so then cut the IBOUND array to end before that column
    new_end_idx = -1
    for a in range(new_st_idx, model_obj.ibound_arr.shape[-1]):
        ibound_act_lay_idxs = [i for i, x in enumerate(model_obj.ibound_arr[:, 0, a].tolist()) if x == 1]
        if ibound_act_lay_idxs != []:
            pass
        else:
            ib_arr = ib_arr[:, :, : a - new_st_idx]
            new_end_idx = a
            break

    #   second step also checks that the end of the model domain ends at the lowest point, start from the end of the model domain
    if new_end_idx != -1:
        offsh_indx = new_end_idx
    else:
        offsh_indx = len(top_lst)

    for b in range(offsh_indx - 11, 10, -10):

        top_right_lst = []
        for c in range(b, b + 10):
            top_right_lay = [j for j, x in enumerate(model_obj.ibound_arr[:, 0, c].tolist()) if x == 1][0]
            top_right_lst.append(top_right_lay)

        top_left_lst = []
        for d in range(b - 10, b):
            top_lay_left = [j for j, x in enumerate(model_obj.ibound_arr[:, 0, d].tolist()) if x == 1][0]
            top_left_lst.append(top_lay_left)

        print(np.mean(top_left_lst), np.mean(top_right_lst))

        #   if the elevation on the right is higher than on the left, continue
        if np.mean(top_left_lst) >= np.mean(top_right_lst) + 1.0:
            pass
        else:
            new_end_idx = b
            #new_end_lay = [j for j, x in enumerate(model_obj.ibound_arr[:, 0, b].tolist()) if x == 1][-1]
            #ib_arr = ib_arr[:new_end_lay, :, :b]
            break

    #   now check that the right side of the model domain is not actually even higher than the left side
    if top_lst[new_end_idx] > top_lst[0]:
        print(top_lst[new_end_idx], top_lst[0])
        new_end_idx = top_lst.index(min(top_lst))
        b = new_end_idx

    new_end_lay_1 = [j for j, x in enumerate(model_obj.ibound_arr[:, 0, new_end_idx].tolist()) if x == 1][-1]
    new_end_lay_2 = new_end_lay_1

    #for c in range(model_obj.cst_idx, new_end_idx):
    for c in range(int(model_obj.cst_idx / 2), new_end_idx):
        try:
            new_end_lay_3 = [j for j, x in enumerate(model_obj.ibound_arr[:, 0, c].tolist()) if x == 1][-1]
            if new_end_lay_3 > new_end_lay_2:
                new_end_lay_2 = new_end_lay_3
        except IndexError:
            pass

    new_end_lay = max(new_end_lay_1, new_end_lay_2)

    if b < new_end_idx:
        ib_arr = ib_arr[:new_end_lay, :, : b]
    else:
        ib_arr = ib_arr[:new_end_lay, :, :]
    """
    #   if the idx_neg is 0 then dont do anything otherwise adjust the model domain and
    if idx_neg == 0:

        #   check for multiple landmasses
        import itertools
        splitted = [list(g) for i, g in itertools.groupby(top_lst,lambda x: x<0)]

        #   sometimes there might be small chunks of below sea level areas, but those are not the real ocean yet
        #   check from the beginning of the splitted list and if there is a short (< 2km) below sea level area
        new_splitted = [x for x in splitted]

        g = 1
        while len(new_splitted[g]) <= 20 and new_splitted[g][0] < 0. or new_splitted[g][0] > 0.:
            new_splitted[0] = new_splitted[0] + splitted[g]
            del new_splitted[g]

        if len(new_splitted) > 2 and new_splitted[0][0] > 0. and new_splitted[1][0] < 0.:
            new_end_idx = top_lst.index(min(new_splitted[1]))
            max_elev = max(top_lst[:new_end_idx])
            if top_lst.index(max_elev) > 0:
                max_elev_idx = top_lst.index(max_elev) - 1
            else:
                max_elev_idx = top_lst.index(max_elev)

            new_x_start = model_obj.x_start
            new_ib_arr = ib_arr[:, :, :new_end_idx]
            new_ncol = new_ib_arr.shape[-1]

        else:
            new_x_start = model_obj.x_start
            new_ib_arr = ib_arr
            new_ncol = ib_arr.shape[-1]
            max_elev = max(top_lst[:])
            max_elev_idx = top_lst.index(max_elev) - 1
            idx_neg = 0

        return new_x_start, new_ib_arr, new_ncol, max_elev_idx, new_end_idx, idx_neg

    else:
    """
    #   find the position of the coastline on the right side of the model domain
    #cst_val  = next((x for x in top_lst[left_cst_idx:] if x < 0.0), None)
    #right_cst_idx = left_cst_idx + top_lst[left_cst_idx:].index(cst_val) - 1
    right_cst_idx = - 1
    #   now find the maximum elevation between these two points
    try:
        max_elev = max(top_lst[left_cst_idx:right_cst_idx])
        max_elev_idx = top_lst.index(max_elev) #- 1 - left_cst_idx
    except ValueError:
        max_elev = max(top_lst[left_cst_idx:])
        max_elev_idx = top_lst.index(max_elev) #- 1 - left_cst_idx

    #   change the
    new_x_start = -(max_elev_idx / 10.) #model_obj.x_start + (max_elev_idx / 10.)
    new_ib_arr = ib_arr[:, :, max_elev_idx : new_end_idx]
    new_end_idx = max_elev_idx + new_ib_arr.shape[-1]
    new_ncol = new_ib_arr.shape[-1]

    return new_x_start, new_ib_arr, new_ncol, max_elev_idx, new_end_idx, idx_neg

def find_lonely_cells(ibound_array):

    col_n = ibound_array.shape[-1]
    row_n = ibound_array.shape[0]

    #   go through the ibound_arr cell by cell
    for j in range(1, col_n - 1):
        for i in range(1, row_n - 1):
            if j == 0:
                left = 0
            else:
                left = ibound_array[i-1,0,j]

            if j == (col_n - 1):
                right = 0
            else:
                right = ibound_array[i+1,0,j]

            if i == 0:
                up = 0
            else:
                up = ibound_array[i,0,j-1]

            if i == (row_n -1):
                down = 0
            else:
                down = ibound_array[i,0,j+1]

            if (left == 0 and right == 0 and up == 0 and down == 0):
                ibound_array[i,0,j] = 0

    return ibound_array

"""
Function that reads a CSV file and returns the lognormal values of MU and STDEV depending
on what COSCAT region and which parameter we are looking for.

coscat_id = 16
subreg_id = 1
csv_dir = r'g:\Water_Nexus\_A4_GUM\_MODEL_input_files\_SUBREG_representative_models_GEOLOGY_stats.csv'
cst_type = 'henry_other'
param_type = 'glhymps_bot'
"""

def read_param_from_csv_subreg(coscat_id, subreg_id, csv_dir, param_type):
    #   read in the csv file with pandas and get the list of column names
    df = pd.read_csv(csv_dir, index_col = False, header = 0)
    col_names = list(df.columns.values)
    #   define the name of the column we are looking for based on the function parameters
    col_param_name = param_type
    match_str = [s for s in col_names if col_param_name in s]
    #   find which string is the mu and which is the stdv
    mu_col = [r for r in match_str if 'mu' in r]
    std_col = [r for r in match_str if 'std' in r]
    #   find the values in the table
    mu_val = df.loc[(df[col_names[0]] == coscat_id) & (df[col_names[1]] == subreg_id), mu_col].values[0]
    std_val = df.loc[(df[col_names[0]] == coscat_id) & (df[col_names[1]] == subreg_id), std_col].values[0]
    #   return the final values
    return mu_val, std_val

def read_param_from_csv(coscat_id, csv_dir, param_type):
    #   read in the csv file with pandas and get the list of column names
    df = pd.read_csv(csv_dir, index_col = False, header = 0)
    col_names = list(df.columns.values)
    #   define the name of the column we are looking for based on the function parameters
    col_param_name = param_type
    match_str = [s for s in col_names if col_param_name in s]
    #   find which string is the mu and which is the stdv
    mu_col = [r for r in match_str if 'mu' in r]
    std_col = [r for r in match_str if 'std' in r]
    #   find the values in the table
    mu_val = df.loc[df[[col_names[0]] == coscat_id], mu_col].values[0]
    std_val = df.loc[df[col_names[0]] == coscat_id, std_col].values[0]
    #   return the final values
    return mu_val, std_val

"""
Function that generates a set of randomly generated values based on a lognormal distribution.
The input variables are the mu_val and std_val create in the previous function.

mu_in = mu_val
std_in = std_val
len_lst = 100
title = 'RCH input values per column'
y_label = 'RCH rate in mm/d'
"""

def generate_ln_rand_lst(rand_seed_val, mu_in, std_in, len_lst, title, y_label, fig_out_dir, plot = True):
    #   set the random seed for this function
    np.random.seed(rand_seed_val)
    #   generate an array of values drawn from the normal distribution
    rand_vals = np.exp(np.random.normal(mu_in, std_in, size = len_lst))
    #   if the plotting is set to True also give the output plots
    if plot is True:
        #   create an X value list
        x_col = np.linspace(0, len_lst - 1, len_lst)
        #   define window size, output and axes
        fig, ax = plt.subplots(figsize=[8,6])
        ax.bar(x_col, rand_vals)
        #   set plot title
        ax.set_title(title)
        #   set x-axis name
        ax.set_xlabel("MODFLOW column number")
        #   set y-axis name
        ax.set_ylabel(y_label)
        ax = plt.gca()
        ax.yaxis.grid()
        #   save the figure
        plt.savefig(fig_out_dir, dpi = 300)
        plt.close()
        del fig
    #   return the list of randomly generated values
    return rand_vals.tolist()

"""
Function that finds and provides the geological input parameters for each COSCAT region.

coscat_id = 1413
csv_dir = r'g:\Water_Nexus\_A2\_models_final\coscat_geology_input.csv'

"""

def read_geo_params(coscat_id, csv_dir):

    #   get the sand and mud % for the given coscat region
    df = pd.read_csv(csv_dir, index_col = False, header = 0)
    col_names = list(df.columns.values)
    sand_val = df.loc[df[col_names[0]] == coscat_id, 'Sand_gl'].values[0]
    mud_val = df.loc[df[col_names[0]] == coscat_id, 'Mud_gl'].values[0]
    y_exact_val = df.loc[df[col_names[0]] == coscat_id, 'Y1'].values[0]
    qs_val = df.loc[df[col_names[0]] == coscat_id, 'QS'].values[0]
    sm_val = df.loc[df[col_names[0]] == coscat_id, 'SM'].values[0]
    y_val = df.loc[df[col_names[0]] == coscat_id, 'Y'].values[0]
    shlf_mud_val = df.loc[df[col_names[0]] == coscat_id, 'Mud_shlf'].values[0]
    slp_mud_val = df.loc[df[col_names[0]] == coscat_id, 'Mud_slp'].values[0]
    #   since the total sum of these two (sand and mud) doesnt amount to 100%
    add_val = (100. - (sand_val + mud_val)) / 2.
    sand_pct, mud_pct = sand_val + add_val, mud_val + add_val
    #   calculate the % for the reworking parameter - basically multiply by 5 since the
    #   highest Y1 value is ~20 (100/20 = 5)
    y_val_out = y_exact_val * 5.

    #   return the final output values
    return qs_val, sm_val, y_val, y_val_out, sand_pct, mud_pct, shlf_mud_val, slp_mud_val

"""
Function that loads in the final output budget file and for each time step computes how much
recharge gets drained for the total model domain.

budget_in = r'g:\Water_Nexus\_A2\_models_testing_1413\coscat_1413_geo_scenario_1_rch_p_min_et\eq_1\_cbc.npy'

rch_csv = r'g:\Water_Nexus\_A2\_models_final\_COSCAT_representative_models_RCH_stats.csv'
rch_param_p_min_et_ln = read_param_from_csv(1413, rch_csv, 'henry_other', 'cs_p_min_et_rch')

np.random.seed(1)
#   generate an array of values drawn from the normal distribution
rch_lst_in = np.exp(np.random.normal(rch_param_p_min_et_ln[0], rch_param_p_min_et_ln[1], size = 217))

"""

def model_res_analysis_rch_drn(budget_in, rch_lst_in):

    #   load the budget dictionary
    dict_in = np.load(budget_in).item()
    #   loop through all the keys of the dictionary and get the RCH and DRN arrays
    for key in dict_in.keys():

        if key != 0:
            rch_arr = dict_in[key]['q_rch'][1][0]
            drn_arr = dict_in[key]['q_drn']

            drn_sum, rch_sum = 0., 0.

            for i in range(len(drn_arr)):
                rch_val = rch_arr[i]
                drn_val = drn_arr[i][1]
                rch_in = rch_lst_in[i] * 0.001

                drn_sum += drn_val
                rch_sum += rch_val

                print(rch_in, rch_val, drn_val)

            print(rch_sum, drn_sum)

#   transform the point water head to fresh water head array
def pt_hd_to_frsh_hd(pt_hd_arr, conc_arr, densref, denseslp, top_elev, bot_lst):

    top_bot_lst = [top_elev] + bot_lst
    frsh_hd_arr = pt_hd_arr * 0.0

    for i in range(pt_hd_arr.shape[0]):
        for j in range(pt_hd_arr.shape[-1]):

            pt_head = pt_hd_arr[i, 0, j]
            conc = denseslp * conc_arr[i, 0, j] + densref
            z = (top_bot_lst[i] + top_bot_lst[i + 1]) / 2

            frsh_hd_arr[i, 0, j] = (conc / densref) * pt_head - ((conc - densref) / densref) * z

    return frsh_hd_arr

#   gets the volumes and % of freshwater in different zones of the model domain
"""
arr_ibound = model.ibound_arr
arr_conc = conc_arr
idx_cst = cst_idx_0  #int(abs(model.x_start) * 10)
idx_shelf_edge = idx_shelf_edge_0  #int((cont_shelf_edge[0] - model.x_start) * 10)

"""
def get_frsh_vol_pct(arr_ibound, arr_conc, idx_cst, idx_shelf_edge):
    #   calculate the volume and % in the inland zone
    tot_inland_cells, fresh_inland_cells = 0, 0
    for a in range(idx_cst):
        ibound_act_lay_idxs = [i for i, x in enumerate(arr_ibound[:, 0, a].tolist()) if x == 1]
        #   loop through the active layers and check whether the concentration is fresh or not
        for x in range(len(ibound_act_lay_idxs)):
            if arr_conc[ibound_act_lay_idxs[x], 0, a] <= 0.1:
                fresh_inland_cells += 1
        tot_inland_cells += len(ibound_act_lay_idxs)

    #   calculate the volume and % in the shelf zone
    tot_shelf_cells, fresh_shelf_cells = 0, 0
    for b in range(idx_cst, min(idx_shelf_edge, arr_conc.shape[-1])):
        ibound_act_lay_idxs = [i for i, x in enumerate(arr_ibound[:, 0, b].tolist()) if x == 1]
        #   loop through the active layers and check whether the concentration is fresh or not
        for y in range(len(ibound_act_lay_idxs)):
            if arr_conc[ibound_act_lay_idxs[y], 0, b] <= 0.1:
                fresh_shelf_cells += 1
        tot_shelf_cells += len(ibound_act_lay_idxs)

    return tot_inland_cells, fresh_inland_cells, tot_shelf_cells, fresh_shelf_cells

#   define a function to plot the GHB boundary and top system summary (DRN and RCH)
"""
ghb_cell_head_lst = model.ghb_head_diff_lst
ghb_cell_head_max_diff = model.ghb_head_check_max_diff
ghb_in_lst = model.ghb_input_lst
rch_lst = rch_p_min_et_lst_m_d
q_rch = qrch
q_drn = qdrains
q_ghb = qghb
delcol = model.del_col
ncol = model.ncol

arr_hk = model.hk_arr
cond_lst = model.ghb_input_lst
index_coast = idx_cst

top_el = model.top_elev
x_st = model.x_start
x_end = model.x_end
index_coast = model.rch_extent
"""

def plot_ghb_drn_rch_summary(index_coast, ghb_cell_head_lst, ghb_cell_head_max_diff, rch_lst, q_rch, q_drn, q_ghb, top_el, x_st, x_end, delcol, ncol, arr_hk, cond_lst, save_dir):

    #index_coast = int(abs(x_st) * 10.)
    fig = plt.figure(figsize = (20, 12))
    #   create a figure object and assign all the parts
    ax1 = plt.subplot2grid((3, 1), (0, 0))  #
    ax2 = plt.subplot2grid((3, 1), (1, 0))  #
    ax3 = plt.subplot2grid((3, 1), (2, 0))  #
    ax2b = ax2.twinx()  # instantiate a second axes that shares the same x-axis
    ax3b = ax3.twinx()  # instantiate a second axes that shares the same x-axis
    #   set the exact positions for each axis
    ax1.set_position([0.05, 0.675, 0.9, 0.25]) # [left, bottom, width, height]
    ax2.set_position([0.05, 0.35, 0.9, 0.25])
    ax3.set_position([0.05, 0.025, 0.9, 0.25])

    #   create the first plot that will show the RCH rates and the GHB difference between assigned and modelled heads
    x_axis_pts = np.linspace(x_st + (delcol / (2 * 1000.)), x_end - (delcol / (2 * 1000.)), ncol)

    #   prepare the individual type of internal flow types for plotting
    qrch_to_plot = q_rch[1].tolist()[0]
    qdrn_to_plot = [round(i[1], 6) for i in q_drn.tolist()] + (ncol - len([round(i[1], 6) for i in q_drn.tolist()])) * [0.]
    qghb_to_plot = len([round(i[1], 6) for i in q_drn.tolist()]) * [0.] + [round(i[1], 6) for i in q_ghb.tolist()][:ncol - len([round(i[1], 6) for i in q_drn.tolist()])]

    tot_lst = qrch_to_plot + qdrn_to_plot + qghb_to_plot

    ax1.bar(x_axis_pts, qrch_to_plot, color = 'blue', width = 0.1, label = 'RCH flow in the cell')
    ax1.bar(x_axis_pts, qdrn_to_plot, color = 'red', width = 0.1, label = 'DRN flow in the cell')
    ax1.bar(x_axis_pts, qghb_to_plot, color = 'green', width = 0.1, label = 'GHB flow in the cell')
    ax1.axis([x_st, x_end, 0., max(list(rch_lst[:index_coast + 1])) * 1000.])
    ax1.set_ylabel('Flow rate (m3/d)')
    ax1.axhline(y = 0., linewidth = 1, color = 'k')
    ax1.set_xlim([x_st, x_end])
    ax1.set_ylim([math.floor(min(tot_lst) * 10.) / 10., math.ceil(max(tot_lst) * 10.) / 10.])
    ax1.set_title('Top system and boundary flow rates in the model domain', fontsize = 14, y = 1.01)
    ax1.legend(loc = 'best')

    rch_to_plot = list([i * 1000. for i in rch_lst[:index_coast]])
    rch_to_plot.extend([0] * (ncol - index_coast))
    ax2.bar(x_axis_pts, rch_to_plot, width = 0.1, label = 'RCH rate')
    #ax2.plot(list(x_axis_pts[:index_coast]), list([i * 1000. for i in rch_lst[:index_coast]]), 'co', markersize = 3, label = 'GEBCO elevation')
    ax2.axis([x_st, x_end, 0., max(list(rch_lst[:index_coast])) * 1000.])
    ax2.set_ylabel('Recharge rate (mm/d)')

    ghb_to_plt_1 = [y[-1] for y in ghb_cell_head_lst if y[2] >= index_coast]
    ghb_to_plt_y = [0] * (ncol - len(ghb_to_plt_1)) + ghb_to_plt_1

    mask1 = np.asarray(ghb_to_plt_y) < 0.0001
    mask2 = np.asarray(ghb_to_plt_y) >= 0.0001
    if x_axis_pts[mask1] != []:
        ax2b.bar(x_axis_pts[mask1], np.asarray(ghb_to_plt_y)[mask1], width = 0.1, color = 'green', label = 'GHB head diff < 0.0001')
    if x_axis_pts[mask2] != []:
        ax2b.bar(x_axis_pts[mask2], np.asarray(ghb_to_plt_y)[mask2], width = 0.1, color = 'red', label = 'GHB head diff > 0.0001')
    ax2b.set_ylim([0., min(.25, round(max(ghb_to_plt_y), 5))])
    ax2b.set_ylabel('GHB head difference (m) between assigned and simulated head')
    ax2.set_title('Recharge rate and GHB head difference (assigned vs. computed) in the model domain', fontsize = 14, y = 1.01)

    lines, labels = ax2.get_legend_handles_labels()
    lines2, labels2 = ax2b.get_legend_handles_labels()
    ax2.legend(lines + lines2, labels + labels2, loc = 0)

    #   get the conductivity of the top active cell in each column
    hk_lst = []
    for  d in range(ncol):
        hk_lst.append(arr_hk[:, 0, d].tolist()[[i for i, x in enumerate(arr_hk[:, 0, d].tolist()) if not math.isnan(x)][0]])
    lst_cond_to_plot = len([round(i[1], 6) for i in q_drn.tolist()]) * [0.] + [i[-1] for i in cond_lst][:ncol - len([round(i[1], 6) for i in q_drn.tolist()])]

    ax3.axis([x_st, x_end, 0., math.ceil(max(hk_lst))])
    ax3.set_ylabel('Hydraulic conductivity (m/d)')
    ax3b.plot(x_axis_pts, lst_cond_to_plot, linewidth = 0.1, color = 'blue', label = 'GHB conductance')
    hk_mask1 = np.asarray(hk_lst) < 0.01
    hk_mask2 = np.asarray(hk_lst) >= 0.01
    if x_axis_pts[hk_mask1] != []:
        ax3.bar(x_axis_pts[hk_mask1], np.asarray([math.ceil(max(hk_lst)) * 0.5] * ncol)[hk_mask1], width = 0.1, color = 'red', label = 'Hk, clay cells')
    if x_axis_pts[hk_mask2] != []:
        ax3.bar(x_axis_pts[hk_mask2], np.asarray(hk_lst)[hk_mask2], width = 0.1, color = 'green', label = 'Hk, non-clay cells')
    ax3b.set_ylabel('Conductance (m2/d)')
    ax3b.set_ylim([0., math.ceil(max(lst_cond_to_plot))])
    ax3.set_title('Horizontal hydraulic conductivity and GHB conductance in the model domain', fontsize = 14, y = 1.01)

    lines, labels = ax3.get_legend_handles_labels()
    lines2, labels2 = ax3b.get_legend_handles_labels()
    ax3.legend(lines + lines2, labels + labels2, loc=0)

    if os.path.exists(save_dir):
        print('Figure   ' + save_dir + '   already exists.')
    else:
        #   save the figure
        plt.savefig(save_dir, dpi = 300)
    plt.close()

"""
Function that wraps up at the end of each model simulation and creates the final output files as netcdf files. The input to the function
is just the directory pointing to the folder system with the model results.

model_dir = r'g:\Water_Nexus\_A2\_models_param_runs\_1103\coscat_1103_param_combo_run_1'
summary_folder = r'g:\Water_Nexus\_A2\_models_param_runs\_1103\coscat_1103_param_combo_run_1\summary_dir'

in_dir = model_dir
summary_dir = os.path.join(summary_folder, '_conc_results')
out_name = 'coscat_1103_param_combo_run_1'

char_dir_conc = os.path.join(char_time_dir, '_conc_results')

dict_ex = np.load(r'g:\Water_Nexus\_A2\_models_param_runs\_0403\all\coscat_0403_param_combo_run_1\eq_00\_conc.npy')

"""

def create_conc_tar(in_dir, summary_dir, char_dir_conc, out_name):

    #   initialize the simulation time
    simu_time = 0
    final_dict = dict()

    #   create lists of folders where the _conc dictionaries should be found
    file_list = os.listdir(in_dir)
    file_list_no_char = [i for i in file_list if 'eq' in i and 'time' not in i]
    file_name_list_no_char = [i.split('_')[-1] for i in file_list_no_char]
    file_name_list_no_char = sorted(file_name_list_no_char)
    #   make a special list for the char times results
    file_list_char = [i for i in file_list if 'eq' in i and 'time' in i]
    file_name_list_char = [i.split('_')[1].split('_')[0] for i in file_list_char]
    file_name_list_char = sorted(file_name_list_char)

    #   loop through the char time list first
    for file in file_name_list_char:
        #   create the conc.npy filename and check if it exists
        dict_dir = os.path.join(in_dir, 'eq_' + file + '_char_time')
        dict_path = os.path.join(dict_dir, '_conc.npy')
        if os.path.exists(dict_path):
            print('dict_dir :       ' + dict_path)
            final_dict_char_time = dict()
            #   load the dictionary and individually read the data inside key by key
            conc_dict = np.load(dict_path)[()]
            #   read each key, change the value of the key according to the time stamp (real time since simulation start)
            for key in conc_dict:
                real_time = simu_time + key
                #data = xr.DataArray(np.round(conc_dict[key]['conc'], 3), dims = ('lay', 'row', 'col'), name = real_time)
                final_dict_char_time[real_time] = {'conc': np.round(conc_dict[key]['conc'], 3)}
            fin_dict_char_time_dir = os.path.join(char_dir_conc, out_name + 'eq_' + file + '_char_time' + '.npy')
            final_dict_char_time_ordered = collections.OrderedDict(sorted(final_dict_char_time.items()))
            #   save the dictionary
            np.save(fin_dict_char_time_dir, final_dict_char_time_ordered)

    #   do the same for the list with no_char_time dictionaries
    for file in file_name_list_no_char:
        #   create the conc.npy filename and check if it exists
        dict_dir = os.path.join(in_dir, 'eq_' + file)
        dict_path = os.path.join(dict_dir, '_conc.npy')
        if os.path.exists(dict_path):
            print('dict_dir :       ' + dict_path)
            #   load the dictionary and individually read the data inside key by key
            conc_dict = np.load(dict_path)[()]
            #   read each key, change the value of the key according to the time stamp (real time since simulation start)
            for key in conc_dict:
                real_time = simu_time + key
                final_dict[real_time] = {'conc': np.round(conc_dict[key]['conc'], 3)}
        simu_time += key

    #fin_dict_dir = r'g:\\Water_Nexus\\_A2\\_models_testing_1413\\cartesius_runs\\coscat_1413_geo_scenarios_GHB_incr_min_100_cond_plots_1\\eq_31_char_time\\_conc_cmp.npy'
    #fin_dict_tar_dir = r'g:\\Water_Nexus\\_A2\\_models_testing_1413\\cartesius_runs\\coscat_1413_geo_scenarios_GHB_incr_min_100_cond_plots_1\\eq_31_char_time\\_conc_cmp.tar.xz'
    fin_dict_dir = os.path.join(summary_dir, out_name + '.npy')
    final_dict_ordered = collections.OrderedDict(sorted(final_dict.items()))
    np.save(fin_dict_dir, final_dict_ordered)
    #fin_dict_tar_dir =  os.path.join(summary_dir, out_name + '.tar.xz')

    #tar = tarfile.open(fin_dict_tar_dir, "w:xz")
    #tar.add(fin_dict_dir)
    #tar.close()
    #os.remove(fin_dict_dir)

"""  Same kind of function, but this time for the heads dictionary   """
def create_head_tar(in_dir, summary_dir, char_dir_head, out_name):

    #   initialize the simulation time
    simu_time = 0
    final_dict = dict()

    #   create lists of folders where the _conc dictionaries should be found
    file_list = os.listdir(in_dir)
    file_list_no_char = [i for i in file_list if 'eq' in i and 'time' not in i]
    file_name_list_no_char = [i.split('_')[-1] for i in file_list_no_char]
    file_name_list_no_char = sorted(file_name_list_no_char)
    #   make a special list for the char times results
    file_list_char = [i for i in file_list if 'eq' in i and 'time' in i]
    file_name_list_char = [i.split('_')[1].split('_')[0] for i in file_list_char]
    file_name_list_char = sorted(file_name_list_char)

    #   loop through the char time list first
    for file in file_name_list_char:
        #   create the conc.npy filename and check if it exists
        dict_dir = os.path.join(in_dir, 'eq_' + file + '_char_time')
        dict_path = os.path.join(dict_dir, '_head.npy')
        if os.path.exists(dict_path):
            print('dict_dir :       ' + dict_path)
            final_dict_char_time = dict()
            #   load the dictionary and individually read the data inside key by key
            head_dict = np.load(dict_path)[()]
            #   read each key, change the value of the key according to the time stamp (real time since simulation start)
            for key in head_dict:
                real_time = simu_time + key
                #data = xr.DataArray(np.round(conc_dict[key]['conc'], 3), dims = ('lay', 'row', 'col'), name = real_time)
                final_dict_char_time[real_time] = {'head': np.round(head_dict[key]['head'], 3)}
            fin_dict_char_time_dir = os.path.join(char_dir_head, out_name + 'eq_' + file + '_char_time' + '.npy')
            final_dict_char_time_ordered = collections.OrderedDict(sorted(final_dict_char_time.items()))
            #   save the dictionary
            np.save(fin_dict_char_time_dir, final_dict_char_time_ordered)

    #   do the same for the list with no_char_time dictionaries
    for file in file_name_list_no_char:
        #   create the conc.npy filename and check if it exists
        dict_dir = os.path.join(in_dir, 'eq_' + file)
        dict_path = os.path.join(dict_dir, '_head.npy')
        if os.path.exists(dict_path):
            print('dict_dir :       ' + dict_path)
            #   load the dictionary and individually read the data inside key by key
            head_dict = np.load(dict_path)[()]
            #   read each key, change the value of the key according to the time stamp (real time since simulation start)
            for key in head_dict:
                real_time = simu_time + key
                final_dict[real_time] = {'head': np.round(head_dict[key]['head'], 3)}
        simu_time += key

    #fin_dict_dir = r'g:\\Water_Nexus\\_A2\\_models_testing_1413\\cartesius_runs\\coscat_1413_geo_scenarios_GHB_incr_min_100_cond_plots_1\\eq_31_char_time\\_conc_cmp.npy'
    #fin_dict_tar_dir = r'g:\\Water_Nexus\\_A2\\_models_testing_1413\\cartesius_runs\\coscat_1413_geo_scenarios_GHB_incr_min_100_cond_plots_1\\eq_31_char_time\\_conc_cmp.tar.xz'
    fin_dict_dir = os.path.join(summary_dir, out_name + '.npy')
    final_dict_ordered = collections.OrderedDict(sorted(final_dict.items()))
    np.save(fin_dict_dir, final_dict_ordered)

"""  Same kind of function, but this time for the CBC dictionary   """
def create_cbc_tar(in_dir, summary_dir, char_dir_cbc, out_name):

    #   initialize the simulation time
    simu_time = 0
    final_dict = dict()

    #   create lists of folders where the _conc dictionaries should be found
    file_list = os.listdir(in_dir)
    file_list_no_char = [i for i in file_list if '_to_' in i]
    file_name_list_no_char = file_list_no_char#[i.split('_')[-1] for i in file_list_no_char]

    #   do the same for the list with no_char_time dictionaries
    for file in file_name_list_no_char:
        #   create the conc.npy filename and check if it exists
        dict_dir = os.path.join(in_dir, file)
        dict_path = os.path.join(dict_dir, '_cbc.npy')
        if os.path.exists(dict_path):
            print('dict_dir :       ' + dict_path)
            #   load the dictionary and individually read the data inside key by key
            cbc_dict = np.load(dict_path, allow_pickle = True)[()]
            #   read each key, change the value of the key according to the time stamp (real time since simulation start)
            for key in cbc_dict:
                real_time = simu_time + key
                final_dict[real_time] = {'q_ghb': cbc_dict[key]['q_ghb'],\
                                         'q_drn': cbc_dict[key]['q_drn'],\
                                         'q_rch': cbc_dict[key]['q_rch']}
        simu_time += key

    #fin_dict_dir = r'g:\\Water_Nexus\\_A2\\_models_testing_1413\\cartesius_runs\\coscat_1413_geo_scenarios_GHB_incr_min_100_cond_plots_1\\eq_31_char_time\\_conc_cmp.npy'
    #fin_dict_tar_dir = r'g:\\Water_Nexus\\_A2\\_models_testing_1413\\cartesius_runs\\coscat_1413_geo_scenarios_GHB_incr_min_100_cond_plots_1\\eq_31_char_time\\_conc_cmp.tar.xz'
    fin_dict_dir = os.path.join(summary_dir, out_name + '.npy')
    final_dict_ordered = collections.OrderedDict(sorted(final_dict.items()))
    np.save(fin_dict_dir, final_dict_ordered)

"""  Same kind of function, but this time for the GHB check dictionary   """
def create_ghb_check_tar(in_dir, summary_dir, char_dir_ghb_check, out_name):

    #   initialize the simulation time
    simu_time = 0
    final_dict = dict()

    #   create lists of folders where the _conc dictionaries should be found
    file_list = os.listdir(in_dir)
    file_list_no_char = [i for i in file_list if '_to_' in i]
    file_name_list_no_char = file_list_no_char#[i.split('_')[-1] for i in file_list_no_char]

    #   do the same for the list with no_char_time dictionaries
    for file in file_name_list_no_char:
        #   create the conc.npy filename and check if it exists
        dict_dir = os.path.join(in_dir, file)
        dict_path = os.path.join(dict_dir, '_ghb_check.npy')
        if os.path.exists(dict_path):
            print('dict_dir :       ' + dict_path)
            #   load the dictionary and individually read the data inside key by key
            ghb_check_dict = np.load(dict_path, allow_pickle = True)[()]
            #   read each key, change the value of the key according to the time stamp (real time since simulation start)
            for key in ghb_check_dict:
                real_time = simu_time + key
                final_dict[real_time] = {'ghb_check': ghb_check_dict[key]['ghb_check'],\
                                         'ghb_head_diff_lst': ghb_check_dict[key]['ghb_head_diff_lst'],\
                                         'ghb_head_diff_max': ghb_check_dict[key]['ghb_head_diff_max'],\
                                         'succ_cnt': ghb_check_dict[key]['succ_cnt'],\
                                         'fail_cnt': ghb_check_dict[key]['fail_cnt']}
        simu_time += key

    #fin_dict_dir = r'g:\\Water_Nexus\\_A2\\_models_testing_1413\\cartesius_runs\\coscat_1413_geo_scenarios_GHB_incr_min_100_cond_plots_1\\eq_31_char_time\\_conc_cmp.npy'
    #fin_dict_tar_dir = r'g:\\Water_Nexus\\_A2\\_models_testing_1413\\cartesius_runs\\coscat_1413_geo_scenarios_GHB_incr_min_100_cond_plots_1\\eq_31_char_time\\_conc_cmp.tar.xz'
    fin_dict_dir = os.path.join(summary_dir, out_name + '.npy')
    final_dict_ordered = collections.OrderedDict(sorted(final_dict.items()))
    np.save(fin_dict_dir, final_dict_ordered)

"""  Same kind of function, but this time for the List dictionary

in_dir = walk_dir
summary_dir = list_dict_dir
char_dir_list = list_dict_dir_ct
out_name = subdir + '_list_dict'

"""
def create_list_tar(in_dir, summary_dir, char_dir_list, out_name):

    #   initialize the simulation time
    simu_time = 0
    final_dict = dict()

    #   create lists of folders where the _conc dictionaries should be found
    file_list = os.listdir(in_dir)
    file_list_no_char = [i for i in file_list if '_to_' in i]
    file_name_list_no_char = file_list_no_char#[i.split('_')[-1] for i in file_list_no_char]

    #   do the same for the list with no_char_time dictionaries
    for file in file_name_list_no_char:
        #   create the conc.npy filename and check if it exists
        dict_dir = os.path.join(in_dir, file)
        dict_path = os.path.join(dict_dir, '_list.npy')
        if os.path.exists(dict_path):
            print('dict_dir :       ' + dict_path)
            #   load the dictionary and individually read the data inside key by key
            list_dict = np.load(dict_path, allow_pickle = True)[()]
            #   read each key, change the value of the key according to the time stamp (real time since simulation start)
            for key in list_dict:
                real_time = simu_time + key
                final_dict[real_time] = {'STORAGE_IN': np.round(list_dict[key]['STORAGE_IN'], 4),\
                                         'CONSTANT_HEAD_IN': np.round(list_dict[key]['CONSTANT_HEAD_IN'], 4),\
                                         'DRAINS_IN': np.round(list_dict[key]['DRAINS_IN'], 4),\
                                         'HEAD_DEP_BOUNDS_IN': np.round(list_dict[key]['HEAD_DEP_BOUNDS_IN'], 4),\
                                         'DCDT_IN': np.round(list_dict[key]['DCDT_IN'], 4),\
                                         'TOTAL_IN': np.round(list_dict[key]['TOTAL_IN'], 4),\
                                         'STORAGE_OUT': np.round(list_dict[key]['STORAGE_OUT'], 4),\
                                         'CONSTANT_HEAD_OUT': np.round(list_dict[key]['CONSTANT_HEAD_OUT'], 4),\
                                         'HEAD_DEP_BOUNDS_OUT': np.round(list_dict[key]['HEAD_DEP_BOUNDS_OUT'], 4),\
                                         'RECHARGE_OUT': np.round(list_dict[key]['RECHARGE_OUT'], 4),\
                                         'DCDT_OUT': np.round(list_dict[key]['DCDT_OUT'], 4),\
                                         'TOTAL_OUT': np.round(list_dict[key]['TOTAL_OUT'], 4),\
                                         'IN-OUT': np.round(list_dict[key]['IN-OUT'], 4),\
                                         'PERCENT_DISCREPANCY': np.round(list_dict[key]['PERCENT_DISCREPANCY'], 4)}
        simu_time += key

    #fin_dict_dir = r'g:\\Water_Nexus\\_A2\\_models_testing_1413\\cartesius_runs\\coscat_1413_geo_scenarios_GHB_incr_min_100_cond_plots_1\\eq_31_char_time\\_conc_cmp.npy'
    #fin_dict_tar_dir = r'g:\\Water_Nexus\\_A2\\_models_testing_1413\\cartesius_runs\\coscat_1413_geo_scenarios_GHB_incr_min_100_cond_plots_1\\eq_31_char_time\\_conc_cmp.tar.xz'
    fin_dict_dir = os.path.join(summary_dir, out_name + '.npy')
    final_dict_ordered = collections.OrderedDict(sorted(final_dict.items()))
    np.save(fin_dict_dir, final_dict_ordered)

"""
function that combines all the csv results into one final csv ouptut file

dir_in = walk_dir
dir_summary = csv_dir
id_sc = sc_id
folder_name_lst = eq_lst
"""
def create_summary_csv_srm(dir_in, dir_summary, folder_name_lst, id_sc):

    #   initialize the simulation time
    simu_time = 0

    #   add 0s to the scenario id to get a good naming structutre
    if id_sc < 10:
        id_sc_str = '00' + str(id_sc)
    elif id_sc < 100 and id_sc >= 10:
        id_sc_str = '0' + str(id_sc)
    else:
        id_sc_str = str(id_sc)

    #   create the output CSV file, the headers will match the headers of the usual individual csv file
    csv_dir_ts = os.path.join(dir_summary, 'sc_' + id_sc_str + '_summary_EQs_output.csv')
    f = open(csv_dir_ts,'w')
    f_headers = 'EQ, time_start, time_end, max_diff_head, min_diff_head, max_diff_conc, min_diff_conc, fresh_cell_diff_(end_start), brackish_cell_diff_(end_start),\
                saline_cell_diff_(end_start), frsh_inl_vol, frsh_inl_pct, frsh_shelf_vol, frsh_shelf_pct, prev_tot_cells, tot_cells, ghb_cells_check_fail, ghb_cells_check_succ,\
                ghb_cell_dif_abs_max, rch_in, ghb_in, tot_in, drn_out, ghb_out, tot_out, run_time (s)'
    f.write(" ".join(f_headers.split()))
    f.write('\n')
    f.close()

    #   do the same for the list with no_char_time dictionaries
    for file in folder_name_lst:
        #   create the conc.npy filename and check if it exists
        dict_dir = os.path.join(dir_in, file)
        dict_path = os.path.join(dict_dir, '_time_steps_DH_DC.csv')

        if os.path.exists(dict_path):
            #   open the csv file
            with open(dict_path, 'r') as f_tmp:
                #   copy line by line, but adjust some of the values - EQ number and the total simulation time
                for cnt, line in enumerate(f_tmp):
                    #   don't copy the header
                    if cnt != 0:
                        #   adjust the first three values in the line
                        end_line = line.split(',')[3:]
                        start_line = str(file) + ',' + str(int(float(line.split(',')[1])) + simu_time) + ',' + str(int(float(line.split(',')[2])) + simu_time)
                        new_line = start_line
                        for i in range(len(end_line)):
                            new_line += ',' + end_line[i]
                        #   write the line into the overall file
                        f = open(csv_dir_ts,'a')
                        #f.write("\n")
                        f.write(new_line)
                f.write("\n")
                f.close()
            simu_time = int(new_line.split(',')[2])

    """
    #   also find the other summary file, which is only with the % of concentrations and just copy it to csv_results
    for dirpath, dirnames, filenames in os.walk(dir_in):
        for filename in [f for f in filenames if "res_coscat_summary" in f]:
            new_filename = 'scenario_' + id_sc_str + '_pct_fresh_summary.csv'
            csv_dir_old = os.path.join(dirpath, filename)
            csv_dir_new = os.path.join(dir_summary, new_filename)
            #   copy the file
            shutil.copy(csv_dir_old, csv_dir_new)
    """
    return csv_dir_ts

def create_summary_csv(dir_in, dir_summary, dir_summary_char_times, id_sc):

    #   initialize the simulation time
    simu_time = 0

    #   add 0s to the scenario id to get a good naming structutre
    if id_sc < 10:
        id_sc_str = '00' + str(id_sc)
    elif id_sc < 100 and id_sc >= 10:
        id_sc_str = '0' + str(id_sc)
    else:
        id_sc_str = str(id_sc)

    #   create the output CSV file, the headers will match the headers of the usual individual csv file
    csv_dir_ts = os.path.join(dir_summary, 'sc_' + id_sc_str + '_summary_EQs_output.csv')
    f = open(csv_dir_ts,'w')
    f_headers = 'EQ, time_start, time_end, max_diff_head, min_diff_head, max_diff_conc, min_diff_conc, fresh_cell_diff_(end_start), brackish_cell_diff_(end_start),\
                saline_cell_diff_(end_start), frsh_inl_vol, frsh_inl_pct, frsh_shelf_vol, frsh_shelf_pct, prev_tot_cells, tot_cells, ghb_cells_check_fail, ghb_cells_check_succ,\
                ghb_cell_dif_abs_max, rch_in, ghb_in, tot_in, drn_out, ghb_out, tot_out, run_time (s)'
    f.write(" ".join(f_headers.split()))
    f.write('\n')
    f.close()

    #   create lists of folders where the _conc dictionaries should be found
    file_list = os.listdir(dir_in)
    file_list_no_char = [i for i in file_list if 'eq' in i and 'time' not in i]
    file_name_list_no_char = [i.split('_')[-1] for i in file_list_no_char]
    file_name_list_no_char = sorted(file_name_list_no_char)
    #   make a special list for the char times results
    file_list_char = [i for i in file_list if 'eq' in i and 'time' in i]
    file_name_list_char = [i.split('_')[1].split('_')[0] for i in file_list_char]
    file_name_list_char = sorted(file_name_list_char)

    #   loop through the char time list first
    for file in file_name_list_char:
        #   create the conc.npy filename and check if it exists
        dict_dir = os.path.join(dir_in, 'eq_' + file + '_char_time')
        dict_path = os.path.join(dict_dir, '_time_steps_DH_DC.csv')
        if os.path.exists(dict_path):
            dst_dir = os.path.join(dir_summary_char_times, 'scenario_' + id_sc_str + '_eq' + dict_path.split('eq')[-1].replace('/', ''))
            print('dict_dir :       ' + dict_path)
            print('dst_dir :       ' + dst_dir)
            shutil.copyfile(dict_path, dst_dir)

    #   do the same for the list with no_char_time dictionaries
    for file in file_name_list_no_char:
        #   create the conc.npy filename and check if it exists
        dict_dir = os.path.join(dir_in, 'eq_' + file)
        dict_path = os.path.join(dict_dir, '_time_steps_DH_DC.csv')
        if os.path.exists(dict_path):

            #   open the csv file
            with open(dict_path, 'r') as f_tmp:
                #   copy line by line, but adjust some of the values - EQ number and the total simulation time
                for cnt, line in enumerate(f_tmp):
                    #   don't copy the header
                    if cnt != 0:
                        #   adjust the first three values in the line
                        end_line = line.split(',')[3:]
                        start_line = str(file) + ',' + str(int(float(line.split(',')[1])) + simu_time) + ',' + str(int(float(line.split(',')[2])) + simu_time)
                        new_line = start_line
                        for i in range(len(end_line)):
                            new_line += ',' + end_line[i]
                        if int(float(line.split(',')[1])) + simu_time == 0:
                            print(line)
                            print(new_line)

                        #   write the line into the overall file
                        f = open(csv_dir_ts,'a')
                        #f.write("\n")
                        f.write(new_line)
                f.write("\n")
                f.close()

        simu_time = int(new_line.split(',')[2])
    """
    #   also find the other summary file, which is only with the % of concentrations and just copy it to csv_results
    for dirpath, dirnames, filenames in os.walk(dir_in):
        for filename in [f for f in filenames if "res_coscat_summary" in f]:
            new_filename = 'scenario_' + id_sc_str + '_pct_fresh_summary.csv'
            csv_dir_old = os.path.join(dirpath, filename)
            csv_dir_new = os.path.join(dir_summary, new_filename)
            #   copy the file
            shutil.copy(csv_dir_old, csv_dir_new)
    """
    return csv_dir_ts

"""   Function that will tar a source folder to a destination

src_dir = r'g:\Water_Nexus\_A2\_models_testing_1413\cartesius_runs\_summary'
dst_dir = r'g:\Water_Nexus\_A2\_models_testing_1413\cartesius_runs\_summ_test.tar.xz'
"""
def tar_dir(src_dir, dst_dir):

    tar = tarfile.open(dst_dir, "w:xz")
    tar.add(src_dir)
    tar.close()

"""
Function to rename and gather all the figures created during the model simulation into one folder

main_dir = r'g:\Water_Nexus\_A2\_models_testing_1413\cartesius_runs\coscat_1413_geo_scenarios_GHB_incr_min_100_cond_plots_1'
fig_type_dir = '_ghb_rch_params_pngs'
splitting_str = '_ghb_rch_params_'
dst_dir = r'g:\Water_Nexus\_A2\_models_testing_1413\cartesius_runs\_summary\_figures'
char_time_dir = r'g:\Water_Nexus\_A2\_models_testing_1413\cartesius_runs\_summary\_char_time\_figures'

"""
def gather_figs(main_dir, fig_type_dir, splitting_str, char_time_dir, dst_dir):

    #   first loop through the model folder and find all the plots () and initialize the simulation time
    simu_time = 0
    dst_char_folder = os.path.join(char_time_dir, fig_type_dir)
    dst_folder = os.path.join(dst_dir, fig_type_dir)
    if os.path.exists(dst_char_folder):
        pass
    else:
        os.makedirs(dst_char_folder)

    if os.path.exists(dst_folder):
        pass
    else:
        os.makedirs(dst_folder)

    #   first make a list of all the dictionaries located with that folder structure
    for dirpath, dirnames, filenames in os.walk(main_dir):
        #   if the dirpath contains the fig_type_dir then enter that folder
        if fig_type_dir in dirpath:

            time_end_lst = []

            for filename in os.walk(dirpath):
                for i in range(len(filename[-1])):
                    if filename[-1][i].endswith(".png"):

                        #   if there is the string "diff" in the filename there are two different time stamps, if not theres only one
                        if 'diff' in filename[-1][i]:
                            #   get the old time stamps
                            str_split = filename[-1][i].split('diff_')[-1].split('_to_')
                            t_start_old = int(str_split[0])
                            t_end_old = int(str_split[-1].split('_yrs')[0])
                            #   create new ones, also add 0s to the string
                            t_start_new = t_start_old + simu_time
                            t_end_new = t_end_old + simu_time
                            time_end_lst.append(t_end_old)

                            if t_start_new < 1000:
                                t_start_str = '000' + str(t_start_new)
                            elif t_start_new >= 1000 and t_start_new < 10000:
                                t_start_str = '00' + str(t_start_new)
                            elif t_start_new >= 10000 and t_start_new < 100000:
                                t_start_str = '0' + str(t_start_new)
                            else:
                                t_start_str = str(t_start_new)

                            if t_end_new < 1000:
                                t_end_str = '000' + str(t_end_new)
                            elif t_end_new >= 1000 and t_end_new < 10000:
                                t_end_str = '00' + str(t_end_new)
                            elif t_end_new >= 10000 and t_end_new < 100000:
                                t_end_str = '0' + str(t_end_new)
                            else:
                                t_end_str = str(t_end_new)

                            #   if there is char_time in the name folder than move those to the correspondant folder
                            if 'char_time' in dirpath:
                                eq_num = int(dirpath.split('eq')[-1].split('char_time/')[0].replace('_', ''))
                                new_name = filename[-1][i].split(str(t_start_old))[0] + t_start_str + '_to_' + t_end_str + '_eq_' + str(eq_num) + '.png'
                                new_dir = os.path.join(dst_char_folder, new_name)
                                old_dir = os.path.join(dirpath, filename[-1][i])
                                #   copy the file
                                shutil.copy(old_dir, new_dir)
                            else:
                                eq_num = int(dirpath.split('eq')[-1].split('/')[0].replace('_', ''))
                                new_name = filename[-1][i].split(str(t_start_old))[0] + t_start_str + '_to_' + t_end_str + '_eq_' + str(eq_num) + '.png'
                                new_dir = os.path.join(dst_folder, new_name)
                                old_dir = os.path.join(dirpath, filename[-1][i])
                                #   copy the file
                                shutil.copy(old_dir, new_dir)

                        #   if there is the string "diff" in the filename there are two different time stamps, if not theres only one
                        else:
                            #   get the old time stamps
                            #splitting_str = fig_type_dir.split('pngs')[0].split('_')[1].replace('_', '')
                            str_split = filename[-1][i].split(splitting_str)[-1].split('_yrs')
                            t_old = int(str_split[0])
                            #   create new ones, also add 0s to the string
                            t_new = t_old + simu_time
                            time_end_lst.append(t_old)

                            if t_new < 1000:
                                t_str = '000' + str(t_new)
                            elif t_new >= 1000 and t_new < 10000:
                                t_str = '00' + str(t_new)
                            elif t_new >= 10000 and t_new < 100000:
                                t_str = '0' + str(t_new)
                            else:
                                t_str = str(t_new)

                            #   if there is char_time in the name folder than move those to the correspondant folder
                            if 'char_time' in dirpath:
                                eq_num = int(dirpath.split('eq')[-1].split('char_time/')[0].replace('_', ''))
                                new_name = filename[-1][i].split(str(t_old))[0] + t_str + '_eq_' + str(eq_num) + '.png'
                                new_dir = os.path.join(dst_char_folder, new_name)
                                old_dir = os.path.join(dirpath, filename[-1][i])
                                #   copy the file
                                print(old_dir, new_dir)
                                shutil.copy(old_dir, new_dir)
                            else:
                                eq_num = int(dirpath.split('eq')[-1].split('/')[0].replace('_', ''))
                                new_name = filename[-1][i].split(str(t_old))[0] + t_str + '_eq_' + str(eq_num) + '.png'
                                new_dir = os.path.join(dst_folder, new_name)
                                old_dir = os.path.join(dirpath, filename[-1][i])
                                #   copy the file
                                print(old_dir, new_dir)
                                shutil.copy(old_dir, new_dir)

            if 'char_time' not in dirpath:
                simu_time += max(time_end_lst)


# Que 4 Part A- Linear Regression with 10-fold validation
# Note: Features used are same as Que 2. But need to use more/other features similar to que 3

import os
import json
import datetime
import numpy
from sklearn import linear_model, neural_network
import utility
from sklearn import linear_model, cross_validation

# Generate training and test data using train_X, train_Y convention for LR
# take no of tweets as train_label & rest of the features as train data or independent vars

path = "../../Datasets/tweets/tweet_data/"
#path = "F:/tweets/"

file_list = []
file_list = os.listdir(path)
#file_list = ["subset.txt"]

window_size = 1

for f in file_list:

    print("\n K fold Linear Regression with best 6 features on file  : ", f)
    print("\n   window size : ", window_size)

    f = path + f

    training_data = utility.generate_training_data(f, 1, False, 0, 0, True)

    training_data.pop(0)
    #print training_data

    X = utility.get_feature_matrix(training_data, window_size)

    #print "matrix"
    #print X
    # numpy.roll( ) is used for circular shifting of elements
    train_label = X[2:, 0]
    train_features = X[1:- 1, [0,1,2,6,10,11]]

    model = linear_model.LinearRegression()
    #model = neural_network.MLPRegressor([10, 40, 5], 'relu', 'adam', 0.0001, 200, 'constant', 0.001, 0.5, 200,
    #                                    True, None, 0.0001, False, False, 0.9, True, False, 0.1, 0.9, 0.999, 1e-08)

    #predicted_tweet_count = cross_validation.cross_val_predict(model, train_features, train_label, 10, 1, 0, None, 0)
    #print ("predicted_tweet_counts : ", predicted_tweet_count)

    # used mean_absolute_error function
    # returns avg diff between predicted and actual tweets, over 10 folds in a window
    # doesnt randomize the input
    scores = cross_validation.cross_val_score(model, train_features, train_label,  cv=10, scoring='mean_absolute_error')
    avg_scores = numpy.average((-scores))
    print("Average Prediciton Error  : ", avg_scores)

#     hashtag_rmse += avg_scores
#
# # Avg error over all windows of a file
# avg_error = float(hashtag_rmse) / float(window_count)
# print("hashtag avg absolute error for file : "+ f +" ", avg_error)


import torch
import torch.nn as nn
import torch.nn.functional as F
from torchvision import transforms
from PIL import Image
from math import sqrt
from torch.autograd import Variable

class DRRN(nn.Module):

    def __init__(self,opt):
        super(DRRN, self).__init__()
        self.opt = opt
        self.block_num = opt.BlockNum
        self.block_size = opt.BlockSize
        self.shuflle_size = 2

        print ("===>block size",self.block_size)
        print("===>block size", self.block_num)

        self.input = nn.Conv2d(in_channels=1, out_channels=128, kernel_size=3, stride=1, padding=1, bias=False)
        # (for DRRN)
        self.conv1 = nn.Conv2d(in_channels=128, out_channels=128, kernel_size=3, stride=1, padding=1, bias=False)
        self.conv2 = nn.Conv2d(in_channels=128, out_channels=128, kernel_size=3, stride=1, padding=1, bias=False)

        # 3 x 3 kernel conv(for dense net)
        self.conv = nn.Conv2d(in_channels=128, out_channels=128, kernel_size=3, stride=1, padding=1, bias=False)
        # 1 x 1 kernel conv
        self.conv_1 = nn.Conv2d(in_channels=128, out_channels=128, kernel_size=1, stride=1, padding=0, bias=False)

        self.conv_cat_local = nn.Conv2d(in_channels=128*(self.block_size+1), out_channels=128, kernel_size=1, stride=1, padding=0, bias=False)
        self.conv_cat_global = nn.Conv2d(in_channels=128*(self.block_num+1), out_channels=128, kernel_size=1,stride=1, padding=0, bias=False)

        #in original DENSE NET , there are 3 channels in output , but here we use 1 channel to fit our code ?Consider: Why we use only 1 channel?
        self.output_3 = nn.Conv2d(in_channels=128, out_channels=3, kernel_size=3, stride=1, padding=1, bias=False)

        self.output = nn.Conv2d(in_channels=128, out_channels=1, kernel_size=3, stride=1, padding=1, bias=False)
        self.relu = nn.ReLU(inplace=True)
        self.conv_bn = torch.nn.BatchNorm2d(128).cuda()
        #ESPCNN
        self.shuffle = torch.nn.PixelShuffle(self.shuflle_size)
        in_channels = 128
        out_channels = in_channels*(self.shuflle_size * self.shuflle_size)
        self.espcn_conv = nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=3, stride=1, padding=1, bias=False)

        self.AvgPool = nn.AvgPool2d(kernel_size=2)

        # weights initialization
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                m.weight.data.normal_(0, sqrt(2. / n))

    def forward(self, x):
        def ESPCN(x):
            ret = self.espcn_conv(x)
            ret = self.shuffle(ret)
            ret = self.AvgPool(ret)
            ret = self.espcn_conv(ret)
            ret = self.shuffle(ret)
            ret = self.AvgPool(ret)
            return ret

        if self.opt.NetStructure == "DRRN":
            # DRRN
            residual = x
            inputs = self.input(self.relu(x))
            out = inputs
            for _ in range(self.opt.DRRNsize):
                if self.opt.batchNormalize:
                    relu_1 = self.relu(self.conv_bn(out))
                    relu_2 = self.relu(self.conv_bn(self.conv1(relu_1)))
                else:
                    relu_1 = self.relu(out)
                    relu_2 = self.relu(self.conv1(relu_1))
                out = self.conv2(relu_2)
                out = torch.add(out, inputs)

            if self.opt.ESPCN :
                out = ESPCN(out)

            out = self.output(self.relu(out))
            out = torch.add(out, residual)
            return out

        else:
            # DENSE NET
            block_size = self.block_size
            block_num = self.block_num

            def residual_block(x):
                out_seq=[]
                input = x

                out_seq.append(input)

                for i in range(block_size):
                    if self.opt.batchNormalize:
                        l1=self.conv(input)
                        out = self.relu(l1)
                        del (l1)
                    else:
                        l1 = self.conv(input)
                        l2 = self.conv_bn(l1)
                        del(l1)
                        out = self.relu(l2)
                        del(l2)
                    out_seq.append(out)
                    input = torch.add(out,input)

                    del (out)
                cat = torch.cat(out_seq,1)
                out = self.relu(self.conv_cat_local(cat))
                out = torch.add(out,x)

                del (out_seq)  # release memory
                del (cat)
                del (input)

                return out

            F_1 = self.relu(self.input(x))
            F0 = self.relu(self.conv(F_1))
            input = F0
            RDBs = []
            for i in range(block_num):
                out = residual_block(input)
                del (input)
                RDBs.append(out)
                input = out
            RDBs.append(out)# output of the last RDB
            cat = torch.cat(RDBs,1)
            F_GF = self.relu(self.conv(self.relu(self.conv_cat_global(cat))))# 1 x 1 and 3 x 3 conv
            F_DF = torch.add(F_GF,F_1)

            if self.opt.ESPCN:
                F_DF = ESPCN(F_DF)
            ret = self.output(F_DF)

            del (RDBs)  # release memory
            del (input)
            del (out)
            del (F0)
            del (F_1)
            del (cat)
            del (F_GF)
            del (F_DF)

            return ret


from telegram.ext import CommandHandler
from telegram import KeyboardButton, ReplyKeyboardMarkup, ReplyKeyboardRemove
from telegram.ext import ConversationHandler, Filters, MessageHandler

from bot_config import dispatcher
from utils import bot_messages, bot_utils
import bot_functions
import locale
from locale import atof
locale.setlocale(locale.LC_ALL, 'pt_BR.UTF-8')

# --- start and help comands ---

def start(update, context):
    context.bot.send_message(chat_id=update.effective_chat.id,
                             text=bot_messages.welcomeMessage)

def help_command(update, context):
    update.message.reply_text(bot_messages.helpMessage)

dispatcher.add_handler(CommandHandler('start', start))
dispatcher.add_handler(CommandHandler('help', help_command))

to_delete = None

# --- plots ---

KM, ELEMENT, LINE, TYPE, PARAM = range(5)
plot_argws = {'by': "", # E: element/line, K: km
              'km': "",
              'element': "", 'line': "",
              'type': "", 'param': ""}

def blanc_plot_argws():
    global plot_argws
    plot_argws = {'by': "",'km': "",'element': "",'line': "",'type': "",'param': ""}
    return plot_argws

# --- plots by line/element ---

def plot_elements(update, context):
    plot_argws = blanc_plot_argws()
    plot_argws['by'] = 'E'

    update.message.reply_text(bot_messages.plot,
                              reply_markup=ReplyKeyboardRemove())
    update.message.reply_text(f"Indique um elemento:")

    return ELEMENT

def get_element(update, context):
    elements = bot_functions.list_elements()
    elements = [l.casefold() for l in elements]
    if not update.message.text.casefold() in elements:
        update.message.reply_text(f"""❌ Não foi possível identificar o elemento: {update.message.text}\n\n
Selecione /elementos para listar todas os elementos disponíveis""")
        return ELEMENT

    plot_argws['element'] = update.message.text.upper()
    global to_delete
    if not to_delete is None:
        to_delete.delete()
        to_delete = None
    update.message.reply_text(f"Indique uma linha:",
                              reply_markup=ReplyKeyboardRemove())

    return LINE

def get_line(update, context):
    lines = bot_functions.list_lines(plot_argws['element'])
    lines = [l.casefold() for l in lines]
    if not update.message.text.casefold() in lines:
        update.message.reply_text(f"""❌ Não foi possível identificar a linha: {update.message.text}\n\n
Selecione /linhas para listar todas as linhas disponíveis""")
        return LINE

    plot_argws['line'] = update.message.text.upper()
    global to_delete
    if not to_delete is None:
        to_delete.delete()
        to_delete = None

    kb = [[KeyboardButton(bot_messages.sensors)],
          [KeyboardButton(bot_messages.contingency)],
          [KeyboardButton(bot_messages.speed_restriction)]]
    kb_markup = ReplyKeyboardMarkup(kb, resize_keyboard=True, one_time_keyboard=True)
    update.message.reply_text("Selecione o tipo:",
                              reply_markup=kb_markup)

    return TYPE

def get_type(update, context):
    plot_argws['type'] = update.message.text

    if update.message.text == bot_messages.sensors:
        kb = [[KeyboardButton(bot_messages.three_seconds)],
             [KeyboardButton(bot_messages.twenty_seconds)]]
        msg = "Selecione o período:"
    elif update.message.text == bot_messages.contingency:
        kb = [[KeyboardButton(bot_messages.severity)],
             [KeyboardButton(bot_messages.abcd_classification)]]
        msg = "Selecione o tipo de contingência:"
    elif update.message.text == bot_messages.speed_restriction:
        kb = [[KeyboardButton(bot_messages.acceleration), KeyboardButton(bot_messages.bodyrock)],
              [KeyboardButton(bot_messages.suspentiontravel), KeyboardButton(bot_messages.bounce)]]
        msg = "Selecione a variável dinâmica:"
    else:
        update.message.reply_text(bot_messages.unknown)
        return TYPE

    kb_markup = ReplyKeyboardMarkup(kb, resize_keyboard=True, one_time_keyboard=True)
    update.message.reply_text(msg,
                              reply_markup=kb_markup)

    return PARAM

def get_param(update, context):
    plot_argws['param'] = update.message.text

    if plot_argws['by'] == 'E':
        msg = '{}\n{}\nLinha: {}\nElemento: {}'.format(plot_argws['type'],
                                                       plot_argws['param'],
                                                       plot_argws['line'],
                                                       plot_argws['element'])
    elif plot_argws['by'] == 'K':
        msg = '{}\n{}\nKm {}'.format(plot_argws['type'],
                                     plot_argws['param'],
                                     ','.join(map(str, plot_argws['km'])))
    else:
        wrong_plot(update, context)

    update.message.reply_text(msg,
                              reply_markup=ReplyKeyboardRemove())
    to_delete = update.message.reply_text(bot_messages.searching)
    table = media = None

    if plot_argws['type'] == bot_messages.sensors:
        media = bot_functions.sensors_plot(plot_argws)

    elif plot_argws['type'] == bot_messages.contingency:
        table = bot_functions.contingency_table(plot_argws)
        media = bot_functions.contingency_plot(plot_argws)

    elif plot_argws['type'] == bot_messages.speed_restriction:
        media = bot_functions.rest_velocidade_plot(plot_argws)

    else:
        wrong_plot(update, context)

    if not to_delete is None:
        to_delete.delete()
        to_delete = None

    if not table is None:
        context.bot.send_photo(chat_id=update.effective_chat.id,
                               photo=table)
    if media is None:
        update.message.reply_text(bot_messages.understand)
    elif isinstance(media, list):
        context.bot.send_media_group(chat_id=update.effective_chat.id,
                                     media=media)
    else:
        context.bot.send_photo(chat_id=update.effective_chat.id,
                               photo=media)

    return ConversationHandler.END

def list_elements(update, context):
    global to_delete

    elements = bot_functions.list_elements()
    elements = '\n'.join(elements)
    to_delete = update.message.reply_text(f"Elementos disponíveis\n\n{elements}")
    update.message.reply_text(f"Indique um elemento:")

    return ELEMENT

def list_lines(update, context):
    global to_delete

    lines = bot_functions.list_lines(plot_argws['element'])
    lines = '\n'.join(lines)
    to_delete = update.message.reply_text(f"Linhas disponíveis\n\n{lines}")
    update.message.reply_text(f"Indique uma linha:")

    return LINE

# --- plots by km ---

def plot_km(update, context):
    plot_argws = blanc_plot_argws()
    plot_argws['by'] = 'K'

    lo, hi = bot_functions.range_km()
    update.message.reply_text(bot_messages.plot,
                              reply_markup=ReplyKeyboardRemove())
    update.message.reply_text("Indique uma ou mais posições entre %.2f e %.2f separadas por vírgula:" % (lo, hi))

    return KM

def get_km(update, context):
    lo, hi = bot_functions.range_km()
    try:
        km = update.message.text
        km = [atof(k.strip().replace('.',',')) for k in km.split(',')]
    except ValueError:
        km = [None]
    for k in km:
        if k is None or k < lo or k > hi:
            update.message.reply_text(f"""❌ Posição {k} incorreta!\n\n
Indique uma ou mais posições entre %.2f e %.2f separadas por vírgula:""" % (lo, hi))
            return KM

    plot_argws['km'] = km

    if len(km) > 1:
        kb = [[KeyboardButton(bot_messages.speed_restriction)]]
    else:
        kb = [[KeyboardButton(bot_messages.sensors)],
              [KeyboardButton(bot_messages.contingency)],
              [KeyboardButton(bot_messages.speed_restriction)]]
    kb_markup = ReplyKeyboardMarkup(kb, resize_keyboard=True, one_time_keyboard=True)
    update.message.reply_text("Selecione o tipo:",
                              reply_markup=kb_markup)

    return TYPE

def wrong_plot(update, context):
    update.message.reply_text(bot_messages.understand,
                              reply_markup=ReplyKeyboardRemove())
    update.message.reply_text(bot_messages.instructions)
    return ConversationHandler.END

def cancel(update, context):
    update.message.reply_text(bot_messages.cancel,
			                  reply_markup=ReplyKeyboardRemove())

    return ConversationHandler.END

plot_conversation_handler = ConversationHandler(
    entry_points=[CommandHandler('elemento', plot_elements),
                  CommandHandler('km', plot_km)],
    states={
        KM: [
            CommandHandler('cancel', cancel),
            MessageHandler(Filters.text, get_km)
        ],
        ELEMENT: [
            CommandHandler('cancel', cancel),
            CommandHandler('elementos', list_elements),
            MessageHandler(Filters.text, get_element)
        ],
        LINE: [
            CommandHandler('cancel', cancel),
            CommandHandler('linhas', list_lines),
            MessageHandler(Filters.text, get_line)
        ],
        TYPE: [
            CommandHandler('cancel', cancel),
            MessageHandler(Filters.text, get_type)
        ],
        PARAM: [
            CommandHandler('cancel', cancel),
            MessageHandler(Filters.text, get_param)
        ]
    },
    fallbacks=[CommandHandler('cancel', cancel)]
)

dispatcher.add_handler(plot_conversation_handler)

# --- unknown comands ---

def unknown(update, context):
    context.bot.send_message(chat_id=update.effective_chat.id,
                             text=bot_messages.unknown,
                             reply_markup=ReplyKeyboardRemove())

dispatcher.add_handler(MessageHandler(Filters.command, unknown))


import datetime as dt
import uvicorn
from enum import Enum
from dataclasses import asdict, dataclass
from dataclasses import field

from settings import settings

from fastapi import FastAPI
from fastapi import status
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import HTMLResponse

@dataclass
class Storage:
    __add_to_dict__ = ['time_left']
    work_delta: int
    pause_delta: int
    pause_start: int = 0
    end_time: int = field(init=False, repr=False)
    is_work: bool = True
    is_paused: bool = False
    pomodoro_cnt: int = 0

    def __post_init__(self):
        self.end_time = int(dt.datetime.now().timestamp()) + self.work_delta

    def _asdict(self):
        return {
                **asdict(self),
                **{a: getattr(self, a) for a in getattr(self, '__add_to_dict__', [])}}

    @property
    def time_left(self) -> int:
        now = int(dt.datetime.now().timestamp())
        if not self.is_paused:
            time_left = self.end_time - now
        else:
            time_left = now - self.pause_start
            time_left = self.end_time - self.pause_start
        return time_left

class Variables(str, Enum):
    IS_WORK = 'is_work'
    IS_PAUSED = 'is_paused'
    TIME = 'end_time'
    PAUSE_START = 'pause_start'
    POMODORO_COUNT = 'pomodoro_count'

class Responses(str, Enum):
    LEFT = '{pomodoro} Time left {minutes}:{seconds}'
    PERIOD_ENDED = 'Period has ended'
    PAUSE = 'PAUSE'
    NO_POMODORO = ''

_MINUTE = 60
storage = None

app = FastAPI()

app.add_middleware(
    CORSMiddleware,
    allow_origins=['*'],
    allow_credentials=True,
    allow_methods=['*'],
    allow_headers=['*'],
)

def init_database():
    global storage

    storage = Storage(
            work_delta=settings.work_delta*_MINUTE,
            pause_delta=settings.pause_delta*_MINUTE,
        )

@app.get('/new')
def new():
    init_database()
    return status.HTTP_200_OK

@app.get('/toggle')
def toggle():

    if storage is None:
        return status.HTTP_200_OK

    if storage.is_paused:
        storage.is_paused = False
        now = int(dt.datetime.now().timestamp())
        paused_time = now - storage.pause_start
        storage.end_time = storage.end_time + paused_time
    else:
        storage.is_paused = True
        storage.pause_start = int(dt.datetime.now().timestamp())

    return status.HTTP_200_OK

@app.get('/time')
def time():

    if storage is None:
        return HTMLResponse(content=Responses.NO_POMODORO)

    if storage.is_paused:
        return HTMLResponse(content=Responses.PAUSE)

    if storage.end_time < int(dt.datetime.now().timestamp()):
        return HTMLResponse(content=Responses.PERIOD_ENDED)

    times_left_in_seconds = storage.time_left

    return HTMLResponse(
            content=Responses.LEFT.value.format(
                pomodoro='ðŸ�…' * storage.pomodoro_cnt,
                minutes=times_left_in_seconds // 60,
                seconds=str(times_left_in_seconds % 60).zfill(2),
            ),
    )

@app.get('/next')
def next():

    if storage is None:
        init_database()
        return status.HTTP_200_OK

    storage.is_paused = False
    now = int(dt.datetime.now().timestamp())

    if storage.is_work:
        storage.pomodoro_cnt += 1
        storage.end_time = now + storage.pause_delta
    else:
        storage.end_time = now + storage.work_delta

    storage.is_work = not storage.is_work

    return status.HTTP_200_OK

@app.get('/previous')
def previous():

    if storage is None or not storage.pomodoro_cnt:
        init_database()
        return status.HTTP_200_OK

    storage.is_paused = False
    now = int(dt.datetime.now().timestamp())

    if storage.is_work:
        storage.end_time = now + storage.pause_delta
        storage.is_work = False
    else:
        storage.is_work = True
        storage.end_time = now + storage.work_delta
        storage.pomodoro_cnt = storage.pomodoro_cnt - 1

    return status.HTTP_200_OK

@app.get('/stop')
def stop():
    global storage
    storage = None
    return status.HTTP_200_OK

# TODO: add response_model but think over @property
@app.get('/time&format=json')
def json():
    return {} if not storage else storage._asdict()

if __name__ == '__main__':
    uvicorn.run(
        app,
        port=settings.port,
        host=settings.host,
    )


from nonebot.adapters.onebot.v11.helpers import CooldownIsolateLevel, Cooldown
from nonebot.adapters.onebot.v11 import GroupMessageEvent, Bot
from nonebot import on_command
from nonebot.log import logger
from nonebot import on_command
from nonebot.matcher import Matcher
from nonebot.params import CommandArg, ArgPlainText
from nonebot.rule import to_me
from nonebot.permission import SUPERUSER
from nonebot.adapters.onebot.v11 import Message

import linecache
import random

from pathlib import Path

group = ["******", "******", "******"]

async def checker(event: GroupMessageEvent) -> bool:
    return str(event.group_id) in group

# 获取文件行数
def getfilelines(filename, eol='\n', buffsize=4096) -> int:
    with open(filename, 'r') as handle:
        linenum = 0
        buffer = handle.read(buffsize)
        while buffer:
            linenum += buffer.count(eol)
            buffer = handle.read(buffsize)

    return linenum + 1

op_source = Path(Path.cwd(), "src/plugins/nonebot_plugin_op", "source.txt")
op_line = getfilelines(op_source)

op_msg = on_command("你说的对", rule=checker, priority=30)

@op_msg.handle(parameterless=[Cooldown(cooldown=60, isolate_level=CooldownIsolateLevel.GROUP_USER)])
async def _(bot: Bot, event: GroupMessageEvent):
    rand_line = random.randint(1, op_line)
    theline = linecache.getline(str(op_source), rand_line)
    await op_msg.finish(theline)
    # await bot.send_group_msg(message=theline, group_id=GroupMessageEvent.group_id)

add_msg = on_command("添加你说的对", rule=(checker & to_me()), priority=27, permission=SUPERUSER)

@add_msg.handle()
async def _(matcher: Matcher, args: Message = CommandArg(), ):
    plain_text = args.extract_plain_text()
    if plain_text:
        matcher.set_arg("plain_text", args)

@add_msg.got("plain_text")
async def _(plain_text: str = ArgPlainText("plain_text")):
    text = plain_text
    with open(op_source, 'a') as f:
        f.write("\n" + text)

    await add_msg.finish("添加语料成功！")

logger.success("你说的对，但是你说的对大全集已经加载完毕。")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri May 1 2020
@author: Charlie Zhang(djogem)
@assignment: COMP 562 final project
"""

import os
import torch
import torch.nn as nn
import torchvision
from torch.utils.data import Dataset

from PIL import Image

def pil_loader(image_path):
    with open(image_path, 'rb') as f:
        return Image.open(f).convert('RGB')

#%%
#Custom Alexnet
class CustomTrainDataset(Dataset):

    def __init__(self, data_dir, transform=None):
        """
        Args:
            data_dir: the dataset folder. The folder should be structured as:
                label1/image/500 .JPEG images
                label2/image/500 .JPEG images
                ...
                label200/image/500 .JPEG images
        """
        self.data_dir = data_dir
        self.transform = transform
        self.samples = []

        #TODO: implement the needed initialization

        self.classes = sorted(os.listdir(data_dir))
        self.class_to_idx = {c: i for i, c in enumerate(self.classes)}

        print(data_dir)
        for _root, _dirs, files in sorted(os.walk(data_dir)):
          for directory in _dirs:
            idx = self.class_to_idx[directory]
            directory_path = os.path.join(data_dir, directory)
            all_images_path = os.path.join(directory_path, 'images')
            print("one of 200 directory paths:", directory_path)
            #directory_path = os.path.join(data_dir, 'images')
            for curr_root, _subdirs, subfiles in sorted(os.walk(all_images_path)):
              #for cur_cur_root, _subsubdir, subsubfile in
              for image_name in subfiles:
                image_path = os.path.join(curr_root, image_name)
                self.samples.append((image_path, idx))
          break

    def __len__(self):
        return len(self.samples)

    def __getitem__(self, idx):
        """Don't forget to apply the transform before returning."""
        path, target = self.samples[idx]
        sample = pil_loader(path)
        if self.transform is not None:
            sample = self.transform(sample)

        return sample, target

class CustomValDataset(Dataset):
    """This is the Validation Dataset.

    Use this as a reference to implement you CustomTrainDataset.
    Remember because the training data and validation data are structured differently,
    you shouldn't directly use the code below. You should design it according to the
    training data folder's structure.
    """
    def __init__(self, data_dir, transform=None):
        self.root = os.path.normpath(data_dir)
        self.transform = transform

        classes_file = os.path.join(self.root, '../wnids.txt')

        self.classes = []
        with open(classes_file, 'r') as f:
            self.classes = f.readlines()
            for i in range(len(self.classes)):
                self.classes[i] = self.classes[i].strip()
        self.classes.sort()

        self.class_to_idx = {c: i for i, c in enumerate(self.classes)}

        images_y_file = os.path.join(self.root, 'val_annotations.txt')
        with open(images_y_file, 'r') as f:
            lines = f.readlines()

        self.class_dict = {}
        for line in lines:
            cols = line.split('\t')
            if len(cols) < 2:
                continue
            img_filename = cols[0].strip()
            img_class = cols[1].strip()
            self.class_dict[img_filename] = img_class

        self.samples = []
        images_dir = os.path.join(self.root, 'images')
        for _root, _dirs, files in sorted(os.walk(images_dir)):
            for image_name in files:
                image_path = os.path.join(_root, image_name)
                c = self.class_dict[image_name]
                idx = self.class_to_idx[c]
                self.samples.append((image_path, idx))

    def __getitem__(self, i):
        path, target = self.samples[i]
        sample = pil_loader(path)
        if self.transform is not None:
            sample = self.transform(sample)

        return sample, target

    def __len__(self):
        return len(self.samples)

#%%
#Custom Alexnet
class CustomAlexnet(nn.Module):
    """Alexnet implementation for comp 590.

    Remember, you are required to implement alexnet using
    nn.Conv2d instead of nn.Linear.
    Failing to do that will lead to 0 points for this task.
    """
    def __init__(self, num_classes=200):
        super(CustomAlexnet, self).__init__()
        self.features = nn.Sequential(
            nn.Conv2d(3, 64, kernel_size=8, stride=2, padding=2),
            nn.BatchNorm2d(64),
            nn.ReLU(inplace=True),
            #nn.MaxPool2d(kernel_size=3, stride=2),
            nn.MaxPool2d(kernel_size=3, stride=1),
            nn.Conv2d(64, 192, kernel_size=5, padding=2),
            nn.BatchNorm2d(192),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=3, stride=2),
            nn.Conv2d(192, 384, kernel_size=3, padding=1),
            nn.BatchNorm2d(384),
            nn.ReLU(inplace=True),
            nn.Conv2d(384, 256, kernel_size=3, padding=1),
            nn.BatchNorm2d(256),
            nn.ReLU(inplace=True),
            nn.Conv2d(256, 256, kernel_size=3, padding=1),
            nn.BatchNorm2d(256),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=3, stride=2),
        )
        self.classifier = nn.Sequential(
            nn.Dropout(),
            nn.Conv2d(256, 4096, kernel_size=6),
            nn.ReLU(inplace=True),
            nn.Dropout(),
            nn.Conv2d(4096, 4096, kernel_size=1),
            nn.ReLU(inplace=True),
            nn.Conv2d(4096, 200, kernel_size=1),
        )

    def forward(self, x):
        x = self.features(x)
        x = self.classifier(x)
        x = torch.flatten(x, 1)
        return x


#python image convert to pico-8
import matplotlib.image as mpimg
import numpy as np
import sys
from math import *

def tohex(val, nbits):
	return hex((val + (1 << nbits)) % (1 << nbits))

def main():
	input_filename = sys.argv[1]

	#input_file = open(input_filename,'r')

	#print(input_filename.strip(".obj"))
	output_filename=input_filename.strip(".obj")+"_converted.txt"

	output_file = open(output_filename,'w')

	image=mpimg.imread(input_filename)

	#write image file to output in hex from 0x00 to 0xff
	count=1
	line=0
	hex_string=tohex(int(floor(1*255)),16)
	hex_string=hex_string[2:]
	hex_string = hex_string.zfill(2)
	print("string: "+hex_string)

	output_file.write("00001011111151555555d5dddddd6d66666676777771090300010000000001010000000100010002080601010000000000000000000000000000000000000000\n01010111151515555d5d5dddd6d6d6666767677777782f1e05000000000002010a0e090100010215302403000000000000000000000000000000000000000000\n")

	for pix in np.nditer(image):
		#if(line>1):
		#pix=count*4
		hex_string=tohex(int(floor(pix)),16)
		hex_string=hex_string[2:]
		hex_string = hex_string.zfill(2)[::-1]

		if(line<126):output_file.write(hex_string)
		#output_file.write("xx")
		count+=1
		if(count==65):
			output_file.write("\n")
			count=1
			line+=1
		#high=flr( (pix*255)/16 )
		#low=flr( (pix*255))%16

	#read until the firt vector

	#with open(input_filename) as f:
	#	output_file.write('model_v="')
	#	for line in f:
	#		if(line[:1]=='v'):
	#			line=line.strip('v ')
	#			#print(line)
	#			#output_file.write('{')
	#			point_text=line.split(' ')
	#			for num_text in point_text:
	#				#print("p: "+num_text)
	#				val=float(num_text)
	#				val=int(floor(val*256))
	#				hex_string=tohex(val, 16)
	#				hex_string=hex_string[2:]
	#				hex_string = hex_string.zfill(4)
	#				output_file.write(hex_string)
	#			#output_file.write('},\n')
	output_file.write('\nsee data below')

	print("\nConversion complete!\nSee output: "+output_filename)

	output_file.close()

main()


from conans import ConanFile, AutoToolsBuildEnvironment, tools
from conans.util import files
import os

class ApacheaprConan(ConanFile):
    description = "Apache Portable Runtime"
    name = "apache-apr"
    version = "1.6.0"
    license = "Apache-2.0"
    url = "https://github.com/pbtrung/conan-apache-apr"
    settings = "os", "compiler", "build_type", "arch"
    options = {"shared": [True, False]}
    default_options = "shared=False"
    install_dir = "install"

    def source(self):
        zip_name = "apr-%s.tar.gz" % self.version
        tools.download("https://codeload.github.com/apache/apr/tar.gz/%s" % self.version, zip_name, verify=False)
        tools.unzip(zip_name)
        os.unlink(zip_name)

    def build(self):
        env_build = AutoToolsBuildEnvironment(self)
        with tools.environment_append(env_build.vars):
            buildconf_command = "./buildconf"
            configure_command = "./configure"
            configure_command += " --prefix=%s%s%s" % (os.getcwd(), os.sep, self.install_dir)
            with tools.chdir("apr-" + self.version):
                files.mkdir(self.install_dir)
                self.run(buildconf_command)
                self.run(configure_command)
                self.run("make -j " + str(max(tools.cpu_count() - 1, 1)))
                self.run("make install")

    def package(self):
        install_path = self.install_dir + os.sep
        self.copy("*.h", dst="include/apr-1", src=install_path + "include/apr-1", keep_path=False)
        if self.settings.os != "Windows":
            self.copy("*.pc", dst="lib/pkgconfig", src=install_path + "lib", keep_path=False)
            if self.options.shared:
                self.copy("*.so*", dst="lib", src=install_path + "lib", keep_path=False)
                self.copy("*.dylib", dst="lib", src=install_path + "lib", keep_path=False)
            else:
                self.copy("*.a", dst="lib", src=install_path + "lib", keep_path=False)
        else:
            if self.options.shared:
                self.copy("*.dll", dst="bin", src=install_path + "bin", keep_path=False)
            else:
                self.copy("*.lib", dst="lib", src=install_path + "lib", keep_path=False)

        self.copy("apr-1-config", dst="bin", src=install_path + "bin", keep_path=False)
        self.copy("*.exp", dst="lib", src=install_path + "lib", keep_path=False)
        self.copy("*", dst="build-1", src=install_path + "build-1", keep_path=False)

    def package_info(self):
        self.cpp_info.includedirs = ["include", "include/apr-1"]
        self.cpp_info.bindirs = ["bin"]
        self.cpp_info.libs = ["apr-1"]


from flask import Flask
import geocoder
import geopy
from geopy.geocoders import Nominatim
from datetime import datetime
from flask import request
from flask import jsonify

app = Flask(__name__)

@app.route("/get_my_ip", methods=["GET"])
def get_my_ip():
    #print(request.environ['REMOTE_ADDR'])
    #request.remote_addr
    return {'ip': request.remote_addr}, 200

@app.route('/get_location')
def get_location():
    ip,_ = get_my_ip()
    #print(ip["ip"])
    try:
        g = geocoder.ip(ip["ip"])
        lat_long=g.latlng
        coord=', '.join(list(map(str,lat_long)))
        geolocator = Nominatim(user_agent="test_project")
        location = geolocator.reverse(coord)

        return location.address ,ip["ip"]
    except:
        return "Sorry..! Could not determine location" ,ip["ip"]

@app.route('/get_shops')
def get_shops():
    return_data = dict()
    curr_address,ip = get_location()
    return_data["location"] = curr_address
    return_data["store_name"] = ["Dummy_Shop1","Dummy_shop2","Dummy_shop3","Dummy_shop4","Dummy_shop5"]
    return_data["date_time"]=str(datetime.now())
    return_data["ip address"] = ip
    return return_data

if __name__ == '__main__':
    app.run(host="0.0.0.0", port="5000")


import numpy as np
import matplotlib.pyplot as plt
import matplotlib.colors as colors

dlc = dict(dlblue = '#0096ff', dlorange = '#FF9300', dldarkred='#C00000', dlmagenta='#FF40FF', dlpurple='#7030A0', dldarkblue =  '#0D5BDC')

def plt_roast(X,Y):
    Y = Y.reshape(-1,)
    colormap = np.array(['r', 'b'])
    fig, ax = plt.subplots(1,1,)
    ax.scatter(X[Y==1,0],X[Y==1,1], s=70, marker='x', c='red', label="Good Roast" )
    ax.scatter(X[Y==0,0],X[Y==0,1], s=100, marker='o', facecolors='none',
               edgecolors=dlc["dldarkblue"],linewidth=1,  label="Bad Roast")
    tr = np.linspace(175,260,50)
    ax.plot(tr, (-3/85) * tr + 21, color=dlc["dlpurple"],linewidth=1)
    ax.axhline(y=12,color=dlc["dlpurple"],linewidth=1)
    ax.axvline(x=175,color=dlc["dlpurple"],linewidth=1)
    ax.set_title(f"Coffee Roasting", size=16)
    ax.set_xlabel("Temperature \n(Celsius)",size=12)
    ax.set_ylabel("Duration \n(minutes)",size=12)
    ax.legend(loc='upper right')
    plt.show()

def plt_prob(ax,fwb):
    """ plots a decision boundary but include shading to indicate the probability """
    #setup useful ranges and common linspaces
    x0_space  = np.linspace(150, 285 , 40)
    x1_space  = np.linspace(11.5, 15.5 , 40)

    # get probability for x0,x1 ranges
    tmp_x0,tmp_x1 = np.meshgrid(x0_space,x1_space)
    z = np.zeros_like(tmp_x0)
    for i in range(tmp_x0.shape[0]):
        for j in range(tmp_x1.shape[1]):
            x = np.array([[tmp_x0[i,j],tmp_x1[i,j]]])
            z[i,j] = fwb(x)

    cmap = plt.get_cmap('Blues')
    new_cmap = truncate_colormap(cmap, 0.0, 0.5)
    pcm = ax.pcolormesh(tmp_x0, tmp_x1, z,
                   norm=colors.Normalize(vmin=0, vmax=1),
                   cmap=new_cmap, shading='nearest', alpha = 0.9)
    ax.figure.colorbar(pcm, ax=ax)

def truncate_colormap(cmap, minval=0.0, maxval=1.0, n=100):
    """ truncates color map """
    new_cmap = colors.LinearSegmentedColormap.from_list(
        'trunc({n},{a:.2f},{b:.2f})'.format(n=cmap.name, a=minval, b=maxval),
        cmap(np.linspace(minval, maxval, n)))
    return new_cmap

def plt_network(X,Y,netf):
    fig, ax = plt.subplots(1,2,figsize=(16,4))
    Y = Y.reshape(-1,)
    plt_prob(ax[0], netf)
    ax[0].scatter(X[Y==1,0],X[Y==1,1], s=70, marker='x', c='red', label="Good Roast" )
    ax[0].scatter(X[Y==0,0],X[Y==0,1], s=100, marker='o', facecolors='none',
                   edgecolors=dlc["dldarkblue"],linewidth=1,  label="Bad Roast")
    ax[0].plot(X[:,0], (-3/85) * X[:,0] + 21, color=dlc["dlpurple"],linewidth=1)
    ax[0].axhline(y= 12, color=dlc["dlpurple"], linewidth=1)
    ax[0].axvline(x=175, color=dlc["dlpurple"], linewidth=1)
    ax[0].set_xlabel("Temperature \n(Celsius)",size=12)
    ax[0].set_ylabel("Duration \n(minutes)",size=12)
    ax[0].legend(loc='upper right')
    ax[0].set_title(f"network probability")

    ax[1].plot(X[:,0], (-3/85) * X[:,0] + 21, color=dlc["dlpurple"],linewidth=1)
    ax[1].axhline(y= 12, color=dlc["dlpurple"], linewidth=1)
    ax[1].axvline(x=175, color=dlc["dlpurple"], linewidth=1)
    fwb = netf(X)
    yhat = (fwb > 0.5).astype(int)
    ax[1].scatter(X[yhat[:,0]==1,0],X[yhat[:,0]==1,1], s=70, marker='x', c='orange', label="Predicted Good Roast" )
    ax[1].scatter(X[yhat[:,0]==0,0],X[yhat[:,0]==0,1], s=100, marker='o', facecolors='none',
                   edgecolors=dlc["dldarkblue"],linewidth=1,  label="Bad Roast")
    ax[1].set_title(f"network decision")
    ax[1].set_xlabel("Temperature \n(Celsius)",size=12)
    ax[1].set_ylabel("Duration \n(minutes)",size=12)
    ax[1].legend(loc='upper right')


#!/usr/bin/env python
"""
Serve your awesome snippet via HTTP
"""
from gevent import monkey; monkey.patch_all()

import inspect
import functools
import importlib

import bottle

###############################################################################

#
# configurations
#
MODULES = ['os.path']

###############################################################################

# import all function from specified modules
modules = [importlib.import_module(m) for m in MODULES]
functions = [f for m in modules for f in vars(m).values() if inspect.isfunction(f)]

#
# helper functions
#
def generate_route(f):
    # get module name for specified function
    modulename = inspect.getmodule(f).__name__

    # get function name
    name = f.__name__

    # done
    return '/%s/%s' % (modulename, name)

#
# routes
#
@bottle.route('/', method=['GET'])
def index():
    """
    Display list of all available functions and their signature
    """
    template = """

Webapizer Server

                Function
                Signature

            % for url, signature in sorted(funclist):

                {{ url }}

{{ signature }}

            % end

    """
    funclist = [(generate_route(f), repr(inspect.getargspec(f).args)) for f in functions]
    return bottle.template(template, funclist=funclist)

def handler(f, *args):
    """
    Invoke function with provided arguments
    """
    return f(*args, **bottle.request.params)

# register route for all functions
for f in functions:
    # generate route for this function
    route = generate_route(f)

    # register
    bottle.route(route, ['GET', 'POST'], functools.partial(handler, f))

###############################################################################

if __name__ == '__main__':
    # run
    bottle.debug(True)
    bottle.run(host='localhost', port=8888, debug=True, server='gevent')


"""Create sitemap for HTML files."""
import os
import jinja2
from datetime import date
import stat

loader = jinja2.FileSystemLoader(os.getcwd())
env = jinja2.Environment(loader=loader)

sitemap_files = []

os.chdir("output")

for root, dirs, files in os.walk(".", topdown=False, followlinks=True):
    for name in files:

        if root.startswith("./theme"):
            continue

        if name.endswith(".html"):
            path = os.path.join(root, name)
            statbuf = os.stat(path)
            fdate = date.fromtimestamp(statbuf[stat.ST_MTIME])
            lastmod = fdate.strftime('%Y-%m-%d')

            # Remove ./output
            path = path[len("."):]

            sitemap_files.append(dict(path=path, lastmod=lastmod))

template = env.get_template('sitemap.xml')
doc = template.render(files=sitemap_files, site="https://operationssecurity.org")

with open("sitemap.xml", "wt") as out:
    out.write(doc)