# coding: utf-8
import os
import glob
import argparse
import re

from logging import getLogger, Formatter, StreamHandler, FileHandler, DEBUG, INFO
import matplotlib.pyplot as plt
import pandas as pd
import dataloader
from indicator_evaluation import CalcIndicator

import indicator_utils
import utils

def main(args):
    # File output handler for save folder
    if args.save_folder:
        log_file_name2 = os.path.join(args.save_folder, '{}.log'.format(__name__))
        file_handler2 = FileHandler(log_file_name2, 'a')
        file_handler2.setLevel(DEBUG)
        file_handler2.setFormatter(formatter)
        logger.addHandler(file_handler2)

    for track in args.track:
        logger.debug('- {}, {} evaluation START -'.format(args.trajection_folder, track))

        # Load data config
        data_config_path = glob.glob(os.path.join(args.ground_truth_folder, '*.ini'))[0]
        conf = dataloader.config(track, args.ground_truth_folder, config_file=data_config_path)

        # Directory path to evaluate
        tra_dname = os.path.join(args.trajection_folder, track)
        logger.debug('tra_dname:{}'.format(tra_dname))

        # Load groundtruth files
        map_size = dataloader.map_size(conf['map_dname'], conf['map_size_fname'])
        map_image = dataloader.map_image(conf['map_dname'], conf['map_image_fname'])
        area_info = dataloader.area_info(conf['map_dname'], conf['area_fname'])
        BLE_info = dataloader.BLE_info(conf['BLE_dname'], conf['BLE_info_fname'])
        map_color = dataloader.map_color(conf['map_obstacle_color'], conf['map_trajectory_color'], conf['map_ref_color'], conf['map_BLE_color'])
        map_makersize = dataloader.map_makersize(conf['map_trajectory_size'], conf['map_ref_size'], conf['map_BLE_size'], conf['map_grid'])

        # Create result save directory
        if args.save_folder:
            result_basedir = os.path.join(args.save_folder, track)
        else:
            result_basedir = os.path.join(tra_dname, 'result')
        indicator_savedir = os.path.join(result_basedir, 'indicator')

        for dir_path in [result_basedir, indicator_savedir]:
            utils.create_dir(dir_path)

        extension_types = ['*.txt', '*.csv']
        tra_files = []
        for ext_type in extension_types:
            keyword = os.path.join(tra_dname, ext_type)
            tra_files.extend([os.path.split(file_path)[-1] for file_path in sorted(glob.glob(keyword, recursive=True))])

        if args.file:
            tra_files = args.file

        logger.debug('trajection files:{}'.format(tra_files))

        # Instance to calculate indicator
        evaluation_indicator = CalcIndicator()
        indicator_holder = indicator_utils.IndicatorHolder()

        for tra_filename in tra_files:
            logger.debug('{} Evaluation START'.format(tra_filename))
            print('{} evaluation progress...'.format(tra_filename))

            # Load trajectory files
            tra_data = dataloader.load_point(tra_dname, tra_filename)
            tra_num = re.sub("\\D", "", tra_filename)

            # Load ground truth files
            ref_point = dataloader.load_point(conf['ref_dname'], conf['ref_fname'].format(tra_num))
            ans_point = dataloader.load_point(conf['ans_dname'], conf['ans_fname'].format(tra_num))
            ALIP_info = dataloader.ALIP_info(conf['ALIP_dname'], conf['ALIP_info_fname'].format(tra_num))

            indicator_holder.add_indicator_percentile('file_name', tra_filename)

            # prepare evaluation point
            if ref_point is None:
                evaluation_point = ans_point
            else:
                evaluation_point = dataloader.drop_ans_duplicated_with_ref(ans_point, ref_point)

            if ALIP_info is None:
                eval_point_ALAP = ans_point
                eval_point_ALIP = pd.DataFrame(index=[], columns=['unixtime', 'x_position_m', 'y_position_m'])
            else:
                eval_point_ALAP = dataloader.filter_evaluation_data_ALIP(evaluation_point, ALIP_info, ALIP_flag=False)
                eval_point_ALIP = dataloader.filter_evaluation_data_ALIP(evaluation_point, ALIP_info, ALIP_flag=True)

            """
            if area_info:
                which_area = indicator_utils.area_of_ans(eval_point_ALIP, area_info)
                which_area_all.extend(which_area)
            """

            # CE
            if 'CE' in args.indicators:
                CE = evaluation_indicator.CE_calculation(tra_data, eval_point_ALAP)
                CE_percentile = indicator_utils.calc_percentile(CE['CE'], args.CE_percentile)
                indicator_holder.add_indicator_percentile(f'CE{args.CE_percentile}', CE_percentile)

                indicator_holder.add_total_indicator(f'CE{args.CE_percentile}', CE['CE'])

                indicator_utils.calc_CE_map(CE, map_size)
                CE_savedir = os.path.join(indicator_savedir, 'CE')
                utils.create_dir(CE_savedir)

                indicator_utils.save_indicator(data=CE, save_dir=CE_savedir, save_filename=f'Traj_No{tra_num}_CE.csv')
                CE_hist = indicator_utils.draw_histgram(data=CE['CE'], indicator_name='CE', percentile=args.CE_percentile)
                indicator_utils.save_figure(CE_hist, save_dir=CE_savedir, save_filename=f'Traj_No{tra_num}_CE_histgram.png')

                CE_map = indicator_utils.draw_CE_map(CE, map_image, map_size, indicator_name='CE')
                indicator_utils.save_figure(CE_map, save_dir=CE_savedir, save_filename=f'Traj_No{tra_num}_CE_map.png')

            # CP
            if 'CP' in args.indicators:
                CP = evaluation_indicator.CP_calculation(tra_data, eval_point_ALAP, band_width=None)
                CP_percentile = indicator_utils.calc_percentile(CP['CP'], args.CP_percentile)
                indicator_holder.add_indicator_percentile(f'CP{args.CP_percentile}', CP_percentile)

                indicator_holder.add_total_indicator(f'CP{args.CP_percentile}', CP['CP'])

                CP_savedir = os.path.join(indicator_savedir, 'CP')
                utils.create_dir(CP_savedir)

                indicator_utils.save_indicator(data=CP, save_dir=CP_savedir, save_filename=f'Traj_No{tra_num}_CP.csv')
                CP_hist = indicator_utils.draw_histgram(data=CP['CP'], indicator_name='CP', percentile=args.CP_percentile)
                indicator_utils.save_figure(CP_hist, save_dir=CP_savedir, save_filename=f'Traj_No{tra_num}_CP_histgram.png')

            # Area-weighted CA
            if 'CA' in args.indicators:
                if args.area_weights is None:
                    area_weights = indicator_utils.get_CA_area_weights(evaluation_point, area_info)
                else:
                    area_weights = dataloader.area_weights_config(track, args.area_weights)

                if area_weights is None:
                    logger.debug('CA is calculated for whole area')
                    if args.CA_hist:
                        CA, CA_fig = evaluation_indicator.CA_2Dhistgram_calculation(tra_data, evaluation_point)
                    else:
                        CA, CA_fig = evaluation_indicator.CA_KernelDensity_calculation(tra_data, evaluation_point, band_width=None)
                    CA_df = pd.DataFrame({'CA': CA}, index=[0])

                else:
                    logger.debug('CA is calculated for each area division')
                    CA, CA_df, CA_fig = evaluation_indicator.Area_weighted_CA_calculation(tra_data, evaluation_point, area_info, area_weights, args.CA_hist)

                indicator_holder.add_indicator_percentile('CA', CA)
                CA_savedir = os.path.join(indicator_savedir, 'CA')
                utils.create_dir(CA_savedir)
                indicator_utils.save_dataframe(CA_savedir, f'Traj_No{tra_num}_CA.csv', CA_df)

                if isinstance(CA_fig, list):
                    for i in range(len(CA_fig)):
                        CA_fig[i].suptitle(f'Traj_No{tra_num}_area{i+1}_CA')
                        indicator_utils.save_figure(CA_fig[i], save_dir=CA_savedir, save_filename=f'Traj_No{tra_num}_area{i+1}_CA.png')
                else:
                    CA_fig.suptitle(f'Traj_No{tra_num}_CA')
                    indicator_utils.save_figure(CA_fig, save_dir=CA_savedir, save_filename=f'Traj_No{tra_num}_CA.png')

            # EAG
            if 'EAG' in args.indicators and not ref_point.empty and not eval_point_ALIP.empty:
                EAG = evaluation_indicator.EAG_calculation(tra_data, ref_point, eval_point_ALIP)

                EAG_percentile = indicator_utils.calc_percentile(EAG['EAG'], args.EAG_percentile)
                indicator_holder.add_indicator_percentile(f'EAG{args.EAG_percentile}', EAG_percentile)

                indicator_holder.add_total_indicator(f'EAG{args.EAG_percentile}', EAG['EAG'])

                EAG_savedir = os.path.join(indicator_savedir, 'EAG')
                utils.create_dir(EAG_savedir)

                indicator_utils.save_indicator(data=EAG, save_dir=EAG_savedir, save_filename=f'Traj_No{tra_num}_EAG.csv')
                EAG_hist = indicator_utils.draw_histgram(data=EAG['EAG'], indicator_name='EAG', percentile=args.EAG_percentile)
                indicator_utils.save_figure(EAG_hist, save_dir=EAG_savedir, save_filename=f'Traj_No{tra_num}_EAG_histgram.png')

            # Requirement for Moving Velocity
            if 'requirement_velocity' in args.indicators:
                moving_velocity_df = evaluation_indicator.requirement_moving_velocity_check(tra_data, args.velocity)
                indicator_holder.add_indicator_percentile('requirement_velocity', moving_velocity_df['velocity'].mean())
                velocity_savedir = os.path.join(indicator_savedir, 'requirement_velocity')
                utils.create_dir(velocity_savedir)
                indicator_utils.save_dataframe(velocity_savedir, f'Traj_No{tra_num}_moving_velocity.csv', moving_velocity_df)

            # Requirement for obstacle avoidance
            if 'requirement_obstacle' in args.indicators:
                obstacle_df = evaluation_indicator.requirement_obstacle_check(tra_data, map_image, map_size)
                if len(obstacle_df) == 0:
                    obstacle_ratio = -1
                else:
                    obstacle_ratio = sum(obstacle_df['obstacle_cordinate_count']) / sum(obstacle_df['check_cordinate_count'])

                indicator_holder.add_indicator_percentile('requirement_obstacle', obstacle_ratio)
                obstacle_savedir = os.path.join(indicator_savedir, 'requirement_obstacle')
                utils.create_dir(obstacle_savedir)
                indicator_utils.save_dataframe(obstacle_savedir, f'Traj_No{tra_num}_obstacle.csv', obstacle_df)

            if 'requirement_coverage' in args.indicators:
                eval_num = len(evaluation_point)
                corr_num = len(evaluation_indicator.extract_correspond_point(tra_data, evaluation_point))
                coverage = corr_num / eval_num * 100
                indicator_holder.add_indicator_percentile('requirement_coverage', coverage)

            logger.debug('{} Evaluation END'.format(tra_filename))

            # Draw trajectory
            Tra_savedir = os.path.join(indicator_savedir, 'Trajectory')
            utils.create_dir(Tra_savedir)
            Trajectory_image = indicator_utils.draw_trajectory(tra_data, map_image, map_size, f'Trajectory_No{tra_num}', ref_point, BLE_info, map_color, map_makersize)
            indicator_utils.save_figure(Trajectory_image, save_dir=Tra_savedir, save_filename=f'Tra_No{tra_num}.png')

        # Show each file's results
        file_indicator_summary = indicator_holder.summarize_file_indicator()
        utils.stdout_dataframe(file_indicator_summary, title='file indicator')
        utils.save_csv(save_file=file_indicator_summary, save_dir=indicator_savedir, save_filename='file_indicator.csv')

        # Show total results
        total_indicator = indicator_holder.summarize_total_indicator()
        utils.stdout_dataframe(total_indicator, title='total indicator')
        utils.save_csv(save_file=total_indicator, save_dir=indicator_savedir, save_filename='total_indicator.csv')

        # Draw histgram and cumulative sum for total CE and EAG
        if 'CE' in args.indicators:
            CE_total = indicator_holder.indicator_total[f'CE{args.CE_percentile}']
            CE_total_hist = indicator_utils.draw_histgram(data=CE_total, indicator_name='CE', percentile=args.CE_percentile)
            indicator_utils.save_figure(CE_total_hist, save_dir=CE_savedir, save_filename='CE_total_histgram.png')
            CE_cumulative_sum = indicator_utils.draw_cumulative_sum(CE_total.values.tolist(), 'CE')
            indicator_utils.save_figure(CE_cumulative_sum, save_dir=CE_savedir, save_filename='CE_total_cumulative_sum.png')

        if 'EAG' in args.indicators:
            EAG_total = indicator_holder.indicator_total[f'EAG{args.EAG_percentile}']
            EAG_total_hist = indicator_utils.draw_histgram(data=EAG_total, indicator_name='EAG', percentile=args.EAG_percentile)
            indicator_utils.save_figure(EAG_total_hist, save_dir=EAG_savedir, save_filename='EAG_total_histgram.png')
            EAG_cumulative_sum = indicator_utils.draw_cumulative_sum(EAG_total.values.tolist(), 'EAG')
            indicator_utils.save_figure(EAG_cumulative_sum, save_dir=EAG_savedir, save_filename='EAG_total_cumulative_sum.png')

        logger.debug('- {}, {} evaluation END -'.format(args.trajection_folder, track))

if __name__ == '__main__':

    parser = argparse.ArgumentParser(description='XDR Challenge Evaluation Setting')

    parser.add_argument('trajection_folder', type=str, help='Set trajection folder name')

    parser.add_argument('ground_truth_folder', type=str, help='Set ground truth folder name')

    parser.add_argument('--save_folder', type=str, help='Set save folder name')

    parser.add_argument('--VDR', action='store_const', dest='track', default=['VDR','PDR'],
                        const=['VDR'],help='Set  VDR track')

    parser.add_argument('--PDR', action='store_const', dest='track', default=['VDR','PDR'],
                        const=['PDR'],help='Set  PDR track')

    parser.add_argument('--file', nargs='*', help='Select file name for evaluation')

    parser.add_argument('--CE', action='append_const', dest='indicators', default=[],
                        const='CE', help='Calculate Circular Error')

    parser.add_argument('--CE_percentile', type=int, default=50,
                        help='Calculate percentile of Circular Error')

    parser.add_argument('--CP', action='append_const', dest='indicators', default=[],
                        const='CP', help='Calculate Circular Precision')

    parser.add_argument('--CP_percentile', type=int, default=50,
                        help='Calculate percentile of Circular Presicion')

    parser.add_argument('--CA', action='append_const', dest='indicators', default=[],
                        const='CA', help='Calculate Area-weighted Circular Error Distribution Deviation')

    parser.add_argument('--area_weights', default=None, help='area weights configuration path')

    parser.add_argument('--CA_hist', action='store_true', help='Calculate CA by 2D hisgram')

    parser.add_argument('--band_width', type=float, default=None, help='band width for kernel density')

    parser.add_argument('--EAG', action='append_const', dest='indicators', default=[],
                        const='EAG', help='Calculate Error Accumulation Gradient')

    parser.add_argument('--EAG_percentile', type=int, default=50,
                        help='Calculate percentile of Error Accumulation Gradient')

    parser.add_argument('--requirement_velocity', action='append_const', dest='indicators', default=[],
                        const='requirement_velocity', help='Calculate requirement for moving velocity')

    parser.add_argument('--velocity', type=float, default=1.5, help='appropriate walking velocity (unit: m/s)')

    parser.add_argument('--requirement_obstacle', action='append_const', dest='indicators', default=[],
                        const='requirement_obstacle', help='Calculate requirement for obstacle avoidance')

    parser.add_argument('--requirement_coverage', action='append_const', dest='indicators', default=[],
                        const='requirement_coverage', help='Calculate requirement for coverage')

    parser.add_argument('--debug', action='store_const', dest='level', default=INFO, const=DEBUG,
                        help='Logger debug mode')

    args = parser.parse_args()
    args.indicators = ['CE', 'CA', 'EAG', 'requirement_velocity', 'requirement_obstacle', 'requirement_coverage'] if not args.indicators else args.indicators

    #　Logger setting
    logger = getLogger(__name__)
    logger.setLevel(DEBUG)

    formatter = Formatter('%(asctime)s - %(name)s - %(message)s')

    # Standard output handler
    stream_handler = StreamHandler()
    stream_handler.setLevel(args.level)
    stream_handler.setFormatter(formatter)

    # File output handler
    log_file_name = os.path.basename(__file__) + '.log'
    file_handler = FileHandler(log_file_name, 'a')
    file_handler.setLevel(DEBUG)
    file_handler.setFormatter(formatter)

    logger.addHandler(stream_handler)
    logger.addHandler(file_handler)

    logger.debug('track:{}, indicators:{}'.format(args.track, args.indicators))
    print('track:{}, indicators:{}'.format(args.track, args.indicators))
    main(args)


#!/bin/env python3

from flask import Flask, jsonify, request, render_template
#app = Flask(__name__)
from napalm import get_network_driver
import datetime, os, json

ipsList = [
    '198.51.100.1', #R1
    '198.51.101.2', #R2
    '172.16.1.3',   #R3
    '198.51.101.4'  #R4
]

def getconfig():
    arr = []

    for i in range(len(ipsList)):
        print("Connecting to: ", ipsList[i])
        driver = get_network_driver('ios')
        ios_router = driver(ipsList[i], 'johnny', 'lab123')
        ios_router.open()

        output = ios_router.get_config()
        output = str(output).replace("\'", "\"")

        now = datetime.datetime.now().isoformat()

        with open('R' + str(i + 1) + '_' + now + '.txt', 'a') as outfile:
            json.dump(output, outfile)
        outfile.close()

        ios_router.close()
        arr.append(os.path.basename(outfile.name))

    strings = ', '.join(arr)
    return "You have saved the files: " + strings


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Shows a simple example about how to use the K-Means algorithm
"""
#https://towardsdatascience.com/understanding-k-means-clustering-in-machine-learning-6a6e67336aa1
#https://towardsdatascience.com/the-5-clustering-algorithms-data-scientists-need-to-know-a36d136ef68

import sys
import argparse
import os
from os import linesep
import pandas as pd
import numpy as np
import pprint
import csv
import yaml

import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages
from numpy.random import seed
from numpy.random import randn
from numpy import mean
from numpy import std

from reportlab.pdfgen import canvas
from reportlab.platypus import *
from reportlab.lib import colors
from reportlab.lib.pagesizes import letter
from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet

styles = getSampleStyleSheet()

from sklearn.cluster import KMeans
#%matplotlib inline

palette = np.array([["red"], # index 0: red
                    ["green"], # index 1: green
                    ["blue"], # index 2: blue
                    ["black"], # index 3: black
                    ["yellow"], # index 4: yellow
                    ["purple"]

                    ])

def get_table_for_special_positions(pos_specials, columns, Y_data):
    specials = "\n"
    specials_data = np.zeros(0, dtype=int)
    for position in pos_specials:

        #print("position: ", position)
        pos = position[0] # number of columns
        #print("pos: ", pos)
        #print("columns: ", columns)
        row = int(pos /  columns)
        #print("row: ", row)
        column = pos - row * columns
        column += 1
        #print("column: ", column)
        '''
        text = "pos, row, column, value    " + str(pos) + "\t"\
            + str(row) + "\t"\
            + str(column) + "\t"\
            + str(Y[pos])
        specials += text
        specials += linesep
        '''
        specials_data = np.append(specials_data, \
            [pos, row, column, int(Y_data[pos])], axis=0)
        #print("outliers_data len: ", len (outliers_data))
        #print("outliers_data # columns: ", outliers_data)
        #break

    return specials_data

def calculate(data, filename, result, df_summary, cutoff_offset_percent):
    os.makedirs("output", exist_ok=True)
   # pdf_out = PdfPages('output/multipage.pdf')

    print("data", linesep, data)

    #split first  row for X
    X = data[:, 0 ]
    print("X", linesep, X)

    # split Y values
    Y = data[:, 1:]
    print("Y", linesep, Y)
    pprint.pprint(Y)

    print("len           X / Y: ", len(X) , len(Y) )

    #duplicate X as many times as needed
    X = np.repeat(X, len(Y[0]))

    # now the array have the same size and noise was removed
    #keep a copy
    X_ORI = X.copy()
    Y_ORI = Y.copy()
    print("X flattened", linesep, X)

    # flatten the data
    X = X.flatten()
    Y = Y.flatten()

    print("len flattened X / Y: ", len(X) , len(Y))
    print("X flattened", linesep, X)
    print("Y flattened", linesep, Y)

    ### Get the data mean and std deviation
    data_mean, data_std = mean(Y), std(Y)#  identify outliers
    print("Data Simple Mean:", linesep, data_mean)
    print("Data Standard Deviation:",  linesep, data_std)

    result[filename] = {}
    result[filename]["mean"] = data_mean
    result[filename]["std_deviaton"] = data_std
    result[filename]["X"] = X
    result[filename]["Y"] = Y
    result[filename]["title"] = filename + " - " + str(len(Y)) + " data points"

    # the cutoff defined as a multiple of the standard deviation
    #cut_off = data_std * 3
    cut_off = data_mean * cutoff_offset_percent / 100
    lower, upper = data_mean - cut_off, data_mean + cut_off

    ### identify outliers
    outliers = [x for x in Y if x < lower or x > upper]
    print('Identified outliers: %d' % len(outliers))
    print("Outliers:", linesep, outliers)

    ### Prepare to display outliers
    Y_outliers = Y.copy()
     # leave only the outliers in the array and display
    Y_outliers[np.logical_and(Y_outliers > lower, Y_outliers < upper)] = np.nan

    print("Y_outliers:", linesep, Y_outliers)
    result[filename]["outliers_Y"] = Y_outliers
    result[filename]["outliers_title"] = "Mean " + str(data_mean)\
                + "\nStd Deviation : " + str(round(data_std, 4))\
               + "\n# Outliers: " + str(len(outliers))\
               + "\nlower: " + str(round(lower, 2)) \
               + "\nupper: " + str(round(upper, 2))

    # get the number of rows and columsn of the original data
    Y_data_rows, columns =  np.shape(Y_ORI)

    ### get the positions of outliers
    # reshape the previously flattened array
    Y_outliers.reshape((Y_data_rows, columns))
    pos_outliers = np.argwhere(~np.isnan(Y_outliers.data))
    print("len pos_outliers: \n", len(pos_outliers))

    outliers_data = get_table_for_special_positions(\
                pos_outliers, columns, Y)

    #print("outliers:\n", outliers_data)
    result[filename]["outliers_values"] = Y_outliers.copy()
    result[filename]["outliers_data"] = outliers_data.copy()

    ### Get the position of  zero values
    Y_zeros = Y.copy()
     # leave only the outliers in the array and display
    Y_zeros[Y_zeros != 0] = np.nan
    Y_zeros.reshape((Y_data_rows, columns))
    pos_zeros = np.argwhere(~np.isnan(Y_zeros.data))
    print("len pos_zeros: \n", len(pos_zeros))

    zeros_data = get_table_for_special_positions(\
                pos_zeros, columns, Y)

    #print("outliers:\n", outliers_data)
    result[filename]["zeros_data"] = zeros_data.copy()

    summary_row1 = pd.Series([filename,\
            len(pos_outliers),\
            len(pos_zeros),\
            round(data_mean, 2),\
            round(data_std, 2),\
            round(lower, 2),\
            round(upper, 2)],\
            index=df_summary.columns)

    df_summary = df_summary.append(summary_row1, ignore_index=True)
    print("df_summary: \n", df_summary)
    return df_summary

def open_file_and_calculate(file_name, result, df_summary, cutoff_offset_percent):
     # open the data file
    file_path = os.path.join(os.getcwd(), file_name)
    test_data = ''
    print("file:", file_path)
    with open(file_path, 'r') as f:
        reader_d = csv.reader(f, delimiter=',')

        # get header from first row
        headers = next(reader_d)
        # get all the rows as a list
        data = list(reader_d)
        # transform data into numpy array
        test_data = np.array(data)

    # take of the first column, it is just empty
    test_data = test_data[:, 1:]

    #convert numbers to integers
    test_data = test_data.astype(np.float)
    #pprint.pprint(test_data)

    df_summary = calculate(test_data, file_name, result, df_summary, cutoff_offset_percent)
    return df_summary

def create_output_pdf_for_special_values(special_data, element, filename):
        # print special data to PDF

        header = Paragraph(filename, styles["Heading1"])
        element.append(header)

        special_data = special_data.reshape(-1, 4)
        print("special data: ", special_data)

        # get number of rows
        rows, coluums = special_data.shape
        txt = Paragraph("Number of findings: " + str(rows), styles["Normal"])
        element.append(txt)

        df = pd.DataFrame(data=special_data, columns=("pos", "row", "column", "value"))
        print("df:\n", df)

        lista = [df.columns[:,].values.astype(str).tolist()] + df.values.tolist()
        t1 = Table(lista)

        element.append(t1)
        return element

def create_output_pdf_summary(df, filebase):
    doc_summary  = SimpleDocTemplate("output/" + filebase + "_summary.pdf", pagesize=letter)

    element = []
    header = Paragraph("\nSummary of Analysis Run", styles["Heading1"])
    element.append(header)

    # sort by standard deviation
    header = Paragraph("\nSorted by Standard Derivation", styles["Heading2"])
    element.append(header)

    df = df.sort_values(by=["std_dev"])
    lista = [df.columns[:,].values.astype(str).tolist()] + df.values.tolist()
    t1 = Table(lista)
    element.append(t1)

    # sort by number of outliers
    header = Paragraph("\nSorted by Number of Outliers", styles["Heading2"])
    element.append(header)

    df = df.sort_values(by=["# outliers"])
    lista = [df.columns[:,].values.astype(str).tolist()] + df.values.tolist()
    t1 = Table(lista)
    element.append(t1)

    # sort by number of zeros
    header = Paragraph("\nSorted by Number of Zeros", styles["Heading2"])
    element.append(header)

    df = df.sort_values(by=["# zeros"])
    lista = [df.columns[:,].values.astype(str).tolist()] + df.values.tolist()
    t1 = Table(lista)
    element.append(t1)

    doc_summary.build(element)

def create_output_pdf(results, df_summary, filebase):
    pdf_plots    = PdfPages("output/" +  filebase + "_plots.pdf")
    doc_outliers = SimpleDocTemplate("output/" + filebase + "_outliers.pdf", pagesize=letter)
    doc_zeros    = SimpleDocTemplate("output/" + filebase + "_zeros.pdf", pagesize=letter)

    element_outliers = []
    element_zeros = []

    for filename in results:
        print("filename: ", filename)

        plt.subplot(1, 2, 1)
        plt.title(filename + " - " + str(len(results[filename]["Y"])) + " data points")
        plt.scatter(results[filename]["X"], results[filename]["Y"], s = 50, c="brown")

        plt.subplot(1, 2, 2)
        plt.title("Outliers")
        plt.xlabel(results[filename]["outliers_title"])
        plt.scatter(results[filename]["X"], results[filename]["outliers_Y"], s = 50, c="blue")
        #plt.figure()
        #plot1 = plt.plot()
        plt.savefig(pdf_plots, format='pdf', bbox_inches='tight')
        plt.plot()
        #break

        # print outliers to PDF
        element_outliers = create_output_pdf_for_special_values(results[filename]["outliers_data"],\
                     element_outliers, filename)

        element_zeros = create_output_pdf_for_special_values(results[filename]["zeros_data"],\
                    element_zeros, filename)

    ## Doen with collecting data, create the documents
    doc_outliers.build(element_outliers)
    doc_zeros.build(element_zeros)

    pdf_plots.close()

'''
The parameters are self explaining

'''
def main(argv):
    parser = argparse.ArgumentParser(description='Training and running a small neural network.')
    parser.add_argument('-i', dest='input_config_file',
                        action='store',
                        required=True,
                        help='The yaml file containing configuration')

    args = parser.parse_args()
    print("args: ", args)

    ### Get the configuration
    configuration = None
    input_file_full_path = os.path.join(os.getcwd(), args.input_config_file)
    with open(input_file_full_path, 'r') as stream:
        configuration = yaml.safe_load(stream)

    print(yaml.safe_dump(configuration, default_flow_style=False, default_style=None))

    read_file_or_dir = configuration["read_file_or_dir"]
    read_full_path = os.path.join(os.getcwd(), read_file_or_dir)

    df_summary = pd.DataFrame(columns=["filename",\
        "# outliers",\
        "# zeros",\
        "mean",\
        "std_dev",\
        "lower_cutoff",\
        "upper_cutoff"])

    results = {}
    # open the data file

    cutoff_offset_percent = configuration["cutoff_offset_percent"]
    if(os.path.isdir(read_full_path)):
        for entry in os.scandir(read_full_path):
            if entry.is_file():
                print(entry.name)
                full_filename = os.path.join(read_file_or_dir, entry.name)
                df_summary = open_file_and_calculate(full_filename,\
                        results, df_summary, cutoff_offset_percent)

    else:
        df_summary = open_file_and_calculate(read_full_path,\
                    results, df_summary, cutoff_offset_percent)

    from datetime import datetime
    now = datetime.now() # current date and time

    date_time = now.strftime("run_%m-%d-%Y--%H-%M")
    create_output_pdf(results, df_summary, date_time)
    create_output_pdf_summary(df_summary, date_time)

if __name__ == '__main__':
    main(sys.argv[1:])


import logging
import argparse

class CalcOptions(object):
    def __init__(self):
        self.log = logging.getLogger(__name__+'.'+self.__class__.__name__)

    def parse_args(self,args):
        parser = self.create_parser()
        opts = parser.parse_args(args)
        self.apply_opts(opts)
        return opts, opts.extra

    def create_parser(self):
        self.log.debug("Creating arg parser")
        parser = argparse.ArgumentParser(description='toptal calculator project')
        parser.add_argument('--debug',
                            help='enable debug-level logging for the given module (default: None)')
        parser.add_argument('extra', nargs='*')

        self.log.debug("arg parser = %s" % parser)
        return parser

    def apply_opts(self,opts):
        self.log.debug("Applying options: %s" % (opts))
        if opts.debug is not None:
            log = logging.getLogger(opts.debug)
            log.setLevel(logging.DEBUG)
            log.info("Enabled debug-level logging on %s" % (opts.debug))


from django.conf.urls import url
import os
from . import views
from django.conf import settings
urlpatterns = [
    url(r'^$', views.index, name='index'),
    url(r'^about/', views.about, name = 'about'),
    url(r'^results/', views.results, name = 'results'),
    url(r'^search/', views.search, name = 'search'),
    url(r'^login', views.auth_view, name = 'login'),
    url(r'^logout', views.log_out, name = 'logout'),
    url(r'^register', views.register, name = 'register'),
    url(r'^filter_listings/$', views.filter_listings, name='filter_listings'),
    url(r'^filter_maps/$', views.filter_maps, name='filter_maps'),
]


import os
import sys
from collections import defaultdict
from Bio.Seq import translate
from Bio.Seq import reverse_complement
from Bio import SearchIO

fastadict = dict()

myfasta = open('all_extractedseq_renamed.fa', 'r')

for line in myfasta:
	if ">" in line:
		fastadict[line.strip()[1:]] = next(myfasta).strip()

mybed = open('all_venom_bed', 'r')
#mybed = open('mytestbed', 'r')

out = open('venom_proteins_sliced.fa', 'w')

for line in mybed:
	info = line.strip().split('\t')

	start = int(info[1])
	end = int(info[2])

	remainder = start%3

	out.write('>' + '|'.join(info)+ '\n')
	out.write(translate('N'*remainder + fastadict[info[0]][start:end]).replace('X','') + '\n')


#!/usr/bin/python
"""Use Amazon EC2 SDK :: boto3."""

import boto3
ec2 = boto3.resource('ec2')

# Check running instances
instances = ec2.instances.filter(
    Filters=[{'Name': 'instance-state-name', 'Values': ['running']}])
for instance in instances:
    print(instance.id, instance.instance_type)

# Service health
for status in ec2.meta.client.describe_instance_status()['InstanceStatuses']:
    print(status)

# Launch new instance
ec2.create_instances(ImageId='<ami-image-id>', MinCount=1, MaxCount=1)

# Stop Instances
ids = ['your_instance_id']
ec2.instances.filter(InstanceIds=ids).stop()

# Terminate instances
ec2.instances.filter(InstanceIds=ids).terminate()


from django.conf.urls import url
from . import views

app_name = 'userProfile'
urlpatterns = [
    # user/
    url(r'^$', views.user, name="user"),
    # user/id
    url(r'^(?P<user_id>\d)/$', views.details, name="details"),
    # addUser
    url(r'^addUser', views.addUser, name="addUser")

]


import logging
from apistar.types import Type
from apistar import validators

from idunn import settings
from idunn.api.kuzzle import kuzzle_client
from apistar.types import Type
from .base import BaseBlock

logger = logging.getLogger(__name__)

class ParticleType(Type):
    value = validators.Number(allow_null=True)
    quality_index = validators.Integer(minimum=1, maximum=5, allow_null=True)

class AirQuality(BaseBlock):
    BLOCK_TYPE = 'air_quality'

    CO = ParticleType
    PM10 = ParticleType
    O3 = ParticleType
    NO2 = ParticleType
    SO2 = ParticleType
    PM2_5 = ParticleType
    quality_index = validators.Integer(minimum=1, maximum=5)
    date = validators.DateTime()
    source = validators.String()
    source_url = validators.String()
    measurements_unit = validators.String(allow_null=True)

    @classmethod
    def from_es(cls, place, lang):
        if not settings['BLOCK_AIR_QUALITY_ENABLED']:
            return None
        if place.PLACE_TYPE != 'admin':
            return None
        if place.get('zone_type') not in ('city', 'city_district', 'suburb'):
            return None

        bbox = place.get_bbox()
        if not bbox:
            return None

        try:
            air_quality = get_air_quality(bbox)
        except Exception:
            logger.warning('Failed to fetch air quality for %s', place.get_id(), exc_info=True)
            return None

        if not air_quality:
            return None

        return cls(**air_quality)

def get_air_quality(geobbox):
    if not kuzzle_client.enabled:
        return None
    return kuzzle_client.fetch_air_quality(geobbox)


#!/usr/bin/env python3

import csv
from util import *
import os
import sys
import time

#-------------------------
#Undergraduates Applicants
#-------------------------
#1. International Applicants : https://apps.knust.edu.gh/admissions/check/Home/ApplicantList?offer=9&streamid=1&page=1         total_pages=4
#2. SSSCE/WASSCE Applicants : https://apps.knust.edu.gh/admissions/check/Home/ApplicantList?offer=1&streamid=1&page=1         total_pages=328
#3. Fee paying Applicants(SSSCE/WASSCE) : https://apps.knust.edu.gh/admissions/check/Home/ApplicantList?offer=3&streamid=1&page=1         total_pages=135
#4. Mature/Other/Ghanaian with foreign results Applicants : https://apps.knust.edu.gh/admissions/check/Home/ApplicantList?offer=8&streamid=1&page=1         total_pages=2
#5. Fee paying Applicants (Mature/Other/Ghanaian with foreign results) : https://apps.knust.edu.gh/admissions/check/Home/ApplicantList?offer=10&streamid=1&page=1         total_pages=9
#6. NMTC Upgrade : https://apps.knust.edu.gh/admissions/check/Home/ApplicantList?offer=15&streamid=13&page=1         total_pages=21

#-----------------------
#Postgraduate Applicants
#-----------------------
#1. International Applicants : https://apps.knust.edu.gh/admissions/check/Home/ApplicantList?offer=11&streamid=2&page=1         total_pages=4
#2. Ghanaian Applicants : https://apps.knust.edu.gh/admissions/check/Home/ApplicantList?offer=12&streamid=2&page=1         total_pages=54

# Import requests
try:
	import requests
	from requests import ConnectionError
except ImportError:
	print("Warning: missing package 'requests' is required")

try:
	import prettytable
except ImportError:
	print("Warning: missing package 'prettytable' is required")

# Import bs4
try:
	from bs4 import BeautifulSoup
except ImportError:
	print("Warning: missing package 'bs4' is required")
	print("Fix: 'sudo pip install -r requirements.txt'")
	sys.exit(1)

# Base url for website
base_url = "https://apps.knust.edu.gh/admissions/check/Home/"

# List of students path urls
students_urls = [
		"ApplicantList?offer=9&streamid=1&page=",
		"ApplicantList?offer=1&streamid=1&page=",
		"ApplicantList?offer=3&streamid=1&page=",
		"ApplicantList?offer=8&streamid=1&page=",
		"ApplicantList?offer=10&streamid=1&page=",
		"ApplicantList?offer=15&streamid=13&page=",
		"ApplicantList?offer=11&streamid=2&page=",
		"ApplicantList?offer=12&streamid=2&page"
]

def students_scrapper(url, end, filename):

	if not os.path.exists(filename):
		os.mknod(filename)

	csvFile = open(filename, "wt+")
	writer = csv.writer(csvFile)
	# Heading of the table
	writer.writerow(["Student ID", "Full Name", "Programme"])

	try:
		print("\033[93m[+] Loading...(Please wait)\033[0m")
		for i in range(1, end+1):
			# url of pages
			full_url = url + "{}".format(i)

			req = requests.get(full_url)
			soup = BeautifulSoup(req.text, "html.parser")

			table = soup.find("table")
			rows = table.find_all("tr")
			sys.stdout.write("#")
			sys.stdout.flush()

			try:
				for row in rows:
					csvRow = []
					for cell in row.find_all(["td", "th"]):
						# Correct UnicodeError
						cell_text = cell.get_text().encode("ascii", "ignore")
						csvRow.append(cell_text.decode("utf-8"))

						if len(csvRow) == 4:
							if csvRow[0].isdigit():
								csvRow.pop(0)

								if csvRow[0].isdigit():
									writer.writerow(csvRow)

			except Exception as e:
				print("No rows found")

			full_url = url

	except ConnectionError:
		print("\033[91m[-]\033[0m Problem connecting to {}".format(base_url)[:-22])

		if os.path.exists(filename):
				os.remove(filename)

	except KeyboardInterrupt:
		print("You pressed Ctrl+C\nExiting...")
		sys.exit(1)

	print("\n\033[92m[*]\033[0m Done...")

	if os.path.exists(filename):
		csvFile.close()

def main():

	clear()
	banner()

	myLoop = True
	mainchoice()

	while myLoop:

		choice = int(input("\033[92mchoice>>\033[0m "))
		if choice == 1:
			url = students_urls[0]
			url = base_url + url
			end = 4
			filename = "students1.csv"
			students_scrapper(url, end, filename)
			myLoop = False

		elif choice == 2:
			url = students_urls[1]
			url = base_url + url
			end = 328
			filename = "students2.csv"
			students_scrapper(url, end, filename)
			myLoop = False

		elif choice == 3:
			url = students_urls[2]
			url = base_url + url
			end = 135
			filename = "students3.csv"
			students_scrapper(url, end, filename)
			myLoop = False

		elif choice == 4:
			url = students_urls[3]
			url = base_url + url
			end = 2
			filename = "students4.csv"
			students_scrapper(url, end, filename)
			myLoop = False

		elif choice == 5:
			url = students_urls[4]
			url = base_url + url
			end = 9
			filename = "students5.csv"
			students_scrapper(url, end, filename)
			myLoop = False

		elif choice == 6:
			url = students_urls[5]
			url = base_url + url
			end = 21
			filename = "students6.csv"
			students_scrapper(url, end, filename)
			myLoop = False

		elif choice == 7:
			url = students_urls[6]
			url = base_url + url
			end = 4
			filename = "students7.csv"
			students_scrapper(url, end, filename)
			myLoop = False

		elif choice == 8:
			url = students_urls[7]
			url = base_url + url
			end = 54
			filename = "students8.csv"
			students_scrapper(url, end, filename)
			myLoop = False

		elif choice == 99:
			print("\033[92m[*]\033[0m Exiting...")
			time.sleep(1)
			sys.exit(1)

		else:
			clear()
			mainchoice()

if __name__ == '__main__':
	main()


from pathlib import Path
from collections.abc import Sequence
import numpy as np
from math import ceil
import uuid

from dask import delayed
from functools import partial
from pintFoam.parareal import (parareal, tabulate)
from pintFoam import (BaseCase, foam, block_mesh)
from pintFoam.vector import (Vector)
from pintFoam.foam import (map_fields)
from pintFoam.utils import (generate_job_name)

fields = ["p", "U"]
case_name = "pipeFlow"

fine_case = BaseCase(Path(case_name + "Fine"), "baseCase", fields=fields)
coarse_case = BaseCase(Path(case_name + "Coarse"), "baseCase", fields=fields)

fine_case.clean()
coarse_case.clean()

block_mesh(fine_case)
block_mesh(coarse_case)

times = np.linspace(0, 2, 21)

@delayed
def gather(*args):
    return list(args)

@delayed
def c2f(x):
    """Coarse to fine.

    Interpolate the underlying field x from the coarse to the fine grid"""
    return map_fields(x, fine_case, map_method="interpolate")

@delayed
def f2c(x):
    """Fine to coarse.

    Interpolate the underlying field x from the fine to the coarse grid"""
    return map_fields(x, coarse_case, map_method="interpolate")

@delayed
def fine(n, x, t_0, t_1):
    """Fine integrator."""
    uid = uuid.uuid4()
    return foam("pimpleFoam", 0.001, x, t_0, t_1,
                job_name=generate_job_name(n, t_0, t_1, uid, "fine"))

@delayed
def coarse(n, x, t_0, t_1):
    """Coarse integrator."""
    uid = uuid.uuid4()
    return foam("pimpleFoam", 0.1, x, t_0, t_1,
                job_name=generate_job_name(n, t_0, t_1, uid, "coarse"))

def time_windows(times, window_size):
    """Split the times vector in a set of time windows of a given size.

    Args:
        times:          The times vector
        window_size:    The number of steps per window (note that n steps
        correspond to n + 1 elements in the window). The last window may
        be smaller.
    """
    n_intervals = len(times) - 1
    n = int(ceil(n_intervals / window_size))
    m = window_size
    return [times[i*m:min(i*m+m+1, len(times))] for i in range(n)]

def solve(init: Vector, times: Sequence[float], max_iter=1) -> list[Vector]:
    # coarse initial integration from fine initial condition
    y = list(map(c2f, tabulate(partial(coarse, 0), f2c(init), times)))
    for n in range(1, max_iter+1):
        y = gather(*parareal(partial(coarse, n), partial(fine, n), c2f, f2c)(y, times))
    return y  # .compute()

def windowed(times, init, window_size):
    windows = time_windows(times, window_size)
    result = [init]
    for w in windows:
        w_result = solve(result[-1], w)
        result.extend(w_result)
    return result

# print(time_windows(np.arange(40), 11))
windows = time_windows(times, 10)
init = fine_case.new_vector()
wf = solve(init, windows[0], 3)
# wf.visualize("parareal.png")
wf.compute(n_workers=4)


# -*- coding: utf-8 -*-
"""
Created on Thu Oct  8 18:45:42 2020

@author: Benjamin
"""

import os
import pickle

from fatiando.vis import mpl #, myv
from fatiando import gridder, mesher, utils
from fatiando.gravmag import prism, imaging, transform
from fatiando.vis.mpl import square

# my own functions
import lib.dEXP as dEXP
from lib.dEXP import _fit
import lib.plot_dEXP as pEXP
import lib.utils_dEXP as uEXP

import notebooks_GRL.load_MALM_model as MALMmod
import notebooks_GRL.load_MALM_real as MALMreal
from mpl_axes_aligner import align

import lib.set_parameters as para

import numpy as np
import matplotlib.pyplot as plt
plt.rcParams['font.size'] = 15

# icsd functions
#from importers.read import load_obs, load_geom

#%% ------------------------------- MALM DATA synthetic

interp_size = 300
smooth = 'CubicSmoothingSpline' #'Hanning+Lowpass'
# smooth = 'CubicSmoothingSpline + interp1d' #'Hanning+Lowpass'
interp = False
file = 'NoAno'

path =  './data/phNO/'
dataset = MALMmod.load_MALM_Landfill_model(path=path,
                                    filename=file,
                                    shape = (300,300),
                                    field=True,
                                    interp = interp,
                                    radius=60) # length of p1p2 profile

coord_xyz, coord_xyz_int = dataset[0:2]
coord_xyz_int
Uload = dataset[2]
coords_liner = dataset[3]
shape, max_elevation = dataset[4:6]

dict_data = dataset[7]
dict_data['AnoBool']

xA = (dict_data['HZ'][0][0]+dict_data['HZ'][0][1])/2
x1 = dict_data['HZ'][0][2]
y1 = dict_data['HZ'][0][0]
y2 = dict_data['HZ'][0][1]

z1 = dict_data['HZ'][1]
z2 = z1 - dict_data['HZ'][2]

p = dataset[6]         # line points
# set imaging pseudo-inversion parameters
parameters = para.set_par(shape=shape,max_elevation=max_elevation)

scaled = parameters[0]
SI = parameters[1]
zp, qorder, nlay = parameters[2:5]
minAlt_ridge, maxAlt_ridge = parameters[5:7]

max_elevation = 30
# nlay = 50

# xp, yp, zp = coord_xyz_int
xp, yp, zp = coord_xyz
# len(xp)
Uini = Uload[0] # U_raw, Ucor, U_int, Ucor_int
p1 , p2 = p

rot = 60
origin=(max(xp), min(yp))
point_torotate = np.array([xp, yp])
xp_r, yp_r = MALMreal.rotate_60(origin, point_torotate, angle_deg=rot, showfig=False)
Xs = xp_r-min(xp_r)
Ys = yp_r-min(yp_r)

prl = 60
# shape = shape  (max(xp)-min(xp))/
shape = (150,150)
xint_scipy, yint_scipy = gridder.regular((min(Xs)-prl, max(Xs)+prl,
                          min(Ys)-prl, max(Ys)+prl),shape=shape)

#%% ------------------------------- MALM DATA real
MainPath= './data/'
# os.chdir(MainPath)
## --------- read MALM measured data file (with electrode positions) --------- ##
# RealData = np.loadtxt("./1_Data_2_plot/to_plot.dat",skiprows=0,delimiter='\t')
out = MALMreal.load_MALM_Porto_real(MainPath,
                          MainPath + 'XYObs_real_f_m3.txt',
                          shape=(100,100),
                          radius=200,
                          rcor=10,
                          rot=0,
                          showfig=False)

# coords_liner = out[3]
p = out[6]         # line points
p1f , p2f = p
coord_xyz, coord_xyz_int = out[0:2]
xf, yf, zf = coord_xyz
# len(xp)

out_r = MALMreal.load_MALM_Porto_real(MainPath,
                          MainPath + 'XYObs_real_f_m3.txt',
                          shape=(100,100),
                          radius=200,
                          rcor=10,
                          rot=60,
                          showfig=False)

# coords_liner = out[3]
p = out_r[6]         # line points
p1f_r , p2f_r = p
coord_xyz_r, coord_xyz_int_r = out_r[0:2]
Uload_r = out_r[2]
uf = Uload_r[1] # U_raw, Ucor, U_int, Ucor_int
# xf_rtmp, yf_rtmp, zf_rtmp = coord_xyz

# plt.figure()
# plt.scatter(xf_rtmp, yf_rtmp, c=uf, cmap='viridis',vmax=0.05)
# plt.colorbar()
# plt.axis('square')
# plt.show()

#%% Mirror field against p1p2

# find point position with respect to line equation defined by p1 and p2
U_af, p_af, bool_abovef, U_bf, p_bf = MALMmod.isabove(xf, yf, uf,
                                  np.array(p1),np.array(p2))
# Compute p1 and p2 line equation ax + by + c = 0
af, bf, cf = MALMmod.slope(p1,p2)

# Mirror points with respect to p1p2 line
Umirrorf, xy_mirrorf = MALMmod.mirrorU_alongLine(U_af,p_af,bool_abovef,af,bf,cf)

U_a_intf = gridder.interp_at(xy_mirrorf[:,0], xy_mirrorf[:,1], Umirrorf, xf, yf, algorithm='cubic',
                        extrapolate=True)
# U_mirror_int = np.copy(U_a_int)
U_mirror_intf = np.copy(uf)
U_mirror_intf[np.where(bool_abovef == True)[0]]= U_a_intf[np.where(bool_abovef == True)[0]]

xf_mirror = np.hstack([xy_mirrorf[:,0],p_af[:,0]])
yf_mirror = np.hstack([xy_mirrorf[:,1],p_af[:,1]])
uf_mirror = np.hstack([Umirrorf,U_af])

#%% choose raw or mirrored field

Uf = np.copy(uf)

Uf  = np.copy(uf_mirror)
xf = xf_mirror
yf = yf_mirror

# %% rotate and rescale all

rot = 60
origin=(max(xp), min(yp))
# origin=(max(Xf), min(Yf))
point_torotate = np.array([xp, yp])
xp_r, yp_r = MALMreal.rotate_60(origin, point_torotate, angle_deg=rot, showfig=False)

point_torotate = np.array([xf, yf])
xf_r, yf_r = MALMreal.rotate_60(origin, point_torotate, angle_deg=rot, showfig=False)
Xfs = xf_r-min(xp_r)
Yfs = yf_r-min(yp_r)

plt.figure()
plt.scatter(xf_r, yf_r, c=Uf, cmap='viridis',vmax=0.0175)
plt.colorbar()
plt.axis('square')
plt.show()

# print(Uf[0:2])

point_torotate = np.array([[dict_data['HZ'][0][0],dict_data['HZ'][0][1]],
                          [dict_data['HZ'][0][2],dict_data['HZ'][0][2]]])
xA_r, yA_r =  MALMreal.rotate_60(origin,point_torotate,angle_deg=rot, showfig=False) #58.511
xA_r = xA_r-min(xp_r)
yA_r = yA_r-min(yp_r)

point_torotate = np.array(coords_liner).T
coords_linerx, coords_linery =  MALMreal.rotate_60(origin,point_torotate,angle_deg=rot, showfig=False)
coords_linerx = coords_linerx-min(xp_r)
coords_linery = coords_linery-min(yp_r)
coords_liner_s = np.array([coords_linerx, coords_linery]).T

plt.figure()
plt.scatter(Xfs, Yfs, c=Uf, cmap='viridis',vmax=0.05)
plt.colorbar()
plt.axis('square')
plt.show()

# %%
# prl = 10
# # shape = shape  (max(xp)-min(xp))/
# shape = (30,30)
# xint_scipy, yint_scipy = gridder.regular((min(Xfs)-prl, max(Xfs)+prl,
#                           min(Yfs)-prl, max(Yfs)+prl),shape=shape)
#%% Solution 1
# extrapolate False and fill with 0 before derivative - mask them later on
U_int_scipyf = gridder.interp_at(Xfs,Yfs,Uf, xint_scipy, yint_scipy, algorithm='cubic', extrapolate=False)
InterpData = np.array([xint_scipy, yint_scipy, U_int_scipyf]).T
where_are_NaNs = np.isnan(InterpData)
InterpData[where_are_NaNs] = 0.0074
xint_scipy, yint_scipy, U_int_scipyf = InterpData.T

#%% Solution 2

# U_int_scipyf = gridder.interp_at(Xfs,Yfs, Uf, xint_scipy, yint_scipy, algorithm='cubic', extrapolate=True)
# len(U_int_scipyf)
# len(Uf)

#%% Smoothing 2d

XFs, YFs = xint_scipy,yint_scipy
UF = U_int_scipyf
UF = dEXP.smooth2d(XFs, YFs, UF, sigma=2)
# plt.savefig('smooth2d' + str(file) + '.png', dpi=450)
UFs = UF

#%%

plt.figure()
plt.scatter(XFs, YFs, c=UF, cmap='viridis')
plt.colorbar()
plt.axis('square')
plt.show()

offset = 255 #260
p1_s = np.array([p1f_r[0] -min(xp_r)+offset,p2f_r[0] -min(xp_r)+offset])
p2_s = np.array([p1f_r[1] -min(yp_r),p2f_r[1] -min(yp_r) ])
ax, plt = pEXP.plot_field(XFs,YFs,UF, shape,
            Vminmax=[0,0.02])
ax.plot(coords_liner_s[2:5,0],coords_liner_s[2:5,1],'k')
ax.plot(p1_s,p2_s,'r')

x_axis = 'x'
# xx, yy, distance, profile, ax, plt = pEXP.plot_line(Xs, Ys, U ,p1_s,p2_s,
#                                                     interp=True, smooth=True,
#                                                     xaxis = x_axis)
p1_s = np.array([p1f_r[0] -min(xp_r)+offset,p1f_r[1] -min(yp_r)])
p2_s = np.array([p2f_r[0] -min(xp_r)+offset,p2f_r[1] -min(yp_r) +
             abs(p1f_r[1] -min(yp_r) - (yA_r[0]+yA_r[1])/2)
             -abs(p2f_r[1] -min(yp_r) - (yA_r[0]+yA_r[1])/2)])
# p1_s = np.array([p1[0] -min(xp_r)+260,0])
# p2_s = np.array([p2[0] -min(xp_r)+260,800])
xx, yy, distance, profile, ax,plt = pEXP.plot_line(XFs, YFs, UF ,p1_s,p2_s,
                                        interp=False,
                                        x_resolution = interp_size,
                                        smooth=smooth,
                                        xaxis = x_axis,
                                        Vminmax=[0,0.2],
                                        limx=[100,650],
                                        limy=[100,650],
                                        showfig=True)
plt.savefig('profile_field.png', dpi=450)

# xA_r_new = [p1_s[0]+xA_r[0]-xA_r[1], p1_s[0]-xA_r[0]+xA_r[1]]

#%% ------------------------------- plot publi mirror

ax, plt = pEXP.plot_field(XFs,YFs,UF, shape,Vminmax=[0,0.009])
ax.plot(coords_liner_s[2:5,0],coords_liner_s[2:5,1],'k')
plt.axis('square')
# plt.xlim(min(Xs),max(Ys))
# plt.ylim(min(Xs),max(Ys))
plt.xlim(300,500)
plt.ylim(300,500)
plt.savefig('publi_mirror_field.png', dpi=450)

#%% SAVE DATA AND PARAMETERS

dict_real_data = { "XYU" : [XFs,YFs,UF, UFs],
            "prl" : 60,
            "shape": shape,
            "coords_liner": coords_liner_s,
            "p12": [p1_s,p2_s]}

afile = open('fig6_data_m' + '.pkl', 'wb')
pickle.dump(dict_real_data, afile)
afile.close()

interp_size
interp
smooth
x_axis
nlay

#%% ------------------------------- Pad the edges of grids

# xp,yp,U, shape = dEXP.pad_edges(xp,yp,U,shape,pad_type=0) # reflexion=5
# pEXP.plot_line(xp, yp,U,p1,p2, interp=interp)

#%% ------------------------------- Plot the derivatives

xderiv = transform.derivx(XFs, YFs, UF, shape,order=0)
yderiv = transform.derivy(XFs, YFs, UF, shape,order=0)
zderiv = transform.derivz(XFs, YFs, UF, shape,order=0)

# interp = True
pEXP.plot_line(XFs, YFs, xderiv ,p1_s,p2_s,title='xderiv',x_resolution= interp_size,
            savefig=False, interp=interp, smooth=smooth,  Xaxis=x_axis)

plt.savefig('xderiv' + str(file) + '.png', dpi=450)

pEXP.plot_line(XFs, YFs, yderiv ,p1_s,p2_s,title='yderiv',x_resolution= interp_size,
            savefig=False, interp=interp, smooth=smooth,  Xaxis=x_axis)

plt.savefig('yderiv' + str(file) + '.png', dpi=450)

pEXP.plot_line(XFs, YFs, zderiv ,p1_s,p2_s,title='zderiv',x_resolution= interp_size,
            savefig=False, interp=interp, smooth=smooth,  Xaxis=x_axis)

plt.savefig('zderiv' + str(file) + '.png', dpi=450)

#%% ------- upward continuation of the field data
p = [p1_s,p2_s]

mesh, label_prop = dEXP.upwc(XFs, YFs, zp, UF, shape,
              zmin=0, zmax=max_elevation, nlayers=nlay,
              qorder=qorder)

plt, cmap = pEXP.plot_xy(mesh, label=label_prop,Xaxis=x_axis,
                     Vminmax=[0,0.0125], p1p2=p)
cbar = plt.colorbar(cmap,shrink=0.25, pad=0.04)
cbar.set_label('upwc voltage (V)')
plt.tight_layout()
plt.savefig('upwc voltage' + str(file) + '.png', dpi=450)

#%% DEXP ratio

# x_axis = 'x'
qratio = [1,0]
mesh_dexp, label_dexp = dEXP.dEXP_ratio(XFs, YFs, zp, UF, shape,
              zmin=0, zmax=max_elevation, nlayers=nlay,
              qorders=qratio)
fig = plt.figure(figsize=(6, 2), dpi=450)
ax = plt.gca()
plt, cmap = pEXP.plot_xy(mesh_dexp, label=label_dexp,
          markerMax=True,qratio=str(qratio),Vminmax=[0,0.075],
          p1p2=np.array([p1_s,p2_s]), ax=ax, Xaxis=x_axis) #, ldg=)
# plt, cmap = pEXP.plot_xy(mesh_dexp, label=label_dexp,
#              markerMax=True,qratio=str(qratio)
#              ax=ax, Xaxis=x_axis) #, ldg=)
cbar = plt.colorbar(cmap,shrink=0.25, pad=0.04)
cbar.set_label('ratio\nvoltage (V)')

# if x_axis=='y':
# square([xA_r_new[0], xA_r_new[1], -z1, -z2])
# else:
# square([yA_r[0], yA_r[1], -z1, -z2])
plt.xlim([200,600])
plt.savefig('fig3b.png', dpi=450)
plt.savefig('fig3b.pdf', dpi=450)
plt.savefig('fig3b.svg', dpi=450)
plt.savefig('fig3b.eps', dpi=450)
# plt.legend('')

# %% ridges identification

dEXP.ridges_minmax_plot(XFs, YFs, mesh, p1_s, p2_s,
                                  label=label_prop,
                                  interp=interp,x_resolution= interp_size,
                                  smooth=smooth,
                                  fix_peak_nb=2,
                                  method_peak='find_peaks',
                                  showfig=True,
                                  Xaxis=x_axis)
#%%
# or  find_peaks or peakdet or spline_roots
# dfI,dfII, dfIII = dEXP.ridges_minmax(Xs, Ys, mesh, p1_s, p2_s,interp=interp,x_resolution= interp_size,
#                                       label=label_prop,fix_peak_nb=2,
#                                       smooth=smooth, # true = low pass, otherwise specify the filter to apply
#                                       method_peak='find_peaks',
#                                       showfig=True,
#                                       Xaxis=x_axis)

D = dEXP.ridges_minmax(XFs, YFs, mesh, p1_s, p2_s,
                                  label=label_prop,
                                  method_peak='find_peaks',
                                  fix_peak_nb=2,
                                  returnAmp=True,
                                  showfig=True,
                                  Xaxis=x_axis,
                                  interp=interp,x_resolution= interp_size,
                                  smooth = smooth,
                                  qorder=qorder)

dfI, dfII, dfIII =  D[0:3]
hI, hII, hIII  = D[3:6]
heights  = D[3:6]

#%% ------------------------------- plot ridges over continuated section

fig = plt.figure(figsize=(6, 2), dpi=450)
ax = plt.gca()
plt, cmap = pEXP.plot_xy(mesh, label=label_prop, ax=ax, Xaxis=x_axis,
        Vminmax=[0,0.0125], p1p2=p)
cbar = plt.colorbar(cmap,shrink=0.25, pad=0.04)
cbar.set_label('upwc\nvoltage (V)')
plt.tight_layout()
pEXP.plot_ridges_harmonic(dfI,dfII,dfIII,ax=ax)
plt.xlim([200,600])
ax.set_xlabel('y (m)')
ax.set_ylabel('elevation\n(m)')
plt.savefig('fig3a.png', dpi=450)
plt.savefig('fig3a.pdf', dpi=450)
plt.savefig('fig3a.svg', dpi=450)
plt.savefig('fig3a.eps', dpi=450)
# plt.savefig('ridges_raw_field.png', dpi=450)

#%% ------------------------------- filter ridges regionally constrainsted)

dfI_f,dfII_f, dfIII_f = dEXP.filter_ridges(dfI,dfII,dfIII,
                                        minDepth=minAlt_ridge,
                                        maxDepth=maxAlt_ridge,
                                        minlength=5,rmvNaN=True,
                                        xmin=100, xmax=700,
                                        Xaxis=x_axis)

df_f = dfI_f, dfII_f, dfIII_f

#%% ------------------------------- plot ridges fitted over continuated section

fig, ax1 = plt.subplots(figsize=(15,3))

plt, cmap = pEXP.plot_xy(mesh, label=label_prop, ax=ax, Xaxis=x_axis,
          Vminmax=[0,0.0125], p1p2=p)
pEXP.plot_ridges_harmonic(dfI_f,dfII_f,dfIII_f,ax=ax,label=True)

df_fit = dEXP.fit_ridges(df_f, rmvOutliers=True) # fit ridges on filtered data

pEXP.plot_ridges_sources(df_fit, ax=ax, z_max_source=-max_elevation*2,
                      ridge_type=[0,1,2],ridge_nb=None)

cbar = plt.colorbar(cmap,shrink=0.25, pad=0.04)
cbar.set_label('upwc voltage (V)')
plt.tight_layout()
pEXP.plot_ridges_harmonic(dfI,dfII,dfIII,ax=ax)
plt.xlim([200,600])

plt.savefig('ridgesfield.png', dpi=450)
# square([y1, y2, z1, z2])
#%%

fig, ax1 = plt.subplots(figsize=(15,3))

plt, cmap = pEXP.plot_xy(mesh, label=label_prop, ax=ax1, Xaxis=x_axis,
          Vminmax=[0,0.0125], p1p2=p)
pEXP.plot_ridges_harmonic(dfI_f,dfII_f,dfIII_f,ax=ax1,label=True)

df_fit = dEXP.fit_ridges(df_f, rmvOutliers=False) # fit ridges on filtered data

ax2 = pEXP.plot_ridges_sources(df_fit, ax=ax1, z_max_source=-max_elevation,
                      ridge_type=[0,1,2],ridge_nb=None)

labels_ax = ax.get_yticks()
labels_ax= labels_ax[labels_ax>0]

labels_ax2 = ax2.get_yticks()
labels_ax2= labels_ax2[labels_ax2<0]

ax.set_yticks(labels_ax)
ax2.set_yticks(labels_ax2)

# Adjust the plotting range of two y axes
org1 = 0.0  # Origin of first axis
org2 = 0.0  # Origin of second axis
pos = 0.2  # Position the two origins are aligned
align.yaxes(ax1, org1, ax2, org2, pos)

cbar = plt.colorbar(cmap,shrink=0.25, pad=0.04)
cbar.set_label('upwc voltage (V)')
plt.tight_layout()
pEXP.plot_ridges_harmonic(dfI,dfII,dfIII,ax=ax1)
plt.xlim([200,600])

plt.savefig('ridgesfield.png', dpi=450)


# main.py
import os
import csv
from statesdata import *
from datetime import datetime

EmpID = []
FirstName = []
LastName = []
DOB = []
SSN = []
State = []
strlast = ""

# inputfile = os.path.join('Resources', strfile)
csvpath = os.path.join('Resources', 'employee_data3.csv')

with open(csvpath, encoding="utf-8", newline='') as csvfile:
	# read file
	csvreader = csv.reader(csvfile, delimiter=',')

	# bypass header
	next(csvreader, None)
	for row in csvreader:
		# print(row[1])
		# add emp id to EmpID
		EmpID.append(row[0])

		# split name column
		strNames = row[1].split(" ")
		# print(str(len(strNames)) + " " +  strNames[0] + "    " + strNames[1])
		# First Name
		FirstName.append(strNames[0])

		# LastName
		# check if more than 1 last name example Michael Rogers Jr.
		if len(strNames) > 2:
			for i in range(1, len(strNames)):
				# print(str(i) + " : " + strNames[i])
				if i>1:
					strlast += " "
				strlast += str(strNames[i])
			LastName.append(strlast)
		else:
			LastName.append(strNames[1])

		# Date of Birth (DD/MM/YYYY) format
		newdate = datetime.strptime(row[2], '%Y-%m-%d').strftime('%d/%m/%Y')
		# print(row[2] + " -> " + newdate)
		DOB.append(newdate)

		# SSN
		ssnlist = list(row[3])
		for i in range(0, 6):
			if ssnlist[i] != '-':
				ssnlist[i] = '*'

		SSN.append(''.join(ssnlist))
		# print(row[3] + " -> " + ''.join(ssn))

		# state
		# stateKey = [key for key, value in states.items() if value == row[4]]
		# print (states.keys()[states.values()])
		# print(row[4] + " -> " + str(stateKey))

		found = ""
		for k,v in states.items():
			if v == row[4]:
				# print(k + " : " + v)
				found = k
				break

		if len(found) > 0:
			State.append(found)
			# print(found)
		else:
			State.append("**Invalid State**")

bossCSV = zip(EmpID,FirstName,LastName,DOB,SSN,State)

# Specify the file to write to
outpath = os.path.join('outpath', 'result_' + csvpath.rsplit('\\', 1)[1].rsplit('.',1)[0] + '.csv' )
# print(outpath)

with open(outpath, 'w', newline="") as csvFile:
        csvWriter = csv.writer(csvFile, delimiter=',')
        csvWriter.writerow(["Emp ID","First Name","Last Name","DOB","SSN","State"])
        csvWriter.writerows(bossCSV)

print("PyBoss ran successfully")


import constants
import Adjust
from GameManager import*

from Animation import*
from Characters import*

#support
import os
import time
import random
import pygame
import math

# initial
page = 0

# screening properties
seeing_size = (constants.SCREEN_WIDTH, constants.SCREEN_HEIGHT)
screen = pygame.display.set_mode(seeing_size)
pygame.display.set_caption("Template")

gm = GameManager(screen)

all_sprites = pygame.sprite.Group()
all_sprites.add(gm)

def game_keys_out(ev,lst,amount):
    out = []
    char = 0
    for i in range(amount):

        player = lst[char]
        out.append(game_keys(ev,player))
        char +=1

    return out

def game_keys(ev, player):

    #adjust.get_event(ev)

    if player.alive():

        #player.change_in_motion()
        if ev.type == pygame.KEYDOWN:
            #KEYS = pygame.key.get_pressed()
            #MODS = pygame.key.get_mods()
            player.key[:] = []

            if ev.key == RIGHT:
                if not player.is_attack_in_motion():
                    player.direction = 'RIGHT'

                    player.motion = 'MOVING'

                player.right = True

            if ev.key == LEFT:
                if not player.is_attack_in_motion():
                    player.direction = 'LEFT'
                    player.motion = 'MOVING'

                player.left = True

            if ev.key == UP:
                player.up = True

            if ev.key == DOWN:
                player.down = True

            if ev.key == JUMP:
                player.is_jump = True
                player.add_key('JUMP')

            '''if ev.key == ATK:
                if player.run_done:
                    player.next_link()
                if player.atk_enable and not player.is_attack_in_motion():
                    player.do_atk = True
                player.add_key('ATK')

            if ev.key == HARD_ATK:
                if player.atk_enable:
                    player.do_hard_atk = True
                player.add_key('HARD')

            if ev.key == GUARD:
                do = None
                player.do_guard = True
                player.add_key('GUARD')

            if ev.key == CHARGE:
                do = None
                player.do_charge = True
                player.charge_count = False'''

        elif ev.type == pygame.KEYUP:

            if ev.key == RIGHT:
                player.motion = 'STOPPED'
                player.right = False
                player.char_direc = 'RIGHT'

            if ev.key == LEFT:
                player.motion = 'STOPPED'
                player.left = False
                player.char_direc = 'LEFT'

            if ev.key == UP:
                player.up = False

            if ev.key == DOWN:
                player.down = False

            if ev.key == JUMP:
                player.is_jump = False

            '''if ev.key == ATK:
                player.do_atk = False

            if ev.key == HARD_ATK:
                player.do_hard_atk = False

            if ev.key == GUARD:
                do = None
                player.do_guard = False

            if ev.key == CHARGE:
                do = None
                player.do_charge = False '''

def page_setup(delay = 0):
    """
    page:
    0 = start screen
    1 = loading screen
    2 = main menu

    - main menu -
    3 = options
    4 = group play
    5 = MUGEN
    6 = solo
    -
    """

    page_list = [gm]

    #if not gm.is_done():
    gm.update()

    #elif not mm.is_done():
    #    gm.clear()
    #    mm.update()
#___________
'''ADD ALL'''
#IKE = Ike()
#link2 = Link()
Player1 = Player()

all_sprites = pygame.sprite.Group()
playing_char = pygame.sprite.Group()
adjusting = pygame.sprite.Group()

playing_char.add(Player1)
all_sprites.add(Player1)

#- - - - - Test - - - - -
counter = 1
lst = []

for charac in playing_char:
    lst.append(charac)

controls = []
count = len(lst)
char = 0
#up_left_plat.set_pos((200, 150))
#up_left_plat.change_spd((1,0))

for player in lst:
    player.move( ( (constants.SCREEN_WIDTH /4)*counter , 0) )
    counter += 1
timer = 0
#___________
clock = pygame.time.Clock()
keep_going = True

while keep_going:
    clock.tick(constants.FRAME_RATE)

    for ev in pygame.event.get():

        gm.get_event(ev)

        if ev.type == QUIT:
            keep_going = False

        #if not adjust.is_typing():
        game_keys_out(ev,lst,len(lst))

         # Get mouse position

        if ev.type == MOUSEMOTION:
            do = None
        if ev.type == MOUSEBUTTONUP:
            do = None
        if ev.type == MOUSEBUTTONDOWN:
            do = None
            #adjust.mos_click(pygame.mouse.get_pos())

    page_setup()
    #screen.fill(screen_fill) dont need for now

    #all_sprites.update()

    #all_sprites.draw(screen)

    pygame.display.flip()

pygame.quit()


import os
import ast
import math
import torch
import stable_nalu
import argparse

# Parse arguments
parser = argparse.ArgumentParser(description='Run either the MNIST counting or MNIST Arithmetic task')
parser.add_argument('--layer-type',
                    action='store',
                    default='NALU',
                    choices=list(stable_nalu.network.SequentialMnistNetwork.UNIT_NAMES),
                    type=str,
                    help='Specify the layer type, e.g. RNN-tanh, LSTM, NAC, NALU')
parser.add_argument('--operation',
                    action='store',
                    default='cumsum',
                    choices=[
                        'cumsum', 'sum', 'cumprod', 'prod', 'cumdiv', 'div'
                    ],
                    type=str,
                    help='Specify the operation to use, sum or count')
parser.add_argument('--regualizer',
                    action='store',
                    default=10,
                    type=float,
                    help='Specify the regualization lambda to be used')
parser.add_argument('--regualizer-z',
                    action='store',
                    default=0,
                    type=float,
                    help='Specify the z-regualization lambda to be used')
parser.add_argument('--regualizer-oob',
                    action='store',
                    default=1,
                    type=float,
                    help='Specify the oob-regualization lambda to be used')
parser.add_argument('--mnist-digits',
                    action='store',
                    default=[0,1,2,3,4,5,6,7,8,9],
                    type=lambda str: list(map(int,str)),
                    help='MNIST digits to use')
parser.add_argument('--mnist-outputs',
                    action='store',
                    default=1,
                    type=int,
                    help='number of MNIST to use, more than 1 adds redundant values')
parser.add_argument('--model-simplification',
                    action='store',
                    default='none',
                    choices=[
                        'none', 'solved-accumulator', 'pass-through'
                    ],
                    type=str,
                    help='Simplifiations applied to the model')

parser.add_argument('--max-epochs',
                    action='store',
                    default=1000,
                    type=int,
                    help='Specify the max number of epochs to use')
parser.add_argument('--batch-size',
                    action='store',
                    default=64,
                    type=int,
                    help='Specify the batch-size to be used for training')
parser.add_argument('--seed',
                    action='store',
                    default=0,
                    type=int,
                    help='Specify the seed to use')

parser.add_argument('--interpolation-length',
                    action='store',
                    default=10,
                    type=int,
                    help='Specify the sequence length for interpolation')
parser.add_argument('--extrapolation-lengths',
                    action='store',
                    default=[100, 1000],
                    type=ast.literal_eval,
                    help='Specify the sequence lengths used for the extrapolation dataset')

parser.add_argument('--softmax-transform',
                    action='store_true',
                    default=False,
                    help='Should a softmax transformation be used to control the output of the CNN model')
parser.add_argument('--nac-mul',
                    action='store',
                    default='none',
                    choices=['none', 'normal', 'safe', 'max-safe', 'mnac'],
                    type=str,
                    help='Make the second NAC a multiplicative NAC, used in case of a just NAC network.')
parser.add_argument('--nac-oob',
                    action='store',
                    default='clip',
                    choices=['regualized', 'clip'],
                    type=str,
                    help='Choose of out-of-bound should be handled by clipping or regualization.')
parser.add_argument('--regualizer-scaling',
                    action='store',
                    default='linear',
                    choices=['exp', 'linear'],
                    type=str,
                    help='Use an expoentational scaling from 0 to 1, or a linear scaling.')
parser.add_argument('--regualizer-scaling-start',
                    action='store',
                    default=10000,
                    type=int,
                    help='Start linear scaling at this global step.')
parser.add_argument('--regualizer-scaling-end',
                    action='store',
                    default=100000,
                    type=int,
                    help='Stop linear scaling at this global step.')
parser.add_argument('--regualizer-shape',
                    action='store',
                    default='linear',
                    choices=['squared', 'linear'],
                    type=str,
                    help='Use either a squared or linear shape for the bias and oob regualizer.')
parser.add_argument('--mnac-epsilon',
                    action='store',
                    default=0,
                    type=float,
                    help='Set the idendity epsilon for MNAC.')
parser.add_argument('--nalu-bias',
                    action='store_true',
                    default=False,
                    help='Enables bias in the NALU gate')
parser.add_argument('--nalu-two-nac',
                    action='store_true',
                    default=False,
                    help='Uses two independent NACs in the NALU Layer')
parser.add_argument('--nalu-two-gate',
                    action='store_true',
                    default=False,
                    help='Uses two independent gates in the NALU Layer')
parser.add_argument('--nalu-mul',
                    action='store',
                    default='normal',
                    choices=['normal', 'safe', 'trig', 'max-safe', 'mnac'],
                    help='Multplication unit, can be normal, safe, trig')
parser.add_argument('--nalu-gate',
                    action='store',
                    default='normal',
                    choices=['normal', 'regualized', 'obs-gumbel', 'gumbel'],
                    type=str,
                    help='Can be normal, regualized, obs-gumbel, or gumbel')

parser.add_argument('--no-cuda',
                    action='store_true',
                    default=False,
                    help=f'Force no CUDA (cuda usage is detected automatically as {torch.cuda.is_available()})')
parser.add_argument('--name-prefix',
                    action='store',
                    default='sequence_mnist',
                    type=str,
                    help='Where the data should be stored')
parser.add_argument('--remove-existing-data',
                    action='store_true',
                    default=False,
                    help='Should old results be removed')
parser.add_argument('--verbose',
                    action='store_true',
                    default=False,
                    help='Should network measures (e.g. gates) and gradients be shown')
args = parser.parse_args()

setattr(args, 'cuda', torch.cuda.is_available() and not args.no_cuda)

# Print configuration
print(f'running')
print(f'  - layer_type: {args.layer_type}')
print(f'  - operation: {args.operation}')
print(f'  - regualizer: {args.regualizer}')
print(f'  - regualizer_z: {args.regualizer_z}')
print(f'  - regualizer_oob: {args.regualizer_oob}')
print(f'  - mnist_digits: {args.mnist_digits}')
print(f'  - mnist_outputs: {args.mnist_outputs}')
print(f'  - model_simplification: {args.model_simplification}')
print(f'  -')
print(f'  - max_epochs: {args.max_epochs}')
print(f'  - batch_size: {args.batch_size}')
print(f'  - seed: {args.seed}')
print(f'  -')
print(f'  - interpolation_length: {args.interpolation_length}')
print(f'  - extrapolation_lengths: {args.extrapolation_lengths}')
print(f'  -')
print(f'  - softmax_transform: {args.softmax_transform}')
print(f'  - nac_mul: {args.nac_mul}')
print(f'  - nac_oob: {args.nac_oob}')
print(f'  - regualizer_scaling: {args.regualizer_scaling}')
print(f'  - regualizer_scaling_start: {args.regualizer_scaling_start}')
print(f'  - regualizer_scaling_end: {args.regualizer_scaling_end}')
print(f'  - regualizer_shape: {args.regualizer_shape}')
print(f'  - mnac_epsilon: {args.mnac_epsilon}')
print(f'  - nalu_bias: {args.nalu_bias}')
print(f'  - nalu_two_nac: {args.nalu_two_nac}')
print(f'  - nalu_two_gate: {args.nalu_two_gate}')
print(f'  - nalu_mul: {args.nalu_mul}')
print(f'  - nalu_gate: {args.nalu_gate}')
print(f'  -')
print(f'  - cuda: {args.cuda}')
print(f'  - name_prefix: {args.name_prefix}')
print(f'  - remove_existing_data: {args.remove_existing_data}')
print(f'  - verbose: {args.verbose}')

# Prepear logging
summary_writer = stable_nalu.writer.SummaryWriter(
    f'{args.name_prefix}/{args.layer_type.lower()}'
    f'{"-nac-" if args.nac_mul != "none" else ""}'
    f'{"n" if args.nac_mul == "normal" else ""}'
    f'{"s" if args.nac_mul == "safe" else ""}'
    f'{"s" if args.nac_mul == "max-safe" else ""}'
    f'{"t" if args.nac_mul == "trig" else ""}'
    f'{"m" if args.nac_mul == "mnac" else ""}'
    f'{"-nalu-" if (args.nalu_bias or args.nalu_two_nac or args.nalu_two_gate or args.nalu_mul != "normal" or args.nalu_gate != "normal") else ""}'
    f'{"b" if args.nalu_bias else ""}'
    f'{"2n" if args.nalu_two_nac else ""}'
    f'{"2g" if args.nalu_two_gate else ""}'
    f'{"s" if args.nalu_mul == "safe" else ""}'
    f'{"s" if args.nalu_mul == "max-safe" else ""}'
    f'{"t" if args.nalu_mul == "trig" else ""}'
    f'{"m" if args.nalu_mul == "mnac" else ""}'
    f'{"r" if args.nalu_gate == "regualized" else ""}'
    f'{"u" if args.nalu_gate == "gumbel" else ""}'
    f'{"uu" if args.nalu_gate == "obs-gumbel" else ""}'
    f'_d-{"".join(map(str, args.mnist_digits))}'
    f'_h-{args.mnist_outputs}'
    f'_op-{args.operation.lower()}'
    f'_oob-{"c" if args.nac_oob == "clip" else "r"}'
    f'_rs-{args.regualizer_scaling}-{args.regualizer_shape}'
    f'_eps-{args.mnac_epsilon}'
    f'_rl-{args.regualizer_scaling_start}-{args.regualizer_scaling_end}'
    f'_r-{args.regualizer}-{args.regualizer_z}-{args.regualizer_oob}'
    f'_m-{"s" if args.softmax_transform else "l"}-{args.model_simplification[0]}'
    f'_i-{args.interpolation_length}'
    f'_e-{"-".join(map(str, args.extrapolation_lengths))}'
    f'_b{args.batch_size}'
    f'_s{args.seed}',
    remove_existing_data=args.remove_existing_data
)

# Set threads
if 'LSB_DJOB_NUMPROC' in os.environ:
    torch.set_num_threads(int(os.environ['LSB_DJOB_NUMPROC']))

# Set seed
torch.manual_seed(args.seed)
torch.backends.cudnn.deterministic = True

# Setup datasets
dataset = stable_nalu.dataset.SequentialMnistDataset(
    operation=args.operation,
    use_cuda=args.cuda,
    seed=args.seed,
    mnist_digits=args.mnist_digits
)
dataset_train = dataset.fork(seq_length=args.interpolation_length, subset='train').dataloader(shuffle=True)
# Seeds are from random.org
dataset_train_full = dataset.fork(seq_length=args.interpolation_length, subset='train',
                                  seed=62379872).dataloader(shuffle=False)
dataset_valid = dataset.fork(seq_length=args.interpolation_length, subset='valid',
                                  seed=47430696).dataloader(shuffle=False)
dataset_test_extrapolations = [
    ( seq_length,
      dataset.fork(seq_length=seq_length, subset='test',
                   seed=88253339).dataloader(shuffle=False)
    ) for seq_length in args.extrapolation_lengths
]

# setup model
model = stable_nalu.network.SequentialMnistNetwork(
    args.layer_type,
    output_size=dataset.get_item_shape().target[-1],
    writer=summary_writer.every(100).verbose(args.verbose),
    mnist_digits=args.mnist_digits,
    mnist_outputs=args.mnist_outputs,
    model_simplification=args.model_simplification,
    softmax_transform=args.softmax_transform,
    nac_mul=args.nac_mul,
    nac_oob=args.nac_oob,
    regualizer_shape=args.regualizer_shape,
    regualizer_z=args.regualizer_z,
    mnac_epsilon=args.mnac_epsilon,
    nalu_bias=args.nalu_bias,
    nalu_two_nac=args.nalu_two_nac,
    nalu_two_gate=args.nalu_two_gate,
    nalu_mul=args.nalu_mul,
    nalu_gate=args.nalu_gate,
)
model.reset_parameters()
if args.cuda:
    model.cuda()
criterion = torch.nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters())

seq_index = slice(None) if dataset.get_item_shape().target[0] is None else -1

def accuracy(y, t):
    return torch.mean((torch.round(y) == t).float())

def test_model(dataloader):
    with torch.no_grad(), model.no_internal_logging(), model.no_random():
        mse_loss = 0
        acc_all = 0
        acc_last = 0
        for x, t in dataloader:
            # forward
            _, y = model(x)
            mse_loss += criterion(y[:,seq_index,:], t[:,seq_index,:]).item() * len(t)
            acc_all += accuracy(y[:,seq_index,:], t[:,seq_index,:]).item() * len(t)
            acc_last += accuracy(y[:,-1,:], t[:,-1,:]).item() * len(t)

        return (
            mse_loss / len(dataloader.dataset),
            acc_all / len(dataloader.dataset),
            acc_last / len(dataloader.dataset)
        )

# Train model
global_step = 0
for epoch_i in range(0, args.max_epochs + 1):
    for i_train, (x_train, t_train) in enumerate(dataset_train):
        global_step += 1
        summary_writer.set_iteration(global_step)
        summary_writer.add_scalar('epoch', epoch_i)

        # Prepear model
        model.set_parameter('tau', max(0.5, math.exp(-1e-5 * global_step)))
        optimizer.zero_grad()

        # Log validation
        if epoch_i % 5 == 0 and i_train == 0:
            (train_full_mse,
             train_full_acc_all,
             train_full_acc_last) = test_model(dataset_train_full)
            summary_writer.add_scalar('metric/train/mse', train_full_mse)
            summary_writer.add_scalar('metric/train/acc/all', train_full_acc_all)
            summary_writer.add_scalar('metric/train/acc/last', train_full_acc_last)

            (valid_mse,
             valid_acc_all,
             valid_acc_last) = test_model(dataset_valid)
            summary_writer.add_scalar('metric/valid/mse', valid_mse)
            summary_writer.add_scalar('metric/valid/acc/all', valid_acc_all)
            summary_writer.add_scalar('metric/valid/acc/last', valid_acc_last)

            for seq_length, dataloader in dataset_test_extrapolations:
                (test_extrapolation_mse,
                 test_extrapolation_acc_all,
                 test_extrapolation_acc_last) = test_model(dataloader)
                summary_writer.add_scalar(f'metric/test/extrapolation/{seq_length}/mse', test_extrapolation_mse)
                summary_writer.add_scalar(f'metric/test/extrapolation/{seq_length}/acc/all', test_extrapolation_acc_all)
                summary_writer.add_scalar(f'metric/test/extrapolation/{seq_length}/acc/last', test_extrapolation_acc_last)

        # forward
        with summary_writer.force_logging(epoch_i % 5 == 0 and i_train == 0):
            mnist_y_train, y_train = model(x_train)
        regualizers = model.regualizer()

        if (args.regualizer_scaling == 'linear'):
            r_w_scale = max(0, min(1, (
                (global_step - args.regualizer_scaling_start) /
                (args.regualizer_scaling_end - args.regualizer_scaling_start)
            )))
        elif (args.regualizer_scaling == 'exp'):
            r_w_scale = 1 - math.exp(-1e-5 * global_step)

        loss_train_criterion = criterion(y_train[:,seq_index,:], t_train[:,seq_index,:])
        loss_train_regualizer = args.regualizer * r_w_scale * regualizers['W'] + regualizers['g'] + args.regualizer_z * regualizers['z'] + args.regualizer_oob * regualizers['W-OOB']
        loss_train = loss_train_criterion + loss_train_regualizer

        # Log loss
        summary_writer.add_scalar('loss/train/accuracy/all', accuracy(y_train[:,seq_index,:], t_train[:,seq_index,:]))
        summary_writer.add_scalar('loss/train/accuracy/last', accuracy(y_train[:,-1,:], t_train[:,-1,:]))
        summary_writer.add_scalar('loss/train/critation', loss_train_criterion)
        summary_writer.add_scalar('loss/train/regualizer', loss_train_regualizer)
        summary_writer.add_scalar('loss/train/total', loss_train)
        if epoch_i % 5 == 0 and i_train == 0:
            summary_writer.add_tensor('MNIST/train',
                                      torch.cat([mnist_y_train[:,0,:], t_train[:,0,:]], dim=1))
            print('train %d: %.5f, full: %.5f, %.3f (acc[last]), valid: %.5f, %.3f (acc[last])' % (
                epoch_i, loss_train_criterion, train_full_mse, train_full_acc_last, valid_mse, valid_acc_last
            ))

        # Optimize model
        if loss_train.requires_grad:
            loss_train.backward()
            optimizer.step()
        model.optimize(loss_train_criterion)

        # Log gradients if in verbose mode
        with summary_writer.force_logging(epoch_i % 5 == 0 and i_train == 0):
            model.log_gradients()

# Write results for this training
print(f'finished:')
print(f'  - loss_train: {loss_train}')
print(f'  - validation: {valid_mse}')

# Use saved weights to visualize the intermediate values.
stable_nalu.writer.save_model(summary_writer.name, model)


# Copyright 2015 The Swarming Authors. All rights reserved.
# Use of this source code is governed by the Apache v2.0 license that can be
# found in the LICENSE file.

"""Task queues for the GCE Backend."""

import json
import logging

from google.appengine.ext import ndb
import webapp2

from components import decorators
from components import machine_provider
from components import net

import models

@ndb.transactional
def uncatalog_instances(instances):
  """Uncatalogs cataloged instances.

  Args:
    instances: List of instance names to uncatalog.
  """
  put_futures = []
  get_futures = [
      models.Instance.generate_key(instance_name).get_async()
      for instance_name in instances
  ]
  while get_futures:
    ndb.Future.wait_any(get_futures)
    instances = [future.get_result() for future in get_futures if future.done()]
    get_futures = [future for future in get_futures if not future.done()]
    for instance in instances:
      if instance.state == models.InstanceStates.CATALOGED:
        # handlers_cron.py sets each Instance's state to
        # CATALOGED before triggering InstanceGroupCataloger.
        logging.info('Uncataloging instance: %s', instance.name)
        instance.state = models.InstanceStates.UNCATALOGED
        put_futures.append(instance.put_async())
      else:
        logging.info('Ignoring already uncataloged instance: %s', instance.name)
  if put_futures:
    ndb.Future.wait_all(put_futures)
  else:
    logging.info('Nothing to uncatalog')

class InstanceGroupCataloger(webapp2.RequestHandler):
  """Worker for cataloging instance groups."""

  @decorators.require_taskqueue('catalog-instance-group')
  def post(self):
    """Reclaim a machine.

    Params:
      dimensions: JSON-encoded string representation of
        machine_provider.Dimensions describing the members of the instance
        group.
      instances: JSON-encoded list of instances in the instance group to
        catalog:
      policies: JSON-encoded string representation of machine_provider.Policies
        governing the members of the instance group.
    """
    dimensions = json.loads(self.request.get('dimensions'))
    instances = json.loads(self.request.get('instances'))
    policies = json.loads(self.request.get('policies'))

    requests = []
    instances_to_uncatalog = set()

    for instance_name in instances:
      instances_to_uncatalog.add(instance_name)
      requests.append({
          'dimensions': dimensions.copy(), 'policies': policies})
      requests[-1]['dimensions']['hostname'] = instance_name

    try:
      responses = machine_provider.add_machines(requests).get('responses', {})
    except net.Error as e:
      logging.warning(e)
      responses = {}

    for response in responses:
      request = response.get('machine_addition_request', {})
      error = response.get('error')
      instance_name = request.get('dimensions', {}).get('hostname')
      if instance_name in instances:
        if not error:
          logging.info('Instance added to Catalog: %s', instance_name)
          instances_to_uncatalog.discard(instance_name)
        elif error == 'HOSTNAME_REUSE':
          logging.warning('Hostname reuse in Catalog: %s', instance_name)
          instances_to_uncatalog.discard(instance_name)
        else:
          logging.warning('Instance not added to Catalog: %s', instance_name)
      else:
        logging.info('Unknown instance: %s', instance_name)

    uncatalog_instances(instances_to_uncatalog)

def create_queues_app():
  return webapp2.WSGIApplication([
      ('/internal/queues/catalog-instance-group', InstanceGroupCataloger),
  ])


#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
#  Facilitate working with dates.
#  2014, SMART Health IT.

import sys
import logging
import isodate
import datetime

logger = logging.getLogger(__name__)

class FHIRDate(object):
    """ Facilitate working with dates.

    - `date`: datetime object representing the receiver's date-time
    """

    def __init__(self, jsonval=None):
        self.date = None
        if jsonval is not None:
            isstr = isinstance(jsonval, str)
            if not isstr and sys.version_info[0] < 3:       # Python 2.x has 'str' and 'unicode'
                isstr = isinstance(jsonval, basestring)
            if not isstr:
                raise TypeError("Expecting string when initializing {}, but got {}"
                    .format(type(self), type(jsonval)))
            try:
                if 'T' in jsonval:
                    self.date = isodate.parse_datetime(jsonval)
                else:
                    self.date = isodate.parse_date(jsonval)
            except Exception as e:
                logger.warning("Failed to initialize FHIRDate from \"{}\": {}"
                    .format(jsonval, e))

        self.origval = jsonval

    def __setattr__(self, prop, value):
        if 'date' == prop:
            self.origval = None
        object.__setattr__(self, prop, value)

    @property
    def isostring(self):
        if self.date is None:
            return None
        if isinstance(self.date, datetime.datetime):
            return isodate.datetime_isoformat(self.date)
        return isodate.date_isoformat(self.date)

    @classmethod
    def with_json(cls, jsonobj):
        """ Initialize a date from an ISO date string.
        """
        isstr = isinstance(jsonobj, str)
        if not isstr and sys.version_info[0] < 3:       # Python 2.x has 'str' and 'unicode'
            isstr = isinstance(jsonobj, basestring)
        if isstr:
            return cls(jsonobj)

        if isinstance(jsonobj, list):
            return [cls(jsonval) for jsonval in jsonobj]

        raise TypeError("`cls.with_json()` only takes string or list of strings, but you provided {}"
            .format(type(jsonobj)))

    @classmethod
    def with_json_and_owner(cls, jsonobj, owner):
        """ Added for compatibility reasons to FHIRElement; "owner" is
        discarded.
        """
        return cls.with_json(jsonobj)

    def as_json(self):
        if self.origval is not None:
            return self.origval
        return self.isostring


from selenium.webdriver.support.select import Select

def fillform1(browser,phone,code,cap_ans):
	"""Fill in the form in <browser>"""
	phoneNumber = browser.find_element_by_id('phoneNumber')
	smsCode = browser.find_element_by_id('reservationCode')
	captcha_ans = browser.find_element_by_id('captchaAnswer')
	submit_button = browser.find_element_by_id('continue')

	phoneNumber.send_keys(phone)
	smsCode.send_keys(code)
	captcha_answer.send_keys(cap_ans)
	submit_button.click()

if __name__ == "__main__":
	from selenium import webdriver
	browser = webdriver.Firefox()
	browser.get('http://176.9.105.210/~afuhost1/4ch7ung2.html')
	fillform(browser,"+85264500366","RESERVATION_CODE","HUILO")


import requests
import os
from dotenv import load_dotenv
from contextlib import suppress
from utils import check_comment_date, get_objects_from_vk_request

BASE_URL = 'https://api.vk.com/method/'

def get_group_id_by_name(access_token, group_name):
    """
    Возвращает ID группы по её имени
    """
    params = {
        'access_token': access_token,
        'v': 5.103,
        'group_id': group_name
    }

    response = requests.get(f"{BASE_URL}/groups.getById", params=params).json()['response']
    return response[0]['id']

def get_user_posts(access_token, owner_id, post_per_page=100, max_count=None):
    """
    Возвращает список постов owner_id. Есть возможность вернуть ограниченное число постов с помощью max_count
    """
    params = {
        'access_token': access_token,
        'v': 5.103,
        'owner_id': -owner_id,
    }

    posts = get_objects_from_vk_request(BASE_URL, 'wall.get', params, post_per_page, max_count)
    return posts

def get_post_comments(access_token, owner_id, post_id, comments_per_page=100, max_count=None):
    """
    Возвращает список комментов post_id. Есть возможость вернуть ограниченное число комметов с помощью max_count.
    """
    params = {
        'access_token': access_token,
        'v': 5.103,
        'owner_id': -owner_id,
        'post_id': post_id
    }

    comments = get_objects_from_vk_request(BASE_URL, 'wall.getComments', params, comments_per_page, max_count)
    return comments

def get_post_likers(access_token ,owner_id, post_id, likers_per_page=100, max_count=None):
    """
    Возвращает список людей, которые лайкнули пост post_id. Есть возможность вернуть ограниченное число лайкнувших
    с помощью max_count.
    """
    params = {
        'access_token': access_token,
        'v': 5.103,
        'type': 'post',
        'owner_id': -owner_id,
        'post_id': post_id
    }

    likers = get_objects_from_vk_request(BASE_URL, 'likes.getList', params, likers_per_page, max_count)
    return likers

def get_commenters(access_token, group_id, posts) -> set:
    """
    Возвращает множество комментаторов из списка постов posts.
    """
    users = set()
    for post in posts:
        comments = get_post_comments(access_token, group_id, post['id'])

        for comment in comments:
            with suppress(KeyError):  # Может не быть from_id, т.к. будет deleted: True
                if check_comment_date(comment['date'], period_days=14) and comment['from_id'] != -16297716:
                    users.add(comment['from_id'])

    return users

def get_likers(access_token, group_id, posts):
    """
    Возвращает множество лайкнувших из списка постов posts.
    """
    return (set(get_post_likers(access_token, group_id, post['id'])) for post in posts)

def get_vk_statistic(access_token, group_name):
    group_id = get_group_id_by_name(access_token, group_name)

    posts = get_user_posts(access_token, group_id)

    commenters = get_commenters(access_token, group_id, posts)
    likers = get_likers(access_token, group_id, posts)

    return commenters.intersection(likers)

if __name__ == '__main__':
    load_dotenv()

    access_token = os.getenv('SERVICE_VK_TOKEN')

    group_name = 'cocacola'
    print(get_vk_statistic(access_token, group_name))