import time
import random
import os
import logging
import hashlib
import json
import glob
import shutil
import argparse
from string import ascii_lowercase
from functools import wraps
from copy import deepcopy
from jinja2 import Environment, FileSystemLoader
from plotly.offline import plot

# Tools
from shared_tools import restore, get_resource_as_string, MainAPIClient, chown_files_in_dir
# from shared_tools import export_plot
from tools.initial_config import load_experiment_description
from logger.default_logger import BRISELogConfigurator

# Plots
from plots.table import table
from plots.repeat_vs_avg import repeat_vs_avg
from plots.improvements import improvements
from plots.box_statistic import box_statistic
from plots.exp_config import exp_description_highlight

# Configuring logging
BRISELogConfigurator()

def build_benchmark_report():
    """ Generate report files from the Experiment class instances.
    """
    folder_with_dumps = './results/serialized/'
    # Container creation performs --volume on `./results/` folder. Change wisely folder_with_resport.
    folder_with_resport = './results/reports/'

    # ------- List with name experiment instances. Default from ./results/serialized/ folder
    experiment_dumps = [f for f in os.listdir(folder_with_dumps) if (f[-4:] == '.pkl')]
    # -------
    logger = logging.getLogger(__name__)
    if experiment_dumps:
        logger.info("Selected Experiment dumps for report: %s" % experiment_dumps)
    else:
        logger.error("Directory '%s' is empty. Terminating." % folder_with_dumps)
        return

    # --- Generate template
    file_loader = FileSystemLoader("./templates")
    env = Environment(loader=file_loader)
    env.globals['get_resource_as_string'] = get_resource_as_string
    template = env.get_template('index.html')

    # --- Restore experiments for benchmarking
    exp_list = restore(*experiment_dumps)

    # --- Generate plot's hooks
    tab = plot(table(exp_list), include_plotlyjs=False, output_type='div')
    impr = plot(improvements(exp_list),
                include_plotlyjs=False,
                output_type='div')
    all_results = plot(box_statistic(exp_list),
                       include_plotlyjs=False,
                       output_type='div')
    rep = ' '.join(plot(repeat_vs_avg(exp), include_plotlyjs=False,
                        output_type='div') for exp in exp_list)
    time_mark = time.strftime('%Y-%m-%d %A', time.localtime())

    # Compose HTML
    html = template.render(
        table=tab,
        impr=impr,
        repeat_vs_avg=rep,
        box_plot=all_results,
        time=time_mark,
        print_config=exp_description_highlight(exp_list)
    )

    # --- Save results
    # Write HTML report
    suffix = ''.join(random.choice(ascii_lowercase) for _ in range(10))
    with open("{}report_{}.html".format(folder_with_resport, suffix), "w", encoding='utf-8') as outf:
        outf.write(html)

    # # Export plots
    # for plt in [table(exp_list), improvements(exp_list), box_statistic(exp_list)]:
    #     export_plot(plot=plt, wight=1200, height=600)

    # Using a host machine User ID to change the owner for the files(initially, the owner was a root).
    chown_files_in_dir(folder_with_resport)

class BRISEBenchmark:
    """
       Class for building and running the benchmarking scenarios.
    """

    def __init__(self, main_api_addr: str, results_storage: str):
        """
            Initializes benchmarking client.
        :param main_api_addr: str. URL of main node API. For example "http://main-node:49152"
        :param results_storage: str. Folder where to store benchmark results (dump files of experiments).
        """
        os.sys.argv.pop()   # Because load_experiment_description will consider 'benchmark' as Experiment Description).
        self._base_experiment_description = load_experiment_description("./Resources/SA/SAExperiment.json")
        self.results_storage = results_storage if results_storage[-1] == "/" else results_storage  + "/"
        self.main_api_client = MainAPIClient(main_api_addr, dump_storage=results_storage)
        self.logger = logging.getLogger(__name__)
        self.counter = 1
        self.experiments_to_be_performed = []   # List of experiment IDs
        self.is_calculating_number_of_experiments = False

    @property
    def base_experiment_description(self):
        return deepcopy(self._base_experiment_description)

    @base_experiment_description.setter
    def base_experiment_description(self, description):
        if not self._base_experiment_description:
            self._base_experiment_description = deepcopy(description)
        else:
            self.logger.error("Unable to update Experiment Description: Read-only property.")

    def benchmarkable(benchmarking_function):
        """
            Decorator that enables a pre calculation of a number of experiments in implemented benchmark scenario
            without actually running them. It is not essential for benchmarking, but could be useful.
        :return: original function, wrapped by wrapper.
        """
        @wraps(benchmarking_function)
        def wrapper(self, *args, **kwargs):
            logging.info("Calculating number of Experiments to perform during benchmark.")
            self.is_calculating_number_of_experiments = True
            logging_level = self.logger.level
            self.logger.setLevel(logging.WARNING)
            benchmarking_function(self, *args, *kwargs)
            self.logger.setLevel(logging_level)
            logging.info("Benchmark is going to run %s unique Experiments (please, take into account also the repetitions)."
                         % len(self.experiments_to_be_performed))
            self.is_calculating_number_of_experiments = False
            benchmarking_function(self, *args, *kwargs)
        return wrapper

    def execute_experiment(self, experiment_description: dict, number_of_repetitions: int = 3, wait_for_results: int=30*60):
        """
             Check how many dumps are available for particular Experiment Description.

         :param experiment_description: Dict. Experiment Description
         :param number_of_repetitions: int. number of times to execute the same Experiment.
         :param wait_for_results:
            If bool ``False`` - client will only send an Experiment Description and return response with
                                the Main node status.
            If bool ``True`` was specified - client will wait until the end of the Experiment.

            If numeric value (int or float) were specified - client will wait specified amount of time (in seconds),
            after elapsing - ``main_stop`` command will be sent to terminate the Main node, current state of Experiment
            will be reported be main node and saved by benchmark.

         :return: int. Number of times experiment dump was found in a storage.
        """
        experiment_id = hashlib.sha1(json.dumps(experiment_description, sort_keys=True).encode("utf-8")).hexdigest()
        dump_file_name = "exp_{0}_{1}".format(
            experiment_description['TaskConfiguration']['Scenario']["ws_file"], experiment_id)
        if self.is_calculating_number_of_experiments:
            self.experiments_to_be_performed.append(experiment_id)
        else:
            number_of_available_repetitions = sum(dump_file_name in file for file in os.listdir(self.results_storage))
            while number_of_available_repetitions < number_of_repetitions:
                if self.main_api_client.perform_experiment(experiment_description, wait_for_results=wait_for_results):
                    number_of_available_repetitions += 1
                    self.logger.info("Executed Experiment #{c} out of {m_c}. ID: {eid}. Repetition: {r}.".format(
                            c=self.counter,
                            m_c=len(self.experiments_to_be_performed) * number_of_repetitions,
                            eid=experiment_id,
                            r=number_of_available_repetitions
                        )
                    )
                    self.counter += 1
            return number_of_repetitions

    def move_redundant_experiments(self, location: str):
        """
            Move all experiment dumps that are not part of current benchmark to separate 'location' folder.
        :param location: (str). Folder path where redundant experiment dumps will be stored.
        """
        os.makedirs(location, exist_ok=True)

        # Mark what to move
        redundant_experiment_files = glob.glob(self.results_storage + "*.pkl")
        for experiment_id in self.experiments_to_be_performed:
            for file in redundant_experiment_files:
                if experiment_id in file:
                    redundant_experiment_files.remove(file)

        # Move
        for file in redundant_experiment_files:
            shutil.move(file, location + os.path.basename(file))

    @benchmarkable
    def benchmark_repeater(self):
        """
            This is an EXAMPLE of the benchmark scenario.

            While benchmarking BRISE, one would like to see the influence of changing some particular parameters on the
            overall process of running BRISE, on the results quality and on the effort.

            In this particular example, the Repeater benchmark described in following way:
                1. Using base Experiment Description for Energy Consumption.
                2. Change ONE parameter of Repeater in a time.
                    2.1. For each Repeater type (Default, Student and Student with enabled model-awareness).
                    2.2. For each Target System Scenario (ws_file).
                3. Execute BRISE with this changed Experiment Description 3 times and save Experiment dump after
                    each execution.

            Do not forget to call your benchmarking scenario in a code block of the `run_benchmark` function,
            highlighted by
            # ---    Add User defined benchmark scenarios execution below

        :return: int, number of Experiments that were executed and experiment dumps are stored.
                Actually you could return whatever you want, here this number is returned only for reporting purposes.
        """
        def_rep_skeleton = {"Repeater": {"Type": "default",
                                          "Parameters": {
                                              "MaxTasksPerConfiguration": 10}}}
        student_rep_skeleton = {"Repeater": {"Type": "student_deviation",
                                             "Parameters": {
                                                 "ModelAwareness": {
                                                     "MaxAcceptableErrors": [50],
                                                     "RatiosMax": [10],
                                                     "isEnabled": True
                                                 },
                                                 "MaxTasksPerConfiguration": 10,
                                                 "MinTasksPerConfiguration": 2,
                                                 "DevicesScaleAccuracies": [0],
                                                 "BaseAcceptableErrors": [5],
                                                 "DevicesAccuracyClasses": [0],
                                                 "ConfidenceLevels": [0.95]}}}

        for ws_file in os.listdir('csv'):
            # Skip ML scenarios, only the Energy consumption scenarios are needed.
            if ws_file in ["taskNB1.csv", "NB_final_result.csv"]:
                continue
            experiment_description = self.base_experiment_description
            experiment_description['TaskConfiguration']['Scenario']['ws_file'] = ws_file
            self.logger.info("Benchmarking next Scenario file(ws_file): %s" % ws_file)

            # benchmarking a default repeater
            experiment_description.update(deepcopy(def_rep_skeleton))
            self.execute_experiment(experiment_description)

            # benchmarking a student repeater with disabled model awareness
            experiment_description.update(deepcopy(student_rep_skeleton))
            experiment_description['Repeater']['Parameters']['ModelAwareness']["isEnabled"] = False
            for BaseAcceptableErrors in [5, 15, 50]:
                experiment_description['Repeater']['Parameters']['BaseAcceptableErrors'] = [BaseAcceptableErrors]
                self.logger.info("Default Repeater: Changing BaseAcceptableErrors to %s" % BaseAcceptableErrors)
                self.execute_experiment(experiment_description)

            # benchmarking a student repeater with enabled model awareness
            experiment_description.update(deepcopy(student_rep_skeleton))
            for BaseAcceptableErrors in [5, 15, 50]:
                experiment_description['Repeater']['Parameters']['BaseAcceptableErrors'] = [BaseAcceptableErrors]
                self.logger.info("Student Repeater: Changing BaseAcceptableErrors to %s" % BaseAcceptableErrors)
                self.execute_experiment(experiment_description)

            experiment_description.update(deepcopy(student_rep_skeleton))
            for MaxAcceptableErrors in [35, 50, 120]:
                experiment_description['Repeater']['Parameters']['ModelAwareness']['MaxAcceptableErrors'] = [MaxAcceptableErrors]
                self.logger.info("Student Repeater: Changing MaxAcceptableErrors to %s" % MaxAcceptableErrors)
                self.execute_experiment(experiment_description)

            experiment_description.update(deepcopy(student_rep_skeleton))
            for RatiosMax in [5, 10, 30]:
                experiment_description['Repeater']['Parameters']['ModelAwareness']['RatiosMax'] = [RatiosMax]
                self.logger.info("Student Repeater: Changing RatiosMax to %s" % BaseAcceptableErrors)
                self.execute_experiment(experiment_description)

        return self.counter

    def benchmark_SA(self):

        scenarios = {
          "trivial": { "variants": 1, "requests": 1, "depth": 1, "resources": 1.0 },
          "small": { "variants": 2, "requests": 1, "depth": 2, "resources": 1.5 },
          "small_hw": { "variants": 2, "requests": 1, "depth": 2, "resources": 5.0 },
          "small_sw": { "variants": 2, "requests": 1, "depth": 5, "resources": 1.5 },
          "medium": { "variants": 10, "requests": 15, "depth": 2, "resources": 1.5 },
          "medium_hw": { "variants": 10, "requests": 15, "depth": 2, "resources": 5.0 },
          "medium_sw": { "variants": 5, "requests": 10, "depth": 5, "resources": 1.5 },
          "large": { "variants": 20, "requests": 20, "depth": 2, "resources": 1.5 },
          "large_hw": { "variants": 20, "requests": 20, "depth": 2, "resources": 5.0 },
          "large_sw": { "variants": 10, "requests": 20, "depth": 5, "resources": 1.5 },
          "huge": { "variants": 50, "requests": 50, "depth": 2, "resources": 1.5 },
          "huge_hw": { "variants": 50, "requests": 50, "depth": 2, "resources": 5.0 },
          "huge_sw": {"variants": 20, "requests": 50, "depth": 5, "resources": 1.5 }
        }

        for s in scenarios:
            self.logger.info("here")
            experiment_description = self.base_experiment_description
            experiment_description['TaskConfiguration']['Scenario']['ws_file'] = "result_v{}_q{}_d{}_r{}.csv".\
                format(scenarios[s]["variants"], scenarios[s]["requests"], scenarios[s]["depth"], str(scenarios[s]["resources"]).replace('.', '_'))
            experiment_description['TaskConfiguration']['Scenario']['numImplementations'] = scenarios[s]["variants"]
            experiment_description['TaskConfiguration']['Scenario']['numRequests'] = scenarios[s]["requests"]
            experiment_description['TaskConfiguration']['Scenario']['componentDepth'] = scenarios[s]["depth"]
            experiment_description['TaskConfiguration']['Scenario']['excessComputeResourceRatio'] = scenarios[s]["resources"]
            self.logger.info("Benchmarking next Scenario file(ws_file): %s" % experiment_description['TaskConfiguration']['Scenario']['ws_file'])
            self.execute_experiment(experiment_description)

        return self.counter

def run_benchmark():
    main_api_address = "http://main-node:49152"
    # Container creation performs --volume on `./results/` folder. Change wisely results_storage.
    results_storage = "./results/serialized/"
    try:
        runner = BRISEBenchmark(main_api_address, results_storage)
        try:
            # ---    Add User defined benchmark scenarios execution below  ---#
            # --- Possible variants: benchmark_repeater, benchmark_SA ---#
            runner.benchmark_repeater()

            # --- Helper method to remove outdated experiments from `./results` folder---#
            # runner.move_redundant_experiments(location=runner.results_storage + "outdated/")

            # ---   Add User defined benchmark scenarios execution above   ---#
        except Exception as err:
            logging.warning("The Benchmarking process interrupted by an exception: %s" % err)
            runner.main_api_client.stop_main()
        finally:
            runner.main_api_client.stop_main()
            chown_files_in_dir(results_storage)
            logging.info("The ownership of dump files was changed, exiting.")
    except Exception as err:
        logging.error("Unable to create BRISEBenchmark instance: %s" % err)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="The entry point of BRISE Benchmark service.")
    parser.add_argument("mode", choices=["analyse", "benchmark"], help="Mode in which Benchmarking functionality should be runned.")
    args = parser.parse_args()

    if args.mode == "analyse":
        build_benchmark_report()
    else:   # args.mode == "benchmark"
        run_benchmark()


from moabb.datasets.base import BaseDataset

from mne.io import read_raw_edf
from mne.channels import read_montage
import numpy as np

from . import download as dl

UPPER_LIMB_URL = 'https://zenodo.org/record/834976/files/'

class UpperLimb(BaseDataset):
    """Upper Limb motor dataset.

    Upper limb dataset :
    http://journals.plos.org/plosone/article?id=10.1371/journal.pone.0182578

    Consist in 6 upper limb movement, recoded over 2 sessions.
    The first session is motor execution, the second session is imagination.

    """

    def __init__(self, imagined=True, executed=False):
        self.imagined = imagined
        self.executed = executed
        event_id = {"right_elbow_flexion": 1536,
                    "right_elbow_extension": 1537,
                    "right_supination": 1538,
                    "right_pronation": 1539,
                    "right_hand_close": 1540,
                    "right_hand_open": 1541,
                    "rest": 1542}

        n_sessions = int(imagined) + int(executed)
        super().__init__(
            subjects=list(range(1, 16)),
            sessions_per_subject=n_sessions,
            events=event_id,
            code='Upper Limb Imagery',
            interval=[2.5, 5],
            paradigm='imagery',
            doi='10.1371/journal.pone.0182578')

    def _get_single_subject_data(self, subject):
        """return data for a single subject"""

        sessions = []
        if self.imagined:
            sessions.append('imagination')

        if self.executed:
            sessions.append('execution')

        out = {}
        for session in sessions:
            paths = self.data_path(subject, session=session)

            eog = ['eog-l', 'eog-m', 'eog-r']
            montage = read_montage('standard_1005')
            data = {}
            for ii, path in enumerate(paths):
                raw = read_raw_edf(path, montage=montage, eog=eog,
                                   misc=range(64, 96), preload=True,
                                   verbose='ERROR')
                # there is nan in the data
                raw._data[np.isnan(raw._data)] = 0
                data['run_%d' % ii] = raw

            out[session] = data
        return out

    def data_path(self, subject, path=None, force_update=False,
                  update_path=None, verbose=None, session=None):
        if subject not in self.subject_list:
            raise(ValueError("Invalid subject number"))

        paths = []

        if session is None:
            sessions = []
            if self.imagined:
                sessions.append('imagination')

            if self.executed:
                sessions.append('execution')
        else:
            sessions = [session]

        for session in sessions:
            for run in range(1, 11):
                url = f"{UPPER_LIMB_URL}motor{session}_subject{subject}_run{run}.gdf"
                p = dl.data_path(url, 'UPPERLIMB', path, force_update,
                                 update_path, verbose)
                paths.append(p)

        return paths


from app import app, lm
from flask import Flask, render_template, request, url_for, flash, redirect
from datetime import datetime
from db import DBAccess
from flask.ext.login import login_user, logout_user, login_required, current_user
from forms import LoginForm, RegisterForm
from user import User
import json

dbManager = DBAccess()

@app.route("/", methods=['GET', 'POST'])
@login_required
def main():
    result = None
    if request.method == 'POST':
        form_data = request.form
        result = dbManager.log_projects(current_user.username, form_data)
        if result == "Hours updated successfully":
            flash(result, 'success')
        elif result == "No projects entered":
            flash(result, 'error')
    sorted_user_projects = {}
    total_hours = 0
    hours_required = 0
    date = datetime.now()
    year = str(date.year)
    month = date.strftime("%B")
    user = load_user(current_user.username)
    user_role = user.get_role()
    user_dict = dbManager.select_user(user.username)
    if year in user_dict and month in user_dict[year]:
        user_projects = user_dict[year][month]['projects']
        for x, y in user_projects.items():
            project = dbManager.select_project(x)
            sorted_user_projects[project['name']] = user_projects[x]
            print(sorted_user_projects)
        if 'total_hours' in user_dict[year][month]:
            total_hours = int(user_dict[year][month]['total_hours'])
        if 'hours_required' in user_dict[year][month]:
            hours_required = int(user_dict[year][month]['hours_required'])
    projects = dbManager.select_all_projects()
    return render_template(
        'staff-returns.html',
        title='home',
        month=month,
        year=year,
        projects=projects,
        user_role=user_role,
        user_projects=sorted_user_projects,
        total_hours=total_hours,
        hours_required=hours_required,
        result=result)

@app.route('/budget-tracking', methods=['GET'])
@login_required
def project_management():
    user = load_user(current_user.username)
    user_role = user.get_role()
    users = dbManager.select_all_users();
    date = datetime.now()
    year = str(date.year)
    month = date.strftime("%B")
    monthNames = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"];
    if user_role == 'Admin' or user_role == 'Delivery Manager':
        projects = dbManager.select_all_projects()
        return render_template('budget-tracking.html',
                               title='budget-tracking',
                               user_role=user_role,
                               projects=projects,
                               monthNames=monthNames,
                               current_month=month,
                               current_year=year,
                               users=users)
    else:
        return render_template('404.html'), 404

@app.route('/user-management', methods=['GET', 'POST'])
@login_required
def user_management():
    result = None
    if request.method == 'POST':
        form_data = request.form
        result = dbManager.update_user(form_data)
        if result == "User updated successfully":
            flash(result, 'success')
    user = load_user(current_user.username)
    user_role = user.get_role()
    if user_role == 'Admin':
        days = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday"]
        roles = dbManager.select_all_roles()
        paygrades = dbManager.select_paygrades()
        users = dbManager.select_all_users()
        return render_template('user-management.html',
                               days=days,
                               paygrades=paygrades,
                               roles=roles,
                               title='user-management',
                               users=users,
                               user_role=user_role,
                               result=result)
    else:
        return render_template('404.html'), 404

@app.route('/load-user/<user_id>', methods=['GET'])
@login_required
def load_user_data(user_id=None):
    user = load_user(current_user.username)
    user_role = user.get_role()
    if user_role == 'Admin' or user_role == 'Delivery Manager':
        user = dbManager.select_user(user_id)
        user.pop('password')
        return json.dumps(user)

@app.route('/load-project/<project_name>', methods=['GET'])
@login_required
def load_project(project_name=None):
    user = load_user(current_user.username)
    user_role = user.get_role()
    if user_role == 'Admin' or user_role == 'Delivery Manager':
        project_id = dbManager.select_project_id_from_name(project_name)
        project = dbManager.select_project(project_id)
        return json.dumps(project)

@app.route('/login', methods=['GET', 'POST'])
def login():
    form = LoginForm(request.form)
    if request.method == 'POST' and form.validate_on_submit():
        user = dbManager.select_user(str.upper(form.username.data))
        if user != 'User not found':
            if user and User.validate_login(user['password'], form.password.data):
                user_obj = User(user['_id'])
                if form.remember.data:
                    login_user(user_obj, remember=True)
                else:
                    login_user(user_obj)
                flash("Logged in successfully", category='success')
                return redirect(request.args.get("next") or url_for('main'))
            else:
                flash("Wrong username or password", category='error')
        else:
            flash("Wrong username or password", category='error')
    return render_template('login.html', title='login', form=form)

@app.route('/logout')
def logout():
    logout_user()
    return redirect(url_for('login'))

@app.route('/register', methods=['GET', 'POST'])
def register():
    if current_user.is_authenticated():
        logout_user()
    form = RegisterForm(request.form)
    days = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday"]
    if request.method == 'POST':
        if form.validate_on_submit() == False:
            flash('All fields are required.', category='error')
        else:
            result = dbManager.insert_user(form)
            if result == 'Success':
                flash('Account created', category='success')
                return redirect(url_for('login'))
            else:
                flash('Username taken', category='error')
    return render_template('register.html', title='register', form=form, days=days)

@app.errorhandler(404)
def page_not_found(e):
    if current_user.is_authenticated():
        user = load_user(current_user.username)
        user_roles = user.get_role()
        return render_template('404.html', user_roles=user_roles), 404
    else:
        return render_template('404.html'), 404

@lm.user_loader
def load_user(user_id):
    u = dbManager.select_user(user_id)
    if not u:
        return None
    return User(u['_id'])

if __name__ == "__main__":
    app.run(debug=True, host='0.0.0.0', port=8080)


# --coding:utf-8--
import os
import sys
import glob
import argparse
import matplotlib.pyplot as plt

from keras import __version__
from keras.applications.inception_v3 import InceptionV3, preprocess_input
#from keras.applications.inception_v3_matt import InceptionV3, preprocess_input

from keras.models import Model
from keras.layers import Dense, GlobalAveragePooling2D
from keras.preprocessing.image import ImageDataGenerator
from keras.optimizers import SGD
from scipy import interp
import csv

os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"

os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2,3"

def get_nb_files(directory):
  """Get number of files by searching directory recursively"""
  if not os.path.exists(directory):
    return 0
  cnt = 0
  for r, dirs, files in os.walk(directory):
    for dr in dirs:
      cnt += len(glob.glob(os.path.join(r, dr + "/*")))
  return cnt
#train_num = get_nb_files('.../train')  2500
#print(train_num)
#input('wait...')

# 数据准备
IM_WIDTH, IM_HEIGHT = 299, 299 #InceptionV3指定的图片尺寸
FC_SIZE = 1024                # 全连接层的节点个数
NB_IV3_LAYERS_TO_FREEZE = 172  # 冻结层的数量

train_dir = '/home/som/lab/yuyao/Inception/yindaojing/train'  # 训练集数据
val_dir = '/home/som/lab/yuyao/Inception/yindaojing/val' # 验证集数据
output_model_file = '/home/som/lab/yuyao/Inception/InceptionV3.model'
nb_classes= 4
nb_epoch = 40
batch_size = 32

nb_train_samples = get_nb_files(train_dir)      # 训练样本个数
nb_classes = len(glob.glob(train_dir + "/*"))  # 分类数
nb_val_samples = get_nb_files(val_dir)       #验证集样本个数
nb_epoch = int(nb_epoch)                # epoch数量
batch_size = int(batch_size)

#　图片生成器
train_datagen =  ImageDataGenerator(
  preprocessing_function=preprocess_input,
  rotation_range=30,
  width_shift_range=0.2,
  height_shift_range=0.2,
  shear_range=0.2,
  zoom_range=0.2,
  horizontal_flip=True
)
test_datagen = ImageDataGenerator(
  preprocessing_function=preprocess_input,
  rotation_range=30,
  width_shift_range=0.2,
  height_shift_range=0.2,
  shear_range=0.2,
  zoom_range=0.2,
  horizontal_flip=True
)

# 训练数据与测试数据
train_generator = train_datagen.flow_from_directory(
train_dir,
target_size=(IM_WIDTH, IM_HEIGHT),
batch_size=batch_size,class_mode='categorical')

validation_generator = test_datagen.flow_from_directory(
val_dir,
target_size=(IM_WIDTH, IM_HEIGHT),
batch_size=batch_size,class_mode='categorical')

# 添加新层
def add_new_last_layer(base_model, nb_classes):
  """
  添加最后的层
  输入
  base_model和分类数量
  输出
  新的keras的model
  """
  x = base_model.output
  x = GlobalAveragePooling2D()(x)
  x = Dense(FC_SIZE, activation='relu')(x) #new FC layer, random init
  predictions = Dense(nb_classes, activation='softmax')(x) #new softmax layer
  model = Model(input=base_model.input, output=predictions)
  return model
# 冻上NB_IV3_LAYERS之前的层
def setup_to_finetune(model):
  """Freeze the bottom NB_IV3_LAYERS and retrain the remaining top layers.
  note: NB_IV3_LAYERS corresponds to the top 2 inception blocks in the inceptionv3 arch
  Args:
    model: keras model
  """
  for layer in model.layers[:NB_IV3_LAYERS_TO_FREEZE]:
     layer.trainable = False
  for layer in model.layers[NB_IV3_LAYERS_TO_FREEZE:]:
     layer.trainable = True
  model.compile(optimizer=SGD(lr=0.0001, momentum=0.9), loss='categorical_crossentropy', metrics=['accuracy'])

# 设置网络结构
model = InceptionV3(weights='imagenet', include_top=False)
model = add_new_last_layer(model, nb_classes)
setup_to_finetune(model)

# 模式二训练
history_ft = model.fit_generator(
train_generator,
samples_per_epoch=nb_train_samples,
nb_epoch=nb_epoch,
validation_data=validation_generator,
nb_val_samples=nb_val_samples,
class_weight='auto1')

# 模型保存
model.save(output_model_file)

# 画图
def plot_training(history):
  acc = history.history['acc']
  val_acc = history.history['val_acc']
  loss = history.history['loss']
  val_loss = history.history['val_loss']
  epochs = range(len(acc))
  plt.plot(epochs, acc, 'r.')
  plt.plot(epochs, val_acc, 'r')
  plt.title('Training and validation accuracy')
  plt.figure()
  plt.plot(epochs, loss, 'r.')
  plt.plot(epochs, val_loss, 'r-')
  plt.title('Training and validation loss')
  plt.show()

# 训练的acc_loss图
plot_training(history_ft)


import csv
import cv2
import numpy as np

import matplotlib.pyplot as plt
import matplotlib.image as mpimg

# Lines in the csv file
lines = []

# Reading the csv file and save each line in lines array
with open('TrainingV3/driving_log.csv') as csvfile:
    reader = csv.reader(csvfile)
    for line in reader:
        lines.append(line)

# Each line contain: Center image, left image, right image, steering value, throttle, brake, speed
# and the tag of each one is found in lines[0], so the data starts from lines[1]
print("First row: ", lines[0])
print("Second row (data): ", lines[1])
print()
print("Lines Number: ", len(lines))

# correction for the left and the right images of the sample data
steering_correction = 0.2
# Images array for the input data
images = []
# Measurements will be the labels for the Images
measurements = []
counter = 0

for line in lines[1:]:
    center_path = line[0]
    left_path = line[1]
    right_path = line[2]
    cent_measurement = float(line[3])
    throttle = float(line[4])
    # print(cent_measurement)

    center_filename = (center_path.split('/')[-1]).split('\\')[-1]
    left_filename = (left_path.split('/')[-1]).split('\\')[-1]
    right_filename = (right_path.split('/')[-1]).split('\\')[-1]

    center_current_path = 'TrainingV3' + '/IMG/' + center_filename
    left_current_path = 'TrainingV3' + '/IMG/' + left_filename
    right_current_path = 'TrainingV3' + '/IMG/' + right_filename

    cent_image = cv2.imread(center_current_path)[..., ::-1]
    left_image = cv2.imread(left_current_path)[..., ::-1]
    right_image = cv2.imread(right_current_path)[..., ::-1]

    left_measurement = cent_measurement + steering_correction
    right_measurement = cent_measurement - steering_correction

    cent_image = cv2.resize(cent_image[40:140, :], (64, 64))
    images.append(cent_image)
    measurements.append(tuple((cent_measurement, throttle)))
    # images.append(cent_image_flipped)

    left_image = cv2.resize(left_image[40:140, :], (64, 64))
    images.append(left_image)
    measurements.append(tuple((left_measurement, throttle)))
    # images.append(left_image_flipped)

    right_image = cv2.resize(right_image[40:140, :], (64, 64))
    images.append(right_image)
    measurements.append(tuple((right_measurement, throttle)))
    counter += 1
    print(counter)

# Convert images and measurements into numpy arrays for keras

# Covert to numpy arrays
images = np.array(images)
print(images.shape)
measurements = np.array(measurements)
print(measurements.shape)
print(measurements[0])
print('data in arrays')

import tensorflow as tf

# Normalization, Mean Centering, and Cropping

# In[37]:

# from keras.models import Sequential
# from keras.layers import Flatten, Dense, Activation, Lambda, Cropping2D
# from keras.layers.convolutional import Convolution2D
# from keras.layers.pooling import MaxPooling2D
# from keras.layers.core import Dropout

from keras.models import Sequential
from keras.layers import Flatten, Dense, Lambda, Convolution2D, MaxPooling2D, Dropout, Cropping2D, Activation
from keras.optimizers import Adam
from keras.layers.advanced_activations import ELU

# We will depend on Nvidia Model

def Nvidia_model(input_shape):
    # Normalizing, Mean Cenetring, Cropping
    model = Sequential()
    model.add(Lambda(lambda x: x / 255.0 - 0.5, name="image_normalization", input_shape=input_shape))

    model.add(Convolution2D(24, 5, 5, name="convolution_1", subsample=(2, 2), activation='relu', border_mode="valid",
                            init='he_normal'))

    # model.add(Convolution2D(nb_filter=36,nb_row=5,nb_co                                                                                l=5,subsample=(2,2),activation='relu'))
    model.add(Convolution2D(36, 5, 5, name="convolution_2", subsample=(2, 2), border_mode="valid", init='he_normal',
                            activation='relu'))

    # model.add(Convolution2D(nb_filter=48,nb_row=5,nb_col=5,subsample=(2,2),activation='relu'))
    model.add(Convolution2D(48, 5, 5, name="convolution_3", subsample=(2, 2), border_mode="valid", init='he_normal',
                            activation='relu'))

    # model.add(Convolution2D(nb_filter=64,nb_row=3,nb_col=3,activation='relu'))
    model.add(Convolution2D(64, 3, 3, name="convolution_4", border_mode="valid", init='he_normal', activation='relu'))

    # model.add(Convolution2D(nb_filter=64,nb_row=3,nb_col=3,activation='relu'))
    model.add(Convolution2D(64, 3, 3, name="convolution_5", border_mode="valid", init='he_normal', activation='relu'))

    model.add(Flatten())

    # Adding Dropout layer
    model.add(Dropout(p=0.2))
    model.add(Activation('relu'))
    model.add(Dense(1000))

    # Adding Dropout layer
    model.add(Dropout(p=0.5))
    model.add(Activation('relu'))
    model.add(Dense(500))

    # Adding Dropout layer
    model.add(Dropout(p=0.5))
    model.add(Activation('relu'))
    model.add(Dense(100))

    # Adding Dropout layer
    model.add(Dropout(p=0.2))
    model.add(Activation('relu'))
    model.add(Dense(10))
    model.add(Dense(2))  # Steering angle action and speed

    return model

# In[38]:

# Training data

X_train = images
print(X_train.shape)
y_train = measurements
print(y_train.shape)

# Run the model
epochs_arr = [20, 30, 40, 50]

for x in range(0, len(epochs_arr)):
    Model = Nvidia_model(input_shape=(64, 64, 3))
    learning_rate = 0.001
    print('created Model')
    # Mean square error and adam optimizer
    adam = Adam(lr=learning_rate)
    Model.compile(optimizer=adam, loss='mse')
    print('compiled Model')
    Model.summary()
    epochs = epochs_arr[x]
    batch_size = 512
    # 15% validation and apply shuffling
    Model.fit(x=X_train, y=y_train, nb_epoch=epochs, batch_size=batch_size,  validation_split=0.15, shuffle=True)
    # Save the model for testing it and give it as a paramter when running drive.py
    Model.save('model_6_str_throttle_' + str(learning_rate) + 'lr_' + str(epochs) + 'epoch.h5')


from django.shortcuts import render

from core.models import Article

def home(request):
    return render(request, "base.html")

def articlelist(request):
    articles = Article.objects.all()
    ctx = {
        "articles": articles
    }
    return render(request, "core/articlelist.html", ctx)

def articledetial(request, pk):  # pk = 아이디 값
    article = Article.objects.get(pk=pk)
    ctx = {
        "article": article
    }
    return render(request, "core/articledetail.html", ctx)

def articlecreate(request):
    if request.method == "POST":
        title = request.POST["title"]
        content = request.POST["content"]
        author = request.POST["author"]

        article = Article.objects.create(title=title, content=content, author=author)  # 인스턴스 만들고 save까지 한 번에 처리

        ctx = {
            "article": article
        }

        return render(request, "core/articlecreate.html", ctx)
    return render(request, "core/articlecreate.html")


#!/usr/bin/env python3
#Mike Moss
#10/14/2016
#Copies a path.

import argparse
import getpass
import superstar
import sys

if __name__=="__main__":
	try:
		#Parse CLI args...
		parser=argparse.ArgumentParser(description="Copies superstar paths.")
		parser.add_argument("-s","--superstar",
			dest="superstar",
			default="robotmoose.com",
			help="Superstar to use (default: robotmoose.com).")
		parser.add_argument("-l","--local",
			action='store_true',
			help="Sets superstar to local superstar.")
		parser.add_argument("-d","--dev",
			action='store_true',
			help="Sets superstar to test.robotmoose.com.")
		parser.add_argument("FROM",
			help="Name of path to copy.")
		parser.add_argument("TO",
			help="New name for the copied path.")
		args=parser.parse_args()

		#Figure out superstar...
		superstar_url=args.superstar
		if args.dev:
			superstar_url="test.robotmoose.com"
		if args.local:
			superstar_url="127.0.0.1:8081"
		ss=superstar.superstar_t(superstar_url)

		#Print errors...
		def onerror(error):
			print(str("Error("+str(error["code"])+") - "+error["message"]))
			exit(1)

		#Print success...
		def onsuccess(result):
			print("Success!")

		#Globals...
		copy_data=None

		#Check if to exists...
		def do_check_to(data):
			if not data:
				print("Path \""+args.FROM+"\" does not exist!")
				exit(1)

			#Global copy data...
			global copy_data
			copy_data=data

			#Get to...
			ss.get(args.TO,do_copy,onerror)
			ss.flush()

		#Do copy...
		def do_copy(data):
			if data:
				answer=""
				while True:
					answer=input("Path \""+args.TO+"\" exists, overwrite (y/n)?  ")
					if answer=="y" or answer=="Y":
						break
					if answer=="n" or answer=="N":
						exit(1)

			#Get auth...
			auth=getpass.getpass(prompt='Enter admin auth:  ')

			#Copy
			global ss
			global copy_data
			ss.set(args.TO,copy_data,auth,onsuccess,onerror)
			ss.flush()

		#Check from==to...
		if ss.pathify(args.FROM)==ss.pathify(args.TO):
			raise Exception("From and to are the same!")

		#Get original...
		ss.get(args.FROM,do_check_to,onerror)
		ss.flush()

	except Exception as error:
		print(error)
		exit(1)
	except KeyboardInterrupt:
		exit(1)


import os
from collections import Counter

import nltk

import tensorflow as tf

root_path='../data1'
file_name='train.txt'

en_corpus_name='en_corpus'
ch_corpus_name='ch_corpus'
en_vocab_name='en_vocab'
ch_vocab_name='ch_vocab'

class T2TDataset():
    def __init__(self, data_path, en_corpus_path, ch_corpus_path, en_vocab_path, ch_vocab_path,
                 en_limit=100000, ch_limit=10000):
        if data_path is None:
            raise ValueError('data_path must not be None')
        if not os.path.exists(data_path):
            raise ValueError("file doesn't exists")

        self._data_path=data_path
        self._en_corpus_path = en_corpus_path
        self._ch_corpus_path = ch_corpus_path
        self._en_vocab_path = en_vocab_path
        self._ch_vocab_path = ch_vocab_path
        self._en_limit = en_limit
        self._ch_limit = ch_limit

        if (not os.path.exists(self._en_corpus_path)) or (not os.path.exists(self._ch_corpus_path)):
            self._export_corpus()

        if (not os.path.exists(self._en_vocab_path)) or (not os.path.exists(self._ch_vocab_path)):
            self._export_vocab()

    def _export_corpus(self):
        count = 0
        print('extract corpus start')
        with tf.gfile.Open(self._data_path, 'r') as f:
            with tf.gfile.Open(self._en_corpus_path, 'w') as en_file:
                with tf.gfile.Open(self._ch_corpus_path, 'w') as ch_file:
                    for sentence in f:
                        arr = sentence.strip().split('\t')
                        en_sentence = arr[2]
                        ch_sentence = arr[3]
                        en_tokens = nltk.word_tokenize(en_sentence)
                        ch_tokens = [token for token in ch_sentence]
                        en_file.write('%s\n' % (' '.join(en_tokens)))
                        ch_file.write('%s\n' % (' '.join(ch_tokens)))
                        count += 1
                        if count % 100000 == 0:
                            print('current process count: %d' % (count))
        print('extract corpus is over')

    def _export_vocab(self):
        if self._en_corpus_path is None or (not os.path.exists(self._en_corpus_path)):
            raise ValueError('en_corpus_path must exists')
        if self._ch_corpus_path is None or (not os.path.exists(self._ch_corpus_path)):
            raise ValueError('ch_corpus_path must exists')

        en_tokens_counter = Counter()
        with tf.gfile.Open(self._en_corpus_path, 'r') as en_file:
            for sentence in en_file:
                for token in sentence.split():
                    en_tokens_counter[token] += 1
        print(en_tokens_counter)
        sorted_en_list = sorted(en_tokens_counter.items(), key=lambda kv: kv[1], reverse=True)
        with open(self._en_vocab_path, 'w') as f:
            for k, v in sorted_en_list[0: self._en_limit]:
                f.write('%s\n' % (k))

        print('english tokens count is %d' % (len(en_tokens_counter)))
        ch_tokens_counter = Counter()
        with tf.gfile.Open(self._ch_corpus_path, 'r') as ch_file:
            for sentence in ch_file:
                for token in sentence.split():
                    ch_tokens_counter[token] += 1
        sorted_ch_list = sorted(ch_tokens_counter.items(), key=lambda kv: kv[1], reverse=True)
        with open(self._ch_vocab_path, 'w') as f:
            for k, v in sorted_ch_list[0: self._ch_limit]:
                f.write('%s\n' % (k))

        print('china tokens count is %d' % (len(ch_tokens_counter)))

if __name__=="__main__":
    data_path = os.path.join(root_path, file_name)
    en_corpus_path = os.path.join(root_path, en_corpus_name)
    ch_corpus_path = os.path.join(root_path, ch_corpus_name)
    en_vocab_path = os.path.join(root_path, en_vocab_name)
    ch_vocab_path = os.path.join(root_path, ch_vocab_name)
    dataset = T2TDataset(data_path, en_corpus_path, ch_corpus_path, en_vocab_path, ch_vocab_path)


"""
Module to find out the qualified name of a class.
"""

import ast
import inspect
import types

from collections import defaultdict

__all__ = ['qualname']

_cache = {}
_file_cache = {}

class _Visitor(ast.NodeVisitor):
    def __init__(self):
        super(_Visitor, self).__init__()
        self.stack = []

        # The keys here are line numbers of a method/function
        self.function_qualnames = {}

        # The keys here are unqualified names of classes
        self.class_qualnames = defaultdict(set)

    def current_qualname(self):
        return ".".join(self.stack)

    def visit_FunctionDef(self, node):
        self.stack.append(node.name)
        self.function_qualnames[node.lineno] = self.current_qualname()
        self.stack.append('<locals>')
        self.generic_visit(node)
        self.stack.pop()
        self.stack.pop()

    def visit_ClassDef(self, node):
        self.stack.append(node.name)
        self.class_qualnames[node.name].add(self.current_qualname())
        self.generic_visit(node)
        self.stack.pop()

def _fallback_to_name(obj):
    name = obj.__name__
    if inspect.isclass(obj):
        _cache[obj] = name
    else:
        try:
            obj.__qualname__ = name
        except (AttributeError, TypeError):
            _cache[obj] = name

    return name

def qualname(obj):
    """Find out the qualified name for a class or function."""

    # For Python 3.3+, this is straight-forward.
    # This attribute is also set where possible on functions processed
    # for the first time as a simple cache
    if hasattr(obj, '__qualname__'):
        return obj.__qualname__

    # This is for objects that can't have an attribute set on them
    # (e.g. builtins) and classes (to prevent inheritance issues)
    # See _fallback_to_name
    if obj in _cache:
        return _cache[obj]

    code = None
    if isinstance(obj, (types.FunctionType, types.MethodType)):
        # Extract function from unbound method (Python 2)
        obj = getattr(obj, 'im_func', obj)
        try:
            code = obj.__code__
        except AttributeError:
            code = obj.func_code

        # Different instances of the same local function share the same code object, so this
        # can be used to look them up in the cache
        if code in _cache:
            return _cache[code]
    elif not (inspect.isclass(obj) or inspect.isroutine(obj)):
        return obj.__qualname__  # This object isn't meant to have a qualname. Raise a sensible error

    # For older Python versions, things get complicated.
    # Obtain the filename where the
    # class/method/function is defined.
    try:
        filename = inspect.getsourcefile(obj)
    except TypeError:
        return _fallback_to_name(obj)

    # Re-parse the source file to figure out what the
    # __qualname__ should be by analysing the abstract
    # syntax tree. Use a cache to avoid doing this more
    # than once for the same file.
    visitor = _file_cache.get(filename)
    if visitor is None:
        with open(filename, 'r') as fp:
            source = fp.read()
        node = ast.parse(source, filename)
        visitor = _Visitor()
        visitor.visit(node)
        _file_cache[filename] = visitor

        # For classes accessible from the top level, directly associate
        # each class with its qualname
        module = inspect.getmodule(obj)
        for k, qname_set in visitor.class_qualnames.items():

            # iterate over a copy since we're going to modify it
            for qname in list(qname_set):
                val = module
                for attr in qname.split('.'):
                    val = getattr(val, attr, None)

                    # Ensure that we're getting the right thing
                    if not (
                            inspect.isclass(val)
                            and val.__name__ == attr
                            and inspect.getmodule(val) == module
                    ):
                        break
                else:
                    _cache[val] = qname
                    qname_set.discard(qname)

        # Check if that worked for the current argument
        if obj in _cache:
            assert _cache[obj].endswith(obj.__name__)
            return _cache[obj]

    if code:
        result = _cache[code] = obj.__qualname__ = visitor.function_qualnames[code.co_firstlineno]
    else:
        results = visitor.class_qualnames[obj.__name__]
        if not results:
            return _fallback_to_name(obj)

        if len(results) == 1:
            result = list(results)[0]
        else:  # This means several local classes in the file have the same short name

            # Since the qualname of a method is unambiguous, the qualname
            # of this class can be guessed pretty reliably from its methods.
            # Since you could theoretically take a method from one class and
            # assign it as an attribute on another class, we look for the most
            # common prefix.
            counts = defaultdict(int)
            for method in obj.__dict__.values():
                if not isinstance(method, types.FunctionType):
                    continue
                method_qualname = qualname(method)
                suffix = '.' + method.__name__
                owner_class_qualname = method_qualname[:-len(suffix)]
                if owner_class_qualname in results:
                    counts[owner_class_qualname] += 1

            if counts:
                result = max(counts.items(), key=lambda item: item[1])[0]
            else:
                result = list(results)[0]

        if '.<locals>.' not in result:  # avoid overloading the cache with local classes
            _cache[obj] = result

    assert result.endswith(obj.__name__)
    return result


# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# example use: python code/invoice.py September 2019
# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------

# LIBRARIES
#python 3.6.8
#tkinter 8.6
#numpy 1.15.4
#pandas 0.24.0
#docxtpl 0.6.3
#docx 0.8.7
#docxcompose 1.0.2

import tkinter as tk
import numpy as np
import pandas as pd
import os

from docxtpl import DocxTemplate
from docx import Document
import datetime
from docxcompose.composer import Composer
from sys import argv

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# COMMAND LINE INPUTS

# check command line args for month and year of invoice
if len(argv) > 2:
    MONTH = argv[1] # first argument is the month
    YEAR = argv[2] # second argument is the year
else:
    exit('Missing MONTH and/or YEAR argument.')

# month set
month_set = set(['January', 'February', 'March', 'April', 'May', 'June', 'July',
              'August', 'September', 'October', 'November', 'December'])
# check for proper month
if MONTH not in month_set:
    exit('Provide a proper month.')

# try for proper dates
try:
    CURR_DATE = datetime.datetime.strptime(MONTH+' ' + YEAR, '%B %Y')
except ValueError: # else exit
    str_exit =  'Improper command line inputs.\n'
    str_exit += 'Proper input looks like:\n'
    str_exit += 'python code/invoice.py September 2019'
    exit(str_exit)

### needed template files
# docs
fpage = 'templates/fpage.docx' # first page
invoice_template = 'templates/invoice_template.docx' # base template
invoice_template_mult = 'templates/invoice_template_mult.docx' # goes with RPind
invoice_template_z = 'templates/invoice_template_z.docx' # goes with Zind
invoice_template_z_gg = 'templates/invoice_template_z_gg.docx' # goes with Zind/ gg
invoice_template_v = 'templates/invoice_template_v.docx' # goes with Vind

# data
invoice_data_template = 'templates/invoice_data_template.xlsx'

# check for needed files
if not os.path.exists(fpage): # fpage template
    exit('Missing fpage template.')
if not os.path.exists(invoice_template): # word template
    exit('Missing invoice word template.')
if not os.path.exists(invoice_template_mult): # word template mult
    exit('Missing invoice word template mult.')
if not os.path.exists(invoice_template_z): # word template z
    exit('Missing invoice word template z.')
if not os.path.exists(invoice_template_z_gg): # word template z_gg
    exit('Missing invoice word template z_gg.')
if not os.path.exists(invoice_template_v): # word template v
    exit('Missing invoice word template v.')
if not os.path.exists(invoice_data_template): # data template
    exit('Missing invoice data template.')

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# IMPORT and AUGMENT DATA

# column list
col_list = ['OWNER', 'DIRECTION', 'FIRST', 'LAST',
            'STREET_ADDRESS', 'CITY_ADDRESS',
            'MY_NOTES',
            'MONTHLY_CHARGE',
            'EMAIL_ADDRESS', 'EMAIL_IND',
            'COND_CHARGE', 'FILT_CHARGE',
            'COND_MONTHS', 'FILT_MONTHS',
            'SEND_REMINDER', 'RP_INDICATOR',
            'Z_INDICATOR', 'V_INDICATOR']
# data type dictionary
types = {'OWNER':str, 'DIRECTION':str, 'FIRST':str, 'LAST':str,
        'STREET_ADDRESS':str, 'CITY_ADDRESS':str,
        'MY_NOTES':str,
        'MONTHLY_CHARGE':float,
        'EMAIL_ADDRESS':str, 'EMAIL_IND':int,
        'COND_CHARGE':float, 'FILT_CHARGE':float,
        'COND_MONTHS':str, 'FILT_MONTHS':str,
        'SEND_REMINDER':int, 'RP_INDICATOR':int,
        'Z_INDICATOR':int, 'V_INDICATOR':int}

# read in data
tabl = pd.read_excel(invoice_data_template,
                     usecols= col_list,
                     dtype=types,
                     skiprows=0)
# fix email improper reading in of NA string values in emails
tabl['EMAIL_ADDRESS'] = tabl['EMAIL_ADDRESS'].astype(str) #'NA' is converted to string 'nan'
# fill in any missing data for string variables
# fill in with a blank ''
na_list = ['FIRST', 'LAST', 'STREET_ADDRESS', 'CITY_ADDRESS', 'MY_NOTES']
for col in na_list:
    tabl[col].fillna(value='', inplace=True)

# number of rows
N = tabl.shape[0]

# add new columns
tabl['ADD_CHARGE'] = np.zeros(N, dtype=float)
tabl['ADD_CHARGE_NOTES'] = ['']*N
tabl['ADD_CHARGE_2'] = np.zeros(N, dtype=float)
tabl['ADD_CHARGE_NOTES_2'] = ['']*N
tabl['CUST_REMINDER'] = ['']*N

# indicator for whether to include rows or not for the outgoing files
included = np.ones(N, dtype=int)

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
## CREATE DIRECTORY
print("Creating directories...")

### month dict: month name -> numerical two digit code
month_dict = {'January':'01', 'February':'02', 'March':'03', 'April':'04',
                'May':'05', 'June':'06', 'July':'07', 'August':'08',
                'September':'09', 'October':'10', 'November':'11', 'December':'12'}
next_month_dict = {'January':'February', 'February':'March', 'March':'April',
                    'April':'May', 'May':'June', 'June':'July', 'July':'August',
                    'August':'September', 'September':'October', 'October':'November',
                    'November':'December', 'December':'January'}

# make directory
# example: invoices/2019/2019_09_September
# this allows invoices to first be sorted by years to avoid eventual clutter, and
# the to be properly sorted numerically
# -- e.g. 2019_08_August is on top of 2019_09_September
path = 'invoices/'+YEAR+'/'+YEAR+'_'+month_dict[MONTH]+'_'+MONTH
try:
    os.mkdir(path)
except OSError:
    exit("Creation of the directory %s failed" % path)

##### sub directories
##### example: invoices/2019/2019_09_September/PP and invoices/2019/2019_09_September/GG
## pp
sub_path_pp = path+'/PP'
try:
    os.mkdir(sub_path_pp)
except OSError:
    exit("Creation of the directory %s failed" % sub_path_pp)
## gg
sub_path_gg = path+'/GG'
try:
    os.mkdir(sub_path_gg)
except OSError:
    exit("Creation of the directory %s failed" % sub_path_gg)

##### make "emails" directories
## pp
emails_path_pp = sub_path_pp+'/emails'
try:
    os.mkdir(emails_path_pp)
except OSError:
    exit("Creation of the directory %s failed" % emails_path_pp)
## gg
emails_path_gg = sub_path_gg+'/emails'
try:
    os.mkdir(emails_path_gg)
except OSError:
    exit("Creation of the directory %s failed" % emails_path_gg)

print("Directories created.")

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# GLOBAL VARS and CONSTANTS

##### global vars
# current row
index = 0
# indicator for whether or not the 'back button' is on in the gui
back_on = False
# indicator for restoring the main gui format upon going back to the main...
# ...program from the exit screen gui format
last_back = False
# error string to attach if we receive improper input
error_str = ''
# back string to infrom user we went back a row
back_str = ''
# indicator for whether we have finished the main invoicing program
final_end = False
# dates error indicator
dates_err_ind = False
# reminder notice indicator
remind_ind = False
##### last row of the gui format
# main -> 0; month charge -> 1;
# add charge -> 2; notes add charge -> 3;
# add charge 2 -> 4; notes add charge 2 -> 5;
# notes -> 6; cust remind -> 7; check -> 8;
# button one and button two -> 9
last_row = 9

##### constants for tkinter display windows
font = 'Times' # font type
bold_ind = 'bold' # bold indicator
size = 18 # font size
col1 = '#14bcfe' #color-> blue-ish
col2 = '#fe9114' #color -> orange-ish
notes_color = 'black' # notes color
reminder_color = 'purple' ## '#cf2b29';'#e34240'; red-ish
amt_color = 'green' # amount color
add_color = '#0a4cbf' # #blue-ish
add_color_2 = '#0a98bf' #light blue-sh'
cust_remind_color = 'red' # cust reminder color
thick = 2 # highlighted thickness
pad = 6 # amount of padding
width = 1000 # width dim -> of the entire window
height = 800 # height dim -> of the entire window
entry_width = 80 # width dim of the user input sections
on_color = 'red' # color of checked button when on
off_color = 'grey' # color of checked button when off

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# WINDOW FUNCTIONS

##### center the window upon opening with a given width and height
def center_window(base, width=300, height=200):
    # get screen width and height
    screen_width = base.winfo_screenwidth()
    screen_height = base.winfo_screenheight()
    # calculate position x and y coordinates
    x = (screen_width/2) - (width/2)
    y = (screen_height/2) - (height/2)
    base.geometry('%dx%d+%d+%d' % (width, height, x, y))
    return

##### close root i.e. the main program
def close_root():
    # global vars
    global root
    global final_end

    # destroy root
    root.destroy()
    # set checkpoint
    final_end = True
    return

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# HELPER FUNCTIONS for navigating

# check value of checkbutton
# - if its clicked, then turn it on to appropriate color
# - else leave it deafault black color
def on_check():
    global var, chk
    if var.get() == 1:
        chk["fg"] = on_color
    else:
        chk["fg"] = off_color
    return

##### function to begin the main program
def begin():
    # global vars
    global btn_one, btn_two
    global monthly_charge_lbl, monthly_charge_entry
    global add_charge_lbl, add_charge_entry
    global add_charge_notes_lbl, add_charge_notes_entry
    global add_charge_lbl_2, add_charge_entry_2
    global add_charge_notes_lbl_2, add_charge_notes_entry_2
    global my_notes_lbl, my_notes_entry
    global cust_reminder_lbl, cust_reminder_entry
    global chk

    ##### main label
    main_lbl.grid_forget()

    ##### first button
    # change text: begin -> next
    # change command
    # if clicked, we move to the next row
    # for proper display, we forget it so that we can place it at the end later
    btn_one.grid_forget()
    btn_one.config(text= 'Next')
    btn_one.config(command= next)

    ##### second button
    # change text: close -> back
    # change command
    # if clicked, we move back to the previous row
    # for proper display, we forget it so that place it at the end later
    btn_two.grid_forget()
    btn_two.config(text= 'Back')
    btn_two.config(command= back)

    ##### pack the labels and entries
    row_inc = 0
    # main label
    main_lbl.grid(row= row_inc, column= 0, pady= pad, columnspan=2)
    row_inc += 1
    # monthly charge
    monthly_charge_lbl.grid(row= row_inc, column= 0, pady= pad)
    monthly_charge_entry.grid(row= row_inc, column= 1, padx= pad)
    row_inc += 1
    # additional charge
    add_charge_lbl.grid(row= row_inc, column= 0, pady= pad)
    add_charge_entry.grid(row= row_inc, column= 1, padx= pad)
    row_inc += 1
    # notes on additional charges
    add_charge_notes_lbl.grid(row= row_inc, column= 0, pady= pad)
    add_charge_notes_entry.grid(row= row_inc, column= 1, padx= pad)
    row_inc += 1
    # additional charge 2
    add_charge_lbl_2.grid(row= row_inc, column= 0, pady= pad)
    add_charge_entry_2.grid(row= row_inc, column= 1, padx= pad)
    row_inc += 1
    # notes on additional charges 2
    add_charge_notes_lbl_2.grid(row= row_inc, column= 0, pady= pad)
    add_charge_notes_entry_2.grid(row= row_inc, column= 1, padx= pad)
    row_inc += 1
    # my notes
    my_notes_lbl.grid(row= row_inc, column= 0, pady= pad)
    my_notes_entry.grid(row= row_inc, column= 1, padx= pad)
    row_inc += 1
    # customer reminders
    cust_reminder_lbl.grid(row= row_inc, column= 0, pady= pad)
    cust_reminder_entry.grid(row= row_inc, column= 1, padx= pad)
    row_inc += 1
    # check button
    chk.grid(row= row_inc, column= 0, pady= pad)
    row_inc += 1

    ##### pack back in the next button, but not the back button just yet
    # - turn the back button on at the second row
    btn_one.grid(row= row_inc, column= 0, pady= pad)

    ##### update the display
    update()
    return

##### update the gui display
def update():
    # global vars
    global index
    global error_str, error_lbl
    global back_on, last_back, back_str
    global btn_one,  btn_two
    global var, chk
    global main_lbl
    global monthly_charge_lbl, monthly_charge_entry
    global add_charge_lbl, add_charge_entry
    global add_charge_notes_lbl, add_charge_notes_entry
    global add_charge_lbl_2, add_charge_entry_2
    global add_charge_notes_lbl_2, add_charge_notes_entry_2
    global my_notes_lbl, my_notes_entry
    global cust_reminder_lbl, cust_reminder_entry
    global CURR_DATE, MONTH
    global dates_err_ind, remind_ind

    ##### clear the entries
    monthly_charge_entry.delete(0, 'end')
    add_charge_entry.delete(0, 'end')
    add_charge_notes_entry.delete(0, 'end')
    add_charge_entry_2.delete(0, 'end')
    add_charge_notes_entry_2.delete(0, 'end')
    my_notes_entry.delete(0, 'end')
    cust_reminder_entry.delete(0, 'end')
    ##### reset checkbox
    var.set(0) # value -> set it back to zero ie off
    chk["fg"] = off_color # color -> set back to off color, not clicked

    ##### error string and label logic
    if error_str != '': # add in the error label and message
        error_lbl.config(text= error_str)
        error_lbl.grid(row= last_row+1, column= 0, pady = 0, columnspan=2)
    else:
        # remove the error lbl if present before
        # if error was not present before, this does nothing
        error_lbl.grid_forget()

    ##### back button logic
    # --------------------------------------------------------------------------
    # add the 'back button' if index > 0; we don't want a back button for the
    # first row becuase there is nothing to go back to; close the window to quit
    if index == 1 and back_on == False:
        # pack the back button
        btn_two.grid(row= last_row, column= 1, padx= pad)
        # update conditional that the 'back button' is on
        back_on = True
    # --------------------------------------------------------------------------
    # remove back button if on index=0 i.e. we got back to the first row and
    # need to remove the back button because there is nothing to go back to
    if index == 0 and back_on == True:
        btn_two.grid_forget() # remove button
        back_on = False # update conditional
    # --------------------------------------------------------------------------
    # going back from the exit - last - screen
    if last_back == True:
        # we want to restore the main gui format
        # -- i.e. the labels and entries followed by the buttons

        ##### forget the buttons
        btn_one.grid_forget()
        btn_two.grid_forget()

        ##### bring back the labels and entries
        # increment row bc we do not forget the main label
        row_inc = 1
        # monthly charge
        monthly_charge_lbl.grid(row= row_inc, column= 0, pady= pad)
        monthly_charge_entry.grid(row= row_inc, column= 1, padx= pad)
        row_inc += 1
        # additional charge
        add_charge_lbl.grid(row= row_inc, column= 0, pady= pad)
        add_charge_entry.grid(row= row_inc, column= 1, padx= pad)
        row_inc += 1
        # additional charge notes
        add_charge_notes_lbl.grid(row= row_inc, column= 0, pady= pad)
        add_charge_notes_entry.grid(row= row_inc, column= 1, padx= pad)
        row_inc += 1
        # additional charge 2
        add_charge_lbl_2.grid(row= row_inc, column= 0, pady= pad)
        add_charge_entry_2.grid(row= row_inc, column= 1, padx= pad)
        row_inc += 1
        # additional charge notes 2
        add_charge_notes_lbl_2.grid(row= row_inc, column= 0, pady= pad)
        add_charge_notes_entry_2.grid(row= row_inc, column= 1, padx= pad)
        row_inc += 1
        # my notes
        my_notes_lbl.grid(row= row_inc, column= 0, pady= pad)
        my_notes_entry.grid(row= row_inc, column= 1, padx= pad)
        row_inc += 1
        # customer reminders
        cust_reminder_lbl.grid(row= row_inc, column= 0, pady= pad)
        cust_reminder_entry.grid(row= row_inc, column= 1, padx= pad)
        row_inc += 1
        # check button
        chk.grid(row= row_inc, column= 0, pady= pad)
        row_inc += 1

        ##### bring back the buttons
        btn_one.grid(row= row_inc, column= 0, pady= pad)
        btn_two.grid(row= row_inc, column= 1, padx= pad)
        # reconfig the first button i.e. the begin button at start, close at end
        btn_one.config(text= 'Next') # update text
        btn_one.config(command= next) # update command

        ###### update last_back conditional i.e. if we press the back button
        # we are no longer on the end screen, coming back from the exit screen
        last_back = False
    ##### END ----------------------------------------------- back button logic

    ##### update logic
    # check if we can update
    # if no more rows left, end
    # if we can update, replace entries with appropriate text
    if index > N-1:
        end() # end screen logic
    else:
        # grab current line
        curr_line = dict(tabl.iloc[index])

        ##### input entries
        ## main label text
        text = ''
        # inform user that we've come back to this row
        if back_str != '':
            text += back_str
            # revert back string
            back_str = ''
        # main info
        text += curr_line['FIRST']+' '+ curr_line['LAST'] + '\n\n' + \
                curr_line['STREET_ADDRESS'] + '\n' + \
                curr_line['CITY_ADDRESS']

        ### ADD TEXT - conditional
        ## cond
        if curr_line['COND_MONTHS'] != 'exclude':
            pot_cond_months = set("".join(curr_line['COND_MONTHS'].split(',')).split())
            ## - proper months
            if len(pot_cond_months - month_set) == 0:
                # check if current month is in potential cond... months
                if MONTH in pot_cond_months:
                    remind_ind = True
                    text += '\n\n' + \
                    "Remember, it's time for cond...." + \
                    ' - $' + str(curr_line['COND_CHARGE'])

                # check if next month is in potential cond... months
                if (curr_line['SEND_REMINDER'] == 1) and (next_month_dict[MONTH] in pot_cond_months):
                    remind_ind = True
                    text += '\n\n' + \
                    "Next month time for cond... send reminder this month."
            ## - improper months
            else:
                dates_err_ind = True
                text += '\n\n' + "Error, improper cond... month"
        ## filt
        if curr_line['FILT_MONTHS'] != 'exclude':
            pot_filt_months = set("".join(curr_line['FILT_MONTHS'].split(',')).split())
            ## - proper months
            if len(pot_filt_months - month_set) == 0:
                # check if current month is in potential filt... months
                if MONTH in pot_filt_months:
                    remind_ind = True
                    text += '\n\n' + \
                    "Remember, it's time for filt...." + \
                    ' - $' + str(curr_line['FILT_CHARGE'])

                # check if next month is in potential cond... months
                if (curr_line['SEND_REMINDER'] == 1) and (next_month_dict[MONTH] in pot_filt_months):
                    remind_ind = True
                    text += '\n\n' + \
                    "Next month time for filt... send reminder this month."
            ## - improper months
            else:
                dates_err_ind = True
                text += '\n\n' + "Error, improper filt... month"

        ##### update the main label
        # color
        if dates_err_ind == True: # first indicator, red if any error
            main_lbl.config(fg= 'red')
        elif remind_ind == True: # if no error, then can set reminder
            main_lbl.config(fg= reminder_color)
        else: # revert to default, black color
            main_lbl.config(fg= 'black')
        # text
        main_lbl.config(text=text)
        ##### revert indicators
        dates_err_ind = False
        remind_ind = False

        ##### update entries
        ## monthly charge entry
        monthly_charge_entry.insert(0, curr_line['MONTHLY_CHARGE'])
        ## my notes
        my_notes_entry.insert(0, curr_line['MY_NOTES'])

        return

##### ending the program
def end():
    # global vars
    global main_lbl
    global btn_one
    global monthly_charge_lbl, monthly_charge_entry
    global add_charge_lbl, add_charge_entry
    global add_charge_notes_lbl, add_charge_notes_entry
    global add_charge_lbl_2, add_charge_entry_2
    global add_charge_notes_lbl_2, add_charge_notes_entry_2
    global my_notes_lbl, my_notes_entry
    global cust_reminder_lbl, cust_reminder_entry
    global chk
    global last_back

    ##### updating labels and buttons
    ## update the text in the main label and its color
    main_lbl.config(text= "You're done.")
    main_lbl.config(fg= 'black')

    ## update button one - originally the begin button then the next button
    btn_one.config(text= 'Close') # update the text of the button
    btn_one.config(command= close_root) # update the button's command
    # no need to change affect of the back button

    ##### forget the other labels and entries
    # monthly charge
    monthly_charge_lbl.grid_forget()
    monthly_charge_entry.grid_forget()
    # additonal charge
    add_charge_lbl.grid_forget()
    add_charge_entry.grid_forget()
    # additional charge notes
    add_charge_notes_lbl.grid_forget()
    add_charge_notes_entry.grid_forget()
    # additonal charge 2
    add_charge_lbl_2.grid_forget()
    add_charge_entry_2.grid_forget()
    # additional charge notes 2
    add_charge_notes_lbl_2.grid_forget()
    add_charge_notes_entry_2.grid_forget()
    # my notes
    my_notes_lbl.grid_forget()
    my_notes_entry.grid_forget()
    # customer reminder
    cust_reminder_lbl.grid_forget()
    cust_reminder_entry.grid_forget()
    # check button
    chk.grid_forget()

    # going back from the end
    last_back = True # update the indicator to restore main gui format
    return

##### next - go to the next row
def next():
    # global vars
    global index, error_str, var
    global monthly_charge_entry
    global add_charge_entry, add_charge_notes_entry
    global add_charge_entry_2, add_charge_notes_entry_2
    global my_notes_entry, cust_reminder_entry

    # grab and try reading in input
    try:
        # monthly charge input
        monthly_in_amt_str = monthly_charge_entry.get().strip()
        if monthly_in_amt_str == '':
            monthly_in_amt_str = '0'
        monthly_in_amt = float(eval(monthly_in_amt_str))
        # additional charge input
        add_in_amt_str = add_charge_entry.get().strip()
        if add_in_amt_str == '':
            add_in_amt_str = '0'
        add_in_amt = float(eval(add_in_amt_str))
        # additional charge notes input
        in_add_charge_notes = str(add_charge_notes_entry.get().strip())

        # additional charge input 2
        add_in_amt_str_2 = add_charge_entry_2.get().strip()
        if add_in_amt_str_2 == '':
            add_in_amt_str_2 = '0'
        add_in_amt_2 = float(eval(add_in_amt_str_2))
        # additional charge notes input 2
        in_add_charge_notes_2 = str(add_charge_notes_entry_2.get().strip())

        # my_notes
        in_my_notes = str(my_notes_entry.get().strip())
        # cust_reminder
        in_cust_reminder = str(cust_reminder_entry.get().strip())

        # update data
        tabl.at[index, 'MONTHLY_CHARGE'] = monthly_in_amt
        tabl.at[index, 'ADD_CHARGE'] = add_in_amt
        tabl.at[index, 'ADD_CHARGE_NOTES'] = in_add_charge_notes
        tabl.at[index, 'ADD_CHARGE_2'] = add_in_amt_2
        tabl.at[index, 'ADD_CHARGE_NOTES_2'] = in_add_charge_notes_2
        tabl.at[index, 'MY_NOTES'] = in_my_notes
        tabl.at[index, 'CUST_REMINDER'] = in_cust_reminder

        ##### checkbox - included update
        if var.get() == 1:
            included[index] = 0

        ##### clear the error string
        # ie we have not run into an error at this point
        error_str = ''

        ##### update index -- bc we have no error
        index += 1

    # reached an error
    except NameError or SyntaxError:
        # we have an error, so update error string
        # and do not move forward with index
        error_str = '\n\n' + \
        'Previous error. Make sure amounts are valid real numbers.'

    ##### update
    # -- if we had no error, we update to the next row
    # -- if we had an error, we re-update the current row with an error message
    update()
    return

##### next - go to the next row
def back():
    # global vars
    global index, back_str, error_str

    # revert index
    index -= 1

    # make sure we revert the included ariable indicator just in case
    included[index] = 1

    # update back string
    back_str = '- WENT BACK - \n\n'

    # update error string - in case we were at an error
    error_str = ''

    # we went back, so now update the main gui display
    update()
    return

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# SET-UP the gui

##### root for gui
root = tk.Tk()
root.title('Invoice: '+MONTH+' '+YEAR)
#root.configure(background='grey')

##### center the window
center_window(root, width, height)

###### main label where we will show input
main_lbl = tk.Label(root, text= 'Click to begin.')
main_lbl.config(font= (font, size, bold_ind))
main_lbl.config(fg= 'black')

main_lbl.grid(row=0, column=0, pady=pad, columnspan=2)

##### error label
error_lbl = tk.Label(root, text= error_str)
error_lbl.config(font= (font, size, bold_ind))
error_lbl.config(fg= 'red')

##### the button to begin -> next -> end program
btn_one = tk.Button(root, text= 'Begin', command= begin,
                highlightbackground= col1,
                highlightthickness= thick)
btn_one.config(font= (font, size, bold_ind))
btn_one.grid(row=1, column=0, pady=pad)

##### the button to close -> back
btn_two = tk.Button(root, text= 'Close', command= close_root,
                    highlightbackground= col2,
                    highlightthickness= thick)
btn_two.config(font= (font, size, bold_ind))
btn_two.grid(row=1, column=1, pady=pad)

##### monthly charge
## label
monthly_charge_lbl = tk.Label(root, text= 'Monthly Charge:')
monthly_charge_lbl.config(font= (font, size, bold_ind))
monthly_charge_lbl.config(fg= amt_color)
## entry
monthly_charge_entry = tk.Entry(root, width= entry_width)
monthly_charge_entry.config(font= (font, size))

##### additional charge
## label
add_charge_lbl = tk.Label(root, text= 'Additional Charge:')
add_charge_lbl.config(font= (font, size, bold_ind))
add_charge_lbl.config(fg= add_color)
## entry
add_charge_entry = tk.Entry(root, width= entry_width)
add_charge_entry.config(font= (font, size))

##### additional charge notes
## label
add_charge_notes_lbl = tk.Label(root, text= 'Notes on additional charge:')
add_charge_notes_lbl.config(font= (font, size, bold_ind))
add_charge_notes_lbl.config(fg= add_color_2)
## entry
add_charge_notes_entry = tk.Entry(root, width= entry_width)
add_charge_notes_entry.config(font= (font, size))

##### additional charge 2
## label
add_charge_lbl_2 = tk.Label(root, text= 'Additional Charge 2:')
add_charge_lbl_2.config(font= (font, size, bold_ind))
add_charge_lbl_2.config(fg= add_color)
## entry
add_charge_entry_2 = tk.Entry(root, width= entry_width)
add_charge_entry_2.config(font= (font, size))

##### additional charge notes
## label
add_charge_notes_lbl_2 = tk.Label(root, text= 'Notes on additional charge 2:')
add_charge_notes_lbl_2.config(font= (font, size, bold_ind))
add_charge_notes_lbl_2.config(fg= add_color_2)
## entry
add_charge_notes_entry_2 = tk.Entry(root, width= entry_width)
add_charge_notes_entry_2.config(font= (font, size))

##### my notes
## label
my_notes_lbl = tk.Label(root, text= 'My notes:')
my_notes_lbl.config(font= (font, size, bold_ind))
my_notes_lbl.config(fg= notes_color)
## entry
my_notes_entry = tk.Entry(root, width= entry_width)
my_notes_entry.config(font= (font, size))

##### reminders
## label
cust_reminder_lbl = tk.Label(root, text= 'Set Customer Reminder:')
cust_reminder_lbl.config(font= (font, size, bold_ind))
cust_reminder_lbl.config(fg= cust_remind_color)
## entry
cust_reminder_entry = tk.Entry(root, width= entry_width)
cust_reminder_entry.config(font= (font, size))

##### include check button
var = tk.IntVar()
chk = tk.Checkbutton(root, text='Do Not Include?', variable=var,
            selectcolor= 'blue', command= on_check, fg= off_color)
chk.config(font= (font, size, bold_ind))
# var.get()

##### main loop
root.mainloop()

# check if we completed the main program all the way through or not
# - if we did not, then exit with an error
if final_end == False:
    exit('Failed to complete full program.')

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# INITIALIZE DOC WRITING

# ADD DATA i.e. total column, and current date for output doc
# create total column
tabl['TOTAL'] = tabl['MONTHLY_CHARGE'] + tabl['ADD_CHARGE'] + tabl['ADD_CHARGE_2']
# add included
tabl['INCLUDED'] = included

# current date
DATE = datetime.datetime.today().strftime('%B %d, %Y')

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# HELPER FUNCTION to grab appropriate data

# grab a row from the table and process it for data to place in doc
def grab_data(row):
    # global data
    global data

    # convert to dict
    data = dict(tabl.iloc[row])
    # add dates
    data['DATE'] = DATE
    data['MONTH'] = MONTH
    data['YEAR'] = YEAR
    # if no additional charge then show nothing, else format float
    if data['ADD_CHARGE'] == 0:
        data['ADD_CHARGE'] = ''
    else:
        data['ADD_CHARGE'] = '${:,.2f}'.format(data['ADD_CHARGE'])

    # if no additional charge then show nothing, else format float
    if data['ADD_CHARGE_2'] == 0:
        data['ADD_CHARGE_2'] = ''
    else:
        data['ADD_CHARGE_2'] = '${:,.2f}'.format(data['ADD_CHARGE_2'])

    # format floats
    data['MONTHLY_CHARGE'] = '${:,.2f}'.format(data['MONTHLY_CHARGE'])
    data['TOTAL'] = '${:,.2f}'.format(data['TOTAL'])
    return

##### set data originally to hold nothining
data = None

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
print("Writing files....")

### first files
doc_pp = DocxTemplate(fpage)
doc_gg = DocxTemplate(fpage)
invoice_doc_path_pp = sub_path_pp+'/pp_invoice_'+MONTH+'_'+YEAR+'.docx'
invoice_doc_path_gg = sub_path_gg+'/gg_invoice_'+MONTH+'_'+YEAR+'.docx'
doc_pp.save(invoice_doc_path_pp)
doc_gg.save(invoice_doc_path_gg)

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# CONCATENATE DOC through loop

##### 'rp' indicator and values --- specific use
rp_inc = 0 # increment
rp_ind = False # rp process completion indicator
rp_months = []
rp_adds = []
rp_notes = []
rp_adds_2 = []
rp_notes_2 = []
#####

# loop
for i in range(N):
    # progress print
    print(i)

    # do not include the current row
    if included[i] == 0:
        continue # skip it

    ##### create temp doc

    # grab data and process it
    grab_data(i)

    ### temp doc
    # rp conditionals
    if tabl['RP_INDICATOR'][i] == 1:
        # increment
        rp_inc += 1
        # have we seen three?
        if rp_inc == 3:
            # first append
            rp_months.append(data['MONTHLY_CHARGE'])
            rp_adds.append(data['ADD_CHARGE'])
            rp_notes.append(data['ADD_CHARGE_NOTES'])
            rp_adds_2.append(data['ADD_CHARGE_2'])
            rp_notes_2.append(data['ADD_CHARGE_NOTES_2'])
            ### new data
            rm = data['CUST_REMINDER'] # cust reminder
            # save data for processing
            data = {}
            data['DATE'] = DATE
            data['MONTH'] = MONTH
            data['YEAR'] = YEAR
            data['M1'] = rp_months[0]
            data['M2'] = rp_months[1]
            data['M3'] = rp_months[2]
            data['A1'] = rp_adds[0]
            data['A2'] = rp_adds[1]
            data['A3'] = rp_adds[2]
            data['A1_NOTES'] = rp_notes[0]
            data['A2_NOTES'] = rp_notes[1]
            data['A3_NOTES'] = rp_notes[2]
            data['A1_2'] = rp_adds_2[0]
            data['A2_2'] = rp_adds_2[1]
            data['A3_2'] = rp_adds_2[2]
            data['A1_NOTES_2'] = rp_notes_2[0]
            data['A2_NOTES_2'] = rp_notes_2[1]
            data['A3_NOTES_2'] = rp_notes_2[2]
            data['CUST_REMINDER'] = rm
            ### loop for total sum
            total_sum = 0
            # month charge
            for ele in rp_months:
                if ele != '':
                    total_sum += float(ele.strip('$'))
            # add charge
            for ele in rp_adds:
                if ele != '':
                    total_sum += float(ele.strip('$'))
            # add charge 2
            for ele in rp_adds_2:
                if ele != '':
                    total_sum += float(ele.strip('$'))
            data['TOTAL'] = '${:,.2f}'.format(total_sum)
            rp_ind = True # finished the 'rp' process, note it
        else:
            # nothing to do yet but to append the data
            rp_months.append(data['MONTHLY_CHARGE'])
            rp_adds.append(data['ADD_CHARGE'])
            rp_notes.append(data['ADD_CHARGE_NOTES'])
            rp_adds_2.append(data['ADD_CHARGE_2'])
            rp_notes_2.append(data['ADD_CHARGE_NOTES_2'])
            continue

    ### temp doc
    if rp_ind: # rp stuff
        temp = DocxTemplate(invoice_template_mult)
        rp_ind = False # revert rp process completion indicator for next round
    elif tabl['Z_INDICATOR'][i] == 1: # Z stuff
        if tabl['DIRECTION'][i].strip() == 'PP':
            temp = DocxTemplate(invoice_template_z)
        else:
            temp = DocxTemplate(invoice_template_z_gg)
    elif tabl['V_INDICATOR'][i] == 1: # v stuff
        temp = DocxTemplate(invoice_template_v)
    else: # default
        temp = DocxTemplate(invoice_template)
    # render the doc
    temp.render(data)

    # if no email indicator, merge to paper invoice
    if tabl['EMAIL_IND'][i] == 0:
        ### merge docs
        # check for DIRECTION
        if tabl['DIRECTION'][i].strip() == 'PP':
            # path
            doc_path = invoice_doc_path_pp
        else:
            # path
            doc_path = invoice_doc_path_gg
        # load master doc
        doc = Document(doc_path)
        # add page break
        doc.add_page_break()
        # compose it
        composer = Composer(doc)
        # merge it with the temp doc
        composer.append(temp)

        # save the final output
        composer.save(doc_path)
    else:
        # check for DIRECTION
        if tabl['DIRECTION'][i].strip() == 'PP':
            # path
            temp_emails_path = emails_path_pp+'/'+tabl['EMAIL_ADDRESS'][i].strip()
        else:
            # path
            temp_emails_path = emails_path_gg+'/'+tabl['EMAIL_ADDRESS'][i].strip()
        # make directory
        os.mkdir(temp_emails_path)
        # temp email paths word doc file name
        temp_emails_path_doc = temp_emails_path+'/invoice_'+MONTH+'_'
        temp_emails_path_doc += '_'.join(tabl['FIRST'][i].strip().lower().split())
        temp_emails_path_doc += '_'
        temp_emails_path_doc += tabl['LAST'][i].strip().lower()
        temp_emails_path_doc += '.docx'
        # save doc
        temp.save(temp_emails_path_doc)

print("Files written.")

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# CREATE EXCEL SHEET for output
# order output table
tabl['AMT_PAID'] = [0]*N
tabl['AMT_LEFT'] = ['']*N
tabl['RECORD'] = [':::::']*N
tabl['REC_DATE'] = [':::::']*N
tabl['OBT_DATE'] = [':::::']*N
tabl['PAY_NOTES'] = [':::::']*N
tabl['CLR_DATE'] = [':::::']*N
tabl = tabl[['DIRECTION','FIRST','LAST','TOTAL',
            'AMT_PAID', 'AMT_LEFT','RECORD', 'REC_DATE', 'OBT_DATE', 'PAY_NOTES', 'CLR_DATE',
            'STREET_ADDRESS','CITY_ADDRESS','MY_NOTES','MONTHLY_CHARGE',
            'ADD_CHARGE','ADD_CHARGE_NOTES','ADD_CHARGE_2','ADD_CHARGE_NOTES_2',
            'CUST_REMINDER','INCLUDED']]

# letters
letters = ['A','B','C','D','E','F','G','H','I','J','K','L','M',
           'N','O','P','Q','R','S','T','U','V','W','X','Y','Z']
let_index = 0

# taken from:
# https://xlsxwriter.readthedocs.io/example_pandas_column_formats.html
# Create a Pandas Excel writer using XlsxWriter as the engine
invoice_data = path+'/data_'+MONTH+'_'+YEAR+'.xlsx' # file path
writer = pd.ExcelWriter(invoice_data, engine='xlsxwriter')
# Convert the dataframe to an XlsxWriter Excel object.
tabl.to_excel(writer, sheet_name='Sheet1', index= False)
# Get the xlsxwriter workbook and worksheet objects.
workbook  = writer.book
worksheet = writer.sheets['Sheet1']
# Set the column width and format.
worksheet.set_column(letters[let_index]+':'+letters[let_index], 9, None) # direction
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 18, None) # first
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 18, None) # last
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 9, None) # total
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 9, None) # amount paid
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 9, None) # amount left
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 9, None) # record
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 9, None) # rec date
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 9, None) # obt date
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 9, None) # pay notes
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 9, None) # clear date
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 18*1.5, None) # street
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 18*1.5, None) # city
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 18*2.5, None) # my notes
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 18, None) # monthly charge
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 18, None) # additional charge
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 18*2.5, None) # add charge notes
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 18, None) # additional charge 2
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 18*2.5, None) # add charge notes 2
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 18, None) # cust reminder
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 9, None) #included

# Close the Pandas Excel writer and output the Excel file.
writer.save()

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# CREATE EXCEL SHEET for shorter output
# 'short' path
short_path = path+'/short_data_'+MONTH+'_'+YEAR+'.xlsx'
# create shorter data
short_tabl = tabl[['FIRST','LAST','STREET_ADDRESS']].copy()
# add month service column
short_tabl[MONTH+' '+YEAR] = tabl['TOTAL'].copy()
# add recrods column
short_tabl['Records'] = ['']*N

# create excel output
short_writer = pd.ExcelWriter(short_path, engine='xlsxwriter')
# Convert the dataframe to an XlsxWriter Excel object.
short_tabl.to_excel(short_writer, sheet_name='Sheet1', index= False)
# Get the xlsxwriter workbook and worksheet objects.
short_workbook  = short_writer.book
short_workbook = short_writer.sheets['Sheet1']
# Set the column width and format.
short_workbook.set_column('A:A', 15, None) # first
short_workbook.set_column('B:B', 15, None) # last
short_workbook.set_column('C:C', 25, None) # street
short_workbook.set_column('D:D', 15, None) # total
short_workbook.set_column('E:E', 15, None) # records
# MARGINS
short_workbook.set_margins(left=0.1, right=0.1, top=0.1, bottom=0.1)
# Close the Pandas Excel writer and output the Excel file.
short_writer.save()

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
## main program done
print('Main program done.')

# total total sum
print('Total: ${:,.2f}'.format(tabl['TOTAL'].values.sum()))

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# ## RUN AUX PROGRAM
# string_exe = 'python code/aux.py ' + .......
# os.system(string_exe)

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------

print('Complete!!!!!!')

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------


#!/usr/bin/python3
""" Your first endpoint (route) will be to return the status of your API """
from api.v1.views import app_views
from flask import jsonify, abort, request
from models import storage
from models.state import State
import json

@app_views.route("/states/", methods=['GET', 'POST'])
@app_views.route("/states", methods=['GET', 'POST'])
def show_states():
    """ returns list of states """
    if request.method == 'GET':
        lista = []
        states = storage.all(State).values()
        for state in states:
            lista.append(state.to_dict())
        return jsonify(lista)
    elif request.method == 'POST':
        if request.json:
            new_dict = request.get_json()
            if "name" in new_dict.keys():
                new_state = State(**new_dict)
                storage.new(new_state)
                storage.save()
                return jsonify(new_state.to_dict()), 201
            else:
                abort(400, description="Missing name")
        else:
            abort(400, description="Not a JSON")

@app_views.route("states/<state_id>/", methods=['GET', 'DELETE', 'PUT'])
@app_views.route("states/<state_id>", methods=['GET', 'DELETE', 'PUT'])
def show_state(state_id):
    """ returns state data """
    if request.method == 'GET':
        states = storage.all(State).values()
        for state in states:
            if state.id == state_id:
                return jsonify(state.to_dict())
        abort(404)
    elif request.method == 'DELETE':
        states = storage.all(State).values()
        for state in states:
            if state.id == state_id:
                state.delete()
                storage.save()
                return jsonify({}), 200
        abort(404)
    elif request.method == 'PUT':
        if request.json:
            new_dict = request.get_json()
            states = storage.all(State).values()
            for state in states:
                if state.id == state_id:
                    state.name = new_dict['name']
                    storage.save()
                    return jsonify(state.to_dict()), 200
            abort(404)
        else:
            abort(400, description="Not a JSON")


import functools
import json
from flask import g
import uuid
import hashlib
from flask_login import current_user

def rollback_errors(db_operation):
    @functools.wraps(db_operation)
    def wrapper(database, *args, **kwargs):

        try:
            return db_operation(database, *args, **kwargs)
        except Exception as ex:
            database.connection.rollback()
            raise ex

    return wrapper

def save_search_event(aliases_data):
    user_id = current_user.user_id
    search_param_string = user_id + json.dumps(aliases_data, sort_keys=True)
    hashed_search_params = hashlib.sha256(search_param_string.encode()).hexdigest()
    _db_insert_search_event(g.database, user_id, hashed_search_params)

@rollback_errors
def _db_insert_search_event(database, user_id, hashed_search_params):
    database.cursor.execute(
        """
        INSERT INTO  SEARCH_RESULTS(search_result_id, user_id, hashed_search_params)
        VALUES ( uuid_generate_v4(), %(user_id)s, %(params)s);
        """,
        {"user_id": uuid.UUID(user_id).hex, "params": hashed_search_params,},
    )


# -*- coding: utf-8 -*-
"""
Created on Sun Apr  8 17:09:25 2018

@author: Debolina
"""

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.cross_validation import cross_val_score
from sklearn.preprocessing import LabelEncoder
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import confusion_matrix
from sklearn.svm import LinearSVC
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import train_test_split
import sklearn.linear_model
import seaborn as sns

data1 = pd.read_csv('Pima.csv')
print('First few observations')
print(data1.head())
data1 = data1.drop('index', axis=1)
number = LabelEncoder()
data1['type'] = number.fit_transform(data1['type'].astype(str))
print('After preprocessing...')
print(data1.head())
train1, test1 = train_test_split(data1, test_size=0.3)
pred = ['npreg', 'glu', 'bp', 'skin', 'bmi', 'ped', 'age']

print('Correlation Matrix')
corr = data1.corr()
plt.figure(figsize=(12, 7))
sns.heatmap(corr, annot=True)

print('Logistic Regression')
model1 = sklearn.linear_model.LogisticRegression()
x_train = train1[pred].values
y_train = train1['type'].values
x_test = test1[pred].values
y_test = test1['type'].values

model1.fit(x_train, y_train)
predicted1 = model1.predict(x_test)
print(confusion_matrix(y_test, predicted1))

modeltest1 = sklearn.linear_model.LogisticRegression()
print('Cross val score')
print(np.mean(cross_val_score(modeltest1, x_train, y_train, cv=5)))

print("\nRANDOM FOREST CLASSIFIER")
model2 = RandomForestClassifier()
model2.fit(x_train, y_train)
prediction2 = model2.predict(x_test)
print(confusion_matrix(y_test, prediction2))
modelchk1 = RandomForestClassifier()
print(np.mean(cross_val_score(modelchk1, x_train, y_train, cv=6)))

print("\nSUPPORT VECTOR MACHINES")
clf1 = LinearSVC(random_state=0)
clf1.fit(x_train, y_train)
prediction3 = clf1.predict(x_test)
print(confusion_matrix(y_test, prediction3))
clftest = LinearSVC(random_state=0)
print(np.mean(cross_val_score(clftest, x_train, y_train, cv=6)))

print("\nMULTILAYERED PERCEPTRON")
clf2 = MLPClassifier(solver='lbfgs')
clf2.fit(x_train, y_train)
prediction4 = clf2.predict(x_test)
print(confusion_matrix(y_test, prediction4))
modelclf1 = MLPClassifier(solver='lbfgs')
print(np.mean(cross_val_score(modelclf1, x_train, y_train, cv=7)))