import datetime
import glob
import math
import os
import traceback

import psutils.custom_errors as cerr
import psutils.globals as psu_globals
import psutils.argtype as psu_at
import psutils.script_action as psu_act
import psutils.string as psu_str
from psutils.print_methods import *

##############################

### Argument globals ###

## Argument strings ("ARGSTR_")
ARGSTR_DSTDIR_GLOBAL = '--dstdir-global'
ARGSTR_TASKLIST = '--tasklist'
ARGSTR_SRCLIST = '--srclist'
ARGSTR_SRCLIST_ROOTED = '--srclist-rooted'
ARGSTR_SRCLIST_PREFIX = '--srclist-prefix'
ARGSTR_SRCLIST_PREFIX_DST = '--srclist-prefix-dst'
ARGSTR_SRCLIST_SUFFIX = '--srclist-suffix'
ARGSTR_SRCLIST_ROOTDIR = '--srclist-rootdir'
ARGSTR_SRCLIST_DELIM = '--srclist-delim'
ARGSTR_SRCLIST_NOGLOB = '--srclist-noglob'
ARGSTR_SRCLIST_IGNORE_MISSING = '--srclist-ignore-missing'
ARGSTR_SYNC_TREE = '--sync-tree'
ARGSTR_TRANSPLANT_TREE = '--transplant-tree'
ARGSTR_COLLAPSE_TREE = '--collapse-tree'

## Argument help info ("ARGHLP_", only when needed outside of argparse)
ARGHLP_SRCLIST_FORMAT = None  # set globally in pre_argparse()
ARGHLP_SRCLIST_ROOTED_FORMAT = None  # set globally in pre_argparse()

## Argument groups ("ARGGRP_" lists of "ARGSTR_" argument strings)
ARGGRP_SRC = [ARGSTR_SRCLIST, ARGSTR_SRCLIST_ROOTED]
ARGGRP_DST = [ARGSTR_DSTDIR_GLOBAL]
ARGGRP_SYNC_MODE = [ARGSTR_SYNC_TREE, ARGSTR_TRANSPLANT_TREE]

## Argument collections ("ARGCOL_" lists of "ARGGRP_" argument strings)
ARGCOL_MUT_EXCL_SET = [
    ARGGRP_SYNC_MODE
]

## Argument modes ("ARGMOD_", used for mutually exclusive arguments that control the same mechanic)
# (It's generally better to use a single argument with multiple choices, but sometimes we want
#  to use multiple `action='store_true'` arguments instead.)
ARGMOD_SYNC_MODE_NULL = 0
ARGMOD_SYNC_MODE_SYNC_TREE = 1
ARGMOD_SYNC_MODE_TRANSPLANT_TREE = 2

## Argument defaults ("ARGDEF_")
ARGDEF_SRCLIST_DELIM = ','
ARGDEF_BUNDLEDIR = os.path.realpath(os.path.join(os.path.expanduser('~'), 'scratch', 'task_bundles'))

##############################

### Custom globals ###

PATH_SEPARATORS_LIST = psu_globals.PATH_SEPARATORS_LIST
PATH_SEPARATORS_CAT = ''.join(PATH_SEPARATORS_LIST)

SYNC_MODE_GLOBAL = None

PATH_TYPE_UNKNOWN = 0
PATH_TYPE_FILE = 1
PATH_TYPE_DIR = 2
PATH_TYPE_DNE = 3

##############################

def pre_argparse(srclist_delim=ARGDEF_SRCLIST_DELIM,
                 argstr_dst=None):
    global ARGHLP_SRCLIST_FORMAT, ARGHLP_SRCLIST_ROOTED_FORMAT

    provided_srclist_delimiter = psu_act.get_script_arg_values(ARGSTR_SRCLIST_DELIM)
    srclist_delimiter = provided_srclist_delimiter if provided_srclist_delimiter is not None else srclist_delim

    ARGHLP_SRCLIST_FORMAT = ' '.join([
        "\n(1) All 'src_path' line items (only when argument {} directory is provided)".format('/'.join([argstr for argstr in [argstr_dst, ARGSTR_DSTDIR_GLOBAL] if argstr is not None])),
        "\n(2) A single 'src_path{}dst_dir' line at top followed by all 'src_path' line items,".format(srclist_delimiter),
        "where 'dst_dir' is a directory used for all items in the list",
        "\n(3) All 'src_path{}dst_path' line items".format(srclist_delimiter)
    ])

    ARGHLP_SRCLIST_ROOTED_FORMAT = ' '.join([
        "A header line is expected containing 'src_rootdir[{}dst_rootdir]' to signify that".format(srclist_delimiter),
        "the folder structure from 'src_rootdir' down to 'src_path' for each 'src_path[{}dst_rootdir]'".format(srclist_delimiter),
        "line item will be replicated within the destination root directory.",
        "\nIf 'src_path' items are absolute paths, each is expected to start with the",
        "'src_rootdir' path EXACTLY as it appears in the header. If a 'src_path' item does not",
        "start with 'src_rootdir', then the 'src_path' item will be treated as a relative path",
        "within 'src_rootdir'.",
    ])

def add_srclist_arguments(parser,
                          srclist_delim=ARGDEF_SRCLIST_DELIM):
    parser.add_argument(
        '-dg', ARGSTR_DSTDIR_GLOBAL,
        type=psu_at.ARGTYPE_PATH(argstr=ARGSTR_DSTDIR_GLOBAL,
            existcheck_fn=os.path.isfile,
            existcheck_reqval=False,
            accesscheck_reqtrue=os.W_OK,
            accesscheck_parent_if_dne=True),
        help=' '.join([
            "Path to directory in which all output files will be created.",
            "This destination directory will override all destination paths in source lists.",
        ])
    )

    parser.add_argument(
        '-tl', ARGSTR_TASKLIST,
        type=psu_at.ARGTYPE_PATH(argstr=ARGSTR_TASKLIST,
            existcheck_fn=os.path.isfile,
            existcheck_reqval=True,
            accesscheck_reqtrue=os.R_OK),
        nargs='+',
        action='append',
        help=' '.join([
            "[write me]"
        ])
    )

    parser.add_argument(
        '-sl', ARGSTR_SRCLIST,
        type=psu_at.ARGTYPE_PATH(argstr=ARGSTR_SRCLIST,
            existcheck_fn=os.path.isfile,
            existcheck_reqval=True,
            accesscheck_reqtrue=os.R_OK),
        nargs='+',
        action='append',
        help=' '.join([
            "Path to textfile list of 'src_path[{}dst_path]' tasks to be performed.".format(ARGSTR_SRCLIST_DELIM),
            ARGHLP_SRCLIST_FORMAT,
        ])
    )
    parser.add_argument(
        '-slr', ARGSTR_SRCLIST_ROOTED,
        type=psu_at.ARGTYPE_PATH(argstr=ARGSTR_SRCLIST_ROOTED,
            existcheck_fn=os.path.isfile,
            existcheck_reqval=True,
            accesscheck_reqtrue=os.R_OK),
        nargs='+',
        action='append',
        help=' '.join([
            "Path to textfile list of 'src_path[{}dst_rootdir]' tasks to be performed.".format(ARGSTR_SRCLIST_DELIM),
            ARGHLP_SRCLIST_ROOTED_FORMAT,
        ])
    )

    parser.add_argument(
        '-slp', ARGSTR_SRCLIST_PREFIX,
        type=psu_at.ARGTYPE_PATH(argstr=ARGSTR_SRCLIST_PREFIX,
            abspath_fn=None,
            existcheck_fn=os.path.isdir,
            existcheck_reqval=True),
        help=' '.join([
            "Directory path to prepend to all source paths in {} and {} textfiles.".format(ARGSTR_SRCLIST, ARGSTR_SRCLIST_ROOTED),
        ])
    )
    parser.add_argument(
        '-slpd', ARGSTR_SRCLIST_PREFIX_DST,
        type=psu_at.ARGTYPE_PATH(argstr=ARGSTR_SRCLIST_PREFIX_DST,
            abspath_fn=None,
            existcheck_fn=os.path.isfile,
            existcheck_reqval=False,
            accesscheck_reqtrue=os.W_OK,
            accesscheck_parent_if_dne=True),
        help=' '.join([
            "Directory path to prepend to all destination paths in {} and {} textfiles.".format(ARGSTR_SRCLIST, ARGSTR_SRCLIST_ROOTED),
        ])
    )
    parser.add_argument(
        '-sls', ARGSTR_SRCLIST_SUFFIX,
        type=str,
        help=' '.join([
            "String to append to all source paths in {} and {} textfiles.".format(ARGSTR_SRCLIST, ARGSTR_SRCLIST_ROOTED),
        ])
    )

    parser.add_argument(
        '-slrd', ARGSTR_SRCLIST_ROOTDIR,
        type=psu_at.ARGTYPE_PATH(argstr=ARGSTR_SRCLIST_PREFIX,
            abspath_fn=None,
            existcheck_fn=os.path.isdir,
            existcheck_reqval=True),
        help=' '.join([
            "Root directory used for source paths in {} textfiles when determining relative".format(ARGSTR_SRCLIST),
            "paths within destination directories. This path will be prepended to all source paths",
            "that do not already start with this absolute path.",
        ])
    )

    parser.add_argument(
        ARGSTR_SRCLIST_DELIM,
        type=str,
        default=srclist_delim,
        help=' '.join([
            "Delimiter used to separate source and destination paths in {} and {} textfiles.".format(ARGSTR_SRCLIST, ARGSTR_SRCLIST_ROOTED),
        ])
    )
    parser.add_argument(
        ARGSTR_SRCLIST_NOGLOB,
        action='store_true',
        help=' '.join([
            "Do not interpret '*' character as a wildcard for path-globbing in {} and {} textfiles.".format(ARGSTR_SRCLIST, ARGSTR_SRCLIST_ROOTED),
        ])
    )
    parser.add_argument(
        '-slim', ARGSTR_SRCLIST_IGNORE_MISSING,
        action='store_true',
        help="[write me]"
    )

    parser.add_argument(
        '-st', ARGSTR_SYNC_TREE,
        action='store_true',
        help=' '.join([
            "Copy contents of source directories directly into destination directories."
            "\nIf neither the {} nor {} options are specified, source directory paths".format(ARGSTR_SYNC_TREE, ARGSTR_TRANSPLANT_TREE),
            "that end with '{}' are automatically treated this way.".format(PATH_SEPARATORS_LIST),
        ])
    )
    parser.add_argument(
        '-tt', ARGSTR_TRANSPLANT_TREE,
        action='store_true',
        help=' '.join([
            "Copy contents of source directories into a folder under destination directories",
            "bearing the name of the source directory (i.e. 'dstdir/srcdir_name/').",
            "\nIf neither the {} nor {} options are specified, source directory paths".format(ARGSTR_SYNC_TREE, ARGSTR_TRANSPLANT_TREE),
            "that do not end with '{}' are automatically treated this way.".format(PATH_SEPARATORS_LIST),
        ])
    )
    parser.add_argument(
        '-ct', ARGSTR_COLLAPSE_TREE,
        action='store_true',
        help=' '.join([
            "Copy all files within source directories into destination directories one level deep,"
            "effectively collapsing the copied source directory tree.",
        ])
    )

def write_task_bundles(task_list, tasks_per_bundle,
                       dstdir, bundle_prefix,
                       task_delim=',', task_fmt='%s'):
    try:
        import numpy as np
        imported_numpy = False
    except ImportError:
        imported_numpy = False
        if task_fmt != '%s':
            raise
    bundle_prefix = os.path.join(dstdir, '{}_{}'.format(bundle_prefix, datetime.now().strftime("%Y%m%d%H%M%S")))
    jobnum_total = int(math.ceil(len(task_list) / float(tasks_per_bundle)))
    jobnum_fmt = psu_str.get_index_fmtstr(jobnum_total)
    bundle_file_list = []
    print("Writing task bundle text files in directory: {}".format(dstdir))
    for jobnum, tasknum in enumerate(range(0, len(task_list), tasks_per_bundle), 1):
        bundle_file = '{}_{}.txt'.format(bundle_prefix, jobnum_fmt.format(jobnum))
        task_bundle = task_list[tasknum:tasknum+tasks_per_bundle]
        if len(task_bundle) == 0:
            with open(bundle_file, 'w'):
                pass
        elif imported_numpy:
            np.savetxt(bundle_file, task_bundle, fmt=task_fmt, delimiter=task_delim)
        else:
            join_task_items = type(task_bundle[0]) in (tuple, list)
            with open(bundle_file, 'w') as bundle_file_fp:
                for task in task_bundle:
                    task_line = str(task) if not join_task_items else task_delim.join([str(arg) for arg in task])
                    bundle_file_fp.write(task_line+'\n')
        bundle_file_list.append(bundle_file)
    return bundle_file_list

def read_task_bundle(bundle_file, args_dtype=str, args_delim=',', header_rows=0,
                     ncol_strict=True, ncol_min=None, ncol_max=None,
                     allow_1d_task_list=True, read_header=False):
    try:
        import numpy as np
        imported_numpy = False
    except ImportError:
        imported_numpy = False

    task_list_ncols_min = None
    task_list_ncols_max = None

    if ncol_strict and imported_numpy:
        loadtxt_dtype = np.dtype(str) if args_dtype is str else args_dtype
        if read_header:
            loadtxt_skiprows = 0
            loadtxt_maxrows = header_rows
        else:
            loadtxt_skiprows = header_rows
            loadtxt_maxrows = None
        try:
            task_list = np.loadtxt(bundle_file, dtype=loadtxt_dtype, delimiter=args_delim,
                                   skiprows=loadtxt_skiprows, max_rows=loadtxt_maxrows,
                                   ndmin=2)
        except ValueError as e:
            if str(e).startswith("Wrong number of columns"):
                traceback.print_exc()
                raise cerr.DimensionError("Inconsistent number of columns in `bundle_file`: {}".format(bundle_file))
            else:
                raise
        task_list_ncols = task_list.shape[1] if task_list.ndim == 2 else task_list.shape[0]
        task_list_ncols_min = task_list_ncols
        task_list_ncols_max = task_list_ncols
        if ncol_min is not None and task_list_ncols is not None and task_list_ncols < ncol_min:
            raise cerr.DimensionError("`bundle_file` line has {} columns, less than required minimum ({}): {}".format(
                                 task_list_ncols, ncol_min, bundle_file))
        if ncol_max is not None and task_list_ncols is not None and task_list_ncols > ncol_max:
            raise cerr.DimensionError("`bundle_file` line has {} columns, more than required maximum ({}): {}".format(
                                 task_list_ncols, ncol_max, bundle_file))
        if allow_1d_task_list and task_list_ncols == 1:
            task_list = task_list[:, 0]

        task_list = task_list.tolist()

    else:
        with open(bundle_file, 'r') as bundle_file_fp:
            if read_header:
                task_list = []
                for i in range(header_rows):
                    header_line = bundle_file_fp.readline().strip()
                    if header_line != '':
                        task_list.append(header_line)
            else:
                task_list = [line for line in bundle_file_fp.read().splitlines() if line.strip() != '']
                if header_rows > 0 and len(task_list) > 0:
                    task_list = task_list[header_rows:]
        if len(task_list) > 0:
            if args_delim is not None:
                if type(args_dtype) in (tuple, list):
                    task_list = [[args_dtype[col_num](arg.strip()) for col_num, arg in enumerate(task.split(args_delim))] for task in task_list]
                else:
                    task_list = [[args_dtype(arg.strip()) for arg in task.split(args_delim)] for task in task_list]

                task_list_ncols = None
                if ncol_min is not None or ncol_max is not None:
                    task_list_ncols = [len(task) for task in task_list]
                    task_list_ncols_min = min(task_list_ncols)
                    task_list_ncols_max = max(task_list_ncols)
                    if task_list_ncols_min == task_list_ncols_max:
                        task_list_ncols = task_list_ncols_min
                elif ncol_strict:
                    first_task_ncols = len(task_list[0])
                    if all(len(task) == first_task_ncols for task in task_list):
                        task_list_ncols = first_task_ncols
                if ncol_strict and task_list_ncols is None:
                    raise cerr.DimensionError("Inconsistent number of columns in `bundle_file`: {}".format(bundle_file))

                if allow_1d_task_list and not read_header:
                    task_list = [task[0] if len(task) == 1 else task for task in task_list]
            elif not allow_1d_task_list or read_header:
                task_list = [[args_dtype(arg.strip())] for arg in task_list]
            else:
                task_list = [args_dtype(arg.strip()) for arg in task_list]

    if ncol_min is not None and task_list_ncols_min is not None and task_list_ncols_min < ncol_min:
        raise cerr.DimensionError("`bundle_file` line has {} columns, less than required minimum ({}): {}".format(
                             task_list_ncols_min, ncol_min, bundle_file))
    if ncol_max is not None and task_list_ncols_min is not None and task_list_ncols_max > ncol_max:
        raise cerr.DimensionError("`bundle_file` line has {} columns, more than required maximum ({}): {}".format(
                             task_list_ncols_max, ncol_max, bundle_file))

    if read_header and len(task_list) == 1 and type(task_list[0]) is list:
        task_list = task_list[0]

    return task_list

class Tasklist2(object):
    def __init__(self, header=None, tasks=None, argname_taskindex_dict=None):
        self.header = header
        self.tasks = tasks
        self.argname_taskindex_dict = argname_taskindex_dict

    def read(self, tasklist_file_or_buff, col_delim=',',
             uniform_dtype=str,
             col_argname_dtype_def=None,  # collection of tuples like ((0, str))
             args=None, col_argstr_def=None,
             ncol_min=None, ncol_max=None, ncol_strict=True,
             expect_header=None, check_header_for_argnames=None,
             allow_missing_header=None, ncol_strict_header_separate=False):

        header_items = None
        task_array = []

        if col_argname_dtype_def is not None and col_argstr_def is not None:
            raise cerr.InvalidArgumentError(
                "`col_argname_dtype_def` and `col_argstr_def` arguments are mutually exclusive"
            )

        if args is None and col_argname_dtype_def is None and col_argstr_def is None:
            if expect_header is None:
                expect_header = False
            if check_header_for_argnames is True or allow_missing_header is True:
                raise cerr.InvalidArgumentError(
                    "`check_header_for_argnames` or `allow_missing_header` can only be True "
                    "when either `col_argname_dtype_def` or `col_argstr_def` are provided"
                )
            check_header_for_argnames = False
            allow_missing_header = False
        else:
            if expect_header is None:
                expect_header = True
            if check_header_for_argnames is None:
                check_header_for_argnames = True
            if col_argname_dtype_def is None and col_argstr_def is None:
                if allow_missing_header is True:
                    raise cerr.InvalidArgumentError(
                        "`allow_missing_header` can only be True "
                        "when either `col_argname_dtype_def` or `col_argstr_def` are provided"
                    )
                allow_missing_header = False
            elif allow_missing_header is None:
                allow_missing_header = True

        col_argname_dtype_def_derived = None

        if col_argname_dtype_def is not None:
            col_argname_dtype_def_derived = col_argname_dtype_def
        elif col_argstr_def is not None:
            if args is None:
                raise cerr.InvalidArgumentError("`args` argument must be provided along with `col_argstr_def`")
            col_argname_dtype_def_derived = [(argstr, args.argstr2argtype[argstr]) for argstr in col_argstr_def]

        with open(tasklist_file_or_buff, 'r') as tasklist_fp:

            first_line = tasklist_fp.readline().strip()
            if first_line == '':
                return
            first_line_items = [item.strip() for item in first_line.split(col_delim)]
            line_num = 1

            if not expect_header:
                task_line = first_line
                task_line_items = first_line_items
            else:
                header_line = first_line
                header_items = first_line_items

                if check_header_for_argnames:
                    assert col_argname_dtype_def is not None or col_argstr_def is not None
                    assert col_argname_dtype_def_derived is not None

                    header_argname_set = set(header_items)
                    col_argname_def_list = [argname for argname, argtype in col_argname_dtype_def_derived]
                    col_argname_def_set = set(col_argname_def_list)

                    if header_argname_set.issubset(col_argname_def_set):
                        argname_dtype_dict = {argname: dtype for argname, dtype in col_argname_dtype_def_derived}
                        col_argname_dtype_def_derived = [(argname, argname_dtype_dict[argname]) for argname in header_items]

                    elif args is not None and header_argname_set.issubset(set(args.all_argstr)):
                        if col_argname_dtype_def is not None:
                            warning(
                                "Argument names in header line ({}) of tasklist file ({}) are not a "
                                "subset of `col_argname_dtype_def` argument names ({}), but are instead "
                                "a subset of script ArgumentPasser argument strings which will be "
                                "leveraged instead.".format(
                                    header_line, tasklist_file_or_buff, col_argname_def_set
                                )
                            )
                        elif col_argstr_def is not None:
                            warning(
                                "Argument strings in header line ({}) of tasklist file ({}) are not "
                                "the expected set of `col_argstr_def` argument strings ({})".format(
                                    header_line, tasklist_file_or_buff, col_argname_def_set
                                )
                            )
                        col_argname_dtype_def_derived = [(argstr, args.argstr2argtype[argstr]) for argstr in header_items]

                    elif allow_missing_header:
                        if col_argname_dtype_def is not None:
                            warning(
                                "Tasklist file ({}) does not have an acceptable header line, "
                                "so it will be assumed that columns follow the provided "
                                "`col_argname_dtype_def` order precisely as given: {}".format(
                                    tasklist_file_or_buff, col_argname_def_list
                                )
                            )
                        elif col_argstr_def is not None:
                            warning(
                                "Tasklist file ({}) does not have an acceptable header line, "
                                "so it will be assumed that columns follow the provided "
                                "`col_argstr_def` order precisely as given: {}".format(
                                    tasklist_file_or_buff, col_argname_def_list
                                )
                            )
                        header_items = None
                    else:
                        raise cerr.ScriptArgumentError("Tasklist file does not have an acceptable header line: {}".format(tasklist_file_or_buff))

                if header_items is not None:
                    ncol_header = len(header_items)
                    if ncol_min is not None and ncol_header < ncol_min:
                        raise cerr.ScriptArgumentError(
                            "Tasklist file header line has {} columns, less than `ncol_min` required minimum ({}): {}".format(
                                ncol_header, ncol_min, tasklist_file_or_buff
                            ))
                    if ncol_max is not None and ncol_header > ncol_max:
                        raise cerr.ScriptArgumentError(
                            "Tasklist file header line has {} columns, more than `ncol_max` required maximum ({}): {}".format(
                                ncol_header, ncol_max, tasklist_file_or_buff
                            ))
                    if ncol_max is not None and ncol_header < ncol_max:
                        raise cerr.ScriptArgumentError(
                            "Tasklist file header line has {} columns, less than `ncol_max` possible maximum ({}): {}".format(
                                ncol_header, ncol_max, tasklist_file_or_buff
                            ))

                if header_items is None:
                    task_line = first_line
                    task_line_items = first_line_items
                else:
                    task_line = tasklist_fp.readline().strip()
                    task_line_items = [item.strip() for item in task_line.split(col_delim)]

                    if len(task_line_items) != len(header_items):
                        errmsg = ("Tasklist file ({}) number of columns mismatch between header ({}) "
                                  "and body ({})".format(tasklist_file_or_buff, len(header_items), len(task_line_items)))
                        if ncol_strict and not ncol_strict_header_separate:
                            raise cerr.ScriptArgumentError(errmsg)
                        else:
                            warning(errmsg)

            if col_argname_dtype_def_derived is not None:
                col_dtype_def_derived = [dtype for argname, dtype in col_argname_dtype_def_derived]
                # if ncol_min is None:
                #     ncol_min = len(col_argname_dtype_def_derived)
                if ncol_max is None:
                    ncol_max = len(col_argname_dtype_def_derived)
            else:
                col_dtype_def_derived = None

            if ncol_strict:
                ncol_body = len(task_line_items)
            ncol_body_max = float('-inf')

            while task_line != '':
                line_num += 1

                ncol_task_line = len(task_line_items)
                if ncol_strict and ncol_task_line != ncol_body:
                    raise cerr.ScriptArgumentError(
                        "Tasklist file {}, line {}: Number of columns ({}) breaks from constant "
                        "number of columns ({}) established prior to this line (`ncol_strict=True`)".format(
                            tasklist_file_or_buff, line_num, ncol_task_line, ncol_body
                        ))
                elif ncol_min is not None and ncol_task_line < ncol_min:
                    raise cerr.ScriptArgumentError(
                        "Tasklist file {}, line {}: Number of columns ({}) is less than `ncol_min` "
                        "required minimum ({})".format(
                            tasklist_file_or_buff, line_num, ncol_task_line, ncol_min
                        ))
                elif ncol_max is not None and ncol_task_line > ncol_max:
                    raise cerr.ScriptArgumentError(
                        "Tasklist file {}, line {}: Number of columns ({}) is more than `ncol_max` "
                        "required maximum ({})".format(
                            tasklist_file_or_buff, line_num, ncol_task_line, ncol_max
                        ))

                if ncol_task_line > ncol_body_max:
                    ncol_body_max = ncol_task_line

                if col_dtype_def_derived is not None:
                    task_line_values = [col_dtype_def_derived[col_idx](item) for col_idx, item in enumerate(task_line_items)]
                else:
                    task_line_values = [uniform_dtype(item) for item in task_line_items]

                task_array.append(task_line_values)

                task_line = tasklist_fp.readline().strip()
                task_line_items = [item.strip() for item in task_line.split(col_delim)]

            if col_argname_dtype_def_derived is None:
                col_argname_dtype_def_derived = [(argname, uniform_dtype) for argname in range(ncol_body_max)]

    def write(self, tasklist_file):
        pass

class Tasklist(object):
    def __init__(self, tasklist_file, args_dtype=str, args_delim=',', header_rows=0,
                 ncol_strict=True, ncol_strict_header_separate=False, ncol_min=None, ncol_max=None,
                 allow_1d_task_list=True, header_dtype=str):
        self.tasklist_file = tasklist_file
        self.header = None
        if header_rows > 0:
            self.header = read_task_bundle(
                tasklist_file, args_dtype=header_dtype, args_delim=args_delim, header_rows=header_rows,
                ncol_strict=ncol_strict, ncol_min=ncol_min, ncol_max=ncol_max,
                allow_1d_task_list=allow_1d_task_list, read_header=True
            )
        self.tasks = read_task_bundle(
            tasklist_file, args_dtype=args_dtype, args_delim=args_delim, header_rows=header_rows,
            ncol_strict=ncol_strict, ncol_min=ncol_min, ncol_max=ncol_max,
            allow_1d_task_list=allow_1d_task_list, read_header=False
        )
        if (    self.header is not None and ncol_strict and not ncol_strict_header_separate
            and len(self.header) > 0 and len(self.tasks) > 0):
                header_ncols = len(self.header[0]) if type(self.header[0]) is list else len(self.header)
                tasks_ncols = len(self.tasks[0]) if type(self.tasks[0]) is list else 1
                if header_ncols != tasks_ncols:
                    raise cerr.DimensionError("Inconsistent number of columns in `tasklist_file`: {}".format(tasklist_file))

def parse_src_args(args, argstr_src, argstr_dst, skip_dir_adjust=False):
    global SYNC_MODE_GLOBAL

    arg_dst = args.get(argstr_dst) if args.get(argstr_dst) is not None else args.get(ARGSTR_DSTDIR_GLOBAL)
    arg_srclist_prefix = args.get(ARGSTR_SRCLIST_PREFIX)
    arg_srclist_prefix_dst = args.get(ARGSTR_SRCLIST_PREFIX_DST)
    arg_srclist_suffix = args.get(ARGSTR_SRCLIST_SUFFIX)
    arg_srclist_rootdir = args.get(ARGSTR_SRCLIST_ROOTDIR)
    arg_srclist_noglob = args.get(ARGSTR_SRCLIST_NOGLOB)

    if args.get(ARGSTR_SYNC_TREE):
        SYNC_MODE_GLOBAL = ARGMOD_SYNC_MODE_SYNC_TREE
    elif args.get(ARGSTR_TRANSPLANT_TREE):
        SYNC_MODE_GLOBAL = ARGMOD_SYNC_MODE_TRANSPLANT_TREE
    else:
        SYNC_MODE_GLOBAL = ARGMOD_SYNC_MODE_NULL

    src_list = []
    srclist_tasklists = []
    srclist_rooted_tasklists = []

    ## Parse and validate multiple source arguments

    if args.get(argstr_src):
        if arg_dst is None:
            args.parser.error("argument {}/{} is required when {} is provided".format(
                argstr_dst, ARGSTR_DSTDIR_GLOBAL, argstr_src))
        src_list.extend(args.get(argstr_src))

    if args.get(ARGSTR_SRCLIST):
        for srclist_file in args.get(ARGSTR_SRCLIST):
            try:
                if arg_dst is None:
                    srclist_header = read_task_bundle(
                        srclist_file, ncol_min=1, ncol_max=2,
                        header_rows=1, read_header=True,
                        args_delim=args.get(ARGSTR_SRCLIST_DELIM)
                    )
                    if len(srclist_header) == 0:
                        warning("{} textfile is empty: {}".format(ARGSTR_SRCLIST, srclist_file))
                        continue
                    elif len(srclist_header) != 2:
                        raise cerr.DimensionError
                tasklist = Tasklist(
                    srclist_file, ncol_min=1, ncol_max=2, ncol_strict=True, ncol_strict_header_separate=True,
                    header_rows=1,
                    args_delim=args.get(ARGSTR_SRCLIST_DELIM)
                )
                if len(tasklist.header) == 0:
                    tasklist.header = None
                elif len(tasklist.tasks) == 0:
                    tasklist.tasks = tasklist.header
                    tasklist.header = None
                elif len(tasklist.tasks) > 0:
                    if len(tasklist.header) == 1:
                        if type(tasklist.tasks[0]) is list and len(tasklist.tasks[0]) == 2:
                            raise cerr.DimensionError
                        tasklist.tasks.insert(0, tasklist.header[0])
                        tasklist.header = None
                    elif len(tasklist.header) == 2:
                        if type(tasklist.tasks[0]) is list and len(tasklist.tasks[0]) == 2:
                            tasklist.tasks.insert(0, tasklist.header)
                            tasklist.header = None
                        elif type(tasklist.tasks[0]) is list and len(tasklist.tasks[0]) == 1:
                            tasklist.tasks.insert(0, [tasklist.header[0]])
                        else:
                            tasklist.tasks.insert(0, tasklist.header[0])
                if type(tasklist.header) is list and len(tasklist.header) == 0:
                    tasklist.header = None
            except cerr.DimensionError as e:
                traceback.print_exc()
                args.parser.error(
                    "{} {}; {} textfiles must be structured in one of the following formats:\n{}".format(
                        ARGSTR_SRCLIST, srclist_file, ARGSTR_SRCLIST, ARGHLP_SRCLIST_FORMAT
                ))

            tasklist_tasks_sanitized = []
            tasklist_src_dne = []
            task_type_is_list = (len(tasklist.tasks) > 0 and type(tasklist.tasks[0]) is list)

            for task in tasklist.tasks:
                task_src = task[0] if task_type_is_list else task

                if arg_srclist_prefix is not None:
                    task_src = os.path.join(arg_srclist_prefix, task_src)
                if arg_srclist_suffix is not None:
                    task_src = task_src + arg_srclist_suffix
                if arg_srclist_rootdir is not None:
                    if not task_src.startswith(arg_srclist_rootdir):
                        task_src = os.path.join(arg_srclist_rootdir, task_src)

                if (not arg_srclist_noglob and '*' in task_src) or os.path.exists(task_src):
                    if args.get(ARGSTR_SRCLIST_IGNORE_MISSING):
                        tasklist_tasks_sanitized.append(task)
                else:
                    tasklist_src_dne.append(task_src)

            if args.get(ARGSTR_SRCLIST_IGNORE_MISSING):
                tasklist.tasks = tasklist_tasks_sanitized
            elif len(tasklist_src_dne) > 0:
                args.parser.error("{} {}; source paths do not exist:\n{}".format(
                    ARGSTR_SRCLIST, srclist_file, '\n'.join(tasklist_src_dne)
                ))

            if len(tasklist.tasks) > 0:
                if arg_srclist_rootdir is not None:
                    tasklist.header = [arg_srclist_rootdir]
                    srclist_rooted_tasklists.append(tasklist)
                else:
                    srclist_tasklists.append(tasklist)

    if args.get(ARGSTR_SRCLIST_ROOTED):
        for srclist_file in args.get(ARGSTR_SRCLIST_ROOTED):
            try:
                if arg_dst is None:
                    srclist_first_two_lines = read_task_bundle(
                        srclist_file, ncol_min=1, ncol_max=2,
                        header_rows=2, read_header=True, allow_1d_task_list=False,
                        args_delim=args.get(ARGSTR_SRCLIST_DELIM)
                    )
                    if len(srclist_first_two_lines) == 0:
                        warning("{} textfile is empty: {}".format(ARGSTR_SRCLIST_ROOTED, srclist_file))
                        continue
                    elif 2 not in [len(line_items) for line_items in srclist_first_two_lines]:
                        raise cerr.DimensionError
                    else:
                        srclist_header = srclist_first_two_lines[0]
                        src_rootdir = srclist_header[0]
                        dst_rootdir = srclist_header[1] if len(srclist_header) == 2 else None
                        if not os.path.isdir(src_rootdir):
                            args.parser.error(
                                "{} {}; source root directory in header must be an existing directory: {}".format(
                                ARGSTR_SRCLIST_ROOTED, srclist_file, src_rootdir
                            ))
                        if dst_rootdir is not None and os.path.isfile(dst_rootdir):
                            args.parser.error(
                                "{} {}; destination root directory in header cannot be an existing file: {}".format(
                                ARGSTR_SRCLIST_ROOTED, srclist_file, dst_rootdir
                            ))
                tasklist = Tasklist(
                    srclist_file, ncol_min=1, ncol_max=2, ncol_strict=True, ncol_strict_header_separate=True,
                    header_rows=1,
                    args_delim=args.get(ARGSTR_SRCLIST_DELIM)
                )
            except cerr.DimensionError as e:
                traceback.print_exc()
                args.parser.error("{} {}; {} textfiles must be structured as follows:\n{}".format(
                    ARGSTR_SRCLIST_ROOTED, srclist_file, ARGSTR_SRCLIST_ROOTED, ARGHLP_SRCLIST_ROOTED_FORMAT
                ))

            tasklist_tasks_sanitized = []
            tasklist_src_dne = []
            task_type_is_list = (len(tasklist.tasks) > 0 and type(tasklist.tasks[0]) is list)

            for task in tasklist.tasks:
                if task_type_is_list:
                    task_src, task_dst_rootdir = task
                    if os.path.isfile(task_dst_rootdir):
                        args.parser.error(
                            "{} {}; destination root directory cannot be an existing file: {}".format(
                            ARGSTR_SRCLIST_ROOTED, tasklist.tasklist_file, task_dst_rootdir
                        ))
                else:
                    task_src = task

                if arg_srclist_prefix is not None:
                    task_src = os.path.join(arg_srclist_prefix, task_src)
                if arg_srclist_suffix is not None:
                    task_src = task_src + arg_srclist_suffix
                if (not arg_srclist_noglob and '*' in task_src) or os.path.exists(task_src):
                    if args.get(ARGSTR_SRCLIST_IGNORE_MISSING):
                        tasklist_tasks_sanitized.append(task)
                else:
                    tasklist_src_dne.append(task_src)

            if args.get(ARGSTR_SRCLIST_IGNORE_MISSING):
                tasklist.tasks = tasklist_tasks_sanitized
            elif len(tasklist_src_dne) > 0:
                args.parser.error("{} {}; source paths do not exist:\n{}".format(
                    ARGSTR_SRCLIST_ROOTED, srclist_file, '\n'.join(tasklist_src_dne)
                ))

            if len(tasklist.tasks) > 0:
                srclist_rooted_tasklists.append(tasklist)

    arg_dst_can_be_file = False
    if args.get(argstr_src) and args.get(argstr_dst) and not os.path.isdir(args.get(argstr_dst)):
        if len(src_list) == 1 and not (args.get(ARGSTR_SRCLIST) or args.get(ARGSTR_SRCLIST_ROOTED)):
            arg_dst_can_be_file = True

    ### Build list of tasks to be performed

    all_task_list = []

    ## Standardize source and destination paths to SYNC-style for file copy tasks

    for src_path in src_list:
        dst_path = arg_dst
        dst_path = adjust_dst_path(src_path, dst_path, arg_dst_can_be_file,
                                   skip_dir_adjust=skip_dir_adjust)
        all_task_list.append((src_path, dst_path))

    for tasklist in srclist_tasklists:

        tasklist_dst_can_be_file = False
        if args.get(ARGSTR_DSTDIR_GLOBAL) is not None:
            tasklist_dst_dir = args.get(ARGSTR_DSTDIR_GLOBAL)
        elif tasklist.header is not None:
            tasklist_dst_dir = tasklist.header[1]
        else:
            tasklist_dst_dir = None
            tasklist_dst_can_be_file = True

        if tasklist_dst_dir is None:
            dst_path_type = PATH_TYPE_UNKNOWN
        else:
            if not os.path.exists(tasklist_dst_dir):
                dst_path_type = PATH_TYPE_DNE
            elif os.path.isdir(tasklist_dst_dir):
                dst_path_type = PATH_TYPE_DIR
            elif os.path.isfile(tasklist_dst_dir):
                dst_path_type = PATH_TYPE_FILE
            else:
                dst_path_type = PATH_TYPE_UNKNOWN

        task_type_is_list = (len(tasklist.tasks) > 0 and type(tasklist.tasks[0]) is list)
        for task in tasklist.tasks:
            if task_type_is_list:
                src_path = task[0]
                if tasklist_dst_dir is not None:
                    dst_path = tasklist_dst_dir
                else:
                    dst_path = task[1]
                    if arg_srclist_prefix_dst is not None:
                        dst_path = os.path.join(arg_srclist_prefix_dst, dst_path)
            else:
                src_path = task
                dst_path = tasklist_dst_dir if tasklist_dst_dir is not None else arg_dst

            if arg_srclist_prefix is not None:
                src_path = os.path.join(arg_srclist_prefix, src_path)
            if arg_srclist_suffix is not None:
                src_path = src_path + arg_srclist_suffix

            if not arg_srclist_noglob and '*' in src_path:
                src_path_glob = glob.glob(src_path)
                if len(src_path_glob) == 0:
                    warning("{} {}; no source files found matching pattern: {}".format(
                        ARGSTR_SRCLIST, tasklist.tasklist_file, src_path
                    ))
                for src_path_single in src_path_glob:
                    dst_path_single = adjust_dst_path(
                        src_path_single, dst_path, dst_can_be_file=False, dst_path_type=dst_path_type,
                        sync_mode_default=ARGMOD_SYNC_MODE_TRANSPLANT_TREE, skip_dir_adjust=skip_dir_adjust
                    )
                    all_task_list.append((src_path_single, dst_path_single))
            else:
                dst_path = adjust_dst_path(
                    src_path, dst_path, tasklist_dst_can_be_file, dst_path_type,
                    skip_dir_adjust=skip_dir_adjust
                )
                all_task_list.append((src_path, dst_path))

    for tasklist in srclist_rooted_tasklists:

        src_rootdir = tasklist.header[0]
        if arg_srclist_prefix is not None:
            src_rootdir = os.path.join(arg_srclist_prefix, src_rootdir)

        if not os.path.isdir(src_rootdir):
            args.parser.error(
                "{} {}; source root directory in header must be an existing directory: {}".format(
                ARGSTR_SRCLIST_ROOTED, tasklist.tasklist_file, src_rootdir
            ))

        if args.get(ARGSTR_DSTDIR_GLOBAL) is not None:
            tasklist_dst_rootdir = args.get(ARGSTR_DSTDIR_GLOBAL)
        elif len(tasklist.header) == 2:
            tasklist_dst_rootdir = tasklist.header[1]
        else:
            tasklist_dst_rootdir = None

        sync_mode = SYNC_MODE_GLOBAL
        if sync_mode == ARGMOD_SYNC_MODE_NULL:
            # Assume user expects the new destination directory to mirror the source directory
            sync_mode = ARGMOD_SYNC_MODE_SYNC_TREE

        src_rootdir_dirname = os.path.basename(src_rootdir.rstrip(PATH_SEPARATORS_CAT))
        if tasklist_dst_rootdir is not None and sync_mode == ARGMOD_SYNC_MODE_TRANSPLANT_TREE:
            tasklist_dst_rootdir = os.path.join(tasklist_dst_rootdir, src_rootdir_dirname)

        task_type_is_list = (len(tasklist.tasks) > 0 and type(tasklist.tasks[0]) is list)
        for task in tasklist.tasks:
            if task_type_is_list:
                src_path = task[0]
                dst_rootdir = tasklist_dst_rootdir if tasklist_dst_rootdir is not None else task[1]
            else:
                src_path = task
                dst_rootdir = tasklist_dst_rootdir if tasklist_dst_rootdir is not None else arg_dst

            if arg_srclist_prefix_dst is not None:
                dst_rootdir = os.path.join(arg_srclist_prefix_dst, dst_rootdir)

            if os.path.isfile(dst_rootdir):
                args.parser.error(
                    "{} {}; destination root directory cannot be an existing file: {}".format(
                    ARGSTR_SRCLIST_ROOTED, tasklist.tasklist_file, dst_rootdir
                ))

            if arg_srclist_prefix is not None:
                src_path = os.path.join(arg_srclist_prefix, src_path)
            if arg_srclist_suffix is not None:
                src_path = src_path + arg_srclist_suffix

            if not src_path.startswith(src_rootdir):
                # Assume source path is relative from the source root directory
                src_path_from_root = src_path
                src_path = os.path.join(src_rootdir, src_path_from_root)

            if not arg_srclist_noglob and '*' in src_path:
                src_path_glob = glob.glob(src_path)
                if len(src_path_glob) == 0:
                    warning("{} {}; no source files found matching pattern: {}".format(
                        ARGSTR_SRCLIST_ROOTED, tasklist.tasklist_file, src_path
                    ))
            else:
                src_path_glob = [src_path]
            for src_path_single in src_path_glob:

                src_path_from_root = src_path_single.replace(src_rootdir, '') if src_path_single.startswith(src_rootdir) else src_path_single
                src_path_from_root = src_path_from_root.lstrip(PATH_SEPARATORS_CAT)
                if tasklist_dst_rootdir is None and sync_mode == ARGMOD_SYNC_MODE_TRANSPLANT_TREE:
                    dst_path_single = os.path.join(dst_rootdir, src_rootdir_dirname, src_path_from_root)
                else:
                    dst_path_single = os.path.join(dst_rootdir, src_path_from_root)

                all_task_list.append((src_path_single, dst_path_single))

    return all_task_list

def adjust_dst_path(src_path, dst_path, dst_can_be_file=False, dst_path_type=PATH_TYPE_UNKNOWN,
                    sync_mode_default=ARGMOD_SYNC_MODE_NULL, skip_dir_adjust=False):
    # global SYNC_MODE_GLOBAL

    if dst_path_type == PATH_TYPE_DIR or (dst_path_type == PATH_TYPE_UNKNOWN and os.path.isdir(dst_path)):
        if os.path.isfile(src_path):
            dst_path = os.path.join(dst_path, os.path.basename(src_path))
        elif not skip_dir_adjust:
            # src_path is a directory
            sync_mode = SYNC_MODE_GLOBAL if SYNC_MODE_GLOBAL != ARGMOD_SYNC_MODE_NULL else sync_mode_default
            if sync_mode == ARGMOD_SYNC_MODE_NULL:
                sync_mode = ARGMOD_SYNC_MODE_SYNC_TREE if psu_str.endswith_one_of_coll(src_path, PATH_SEPARATORS_LIST) else ARGMOD_SYNC_MODE_TRANSPLANT_TREE
            if sync_mode == ARGMOD_SYNC_MODE_TRANSPLANT_TREE:
                dst_path = os.path.join(dst_path, os.path.basename(src_path.rstrip(PATH_SEPARATORS_CAT)))
            if not psu_str.endswith_one_of_coll(dst_path, PATH_SEPARATORS_LIST):
                dst_path = dst_path+os.path.sep

    elif dst_path_type == PATH_TYPE_FILE or (dst_path_type == PATH_TYPE_UNKNOWN and os.path.isfile(dst_path)):
        if os.path.isdir(src_path):
            raise cerr.ScriptArgumentError(
                "source directory ({}) cannot overwrite existing destination file ({})".format(src_path, dst_path)
            )
        else:
            # src_path is a file
            pass

    else:
        # dst_path does not yet exist
        if os.path.isfile(src_path):
            if dst_can_be_file and not psu_str.endswith_one_of_coll(dst_path, PATH_SEPARATORS_LIST):
                # dst_path will be the exact path of the file copy
                pass
            else:
                dst_path = os.path.join(dst_path, os.path.basename(src_path))
        elif not skip_dir_adjust:
            # src_path is a directory; dst_path will be a new destination directory
            sync_mode = SYNC_MODE_GLOBAL if SYNC_MODE_GLOBAL != ARGMOD_SYNC_MODE_NULL else sync_mode_default
            if sync_mode == ARGMOD_SYNC_MODE_NULL:
                if (    not psu_str.endswith_one_of_coll(src_path, PATH_SEPARATORS_LIST)
                    and     psu_str.endswith_one_of_coll(dst_path, PATH_SEPARATORS_LIST)):
                    # Assume user expects the new destination directory to contain the mirrored source directory
                    sync_mode = ARGMOD_SYNC_MODE_TRANSPLANT_TREE
                else:
                    # Assume user expects the new destination directory to mirror the source directory
                    sync_mode = ARGMOD_SYNC_MODE_SYNC_TREE
            if sync_mode == ARGMOD_SYNC_MODE_TRANSPLANT_TREE:
                dst_path = os.path.join(dst_path, os.path.basename(src_path.rstrip(PATH_SEPARATORS_CAT)))
            if not psu_str.endswith_one_of_coll(dst_path, PATH_SEPARATORS_LIST):
                dst_path = dst_path+os.path.sep

    return dst_path


from __future__ import print_function
import os
import re
import codecs
import copy
import numpy as np
np.random.seed(0)
import random
random.seed(0)
from collections import Counter
import torch

def create_dico(item_list):
    """
    Create a dictionary of items from a list of list of items.
    """
    assert type(item_list) is list
    dico = {}
    for items in item_list:
        for item in items:
            if item not in dico:
                dico[item] = 1
            else:
                dico[item] += 1
    return dico

def create_mapping(dico):
    """
    Create a mapping (item to ID / ID to item) from a dictionary.
    Items are ordered by decreasing frequency.
    """
    sorted_items = sorted(dico.items(), key=lambda x: (-x[1], x[0]))
    id_to_item = {i: v[0] for i, v in enumerate(sorted_items)}
    item_to_id = {v: k for k, v in id_to_item.items()}
    return item_to_id, id_to_item

def augment_with_pretrained(dictionary, ext_emb_path, words):
    """
    Augment the dictionary with words that have a pretrained embedding.
    If `words` is None, we add every word that has a pretrained embedding
    to the dictionary, otherwise, we only add the words that are given by
    `words` (typically the words in the development and test sets.)
    """
    print('Loading pretrained embeddings from %s...' % ext_emb_path)
    assert os.path.isfile(ext_emb_path)

    # Load pretrained embeddings from file
    pretrained = set([
        line.rstrip().split()[0].strip()
        for line in codecs.open(ext_emb_path, 'r', 'utf-8')
        if len(ext_emb_path) > 0
    ])

    if words is None:
        for word in pretrained:
            if word not in dictionary:
                dictionary[word] = 0
    else:
        for word in words:
            if any(x in pretrained for x in [
                word,
                word.lower(),
                re.sub('\d', '0', word.lower())
            ]) and word not in dictionary:
                dictionary[word] = 0

    word_to_id, id_to_word = create_mapping(dictionary)
    return dictionary, word_to_id, id_to_word

def tag_mapping(dataset):
    """
    Create a dictionary and a mapping of tags, sorted by frequency.
    """
    tags = [s[1] for s in dataset]
    dico = Counter(tags)
    tag_to_id, id_to_tag = create_mapping(dico)
    print("Found %i unique named entity tags" % len(dico))
    return dico, tag_to_id, id_to_tag

def prepare_dataset(dataset, word_to_id, tag_to_id):
    """
    Prepare the dataset. Return a list of lists of dictionaries containing:
        - word indexes
        - word char indexes
        - tag indexes
    """
    def f(x): return x.lower()
    data = []
    for s in dataset:
        str_words = [w for w in s[0].split()]
        words = [word_to_id[f(w) if f(w) in word_to_id else '<UNK>']
                 for w in str_words]
        tag = tag_to_id[s[1]]
        data.append({
            'str_words': str_words,
            'words': words,
            'tag': tag,
        })
    return data

def pad_seq(seq, max_length, PAD_token=0):

    seq += [PAD_token for i in range(max_length - len(seq))]
    return seq

def create_batches(dataset, batch_size, order='keep'):

    newdata = copy.deepcopy(dataset)
    if order=='sort':
        newdata.sort(key = lambda x:len(x['words']))
    elif order=='random':
        random.shuffle(newdata)

    newdata = np.array(newdata)
    batches = []
    num_batches = np.ceil(len(dataset)/float(batch_size)).astype('int')

    for i in range(num_batches):
        batch_data = newdata[(i*batch_size):min(len(dataset),(i+1)*batch_size)]

        words_seqs = [itm['words'] for itm in batch_data]
        target_seqs = [itm['tag'] for itm in batch_data]
        str_words_seqs = [itm['str_words'] for itm in batch_data]

        seq_pairs = sorted(zip(words_seqs, target_seqs, str_words_seqs,
                               range(len(words_seqs))), key=lambda p: len(p[0]), reverse=True)

        words_seqs, target_seqs, str_words_seqs, sort_info = zip(*seq_pairs)

        words_lengths = np.array([len(s) for s in words_seqs])
        words_padded = np.array([pad_seq(s, np.max(words_lengths)) for s in words_seqs])
        words_mask = (words_padded!=0).astype('int')

        outputdict = {'words':words_padded, 'tags': target_seqs, 'wordslen': words_lengths,
                      'tagsmask':words_mask, 'str_words': str_words_seqs, 'sort_info': sort_info}

        batches.append(outputdict)

    return batches

def log_gaussian(x, mu, sigma):
    return float(-0.5 * np.log(2 * np.pi) - np.log(np.abs(sigma))) - (x - mu)**2 / (2 * sigma**2)

def log_gaussian_logsigma(x, mu, logsigma):
    return float(-0.5 * np.log(2 * np.pi)) - logsigma - (x - mu)**2 / (2 * torch.exp(logsigma)**2)

def bayes_loss_function(l_pw, l_qw, l_likelihood, n_batches, batch_size):
    return ((1./n_batches) * (l_qw - l_pw) - l_likelihood).sum() / float(batch_size)


from http.server import BaseHTTPRequestHandler, HTTPServer
import logging
import json
from controller import *

def checkParam(path, newPath):
  pathArray = path.split('/')
  newPathArray = newPath.split('/')
  paramsMap = {}
  if len(pathArray) != len(newPathArray):
    return False
  for index in range(len(pathArray)):
    if pathArray[index] != newPathArray[index]:
      if len(pathArray[index]) == 0 or pathArray[index][0] != ':':
        return False
      paramsMap[pathArray[index]] = newPathArray[index]
  return paramsMap

class S(BaseHTTPRequestHandler):
    def _set_response(self):
        self.send_response(200)
        self.send_header('Content-type', 'text/html')
        self.end_headers()
    def do_GET(self):
      logging.info("GET request,\nPath: %s\nHeaders:\n%s\n", str(self.path), str(self.headers))
      self._set_response()
      if self.path == '/sectors':
        sectors = json.dumps(getAllSectors())
        self.wfile.write(sectors.encode('utf-8'))
        return
      params = checkParam('/companies/:sector/:name', self.path)
      if params:
        sector = params[":sector"].replace("%20", " ")
        name = params[":name"].replace("%20", " ")
        companies = json.dumps(getBestCompanies(sector, name))
        self.wfile.write(companies.encode('utf-8'))
        return
      params = checkParam('/company/:cmp', self.path)
      if params:
        sector = params[":cmp"].replace("%20", " ")
        companies = json.dumps(getCompanyDataByAbreviation(sector))
        self.wfile.write(companies.encode('utf-8'))
        return
      self.wfile.write(json.dumps({"Error": "Wrong endpoint!"}).encode('utf-8'))
      return

    def do_POST(self):
      content_length = int(self.headers['Content-Length'])
      post_data = self.rfile.read(content_length)
      logging.info("POST request,\nPath: %s\nHeaders:\n%s\n\nBody:\n%s\n",
              str(self.path), str(self.headers), post_data.decode('utf-8'))
      self._set_response()
      self.wfile.write("POST request for {}".format(self.path).encode('utf-8'))

def run(server_class=HTTPServer, handler_class=S, port=8080):
  logging.basicConfig(level=logging.INFO)
  server_address = ('', port)
  httpd = server_class(server_address, handler_class)
  logging.info('Starting httpd...\n')
  try:
      httpd.serve_forever()
  except KeyboardInterrupt:
      pass
  httpd.server_close()
  logging.info('Stopping httpd...\n')

if __name__ == '__main__':
    from sys import argv
    if len(argv) == 2:
        run(port=int(argv[1]))
    else:
        run()


import argparse
import logging
import os
import signal
import config
from multiprocessing import Process, Event as MEvent
from threading import Thread, Event
from time import sleep
from typing import List, Union

from kafka.coordinator.assignors.range import RangePartitionAssignor
from kafka.coordinator.assignors.roundrobin import RoundRobinPartitionAssignor

from consumer.common.message_handler import MessageHandler
from consumer.concurrent.concurrent_consumer import ConcurrentKafkaConsumer
from consumer.parallel.parallel_consumer import ParallelKafkaConsumer

parser = argparse.ArgumentParser()
parser.add_argument('--mode', choices=['concurrent', 'parallel'])

logging.basicConfig(level=logging.DEBUG,
                    format="[%(asctime)s.%(msecs)03d][%(threadName)s][%(levelname)s][%(name)s] - %(message)s",
                    datefmt='%H:%M:%S')

config = {
    'bootstrap_servers': config.KAFKA_BOOTSTRAP,
    'api_version': (2, 1, 0),
    'group_id': 'my_consumer',
    'client_id': 'my_client_id',
    'enable_auto_commit': False,
    'auto_offset_reset': 'earliest',
    'partition_assignment_strategy': (RangePartitionAssignor, RoundRobinPartitionAssignor)
}

def start_concurrent_consumer(topic, app_reference):
    logging.getLogger('kafka').setLevel(logging.WARNING)
    c = ConcurrentKafkaConsumer(
        kafka_config=config,
        event_handlers={'Event': MessageHandler()},
        max_workers=2,
        topic=topic)
    app_reference.append(c)
    c.start()

def start_parallel_consumer(topic):
    logging.basicConfig(level=logging.DEBUG,
                        format="[%(asctime)s.%(msecs)03d][%(threadName)s][%(levelname)s][%(name)s] - %(message)s",
                        datefmt='%H:%M:%S')
    logging.getLogger('kafka').setLevel(logging.WARNING)
    logging.getLogger().info('Starting topic %s... on PID: %s', topic, os.getpid())
    c = ParallelKafkaConsumer(
        kafka_config=config,
        event_handlers={'Event': MessageHandler()},
        max_workers=2,
        topic=topic)
    signal.signal(signal.SIGTERM, c.stop)
    signal.signal(signal.SIGINT, c.stop)
    c.start()

thread_stop_event = Event()
process_stop_event = MEvent()

def start_and_wait(pool: List[Union[Thread, Process]], stop_event):
    [p.start() for p in pool]

    while not stop_event.wait(5):
        for p in pool:
            if not p.is_alive():
                logging.getLogger().warning("%s crashed", p.name)
                # signal.raise_signal(signal.SIGINT)

    # Wait for graceful shutdown
    while all([p.is_alive() for p in pool]):
        logging.info('Waiting for shutdown... %s', [p.is_alive() for p in pool])
        sleep(1)

def _handle_stop_thread(*args):
    logging.info('Setting Threading stop event... %s', args)
    thread_stop_event.set()
    logging.info('Stopping consumers %s', [c for c in consumer_references])
    [c.stop() for c in consumer_references]

def _handle_stop_process(*args):
    logging.info('Setting Processing stop event...')
    process_stop_event.set()

if __name__ == '__main__':
    program_args = parser.parse_args()

    if program_args.mode == 'concurrent':
        signal.signal(signal.SIGTERM, _handle_stop_thread)
        signal.signal(signal.SIGINT, _handle_stop_thread)

        consumer_references = []

        threads = [
            # ConsumerThread-T1 will subscribe and handle all partitions of topic T1
            Thread(target=start_concurrent_consumer, args=('T1', consumer_references), daemon=False, name='ConsumerThread-T1'),
            # ConsumerThread-T2 will subscribe and handle all partitions of topic T2
            Thread(target=start_concurrent_consumer, args=('T2', consumer_references), daemon=False, name='ConsumerThread-T2')
        ]

        start_and_wait(threads, thread_stop_event)

    else:
        signal.signal(signal.SIGTERM, _handle_stop_process)
        signal.signal(signal.SIGINT, _handle_stop_process)

        processes = [
            # ConsumerProcess-T1 will subscribe and handle all partitions of topic T1
            Process(target=start_parallel_consumer, args=('T1',), daemon=False, name='ConsumerProcess-T1'),
            # ConsumerProcess-T2 will subscribe and handle all partitions of topic T2
            Process(target=start_parallel_consumer, args=('T2',), daemon=False, name='ConsumerProcess-T2')
        ]

        start_and_wait(processes, process_stop_event)


#!/usr/bin/python3
"""
This module defines the BaseModel class.
"""
import uuid
from datetime import datetime

class BaseModel:
    """
    A base class that defines common attributes
    and methods for other classes.
    """

    def __init__(self, *args, **kwargs):
        """Initialize instance attributes."""
        if kwargs:
            for key, value in kwargs.items():
                if key == 'created_at' or key == 'updated_at':
                    setattr(self, key, datetime.strptime(
                        value, '%Y-%m-%dT%H:%M:%S.%f'))
                elif key != '__class__':
                    setattr(self, key, value)
        else:
            self.id = str(uuid.uuid4())
            self.created_at = datetime.now()
            self.updated_at = datetime.now()

    def __str__(self):
        """Return a string representation of the instance."""
        class_name = self.__class__.__name__
        return "[{}] ({}) {}".format(class_name, self.id, self.__dict__)

    def save(self):
        """
        Update the public instance attribute
        'updated_at' with the current datetime.
        """
        self.updated_at = datetime.now()

    def to_dict(self):
        """Return a dictionary containing all keys/values of the instance."""
        instance_dict = self.__dict__.copy()
        instance_dict['__class__'] = self.__class__.__name__
        instance_dict['created_at'] = self.created_at.isoformat()
        instance_dict['updated_at'] = self.updated_at.isoformat()
        return instance_dict

if __name__ == "__main__":
    my_model = BaseModel()
    my_model.name = "My First Model"
    my_model.my_number = 89
    print(my_model)
    my_model.save()
    print(my_model)
    my_model_json = my_model.to_dict()
    print(my_model_json)
    print("JSON of my_model:")
    for key in my_model_json.keys():
        print("\t{}: ({}) - {}".format(key, type(my_model_json[key]),
                                       my_model_json[key]))


import collections
import contextlib
import cStringIO
import os
import sys
import textwrap

from googlecloudsdk.api_lib.storage import storage_util
from googlecloudsdk.command_lib.ml import flags
from googlecloudsdk.command_lib.ml import uploads
from googlecloudsdk.core import exceptions
from googlecloudsdk.core import execution_utils
from googlecloudsdk.core import log
from googlecloudsdk.core.util import files

DEFAULT_SETUP_FILE = """\
from setuptools import setup

if __name__ == '__main__':
    setup(name='{package_name}', packages=['{package_name}'])
"""

_NO_PACKAGES_ERROR_MSG = (
    'If `--package-path` is not specified, at least one Python package '
    'must be specified via `--packages`.')

class UploadFailureError(exceptions.Error):
  """Generic error with the packaging/upload process."""
  pass

class SetuptoolsFailedError(UploadFailureError):
  """Error indicating that setuptools itself failed."""

  def __init__(self, output, generated):
    msg = ('Packaging of user Python code failed with message:\n\n'
           '{}\n\n').format(output)
    if generated:
      msg += ('Try manually writing a setup.py file at your package root and '
              'rerunning the command.')
    else:
      msg += ('Try manually building your Python code by running:\n'
              '  $ python setup.py sdist\n'
              'and providing the output via the `--packages` flag (for '
              'example, `--packages dist/package.tar.gz,dist/package2.whl)`')
    super(SetuptoolsFailedError, self).__init__(msg)

class SysExecutableMissingError(UploadFailureError):
  """Error indicating that sys.executable was empty."""

  def __init__(self):
    super(SysExecutableMissingError, self).__init__(textwrap.dedent("""\
        No Python executable found on path. A Python executable with setuptools
        installed on the PYTHONPATH is required for building Cloud ML training
        jobs.
        """))

class MissingInitError(UploadFailureError):
  """Error indicating that the package to build had no __init__.py file."""

  def __init__(self, package_dir):
    super(MissingInitError, self).__init__(textwrap.dedent("""\
        [{}] is not a valid Python package because it does not contain an \
        `__init__.py` file. Please create one and try again.
        """).format(package_dir))

class UncopyablePackageError(UploadFailureError):
  """Error with copying the package."""

class DuplicateEntriesError(UploadFailureError):
  """Error indicating that multiple files with the same name were provided."""

  def __init__(self, duplicates):
    super(DuplicateEntriesError, self).__init__(
        'Cannot upload multiple packages with the same filename: [{}]'.format(
            ', '.join(duplicates)))

class NoStagingLocationError(UploadFailureError):
  """No staging location was provided but one was required."""

def _CopyIfNotWritable(source_dir, temp_dir):
  """Returns a writable directory with the same contents as source_dir.

  If source_dir is writable, it is used. Otherwise, a directory 'dest' inside of
  temp_dir is used.

  Args:
    source_dir: str, the directory to (potentially) copy
    temp_dir: str, the path to a writable temporary directory in which to store
      any copied code.

  Returns:
    str, the path to a writable directory with the same contents as source_dir
      (i.e. source_dir, if it's writable, or a copy otherwise).

  Raises:
    UploadFailureError: if the command exits non-zero.
  """
  if files.HasWriteAccessInDir(source_dir):
    return source_dir

  if files.IsDirAncestorOf(source_dir, temp_dir):
    raise UncopyablePackageError(
        'Cannot copy directory since working directory [{}] is inside of '
        'source directory [{}].'.format(temp_dir, source_dir))

  dest_dir = os.path.join(temp_dir, 'dest')
  log.debug('Copying local source tree from [%s] to [%s]', source_dir, dest_dir)
  try:
    files.CopyTree(source_dir, dest_dir)
  except OSError:
    raise UncopyablePackageError(
        'Cannot write to working location [{}]'.format(dest_dir))
  return dest_dir

def _GenerateSetupPyIfNeeded(setup_py_path, package_name):
  """Generates a temporary setup.py file if there is none at the given path.

  Args:
    setup_py_path: str, a path to the expected setup.py location.
    package_name: str, the name of the Python package for which to write a
      setup.py file (used in the generated file contents).

  Returns:
    bool, whether the setup.py file was generated.
  """
  log.debug('Looking for setup.py file at [%s]', setup_py_path)
  if os.path.isfile(setup_py_path):
    log.info('Using existing setup.py file at [%s]', setup_py_path)
    return False

  setup_contents = DEFAULT_SETUP_FILE.format(package_name=package_name)
  log.info('Generating temporary setup.py file:\n%s', setup_contents)
  with open(setup_py_path, 'w') as setup_file:
    setup_file.write(setup_contents)
  return True

@contextlib.contextmanager
def _TempDirOrBackup(default_dir):
  """Yields a temporary directory or a backup temporary directory.

  Prefers creating a temporary directory (which will be cleaned up when the
  context manager is closed), but falls back to default_dir. There are systems
  where users can't write to temp, but we still need to copy.

  Args:
    default_dir: str, the backup temporary directory.

  Yields:
    str, the temporary directory.
  """

  try:
    with files.TemporaryDirectory() as temp_dir:
      yield temp_dir
  except OSError:
    # Some systems don't allow access to '/tmp'
    yield default_dir

def _RunSetupTools(package_root, setup_py_path, output_dir):
  """Executes the setuptools `sdist` command.

  Specifically, runs `python setup.py sdist` (with the full path to `setup.py`
  given by setup_py_path) with arguments to put the final output in output_dir
  and all possible temporary files in a temporary directory. package_root is
  used as the working directory.

  package_root must be writable, or setuptools will fail (there are
  temporary files from setuptools that get put in the CWD).

  Args:
    package_root: str, the directory containing the package (that is, the
      *parent* of the package itself).
    setup_py_path: str, the path to the `setup.py` file to execute.
    output_dir: str, path to a directory in which the built packages should be
      created.

  Returns:
    list of str, the full paths to the generated packages.

  Raises:
    SysExecutableMissingError: if sys.executable is None
    RuntimeError: if the execution of setuptools exited non-zero.
  """
  if not sys.executable:
    raise SysExecutableMissingError()

  # We could just include the 'sdist' command and its flags here, but we want
  # to avoid leaving artifacts in the setup directory. That's what the
  # 'egg_info' and 'build' options do (these are both invoked as subcommands
  # of 'sdist').
  # Unfortunately, there doesn't seem to be any easy way to move *all*
  # temporary files out of the current directory, so we'll fail here if we
  # can't write to it.
  with _TempDirOrBackup(package_root) as working_dir:
    args = [sys.executable, setup_py_path,
            'egg_info', '--egg-base', working_dir,
            'build', '--build-base', working_dir, '--build-temp', working_dir,
            'sdist', '--dist-dir', output_dir]
    out = cStringIO.StringIO()
    if execution_utils.Exec(args, no_exit=True, out_func=out.write,
                            err_func=out.write, cwd=package_root):
      # If the first execution doesn't work, it might be because setup.py is
      # using distutils. We still want to try it that way because it leaves
      # fewer droppings (that is, leftover files) on disk.
      args = [sys.executable, setup_py_path,
              'sdist', '--dist-dir', output_dir]
      out = cStringIO.StringIO()
      if execution_utils.Exec(args, no_exit=True, out_func=out.write,
                              err_func=out.write, cwd=package_root):
        raise RuntimeError(out.getvalue())

  local_paths = [os.path.join(output_dir, rel_file)
                 for rel_file in os.listdir(output_dir)]
  log.debug('Python packaging resulted in [%s]', ', '.join(local_paths))
  return local_paths

def BuildPackages(package_path, output_dir):
  """Builds Python packages from the given package source.

  That is, builds Python packages from the code in package_path, using its
  parent directory (the 'package root') as its context using the setuptools
  `sdist` command.

  If there is a `setup.py` file in the package root, use that. Otherwise,
  use a simple, temporary one made for this package.

  We try to be as unobstrustive as possible (see _RunSetupTools for details):

  - setuptools writes some files to the package root--we move as many temporary
    generated files out of the package root as possible
  - the final output gets written to output_dir
  - any temporary setup.py file is written outside of the package root.
  - if the current directory isn't writable, we silenly make a temporary copy

  Args:
    package_path: str. Path to the package. This should be the path to
      the directory containing the Python code to be built, *not* its parent
      (which optionally contains setup.py and other metadata).
    output_dir: str, path to a long-lived directory in which the built packages
      should be created.

  Returns:
    list of str. The full local path to all built Python packages.

  Raises:
    SetuptoolsFailedError: If the setup.py file fails to successfully build.
    MissingInitError: If the package doesn't contain an `__init__.py` file.
  """
  package_path = os.path.abspath(package_path)
  package_root = os.path.dirname(package_path)
  with _TempDirOrBackup(package_path) as working_dir:
    package_root = _CopyIfNotWritable(package_root, working_dir)
    if not os.path.exists(os.path.join(package_path, '__init__.py')):
      # We could drop `__init__.py` in here, but it's pretty likely that this
      # indicates an incorrect directory or some bigger problem and we don't
      # want to obscure that.
      #
      # Note that we could more strictly validate here by checking each package
      # in the `--module-name` argument, but this should catch most issues.
      raise MissingInitError(package_path)

    setup_py_path = os.path.join(package_root, 'setup.py')
    package_name = os.path.basename(package_path)
    generated = _GenerateSetupPyIfNeeded(setup_py_path, package_name)
    try:
      return _RunSetupTools(package_root, setup_py_path, output_dir)
    except RuntimeError as err:
      raise SetuptoolsFailedError(str(err), generated)
    finally:
      if generated:
        # For some reason, this artifact gets generated in the package root by
        # setuptools, even after setting PYTHONDONTWRITEBYTECODE or running
        # `python setup.py clean --all`. It's weird to leave someone a .pyc for
        # a file they never knew they had, so we clean it up.
        pyc_file = os.path.join(package_root, 'setup.pyc')
        for path in (setup_py_path, pyc_file):
          try:
            os.unlink(path)
          except OSError:
            log.debug(
                "Couldn't remove file [%s] (it may never have been created).",
                pyc_file)

def _UploadFilesByPath(paths, staging_location):
  """Uploads files after validating and transforming input type."""
  if not staging_location:
    raise NoStagingLocationError()
  counter = collections.Counter(map(os.path.basename, paths))
  duplicates = [name for name, count in counter.iteritems() if count > 1]
  if duplicates:
    raise DuplicateEntriesError(duplicates)

  upload_pairs = [(path, os.path.basename(path)) for path in paths]
  return uploads.UploadFiles(upload_pairs, staging_location.bucket_ref,
                             staging_location.name)

def UploadPythonPackages(packages=(), package_path=None, staging_location=None):
  """Uploads Python packages (if necessary), building them as-specified.

  A Cloud ML job needs one or more Python packages to run. These Python packages
  can be specified in one of three ways:

    1. As a path to a local, pre-built Python package file.
    2. As a path to a Cloud Storage-hosted, pre-built Python package file (paths
       beginning with 'gs://').
    3. As a local Python source tree (the `--package-path` flag).

  In case 1, we upload the local files to Cloud Storage[1] and provide their
  paths. These can then be given to the Cloud ML API, which can fetch these
  files.

  In case 2, we don't need to do anything. We can just send these paths directly
  to the Cloud ML API.

  In case 3, we perform a build using setuptools[2], and upload the resulting
  artifacts to Cloud Storage[1]. The paths to these artifacts can be given to
  the Cloud ML API. See the `BuildPackages` method.

  These methods of specifying Python packages may be combined.

  [1] Uploads are to a specially-prefixed location in a user-provided Cloud
  Storage staging bucket. If the user provides bucket `gs://my-bucket/`, a file
  `package.tar.gz` is uploaded to
  `gs://my-bucket/<job name>/<checksum>/package.tar.gz`.

  [2] setuptools must be installed on the local user system.

  Args:
    packages: list of str. Path to extra tar.gz packages to upload, if any. If
      empty, a package_path must be provided.
    package_path: str. Relative path to source directory to be built, if any. If
      omitted, one or more packages must be provided.
    staging_location: storage_util.ObjectReference. Cloud Storage prefix to
      which archives are uploaded. Not necessary if only remote packages are
      given.

  Returns:
    list of str. Fully qualified Cloud Storage URLs (`gs://..`) from uploaded
      packages.

  Raises:
    ValueError: If packages is empty, and building package_path produces no
      tar archives.
    SetuptoolsFailedError: If the setup.py file fails to successfully build.
    MissingInitError: If the package doesn't contain an `__init__.py` file.
    DuplicateEntriesError: If multiple files with the same name were provided.
    ArgumentError: if no packages were found in the given path or no
      staging_location was but uploads were required.
  """
  remote_paths = []
  local_paths = []
  for package in packages:
    if storage_util.ObjectReference.IsStorageUrl(package):
      remote_paths.append(package)
    else:
      local_paths.append(package)

  if package_path:
    package_root = os.path.dirname(os.path.abspath(package_path))
    with _TempDirOrBackup(package_root) as working_dir:
      local_paths.extend(BuildPackages(package_path,
                                       os.path.join(working_dir, 'output')))
      remote_paths.extend(_UploadFilesByPath(local_paths, staging_location))
  elif local_paths:
    # Can't combine this with above because above requires the temporary
    # directory to still be around
    remote_paths.extend(_UploadFilesByPath(local_paths, staging_location))

  if not remote_paths:
    raise flags.ArgumentError(_NO_PACKAGES_ERROR_MSG)
  return remote_paths

def GetStagingLocation(job_id=None, staging_bucket=None, job_dir=None):
  """Get the appropriate staging location for the job given the arguments."""
  staging_location = None
  if staging_bucket:
    staging_location = storage_util.ObjectReference(staging_bucket,
                                                    job_id)
  elif job_dir:
    staging_location = storage_util.ObjectReference(
        job_dir.bucket_ref, '/'.join((job_dir.name.rstrip('/'), 'packages')))
  return staging_location


#!/usr/bin/env python
from openpyxl import load_workbook
from pdb import set_trace as debug
import re
import os
from bs4 import BeautifulSoup as bs4
import sys
import logging as log
import traceback
os.chdir(os.path.dirname(os.path.abspath(sys.argv[0])))
log.basicConfig(filename='mailmerge.log',level=log.INFO)

log.info(os.getcwd())
log.info(os.path.abspath(__name__))

import smtplib
from email.message import EmailMessage

import datetime
from jinja2 import Template
from nameparser import HumanName
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from html2markdown import convert as html2text
from argparse import ArgumentParser

parser = ArgumentParser(description=u"""Send emails based on data in an excel spreadsheet called data.xlsx.  Emails are
sent to the address listed in the 'email' column and data. The email is sent as HTML with a text part autogenerated from
the HTML. Put the HTML in 'template.html. Subject comes from the <title></title> tag in the head of the template.""")

parser.add_argument("-s", "--stylesheet", action="store_true", help="include stylesheet.css in each emails. This may prevent the text part from being automatically generated. ")

ns = parser.parse_args()

if ns.stylesheet:
    try:
        stylesheet = open('stylesheet.css', 'rb').read().decode('utf-8')
    except Exception as e:
        log.info("No stylesheet found.")
        stylesheet = u""
else:
    stylesheet = u""

wb = load_workbook("data.xlsx")
wsname = wb.sheetnames[0]
ws = wb[wsname]
headers = [cell.value for cell in ws[1]]
headers = {header: headers.index(header) for header in headers}

s = smtplib.SMTP('smtp.snl.salk.edu')

def send(To="", From="", Subject="", Html="", Text=""):
    message = MIMEMultipart('alternative')
    message['From'] = From
    message['To'] = To
    message['Subject'] = Subject

    if not Text:
        Text = html2text(Html)

    # Record the MIME types of both parts - text/plain and text/html.
    part1 = MIMEText(Text, 'text')
    part2 = MIMEText(Html, 'html')

    # Attach parts into message container.
    # According to RFC 2046, the last part of a multipart message, in this case
    # the HTML message, is best and preferred.
    message.attach(part1)
    message.attach(part2)

    try:
        s.send_message(message)
        log.info("Emailed {0} -> {1}".format(To, Text))
    except Exception as e:
        log.warning("Failed to send message to {0}".format(To))
        tb = traceback.format_exc()
        print(tb)

    return()

def clean_template(html):
    soup = bs4(html, "lxml")
    try:
        #Subject might be in the first line of the body.
        Subject = soup.p.text.split("Subject:")[1].strip()
        #Now delete the subject from the template
        soup.p.extract()
    except IndexError:
        #We couldn't find a subject in the template, let's use the name of the parent folder unless it's "mailmerge"
        Subject = os.path.split(os.getcwd())[1]
        if Subject == "mailmerge":
            print("Could not find a Subject. Put a Subject: line as the first line of your template or rename the mailmerge folder to be the subject.")
            sys.exit()

    #Remove meta and style sections of the head
    [meta.extract() for meta in soup('meta')]
    [style.extract() for style in soup('style')]
    htmlTemplate = str(soup)
    return(htmlTemplate, Subject)

htmlTemplate = open("template.html")
htmlTemplate, Subject = clean_template(htmlTemplate)

message_count = 0

rownumber = 1
for row in ws.iter_rows(min_row=2):
    rownumber += 1
    rowDict = {}
    for header in headers:
        rowDict[header] = row[headers[header]].value

    if not rowDict['to'] or not rowDict['from']:
        continue

    skip = str(rowDict.get("skiprow", ""))
    if skip and skip.lower() in ["skip", "yes", "true", "1"]:
        print("skipping row {0}.".format(rownumber))
        continue

    #If the spreadsheet defines a new templatename, then re-render the template. Otherwise, use the old one.
    templatefile = rowDict.get("template")
    if templatefile:
        try:
            htmlTemplate = open(templatefile, 'rb').read()
            htmlTemplate, Subject = clean_template(htmlTemplate)
        except IOError:
            print(u"You have specified a template file {0} that doesn't exist. Exiting...")
            sys.exit(1)

    name = rowDict.get('Name')
    if name:
    #Guess at the first and last name based on the full name. Put them into the dictionary for use in the template.
        try:
            nameparser = HumanName(name)
            rowDict['Firstname'] = nameparser.first
            rowDict['Lastname'] = nameparser.last
        except TypeError: #Occurs when name None.
            pass

    rowDict['stylesheet'] = stylesheet  #Ususally, this does nothing.
    template = Template(htmlTemplate)
    html = template.render(rowDict)

    redirect = rowDict.get('redirect')
    if redirect:
        print("Redirecting {0} to {1}".format(rowDict['to'], redirect))
        rowDict['to'] = redirect

    send(To=rowDict['to'], From=rowDict['from'], Subject=Subject, Html=html)
    logmessage = u"{2} : {1} ==> {0}".format(rowDict['to'], Subject, datetime.datetime.now())
    print(logmessage)
    log.info(logmessage)
    message_count += 1

print("Sent {0} messages".format(message_count))


"""
tianyi: 天一是三渣的早期作品:<贩罪>的主角，是像猫一样生活的男人，并且善于计算人心/性，做出各种冷静的数据分析
这里存一些数据分析相关的东西
"""
import decimal
from typing import List

import pendulum
from django.conf import settings
from django.db import models

from apps import utils
from apps.proxy import models as pm
from apps.sspanel import models as sm
from apps.stats.models import DailyStats

class DashBoardManger:
    """
    无情的dashboard生成器
    """

    def __init__(self, dt_list: List[pendulum.DateTime]):
        self.dt_list = dt_list
        self.log_dict = DailyStats.get_date_str_dict(dt_list)

    def _get_by_dt(self, dt: pendulum.DateTime):
        return self.log_dict[str(dt.date())]

    def get_node_status(self):
        def gen_bar_config(dt_list):
            node_total_traffic = pm.ProxyNode.calc_total_traffic()
            return {
                "title": f"所有节点当月共消耗:{node_total_traffic}",
                "labels": ["{}-{}".format(t.month, t.day) for t in dt_list],
                "data": [self._get_by_dt(dt).total_used_traffic for dt in dt_list],
                "data_title": "每日流量(GB)",
                "x_label": f"最近{len(dt_list)}天",
                "y_label": "单位:GB",
            }

        def gen_doughnut_config():
            active_nodes = pm.ProxyNode.get_active_nodes()
            labels = [node.name for node in active_nodes]
            return {
                "title": f"总共{len(labels)}条节点",
                "labels": labels,
                "data": [
                    round(node.used_traffic / settings.GB, 2) for node in active_nodes
                ],
                "data_title": "节点流量",
            }

        return {
            "doughnut_config": gen_doughnut_config(),
            "bar_config": gen_bar_config(self.dt_list),
        }

    def get_user_status_data(self):
        """统计用户信息"""

        def gen_line_configs(dt_list):
            active_user_count = [
                self._get_by_dt(dt).active_user_count for dt in dt_list
            ]
            active_user_line_config = {
                "title": f"最近{len(dt_list)}天 总活跃人数为{sum(active_user_count)}人",
                "labels": ["{}-{}".format(t.month, t.day) for t in dt_list],
                "data": active_user_count,
                "data_title": "活跃用户",
                "x_label": f"最近{len(dt_list)}天",
                "y_label": "活跃用户数",
            }
            new_user_count = [self._get_by_dt(dt).new_user_count for dt in dt_list]
            new_user_line_config = {
                "title": f"最近{len(dt_list)}天 新增人数为{sum(new_user_count)}人",
                "labels": ["{}-{}".format(t.month, t.day) for t in dt_list],
                "data": new_user_count,
                "data_title": "新增用户",
                "x_label": f"最近{len(dt_list)}天",
                "y_label": "新用户数",
            }
            return {
                "active_user_line_config": active_user_line_config,
                "new_user_line_config": new_user_line_config,
            }

        def gen_doughnut_config():
            user_status = [
                pm.UserTrafficLog.get_all_node_online_user_count(),
                sm.User.get_today_register_user().count(),
                sm.UserCheckInLog.get_checkin_user_count(
                    utils.get_current_datetime().date()
                ),
            ]
            return {
                "title": f"总用户数量{sm.User.objects.all().count()}人",
                "labels": ["在线人数", "今日注册", "今日签到"],
                "data": user_status,
                "data_title": "活跃用户",
            }

        data = {"doughnut_config": gen_doughnut_config()}
        data.update(gen_line_configs(self.dt_list))
        return data

    def get_userorder_status_data(self):
        """获取的订单统计数据"""

        def gen_bar_config(dt_list):
            success_order_count = [self._get_by_dt(dt).order_count for dt in dt_list]
            return {
                "title": f"总订单数量:{sum(success_order_count)}",
                "labels": [f"{date.month}-{date.day}" for date in dt_list],
                "data": success_order_count,
                "data_title": "每日订单数量",
                "x_label": f"最近{len(dt_list)}天",
                "y_label": "订单数量",
            }

        def gen_doughnut_config(dt_list):
            now = utils.get_current_datetime()
            today_amount = (
                sm.UserOrder.objects.filter(
                    status=sm.UserOrder.STATUS_FINISHED,
                    created_at__range=[
                        now.start_of("day"),
                        now.end_of("day"),
                    ],
                ).aggregate(amount=models.Sum("amount"))["amount"]
                or "0"
            )
            total_amount = (
                sm.UserOrder.objects.filter(
                    status=sm.UserOrder.STATUS_FINISHED,
                    created_at__range=[
                        dt_list[0].start_of("day"),
                        dt_list[-1].end_of("day"),
                    ],
                ).aggregate(amount=models.Sum("amount"))["amount"]
                or "0"
            )
            return {
                "title": f"总收入{total_amount}元",
                "labels": ["总收入", "今日收入"],
                "data": [
                    int(decimal.Decimal(total_amount)),
                    int(decimal.Decimal(today_amount)),
                ],
                "data_title": "收入分析",
            }

        def gen_line_config(dt_list):
            amount_data = [self._get_by_dt(dt).order_amount for dt in dt_list]
            return {
                "title": f"最近{len(dt_list)}天 总收益为{sum(amount_data)}元",
                "labels": ["{}-{}".format(t.month, t.day) for t in dt_list],
                "data": amount_data,
                "data_title": "收益",
                "x_label": f"最近{len(dt_list)}天",
                "y_label": "金额/元",
            }

        return {
            "bar_config": gen_bar_config(self.dt_list),
            "doughnut_config": gen_doughnut_config(self.dt_list),
            "line_config": gen_line_config(self.dt_list),
        }

    @classmethod
    def gen_traffic_line_chart_configs(cls, user_id, node_id, dt_list):
        proxy_node = pm.ProxyNode.get_or_none(node_id)  # node must exists
        user_total_traffic = pm.UserTrafficLog.calc_user_total_traffic(
            proxy_node, user_id
        )
        dt_list = sorted(dt_list)
        return {
            "title": "节点 {} 当月共消耗：{}".format(proxy_node.name, user_total_traffic),
            "labels": ["{}-{}".format(t.month, t.day) for t in dt_list],
            "data": [
                pm.UserTrafficLog.calc_traffic_by_datetime(dt, user_id, proxy_node)
                for dt in dt_list
            ],
            "data_title": proxy_node.name,
            "x_label": f"最近{len(dt_list)}天",
            "y_label": "单位:GB",
        }

    @classmethod
    def gen_ref_log_bar_chart_configs(cls, user_id, dt_list):
        """set register_count to 0 if the query date log not exists"""
        dt_list = sorted(dt_list)
        logs = {
            log.date: log.register_count
            for log in sm.UserRefLog.list_by_user_id_and_date_list(user_id, dt_list)
        }
        return {
            "title": f"总共邀请用户:{sm.UserRefLog.calc_user_total_ref_count(user_id)}人",
            "labels": [f"{date.month}-{date.day}" for date in dt_list],
            "data": [logs.get(date, 0) for date in dt_list],
            "data_title": "每日邀请注册人数",
            "x_label": f"最近{len(dt_list)}天",
            "y_label": "人",
        }


from boggle import Boggle
from flask import Flask,session,render_template,redirect,Request,request,jsonify

boggle_game = Boggle()

app = Flask(__name__)
app.config['SECRET_KEY'] = 'the random string'
boggle = Boggle()
@app.route('/')
def start_game():
    board = boggle.make_board()
    session['board'] = board
    return render_template('Start.html',board = board)

@app.route("/check-word")
def check_word():

    guess = request.args["guess"]
    board = session["board"]
    response = boggle_game.check_valid_word(board, guess)

    return jsonify({'result': response})

@app.route('/save-score', methods=['POST'])
def save_score():
    new_score = request.json['scores']
    if 'score' in session and session['score'] < new_score:
        session['score'] = new_score
    elif 'score' not in session:
        session['score'] = new_score
    return jsonify({'scores': session['score']})


"""ARP Scanner."""
# In future, we may want to import asyncio
import logging

import click
from scapy import all as sc
from scapy import config as conf

conf.verb = 0
logger = logging.getLogger(__name__)

@click.command('arp')
# @click.option('--quiet', default=False, is_flag=True)
@click.option('-t', '--target', required=True, type=str)
def scan_local_network(target):
    """Scan local network.

    Args:
        target (str): Destintaion IP
    """
    arp_layer = sc.ARP(pdst=target)
    ether_layer = sc.Ether(dst='ff:ff:ff:ff:ff:ff')
    arp_request = ether_layer / arp_layer

    answered, _ = sc.srp(arp_request, timeout=10)

    if answered:
        print(f'host {target} available')
        logger.setLevel(logging.INFO)
    else:
        print(f'host {target} not available')
        logger.setLevel(logging.ERROR)

if __name__ == '__main__':
    scan_local_network()


from communities import views
from django.conf.urls import url

urlpatterns = [
    url(r'^$', views.GroupListView.as_view(), name='list'),
    url(r'^create/$', views.GroupCreateView.as_view(), name='create'),
    url(r'^(?P<pk>\d+)/$', views.GroupDetailView.as_view(), name='detail'),
    url(r'^(?P<pk>\d+)/edit/$', views.GroupUpdateView.as_view(), name='update'),

]


# 特定のディレクトリにある画像を別のディレクトリへ移動させる
import os
import argparse
from typing import Tuple
import shutil

def count_images(path: str) -> int:
  """ディレクトリ内の画像数を調べる

  Args:
      path (str): 画像数を調べるフォルダのパス

  Returns:
      int: 画像数
  """

  return sum(os.path.isfile(os.path.join(path, name)) for name in os.listdir(path))

def get_copy_dir() -> tuple:
  """コピー元とコピー先ディレクトリ名を取得する

  Returns:
      Tuple[str, str]: [0]コピー元, [1]コピー先ディレクトリ名
  """
  parser: argparse.ArgumentParser = argparse.ArgumentParser()
  parser.add_argument('origin')
  parser.add_argument('copy_goal')
  args: argparse.Namespace = parser.parse_args()

  return (args.origin, args.copy_goal)

def copy_images(image_num: int, origin_dir: str, copy_dir: str):
  """画像を指定のディレクトリに移動する

  Args:
      image_num (int): 画像数（ファイル名の重複を防ぐため）
      origin_dir (str): コピー元ディレクトリパス
      copy_dir (str): コピー先ディレクトリパス
  """
  index: int = 0
  for img in os.listdir(origin_dir):
    filename: str = os.path.splitext(os.path.basename(img))[0]
    new_filepath = os.path.join(copy_dir, f"{filename}-{image_num + index}.jpg")
    shutil.move(os.path.join(origin_dir, img), new_filepath)
    index += 1

if __name__ == "__main__":
  dir: Tuple[str, str] = get_copy_dir()
  img_num: int = count_images(dir[0])
  copy_images(img_num, dir[0], dir[1])
  os.rmdir(dir[0])


import asyncio
import time
from threading import Event
import signal

from libs.utils import get_logger
from libs.senzing_libs.senzing_init import SenzingInit

class Worker(SenzingInit):

    def __init__(self, ex_ev):
        self.ex_ev = ex_ev
        self.log = get_logger()
        super(Worker, self).__init__(self.log)
        self.threads = 5

    async def work(self):

        while not self.ex_ev.is_set():
            cnt = self.g2_engine.countRedoRecords()
            if not cnt:
                await asyncio.sleep(10)
                continue

            response_bytearray = bytearray()
            try:
                self.g2_engine.processRedoRecord(response_bytearray)
            except Exception as e:
                self.log.error(e)
                active_config_id_bytearray = bytearray()
                default_config_id_bytearray = bytearray()
                self.g2_engine.getActiveConfigID(active_config_id_bytearray)
                self.g2_config_mgr.getDefaultConfigID(
                    default_config_id_bytearray)
                try:
                    if active_config_id_bytearray != \
                            default_config_id_bytearray:
                        self.g2_engine.reinit(default_config_id_bytearray)
                        self.log.info('G2Engine reinitialised')
                except Exception as err:
                    self.log.error(err)

        if self.ex_ev.is_set():
            self.log.info('Stopping daemons.')

    def run(self):
        loop = asyncio.get_event_loop()
        tasks = [loop.create_task(self.work()) for _ in
                 range(self.threads)]
        group = asyncio.gather(*tasks)
        loop.run_until_complete(group)

if __name__ == '__main__':
    time.sleep(10)
    exit_event = Event()
    app = Worker(exit_event)

    def signal_handler(sig, frame):
        app.ex_ev.set()

    signal.signal(signal.SIGTERM, signal_handler)
    signal.signal(signal.SIGINT, signal_handler)
    app.run()