#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Command line interface for ms3. This module includes the argument parsers for the various commands and has one function
<command>_cmd() per <command> which takes the parsed arguments namespace and dispatches them to the appropriate function
in the ms3.operations module, using a couple of helper functions to that aim.
"""

import argparse
import os
import re
from collections import defaultdict
from typing import List, Optional

from ms3 import Parse, compute_path_from_file, make_coloring_reports_and_warnings
from ms3._version import __version__
from ms3.logger import get_logger, inspect_loggers
from ms3.operations import (
    check,
    compare,
    extract,
    insert_labels_into_score,
    store_scores,
    transform_to_package,
    transform_to_resources,
    update,
)
from ms3.utils import (
    capture_parse_logs,
    convert_folder,
    resolve_dir,
    write_warnings_to_file,
)

__author__ = "johentsch"
__copyright__ = "École Polytechnique Fédérale de Lausanne"
__license__ = "gpl3"

def gather_extract_params(args) -> List[str]:
    params = [
        name
        for name, arg in zip(
            (
                "measures",
                "notes",
                "rests",
                "labels",
                "expanded",
                "form_labels",
                "events",
                "chords",
            ),
            (
                args.measures,
                args.notes,
                args.rests,
                args.labels,
                args.expanded,
                args.form_labels,
                args.events,
                args.chords,
            ),
        )
        if arg
    ]
    if args.metadata is not None and not (args.metadata == False):  # noqa
        params.append("metadata")
    return params

def add_cmd(args, parse_obj: Optional[Parse] = None) -> Parse:
    logger = get_logger("ms3.add", level=args.level)
    if parse_obj is None:
        p = make_parse_obj(args)
    else:
        p = parse_obj

    insert_labels_into_score(
        p,
        facet=args.use,
        ask_for_input=args.ask,
        replace=args.replace,
    )
    corpus2paths = store_scores(
        p, root_dir=args.out, folder=".", simulate=args.test, suffix=args.suffix
    )
    changed = sum(map(len, corpus2paths.values()))
    logger.info(
        f"Operation resulted in {changed} annotated score{'s' if changed != 1 else ''}."
    )

def check_cmd(args, parse_obj: Optional[Parse] = None) -> Parse:
    if parse_obj is None:
        p = make_parse_obj(args)
    else:
        p = parse_obj
    _ = check(
        p,
        ignore_labels=args.ignore_labels,
        ignore_scores=args.ignore_scores,
        assertion=args.fail,
        ignore_metronome=args.ignore_metronome,
    )
    return p

def compare_cmd(args, parse_obj=None, output: bool = True, logger=None) -> None:
    if logger is None:
        logger = get_logger("ms3.compare", level=args.level)
    if parse_obj is None:
        p = make_parse_obj(args)
    else:
        p = parse_obj
    revision = None if args.compare == "" else args.compare
    try:
        n_changed, n_unchanged = compare(
            p,
            facet=args.use,
            ask=args.ask,
            revision_specifier=revision,
            flip=args.flip,
            logger=p.logger,
        )
    except TypeError:
        logger.error(
            "No files to compare. Try adding the flag -a to parse files even if they are not included in a "
            "metdata.tsv file."
        )
        return
    logger.debug(
        f"{n_changed} files changed labels during comparison, {n_unchanged} didn't."
    )
    if output:
        corpus2paths = store_scores(
            p, root_dir=args.out, simulate=args.test, suffix=args.suffix
        )
        changed = sum(map(len, corpus2paths.values()))
        logger.info(
            f"Operation resulted in {changed} comparison file{'s' if changed != 1 else ''}."
        )
    else:
        logger.info(
            f"Operation resulted in colored labels within {n_changed} score{'s' if n_changed != 1 else ''}."
        )

def convert_cmd(args):
    # assert target[:len(
    #    dir)] != dir, "TARGET_DIR cannot be identical with nor a subfolder of DIR.\nDIR:        " + dir +
    #    '\nTARGET_DIR: ' + target
    update_logger = get_logger("ms3.convert", level=args.level)
    for argument_name, argument, default in (
        ("-a/--all", args.all, False),
        ("-e/--exclude", args.exclude, None),
        ("-f/--folders", args.folders, None),
        ("--reviewed", args.reviewed, False),
        ("-t/--test", args.test, False),
        ("-v/--verbose", args.verbose, False),
    ):
        if argument != default:
            update_logger.info(
                f"Argument '{argument_name}' is currently being ignored."
            )
    out_dir = os.getcwd() if args.out is None else resolve_dir(args.out)
    ms = "auto" if args.musescore is None else args.musescore
    convert_folder(
        directory=args.dir,
        file_paths=args.files,
        target_dir=out_dir,
        extensions=args.extensions,
        target_extension=args.format,
        regex=args.include,
        suffix=args.suffix,
        recursive=not args.nonrecursive,
        ms=ms,
        overwrite=args.safe,
        parallel=not args.iterative,
        logger=update_logger,
    )

def empty(args, parse_obj: Optional[Parse] = None):
    if parse_obj is None:
        p = make_parse_obj(args, parse_scores=True)
    else:
        p = parse_obj
    p.detach_labels()
    p.store_parsed_scores(
        only_changed=True,
        root_dir=args.out,
        suffix=args.suffix,
        overwrite=True,
    )

def extract_cmd(args, parse_obj: Optional[Parse] = None):
    if parse_obj is None:
        p = make_parse_obj(args)
    else:
        p = parse_obj
    params = gather_extract_params(args)
    if len(params) == 0:
        print(
            "In order to extract DataFrames, pass at least one of the following arguments: -M (measures), -N (notes), "
            "-R (rests), -L (labels), -X (expanded), -F (form_labels), -E (events), -C (chords), -D (metadata)"
        )
        return
    extract(
        p,
        root_dir=args.out,
        notes_folder=args.notes,
        labels_folder=args.labels,
        measures_folder=args.measures,
        rests_folder=args.rests,
        events_folder=args.events,
        chords_folder=args.chords,
        expanded_folder=args.expanded,
        form_labels_folder=args.form_labels,
        metadata_suffix=args.metadata,
        simulate=args.test,
        parallel=not args.iterative,
        unfold=args.unfold,
        interval_index=args.interval_index,
        corpuswise=args.corpuswise,
    )

def metadata(args, parse_obj: Optional[Parse] = None):
    """Update MSCX files with changes made in metadata.tsv (created via ms3 extract -D). In particular,
    add the values from (new?) columns to the corresponding fields in the MuseScore files' "Score info".
    """
    if parse_obj is None:
        p = make_parse_obj(args)
    else:
        p = parse_obj
    print("PARSING SCORES ONE AFTER THE OTHER...")
    p.parse_scores(parallel=False)
    modified_score_files = p.update_score_metadata_from_tsv(
        write_empty_values=args.empty,
        remove_unused_fields=args.remove,
        write_text_fields=args.prelims,
        update_instrumentation=args.instrumentation,
    )
    if len(modified_score_files) == 0:
        print("Nothing to update.")
        return
    corpus2paths = store_scores(
        p,
        only_changed=True,
        root_dir=args.out,
        folder=".",
        simulate=args.test,
        suffix=args.suffix,
    )
    changed = sum(map(len, corpus2paths.values()))
    print(
        f"Operation resulted in {changed} updated score{'s' if changed != 1 else ''}."
    )

# def repair(args):
#     print("Sorry, the command has not been implemented yet.")
#     print(args.dir)

def transform_cmd(args):
    params = gather_extract_params(args)
    if len(params) == 0:
        print(
            "Pass at least one of the following arguments: -M (measures), -N (notes), -R (rests), -L (labels), "
            "-X (expanded), -F (form_labels), -E (events), -C (chords), -D (metadata)"
        )
        return
    parse_obj = make_parse_obj(args, parse_tsv=True, facets=params)
    filename = os.path.basename(args.dir)
    func = transform_to_resources if args.resources else transform_to_package
    # noinspection PyTypeChecker
    func(
        ms3_object=parse_obj,
        facets=params,
        filename=filename,
        output_folder=args.out,
        choose="auto",
        interval_index=args.interval_index,
        unfold=args.unfold,
        test=args.test,
        zipped=not args.uncompressed,
        overwrite=args.safe,
        log_level=args.level,
    )

def update_cmd(args, parse_obj: Optional[Parse] = None):
    if parse_obj is None:
        p = make_parse_obj(args, parse_scores=True)
    else:
        p = parse_obj
    changed_paths = update(
        p,
        root_dir=args.out,
        suffix="" if args.suffix is None else args.suffix,
        overwrite=True,
        staff=int(args.staff),
        harmony_layer=args.type,
        above=args.above,
        safe=args.safe,
    )
    if len(changed_paths) > 0:
        print(f"Paths of scores with updated labels:\n{changed_paths}")
    else:
        print("Nothing to do.")

def check_and_create(d):
    """Turn input into an existing, absolute directory path."""
    if not os.path.isdir(d):
        d = resolve_dir(os.path.join(os.getcwd(), d))
        if not os.path.isdir(d):
            if input(d + " does not exist. Create? (y|n)") == "y":
                os.mkdir(d)
            else:
                raise argparse.ArgumentTypeError(
                    d + " needs to be an existing directory"
                )
    return resolve_dir(d)

def check_dir(d):
    if not os.path.isdir(d):
        d = resolve_dir(os.path.join(os.getcwd(), d))
        if not os.path.isdir(d):
            raise argparse.ArgumentTypeError(d + " needs to be an existing directory")
    return resolve_dir(d)

def review_cmd(
    args,
    parse_obj: Optional[Parse] = None,
) -> None:
    """This command combines the functionalities check-extract-compare and additionally colors non-chord tones."""
    logger = get_logger("ms3.review", level=args.level)
    if parse_obj is None:
        p = make_parse_obj(args)
    else:
        p = parse_obj
    accumulated_warnings = []

    # call ms3 check, collecting all warnings issued while parsing
    if args.ignore_scores + args.ignore_labels < 2:
        what = "" if args.ignore_scores else "SCORES"
        if args.ignore_labels:
            what += "LABELS" if what == "" else "AND LABELS"
        print(f"CHECKING {what}...")
        warning_strings = check(
            p,
            ignore_labels=args.ignore_labels,
            ignore_scores=args.ignore_scores,
            assertion=False,
            parallel=False,
            ignore_metronome=args.ignore_metronome,
        )
        accumulated_warnings.extend(warning_strings)
        test_passes = len(warning_strings) == 0
        p.parse_tsv()
    else:
        test_passes = True
        p.parse(parallel=False)
    if p.n_parsed_scores == 0:
        msg = "NO SCORES PARSED, NOTHING TO DO."
        if not args.all:
            msg += (
                "\nI was disregarding all files whose file names are not listed in the column 'piece' of a "
                "'metadata.tsv' file. "
                "Add -a if you want me to include all scores."
            )
        print(msg)
        return

    # call ms3 extract
    params = gather_extract_params(args)
    if len(params) > 0:
        print("EXTRACTING FACETS...")
        extract_cmd(args, p)

    # color out-of-label tones
    print("COLORING OUT-OF-LABEL TONES...")
    with capture_parse_logs(p.logger) as captured_warnings:
        test_passes = (
            make_coloring_reports_and_warnings(
                p, out_dir=args.out, threshold=args.threshold
            )
            and test_passes
        )
        accumulated_warnings.extend(captured_warnings.content_list)

    # attribute warnings to pieces based on logger names
    logger2pieceID = {
        corpus.logger_names[piece]: (c, piece)
        for c, corpus in p.corpus_objects.items()
        for piece in corpus.keys()
    }
    piece2warnings = defaultdict(list)
    for warning_strings in accumulated_warnings:
        warning_lines = warning_strings.splitlines()
        first_line = warning_lines[0]
        match = re.search(r"(ms3\.Parse\..+) --", first_line)
        if match is None:
            logger.warning(
                f"This warning contains no ms3 logger name, skipping: {warning_strings}"
            )
            continue
        warning_lines[0] = first_line[
            : match.end(1)
        ]  # cut off the warning's header everything following the logger
        # name because paths to source code are system-dependent
        pieceID = logger2pieceID[match.group(1)]
        piece2warnings[pieceID].append("\n".join(warning_lines))

    # write all warnings to piece-specific warnings files and remove existing files where no warnings were captured
    for pieceID, score_files in p.get_files(
        "scores", unparsed=False, flat=True, include_empty=False
    ).items():
        file = score_files[0]
        warnings_path = compute_path_from_file(
            file, root_dir=args.out, folder="reviewed"
        )
        warnings_file = os.path.join(
            warnings_path, file.piece + file.suffix + ".warnings"
        )
        write_warnings_to_file(
            warnings_file=warnings_file, warnings=piece2warnings[pieceID], logger=logger
        )

    # call ms3 compare
    if args.compare is not None:
        revision = None if args.compare == "" else args.compare
        commit_str = "" if revision is None else f" @ {revision}"
        expanded = "EXPANDED " if args.use == "expanded" else ""
        print(
            f"COMPARING CURRENT {expanded}LABELS WITH PREVIOUS ONES FROM THE TSV FILES{commit_str}..."
        )
        n_changed, n_unchanged = compare(
            p,
            facet=args.use,
            ask=args.ask,
            revision_specifier=revision,
            flip=args.flip,
            logger=logger,
        )
        logger.debug(
            f"{n_changed} files changed labels during comparison, {n_unchanged} didn't."
        )
    corpus2paths = store_scores(p, root_dir=args.out, simulate=args.test)
    changed = sum(map(len, corpus2paths.values()))
    logger.info(
        f"Operation resulted in {changed} review file{'s' if changed != 1 else ''}."
    )
    if test_passes:
        logger.info("Parsed scores passed all tests.")
    else:
        msg = "Not all tests have passed. Please check the relevant warnings in the generated .warnings files."
        if args.fail:
            assert test_passes, msg
        else:
            logger.info(msg)
    out_dir = "." if args.out is None else args.out
    warnings_file = os.path.join(out_dir, "warnings.log")
    if os.path.isfile(warnings_file):
        logger.info(f"Removing deprecated warnings file: {warnings_file}")
        os.remove(warnings_file)

def make_parse_obj(args, parse_scores=False, parse_tsv=False, facets=None):
    labels_cfg = {}
    if hasattr(args, "positioning"):
        labels_cfg["positioning"] = args.positioning
    if hasattr(args, "raw"):
        labels_cfg["decode"] = args.raw

    logger_cfg = {
        "level": args.level,
        "path": args.log,
    }
    ms = args.musescore if hasattr(args, "musescore") else None
    file_re_str = None if args.include is None else f"'{args.include}'"
    folder_re_str = None if args.folders is None else f"'{args.folders}'"
    exclude_re_str = None if args.exclude is None else f"'{args.exclude}'"
    ms_str = None if ms is None else f"'{ms}'"
    print(
        f"""CREATING PARSE OBJECT WITH THE FOLLOWING PARAMETERS:
Parse('{args.dir}',
     recursive={not args.nonrecursive},
     only_metadata_pieces={not args.all},
     include_convertible={ms is not None},
     exclude_review={not args.reviewed},
     file_re={file_re_str},
     folder_re={folder_re_str},
     exclude_re={exclude_re_str},
     file_paths={args.files},
     labels_cfg={labels_cfg},
     ms={ms_str},
     **{logger_cfg})
"""
    )
    parse_obj = Parse(
        args.dir,
        recursive=not args.nonrecursive,
        only_metadata_pieces=not args.all,
        include_convertible=ms is not None,
        exclude_review=not args.reviewed,
        file_re=args.include,
        folder_re=args.folders,
        exclude_re=args.exclude,
        file_paths=args.files,
        labels_cfg=labels_cfg,
        ms=ms,
        **logger_cfg,
    )
    if facets is not None:
        facets = [f"{f}$" for f in facets if f != "metadata"]
        if len(facets) > 0:
            parse_obj.view.include("facets", *facets)
    if parse_scores:
        mode = "ONE AFTER THE OTHER" if args.iterative else "IN PARALLEL"
        print(f"PARSING SCORES {mode}...")
        parse_obj.parse_scores(parallel=not args.iterative)
    if parse_tsv and facets != []:
        print("PARSING TSV FILES...")
        parse_obj.parse_tsv()
    info_str = parse_obj.info(show_discarded=args.verbose, return_str=True).replace(
        "\n", "\n\t"
    )
    print(f"RESULTING PARSE OBJECT:\n\t{info_str}")
    if args.verbose:
        print(inspect_loggers())
    return parse_obj

def get_arg_parser():
    # reusable argument sets
    parse_args = argparse.ArgumentParser(add_help=False)
    parse_args.add_argument(
        "-d",
        "--dir",
        metavar="DIR",
        default=os.getcwd(),
        type=check_dir,
        help="Folder(s) that will be scanned for input files. Defaults to current working directory if no individual "
        "files are passed via -f.",
    )
    parse_args.add_argument(
        "-o",
        "--out",
        metavar="OUT_DIR",
        type=check_and_create,
        help="Output directory.",
    )
    parse_args.add_argument(
        "-n",
        "--nonrecursive",
        action="store_true",
        help="Treat DIR as single corpus even if it contains corpus directories itself.",
    )
    parse_args.add_argument(
        "-a",
        "--all",
        action="store_true",
        help="By default, only files listed in the 'piece' column of a 'metadata.tsv' file are parsed. With "
        "this option, all files will be parsed.",
    )
    parse_args.add_argument(
        "-i",
        "--include",
        metavar="REGEX",
        help="Select only files whose names include this string or regular expression.",
    )
    parse_args.add_argument(
        "-e",
        "--exclude",
        metavar="REGEX",
        help="Any files or folders (and their subfolders) including this regex will be disregarded."
        "By default, files including '_reviewed' or starting with . or _ or 'concatenated' are excluded.",
    )
    parse_args.add_argument(
        "-f",
        "--folders",
        metavar="REGEX",
        help="Select only folders whose names include this string or regular expression.",
    )
    parse_args.add_argument(
        "-m",
        "--musescore",
        metavar="PATH",
        nargs="?",
        const="auto",
        help="""Command or path of your MuseScore 3 executable. -m by itself will set 'auto' (attempt to use standard
        path for your system).
        Other shortcuts are -m win, -m mac, and -m mscore (for Linux).""",
    )
    parse_args.add_argument(
        "--reviewed",
        action="store_true",
        help="By default, review files and folder are excluded from parsing. With this option, "
        "they will be included, too.",
    )
    parse_args.add_argument(
        "--files",
        metavar="PATHs",
        nargs="+",
        help="(Deprecated) The paths are expected to be within DIR. They will be converted into a view "
        "that includes only the indicated files. This is equivalent to specifying the file names as "
        "a regex via --include (assuming that file names are unique amongst corpora.",
    )
    parse_args.add_argument(
        "--iterative",
        action="store_true",
        help="Do not use all available CPU cores in parallel to speed up batch jobs.",
    )
    parse_args.add_argument(
        "-l",
        "--level",
        metavar="{c, e, w, i, d}",
        default="i",
        help="Choose how many log messages you want to see: c (none), e, w, i, d (maximum)",
    )
    parse_args.add_argument(
        "--log",
        nargs="?",
        const=".",
        help="Can be a file path or directory path. Relative paths are interpreted relative to the current directory.",
    )
    parse_args.add_argument(
        "-t", "--test", action="store_true", help="No data is written to disk."
    )
    parse_args.add_argument(
        "-v",
        "--verbose",
        action="store_true",
        help="Show more output such as files discarded from parsing.",
    )

    check_args = argparse.ArgumentParser(add_help=False)
    check_args.add_argument(
        "--ignore_scores",
        action="store_true",
        help="Don't check scores for encoding errors.",
    )
    check_args.add_argument(
        "--ignore_labels",
        action="store_true",
        help="Don't check DCML labels for syntactic correctness.",
    )
    check_args.add_argument(
        "--fail",
        action="store_true",
        help="If you pass this argument the process will deliberately fail with an AssertionError "
        "when there are any mistakes.",
    )
    check_args.add_argument(
        "--ignore_metronome",
        action="store_true",
        help="Pass this flag if you want the check to pass (not fail) even if there is a warning about a missing "
        "metronome mark in the first bar of the score.",
    )

    compare_args = argparse.ArgumentParser(add_help=False)
    compare_args.add_argument(
        "--flip",
        action="store_true",
        help="Pass this flag to treat the annotation tables as if updating the scores instead of the other way around, "
        "effectively resulting in a swap of the colors in the output files.",
    )
    compare_args.add_argument(
        "--safe", action="store_false", help="Don't overwrite existing files."
    )

    extract_args = argparse.ArgumentParser(add_help=False)
    extract_args.add_argument(
        "-M",
        "--measures",
        metavar="folder",
        nargs="?",
        const="../measures",
        help="Folder where to store TSV files with measure information needed for tasks such as unfolding repetitions.",
    )
    extract_args.add_argument(
        "-N",
        "--notes",
        metavar="folder",
        nargs="?",
        const="../notes",
        help="Folder where to store TSV files with information on all notes.",
    )
    extract_args.add_argument(
        "-R",
        "--rests",
        metavar="folder",
        nargs="?",
        const="../rests",
        help="Folder where to store TSV files with information on all rests.",
    )
    extract_args.add_argument(
        "-L",
        "--labels",
        metavar="folder",
        nargs="?",
        const="../labels",
        help="Folder where to store TSV files with information on all annotation labels.",
    )
    extract_args.add_argument(
        "-X",
        "--expanded",
        metavar="folder",
        nargs="?",
        const="../harmonies",
        help="Folder where to store TSV files with expanded DCML labels.",
    )
    extract_args.add_argument(
        "-F",
        "--form_labels",
        metavar="folder",
        nargs="?",
        const="../form_labels",
        help="Folder where to store TSV files with all form labels.",
    )
    extract_args.add_argument(
        "-E",
        "--events",
        metavar="folder",
        nargs="?",
        const="../events",
        help="Folder where to store TSV files with all events (chords, rests, articulation, etc.) without further "
        "processing.",
    )
    extract_args.add_argument(
        "-C",
        "--chords",
        metavar="folder",
        nargs="?",
        const="../chords",
        help="Folder where to store TSV files with <chord> tags, "
        "i.e. groups of notes in the same voice with identical onset and duration. "
        "The tables include lyrics, dynamics, articulation, staff- and system texts, tempo marking, spanners, "
        "and thoroughbass figures.",
    )
    extract_args.add_argument(
        "-J",
        "--joined_chords",
        metavar="folder",
        nargs="?",
        const="../chords",
        help="Like -C except that all Chords are substituted with the actual Notes they contain. This is useful, "
        "for example, "
        "for relating slurs to the notes they group, or bass figures to their bass notes. ",
    )
    extract_args.add_argument(
        "-D",
        "--metadata",
        metavar="suffix",
        nargs="?",
        const="",
        help="Set -D to update the 'metadata.tsv' files of the respective corpora with the parsed scores. "
        "Add a suffix if you want to update 'metadata{suffix}.tsv' instead.",
    )
    extract_args.add_argument(
        "-p",
        "--positioning",
        action="store_true",
        help="When extracting labels, include manually shifted position coordinates in order to restore them when "
        "re-inserting.",
    )
    extract_args.add_argument(
        "--raw",
        action="store_false",
        help="When extracting labels, leave chord symbols encoded instead of turning them into a single column of "
        "strings.",
    )
    extract_args.add_argument(
        "-u",
        "--unfold",
        action="store_true",
        help="Unfold the repeats for all stored DataFrames.",
    )
    extract_args.add_argument(
        "--interval_index",
        action="store_true",
        help="Prepend a column with [start, end) intervals to the TSV files.",
    )
    extract_args.add_argument(
        "--corpuswise",
        action="store_true",
        help="Parse one corpus after the other rather than all at once.",
    )

    select_facet_args = argparse.ArgumentParser(add_help=False)
    select_facet_args.add_argument(
        "--ask",
        action="store_true",
        help="If several files are available for the selected facet (default: 'expanded', see --use), I will pick one "
        "automatically. Add --ask if you want me to have you select which ones to compare with the scores.",
    )
    select_facet_args.add_argument(
        "--use",
        default="expanded",
        choices=["expanded", "labels"],
        help="Which type of labels you want to compare with the ones in the score. Defaults to 'expanded', "
        "i.e., DCML labels. Set --use labels to use other labels available as TSV and set --ask if several "
        "sets of labels are available that you want to choose from.",
    )

    # main argument parser
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description="""\
--------------------------
| Welcome to ms3 parsing |
--------------------------

The library offers you the following commands. Add the flag -h to one of them to learn about its parameters.
""",
    )
    parser.add_argument("--version", action="version", version=__version__)
    subparsers = parser.add_subparsers(
        help="The action that you want to perform.", dest="action"
    )

    add_parser = subparsers.add_parser(
        "add",
        help="Add labels from annotation tables to scores.",
        parents=[select_facet_args, parse_args],
    )
    # TODO: staff parameter needs to accept one to several integers including negative
    # add_parser.add_argument('-s', '--staff',
    #                            help="Remove labels from selected staves only. 1=upper staff; -1=lowest staff (
    #                            default)")
    # add_parser.add_argument('--type', default=1,
    #                            help="Only remove particular types of harmony labels.")
    add_parser.add_argument(
        "-s",
        "--suffix",
        metavar="SUFFIX",
        default="_annotated",
        help="Suffix of the new scores with inserted labels. Defaults to _annotated.",
    )
    add_parser.add_argument(
        "--replace",
        action="store_true",
        help="Remove existing labels from the scores prior to adding. Like calling ms3 empty first.",
    )
    add_parser.set_defaults(func=add_cmd)

    check_parser = subparsers.add_parser(
        "check",
        help="""Parse MSCX files and look for errors.
In particular, check DCML harmony labels for syntactic correctness.""",
        parents=[check_args, parse_args],
    )
    check_parser.set_defaults(func=check_cmd)

    compare_parser = subparsers.add_parser(
        "compare",
        help="For MSCX files for which annotation tables exist, create another MSCX file with a coloured label "
        "comparison if differences are found.",
        parents=[select_facet_args, compare_args, parse_args],
    )
    compare_parser.add_argument(
        "-c",
        "--compare",
        metavar="GIT_REVISION",
        default="",
        help="By default, the _reviewed file displays removed labels in red and added labels in green, compared to "
        "the version currently "
        "represented in the present TSV files, if any. If instead you want a comparison with the TSV files from "
        "another Git commit, "
        "pass its specifier, e.g. 'HEAD~3', <branch-name>, <commit SHA> etc.",
    )
    compare_parser.add_argument(
        "-s",
        "--suffix",
        metavar="SUFFIX",
        default="_compared",
        help="Suffix of the newly created comparison files. Defaults to _compared",
    )
    compare_parser.set_defaults(func=compare_cmd)

    convert_parser = subparsers.add_parser(
        "convert",
        help="Use your local install of MuseScore to convert MuseScore files.",
        parents=[parse_args],
    )
    convert_parser.add_argument(
        "-s",
        "--suffix",
        metavar="SUFFIX",
        default="",
        help="Suffix of the converted files. Defaults to " ".",
    )
    convert_parser.add_argument(
        "--format",
        default="mscx",
        help="Output format of converted files. Defaults to mscx. Other options are "
        "{png, svg, pdf, mscz, wav, mp3, flac, ogg, xml, mxl, mid}",
    )
    convert_parser.add_argument(
        "--extensions",
        nargs="+",
        default=["mscx", "mscz"],
        help="Those file extensions that you want to be converted, separated by spaces. Defaults to mscx mscz",
    )
    convert_parser.add_argument(
        "--safe", action="store_false", help="Don't overwrite existing files."
    )
    convert_parser.set_defaults(func=convert_cmd)

    empty_parser = subparsers.add_parser(
        "empty",
        help="Remove harmony annotations and store the MuseScore files without them.",
        parents=[parse_args],
    )
    # TODO: staff parameter needs to accept one to several integers including negative
    # empty_parser.add_argument('-s', '--staff',
    #                            help="Remove labels from selected staves only. 1=upper staff; -1=lowest staff (
    #                            default)")
    # empty_parser.add_argument('--type', default=1,
    #                            help="Only remove particular types of harmony labels.")
    empty_parser.add_argument(
        "-s",
        "--suffix",
        metavar="SUFFIX",
        default="_clean",
        help="Suffix of the new scores with removed labels. Defaults to _clean.",
    )
    empty_parser.set_defaults(func=empty)

    extract_parser = subparsers.add_parser(
        "extract",
        help="Extract selected information from MuseScore files and store it in TSV files.",
        parents=[extract_args, parse_args],
    )
    extract_parser.set_defaults(func=extract_cmd)

    metadata_parser = subparsers.add_parser(
        "metadata",
        help="Update MSCX files with changes made to metadata.tsv (created via ms3 extract -D [-a]).",
        parents=[parse_args],
    )
    metadata_parser.add_argument(
        "-s",
        "--suffix",
        metavar="SUFFIX",
        help="Suffix of the new scores with updated metadata fields.",
    )
    metadata_parser.add_argument(
        "-p",
        "--prelims",
        action="store_true",
        help="Pass this flag if, in addition to updating metadata fields, you also want score headers to be updated "
        "from the columns title_text, subtitle_text, composer_text, lyricist_text, part_name_text.",
    )
    metadata_parser.add_argument(
        "--instrumentation",
        action="store_true",
        help="Pass this flag to update the score's instrumentation based on changed values from "
        "'staff_<i>_instrument' columns.",
    )
    metadata_parser.add_argument(
        "--empty",
        action="store_true",
        help="Set this flag to also allow empty values to be used for overwriting existing ones.",
    )
    metadata_parser.add_argument(
        "--remove",
        action="store_true",
        help="Set this flag to remove non-default metadata fields that are not columns in the metadata.tsv file "
        "anymore.",
    )
    metadata_parser.set_defaults(func=metadata)
    #
    # repair_parser = subparsers.add_parser('repair',
    #                                       help="Apply automatic repairs to your uncompressed MuseScore files.",
    #                                       parents=[parse_args])
    # repair_parser.set_defaults(func=repair)

    review_parser = subparsers.add_parser(
        "review",
        help="Extract facets, check labels, and create _reviewed files.",
        parents=[check_args, select_facet_args, compare_args, extract_args, parse_args],
    )
    review_parser.add_argument(
        "-c",
        "--compare",
        nargs="?",
        metavar="GIT_REVISION",
        const="",
        help="Pass -c if you want the _reviewed file to display removed labels in red and added labels in green, "
        "compared to the version currently "
        "represented in the present TSV files, if any. If instead you want a comparison with the TSV files from "
        "another Git commit, additionally "
        "pass its specifier, e.g. 'HEAD~3', <branch-name>, <commit SHA> etc.",
    )
    review_parser.add_argument(
        "--threshold",
        default=0.6,
        type=float,
        help="Harmony segments where the ratio of non-chord tones vs. chord tones lies above this threshold "
        "will be printed in a warning and will cause the check to fail if the --fail flag is set. Defaults to "
        "0.6 (3:2).",
    )
    review_parser.set_defaults(func=review_cmd)

    transform_parser = subparsers.add_parser(
        "transform",
        help="Concatenate and transform TSV data from one or several corpora. "
        "Available transformations are unfolding repeats and adding an interval index.",
        parents=[parse_args],
    )
    transform_parser.add_argument(
        "-M",
        "--measures",
        action="store_true",
        help="Concatenate measures TSVs for all selected pieces.",
    )
    transform_parser.add_argument(
        "-N",
        "--notes",
        action="store_true",
        help="Concatenate notes TSVs for all selected pieces.",
    )
    transform_parser.add_argument(
        "-R",
        "--rests",
        action="store_true",
        help="Concatenate rests TSVs for all selected pieces (use ms3 extract -R to create those).",
    )
    transform_parser.add_argument(
        "-L",
        "--labels",
        action="store_true",
        help="Concatenate raw harmony label TSVs for all selected pieces (use ms3 extract -L to create those).",
    )
    transform_parser.add_argument(
        "-X",
        "--expanded",
        action="store_true",
        help="Concatenate expanded DCML label TSVs for all selected pieces.",
    )
    transform_parser.add_argument(
        "-F",
        "--form_labels",
        metavar="folder",
        nargs="?",
        const="../form_labels",
        help="Concatenate form label TSVs for all selected pieces.",
    )
    transform_parser.add_argument(
        "-E",
        "--events",
        action="store_true",
        help="Concatenate events TSVs (notes, rests, articulation, etc.) for all selected pieces (use ms3 extract -E "
        "to create those).",
    )
    transform_parser.add_argument(
        "-C",
        "--chords",
        action="store_true",
        help="Concatenate chords TSVs (<chord> tags group notes in the same voice with identical onset and duration) "
        "including "
        "lyrics, dynamics, articulation, staff- and system texts, tempo marking, spanners, and thoroughbass "
        "figures, "
        "for all selected pieces (use ms3 extract -C to create those).",
    )
    transform_parser.add_argument(
        "-D",
        "--metadata",
        action="store_true",
        help="Output 'concatenated_metadata.tsv' with one row per selected piece.",
    )
    transform_parser.add_argument(
        "-s",
        "--suffix",
        nargs="*",
        metavar="SUFFIX",
        help="Pass -s to use standard suffixes or -s SUFFIX to choose your own. In the latter case they will be "
        "assigned to the extracted aspects in the order "
        "in which they are listed above (capital letter arguments).",
    )
    transform_parser.add_argument(
        "-u",
        "--unfold",
        action="store_true",
        help="Unfold the repeats for all concatenated DataFrames.",
    )
    transform_parser.add_argument(
        "--interval_index",
        action="store_true",
        help="Prepend a column with [start, end) intervals to the TSV files.",
    )
    transform_parser.add_argument(
        "--resources",
        action="store_true",
        help="Store the concatenated DataFrames as TSV files with resource descriptors rather than in a ZIP with a "
        "package descriptor.",
    )
    transform_parser.add_argument(
        "--safe", action="store_false", help="Don't overwrite existing files."
    )
    transform_parser.add_argument(
        "--uncompressed",
        action="store_true",
        help="Store the transformed files as uncompressed TSVs rather than writing them into a ZIP file.",
    )
    transform_parser.set_defaults(func=transform_cmd)

    update_parser = subparsers.add_parser(
        "update",
        help="Convert MSCX files to the latest MuseScore version and move all chord annotations "
        "to the Roman Numeral Analysis layer. This command overwrites existing files!!!",
        parents=[parse_args],
    )
    # update_parser.add_argument('-a', '--annotations', metavar='PATH', default='../harmonies',
    #                             help='Path relative to the score file(s) where to look for existing annotation
    #                             tables.')
    update_parser.add_argument(
        "-s",
        "--suffix",
        metavar="SUFFIX",
        help="Add this suffix to the filename of every new file.",
    )
    update_parser.add_argument(
        "--above", action="store_true", help="Display Roman Numerals above the system."
    )
    update_parser.add_argument(
        "--safe",
        action="store_true",
        help="Only moves labels if their temporal positions stay intact.",
    )
    update_parser.add_argument(
        "--staff",
        default=-1,
        help="Which staff you want to move the annotations to. 1=upper staff; -1=lowest staff (default)",
    )
    update_parser.add_argument(
        "--type",
        default=1,
        help="defaults to 1, i.e. moves labels to Roman Numeral layer. Other types have not been tested!",
    )
    update_parser.set_defaults(func=update_cmd)

    return parser

def run():
    parser = get_arg_parser()
    args = parser.parse_args()
    if "func" not in args:
        parser.print_help()
        return
    if args.files is not None:
        args.files = [resolve_dir(path) for path in args.file]
    args.func(args)

if __name__ == "__main__":
    run()


from __future__ import annotations

from types import SimpleNamespace
from typing import TYPE_CHECKING, Any, Callable, Generic, Type, TypeVar

from ...commands import Command, Group
from ...commands.application.command import ApplicationCommand
from ...events.event_manager import Event
from ...missing import MISSING, MissingEnum
from ...types import AsyncFunc

if TYPE_CHECKING:
    from ...bot import Bot

T = TypeVar('T')
AF = TypeVar('AF', AsyncFunc)

class BaseContext(SimpleNamespace):
    ...

ContextT = TypeVar('ContextT', bound=BaseContext)

class Gear(Generic[ContextT]):
    """
    The Gear. A set of commands & events separate from Bot.
    Useful for multi-file separation for huge bots.

    Parameters
    ----------
    name: :class:`str`
        The name of this Gear.

    Attributes
    ----------
    bot: Union[:class:`pycord.Bot`, None]
        The bot this Gear is attached to.
    """

    ctx: ContextT

    __slots__ = ('_listener_functions', '_commands', 'name', 'ctx', 'bot')

    def __init__(self, name: str, ctx: ContextT | None = None) -> None:
        self.name = name
        self._listener_functions: dict[Type[Event], list[AsyncFunc]] = {}
        self.bot: Bot
        self._commands: list[Command | Group] = []
        if ctx is None:
            self.ctx = BaseContext()
        else:
            self.ctx = ctx

    async def on_attach(self, *args, **kwargs) -> None:
        ...

    def listen(self, name: str) -> T:
        """
        Listen to an event

        Parameters
        ----------
        name: :class:`str`
            The name of the event to listen to.
        """

        def wrapper(func: T) -> T:
            if self._listener_functions.get(name):
                self._listener_functions[name].append(func)
            else:
                self._listener_functions[name] = [func]
            return func

        return wrapper

    def command(
        self,
        name: str | MissingEnum = MISSING,
        cls: Type[T] = ApplicationCommand,
        **kwargs: Any,
    ) -> T:
        """
        Create a command within the Gear

        Parameters
        ----------
        name: :class:`str`
            The name of the Command.
        cls: type of :class:`.commands.Command`
            The command type to instantiate.
        kwargs: dict[str, Any]
            The kwargs to entail onto the instantiated command.
        """

        def wrapper(func: AsyncFunc) -> Command:
            command = cls(func, state=None, name=name, **kwargs)
            self._commands.append(command)
            return command

        return wrapper

    def group(self, name: str, cls: Type[Group], **kwargs: Any) -> Callable[[AF], AF]:
        """
        Create a brand-new Group of Commands

        Parameters
        ----------
        name: :class:`str`
            The name of the Group.
        cls: type of :class:`.commands.Group`
            The group type to instantiate.
        kwargs: dict[str, Any]
            The kwargs to entail onto the instantiated group.
        """

        def wrapper(func: AF) -> AF:
            # I know this partially ruins typing, but
            # Gears are loaded before events are taken in, so
            # theoretically nothing can break with state being None.
            r = cls(func, name, None, **kwargs)  # type: ignore
            self._commands.append(r)
            return r

        return wrapper

    def attach(self, bot: Bot) -> None:
        """
        Attaches this Gear to a bot.

        Parameters
        ----------

        bot: :class:`pycord.Bot`
            The bot to attach onto
        """

        for name, funcs in self._listener_functions.items():
            for func in funcs:
                bot._state.event_manager.add_event(name, func)

        for cmd in self._commands:
            if isinstance(cmd, Command):
                cmd._state = bot._state
                cmd._state.commands.append(cmd)
            elif isinstance(cmd, Group):
                cmd._state = bot._state

        self.bot = bot

        self.bot._state.gears.append(self)


import pandas as pd
import torch
import numpy as np
import random
import matplotlib.pyplot as plt
import os
import time
from model import *
from Modules import *
from sklearn.manifold import TSNE
import timeit
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
torch.cuda.set_device(0)
#device=torch.device('cuda')
#将数据导入
def load_tensor(file_name, dtype):
    result=[]
    np.load.__defaults__=(None, True, True, 'ASCII')
    for d in np.load(file_name):
        #print(d)
        #temp=dtype(d).to(device)
        result.append(d)
    np.load.__defaults__=(None, False, True, 'ASCII')
    return result
#打乱数据集
def shuffle_dataset(dataset, seed):
    np.random.seed(seed)
    np.random.shuffle(dataset)
    return dataset

#将数据集拆分为训练集和测试集
def split_dataset(dataset, ratio):
    n = int(ratio * len(dataset))
    dataset_1, dataset_2 = dataset[:n], dataset[n:]
    return dataset_1, dataset_2

if __name__ == "__main__":
    SEED = 1
    random.seed(SEED)
    torch.manual_seed(SEED)
    # torch.backends.cudnn.deterministic = True
    DATASET='Human,C.elegans'

    """Load preprocessed data."""

    #dir_input = ('/mnt/sdb/home/hjy/exp1/dataset/' + DATASET+'/Human,C.elegans')
    dir_input='G:/序列预测相关/模型/DrugormerDTI/dataset/Davis/'
    drug = load_tensor(dir_input + 'atom_feat.npy', torch.FloatTensor)
    bond_adj = load_tensor(dir_input + 'bond_adj.npy', torch.FloatTensor)
    dist_adj = load_tensor(dir_input+ 'dist_adj.npy', torch.FloatTensor)
    proteins = load_tensor(dir_input + 'proteins.npy', torch.FloatTensor)
    interactions = load_tensor(dir_input + 'interactions.npy', torch.LongTensor)
    """Create a dataset and split it into train/dev/test."""

    dataset = list(zip(drug, bond_adj, dist_adj, proteins, interactions))
    dataset = shuffle_dataset(dataset,1234)
    dataset_train, dataset_dev = split_dataset(dataset, 0.8)
    print('dataset created!')
    """ create model ,trainer and tester """
    protein_dim = 100#蛋白质的输入维度
    atom_dim = 128
    hid_dim = 128
    n_layers = 3
    n_heads = 8
    pf_dim =256
    dropout = 0.1
    batch = 4
    lr = 1e-4
    weight_decay = 1e-4
    decay_interval = 10
    lr_decay = 1.0
    iteration = 100
    kernel_size = 7

    encoder = Encoder(protein_dim, hid_dim, n_layers, kernel_size, dropout, device)
    graphormer =DrugEncoder(hid_dim)
    decoder = Decoder(atom_dim, hid_dim, n_layers, n_heads, pf_dim, DecoderLayer, MultiHeadAttention, PositionwiseFeedforward, dropout, device)
    model = Predictor(encoder,graphormer,decoder,device,atom_dim)
    encoder=encoder.cuda()
    graphormer=graphormer.cuda()
    decoder=decoder.cuda()
    model=model.cuda()
    model.load_state_dict(torch.load("G:\序列预测相关\模型\DrugormerDTI\Davis"))
    #model.to(device)
    trainer = Trainer(model, lr, weight_decay, batch)
    tester = Tester(model)

    """Output files."""
    file_AUCs = 'G:\序列预测相关\模型\DrugormerDTI\Davis6.txt'
    file_model = 'G:\序列预测相关\模型\DrugormerDTI\Davis'
    AUCs = ('Epoch\tTime(sec)\tLoss_train\tAUC_dev\tPRC_dev\tRecall_dev\tACC_dev\tPrecision_dev\tF1_dev')
    with open(file_AUCs, 'w') as f:
        f.write(AUCs + '\n')

    """Start training."""
    print('Training...')
    print(AUCs)
    start = timeit.default_timer()

    max_AUC_dev = 0
    for epoch in range(1, iteration+1):
        if epoch % decay_interval == 0:
            trainer.optimizer.param_groups[0]['lr'] *= lr_decay
        loss_train = trainer.train(dataset_train, device)
        #print(loss_train)
        print('Testing……')
        AUC_dev, PRC_dev,recall_dev,accuracy_dev,precision_dev,F1_dev,labels,embeddings,attention = tester.test(dataset_dev)
        end = timeit.default_timer()
        time = end - start
        AUCs = [epoch, time, loss_train, AUC_dev,PRC_dev,recall_dev,accuracy_dev,precision_dev,F1_dev]
        tester.save_AUCs(AUCs, file_AUCs)
        if AUC_dev > max_AUC_dev:
            tester.save_model(model, file_model)
            max_AUC_dev = AUC_dev
        print('\t'.join(map(str, AUCs)))
        print('saving……')
        #print(labels)
        #print(attention.shape)
        emd=[]
        for i in range(len(embeddings)):
            emd.append(embeddings[i][0])
        emd_df=pd.DataFrame(emd)
        emd_df.to_csv(r'G:\序列预测相关\模型\DrugormerDTI\file\embeddings_davis.csv', encoding='utf_8_sig')  # 防止中文乱码
        '''
       #print(attention[-1])

        attentions= pd.DataFrame()
        for i in range(len(attention)):
            #print(attention[i].shape)
            if attention[i].shape[1]<300:
                attentions=pd.DataFrame(attention[i].cpu())
            #attentions.append(attention[i][-1].cpu())
        #attention_df = pd.DataFrame(attentions)
        attentions.to_csv('/mnt/sdb/home/hjy/exp1/file/attention.csv', encoding='utf_8_sig')  # 防止中文乱码

        file = open('/mnt/sdb/home/hjy/exp1/file/label.txt', 'w', encoding='utf-8')
        for i in range(len(labels)):
            #print(len(labels[i]))
            file.write(str(labels[i]) + '\n')
        file.close()
        file = open('/mnt/sdb/home/hjy/exp1/file/embeddings.txt', 'w', encoding='utf-8')
        for i in range(len(embeddings)):
            print(embeddings[i])
            print("===============================")
            file.write(str(embeddings[i]) + '\n')
        file.close()
        '''


import pandas as pd
from io import StringIO

data = 'Airline Code;DelayTimes;FlightCodes;To_From\nAir Canada (!);[21, 40];20015.0;WAterLoo_NEWYork\n<Air France> (12);[];;Montreal_TORONTO\n(Porter Airways. );[60, 22, 87];20035.0;CALgary_Ottawa\n12. Air France;[78, 66];;Ottawa_VANcouvER\n""".\\.Lufthansa.\\.""";[12, 33];20055.0;london_MONTreal\n'

StringData = StringIO("""{}""".format(data))

df = pd.read_csv(StringData, sep=";")
# print("Intial Data:\n", df)

# 1 Find null rows, pandas series groupby (series.diff compares current and previous row)
nul = df.FlightCodes.isnull()
df.FlightCodes = nul.groupby(
    (nul.diff() == True).cumsum()).sum()*10 + df.FlightCodes.ffill()
df.FlightCodes = df.FlightCodes.astype('int')
# print("Cleaned FlightCodes:\n", df)

# 2
df[['To', 'From']] = df.To_From.str.split('_', expand=True)
df.To = df.To.str.upper()
df.From = df.From.str.upper()
df = df.drop(['To_From'], axis=1)
# print("Cleaned To_From:\n", df)

# 3
df['Airline Code'] = df['Airline Code'].str.replace(
    r'[^a-zA-Z]+', " ", regex=True)
df['Airline Code'] = df['Airline Code'].str.strip()
# print("Cleaned Airline Code:\n", df)

df.to_csv('cleanedData', sep=';', index=False)
print("Cleaned data:\n", df)


import json
from django.conf import settings
from django.http import HttpResponse
from django.views.generic import View
from haystack.query import SearchQuerySet
from haystack.inputs import Raw
from djinn_core.utils import object_to_urn
from djinn_contenttypes.registry import CTRegistry

def map_swappable_ct(ct_name):

    """ Is the model swappable? If so, what is the underlying model """

    result = ct_name

    try:
        swapped_ct_name = CTRegistry.get(ct_name)['class']._meta.swappable

        if swapped_ct_name:
            result = getattr(settings, swapped_ct_name, ct_name).lower()
    except:
        pass

    return result

class RelateSearch(View):

    """ Find related stuff, using autocomplete """

    def get(self, request, term=None):

        term = term or request.GET.get('term', '')

        if not term:
            return HttpResponse(
                json.dumps([]),
                content_type='application/json')

        search_field = "%s__contains" % request.GET.get("searchfield",
                                                        "title")

        ct_parameter = request.GET.get("ct_searchfield", "meta_ct")

        ct_search_field = "%s__in" % ct_parameter

        content_types = request.GET.get('content_type', '').split(",")

        # MJB map
        # content_types = map(map_swappable_ct, content_types)
        content_types = [map_swappable_ct(ctn) for ctn in content_types]

        _filter = {search_field: Raw("*%s*" % term)}

        if content_types:
            _filter[ct_search_field] = content_types

        parentusergroup = request.GET.get("parentusergroup", None)
        if parentusergroup and parentusergroup != '-1':
            _filter['parentusergroup'] = parentusergroup

        sqs = SearchQuerySet().filter(**_filter)

        results = []

        for res in [res for res in sqs if res.object]:
            results.append({"label": str(res.object),
                            "value": object_to_urn(res.object)
                            })

        if len(results) == 0:
            results.append({"label": "Geen resultaat gevonden",
                            "value": ''})

        return HttpResponse(json.dumps(results), content_type='application/json')


from gi.repository import GLib

from gnome_appfolders_manager.constants import MISSING_ICON_NAME
from gnome_appfolders_manager.functions import get_pixbuf_from_icon_name
from gnome_appfolders_manager.models.abstract import ModelAbstract

class ModelApplications(ModelAbstract):
    COL_TITLE = 1
    COL_COMMENT = 2
    COL_DESCRIPTION = 3
    COL_ICON = 4
    COL_VALID = 5
    COL_VISIBLE = 6

    def __init__(self, model):
        super(self.__class__, self).__init__(model)
        self.items = {}

    def add_data(self, item):
        """Add a new row to the model if it doesn't exist"""
        super(self.__class__, self).add_data(item)
        if item.filename not in self.rows:
            icon = None
            if item.icon_name:
                icon = get_pixbuf_from_icon_name(item.icon_name, 48)
            # Use a fallback icon if the icon was not found, or it was missing
            if not icon:
                icon = get_pixbuf_from_icon_name(MISSING_ICON_NAME, 48)
            new_row = self.model.append((
                item.filename,
                item.name,
                item.description,
                '{name}\n'
                '{description}\n'
                '{filename}'.format(**{
                    'name': item.name,
                    'description': item.description,
                    'filename': GLib.markup_escape_text(item.filename)}),
                icon,
                item.valid,
                item.valid))
            self.rows[item.filename] = new_row
            self.items[item.filename] = item
            return new_row

    def get_title(self, treeiter):
        """Get the title from a TreeIter"""
        return self.model[treeiter][self.COL_TITLE]

    def get_description(self, treeiter):
        """Get the description from a TreeIter"""
        return self.model[treeiter][self.COL_DESCRIPTION]

    def get_icon(self, treeiter):
        """Get the icon from a TreeIter"""
        return self.model[treeiter][self.COL_ICON]

    def set_all_rows_visibility(self, visible):
        """Set all items visible or restore the valid column for visibility"""
        for treeiter in self.model:
            self.model[treeiter.path][self.COL_VISIBLE] = (
                # All rows are visible
                True if visible
                # Only valid rows are visible
                else self.model[treeiter.path][self.COL_VALID])


import numpy as np
import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import MultinomialNB
from sklearn.feature_extraction.text import TfidfVectorizer
import os
import sys
import requests
from PyLyrics import *
import requests
import urllib.parse

#https://github.com/shreyans29/thesemicolon/blob/master/Text%20Analytics%20tfidf.ipynb
if __name__ == '__main__':

    df=pd.read_csv("finaldset.csv")

    df.head()

    #print(len(df))
    '''
    print("0-> ", len(df[df.mood==0]))
    print("1-> ", len(df[df.mood==1]))
    print("2-> ", len(df[df.mood == 2]))
    print("3-> ", len(df[df.mood == 3]))
    '''
    #df.loc[df["Status"]=='ham',"Status",]=1

    #df.loc[df["Status"]=='spam',"Status",]=0

    #df.head()

    df_x=df["lyrics"]
    df_y=df["mood"]

    cv = TfidfVectorizer(min_df=1,stop_words='english')

    x_train, x_test, y_train, y_test = train_test_split(df_x, df_y, test_size=0.20, random_state=22)

    #print(x_train.head())

    cv = TfidfVectorizer(min_df=1,stop_words='english')
    '''
    x_traincv = cv.fit_transform(["Hi How are you How are you doing","Hi what's up","Wow that's awesome"])

    x_traincv.toarray()

    cv.get_feature_names()
    '''
    cv1 = TfidfVectorizer(min_df=1,stop_words='english')

    x_traincv = cv1.fit_transform(x_train)

    a=x_traincv.toarray()

    print(a[0])
    print(cv1.get_feature_names())
    cv1.inverse_transform(a[0])

    print(x_train.iloc[1])
    print("x-test--------------",x_test)
    x_testcv=cv1.transform(x_test)

    x_testcv.toarray()

    mnb = MultinomialNB()

    y_train=y_train.astype('int')

    #print(y_train)

    mnb.fit(x_traincv,y_train)

    #testmessage=x_test.iloc[0]

    #print(testmessage)
    print("---input format---", x_testcv)

    predictions=mnb.predict(x_testcv)

    #print(predictions)

    a=np.array(y_test)

    #print(a)

    count=0

    for i in range (len(predictions)):
        if predictions[i]==a[i]:
            count=count+1

    out=len(predictions)
    accuracy=(count/out)*100
    print("accuracy-> ", accuracy)

    #get percentage'''

    n_testcv = cv1.fit_transform(["Hi How are you How are you doing"])

    n_testcv.toarray()
    print(n_testcv)
    predict_val = mnb.predict(n_testcv)
    print(predict_val)


import requests
from bs4 import BeautifulSoup
import Executando

def obtendoResposta(pergunta,Swan):
    print(pergunta)
    headers = {'User-Agent':"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36"}
    r = requests.get(f'https://www.google.com/search?q={pergunta}',headers=headers)
    soup = BeautifulSoup(r.content, 'html.parser')
    texto = soup.find(class_='kno-rdesc')
    filtrando = texto
    if filtrando:
        texto2 = filtrando.find('span').get_text()
        print("XXXX")
        print(texto2)
        return texto2
    else:
        Executando.Executar(Swan,'pesquisa','')

if __name__ == '__main__':
    obtendoResposta('Quantos habitantes o brasil tem', '')


import os
import datetime
import time
#Local imports
import cfg
import rf_uart as ser
import json
import logging as log
import requests

on_broadcast = None
on_message = None
on_cmd_response = None

config = cfg.configure_log(__file__)
nodes_config_file = '/home/pi/nRF52_Mesh/nodes.json'
if("config" in config["mesh"]):
    nodes_config_file = config["mesh"]["config"]
log.info("mesh> using NODES_CONFIG : %s",nodes_config_file)
if(nodes_config_file.startswith("http")):
    response = requests.get(nodes_config_file)
    nodes = response.json()
    log.info(f" => {len(nodes)} nodes")
else:
    nodes = cfg.get_local_nodes(nodes_config_file)

pid = {
    "exec_cmd"      : 0xEC,
    "ping"          : 0x01,
    "request_pid"   : 0x02,
    "chan_switch"   : 0x03,
    "reset"         : 0x04,
    "alive"         : 0x05,
    "button"        : 0x06,
    "light"         : 0x07,
    "temperature"   : 0x08,
    "heat"          : 0x09,
    "bme280"        : 0x0A,
    "rgb"           : 0x0B,
    "magnet"        : 0x0C,
    "dimmer"        : 0x0D,
    "light_rgb"     : 0x0E,
    "gesture"       : 0x0F,
    "proximity"     : 0x10,
    "humidity"      : 0x11,
    "pressure"      : 0x12,
    "acceleration"  : 0x13,
    "light_n"       : 0x14,
    "voltage"       : 0x15,
    "text"          : 0x16,
    "test_rf_resp"  : 0x30
}

inv_pid = {v: k for k, v in pid.items()}

exec_cmd = {
    "set_node_id"   : 0x01,
    "get_node_id"   : 0x02,
    "set_channel"   : 0x03,
    "get_channel"   : 0x04,
    "set_tx_power"  : 0x05,
    "get_tx_power"  : 0x06,
    "set_param"     : 0x07,
    "get_param"     : 0x08,
}

set_rx = {
    "sniff" : 0x00,
    "bcast" : 0x01,
    "msg"   : 0x02,
    "resp"  : 0x03
}
mode = {
    "power_down"    : 0x01,
    "standby"       : 0x02,
    "tx_tdby2"      : 0x03,
    "rx"            : 0x04
}
inv_mode = {v: k for k, v in mode.items()}

msg = {
    "size":0,
    "payload":[]
}

def is_broadcast(hex_byte):
    return hex_byte.startswith("0x8")

def parse_pid(byte):
    return inv_pid[byte]

def parse_payload(data):
    res = ""
    if(data[2] == pid["light_n"]):
        light = int.from_bytes(bytearray(data[4:6]),'little',signed=False)
        res = "%d" %(light)
    elif(data[2] == pid["temperature"]):
        val = float(int.from_bytes(bytearray(data[4:8]),'big',signed=True)) / 100
        res = '{:02.2f}'.format(val)
    elif(data[2] == pid["humidity"]):
        val = float(int.from_bytes(bytearray(data[4:8]),'big',signed=True)) / 1024
        res = '{:02.2f}'.format(val)
    elif(data[2] == pid["pressure"]):
        val = float(int.from_bytes(bytearray(data[4:8]),'big',signed=True)) / (256*100)
        res = '{:02.2f}'.format(val)
    elif(data[2] == pid["acceleration"]):
        accel_x = float(int.from_bytes(bytearray(data[4:6]),'big',signed=True)) / 16384
        accel_y = float(int.from_bytes(bytearray(data[6:8]),'big',signed=True)) / 16384
        accel_z = float(int.from_bytes(bytearray(data[8:10]),'big',signed=True)) / 16384
        res = '(g) X {:02.2f} ; Y {:02.2f} ; Z {:02.2f}'.format(accel_x,accel_y,accel_z)
    elif(data[2] == pid["voltage"]):
        bat_v = float(int.from_bytes(bytearray(data[4:6]),'big',signed=True)) / 1000
        res = 'voltage {:02.3f} V'.format(bat_v)
    elif(data[2] == pid["button"]):
        if(data[4] == 0):
            res = 'release'
        else:
            res = 'press'
    if(data[2] == pid["light_rgb"]):
        light = int.from_bytes(bytearray(data[4:6]),'big',signed=False)
        red   = int.from_bytes(bytearray(data[6:8]),'big',signed=False)
        green = int.from_bytes(bytearray(data[8:10]),'big',signed=False)
        blue  = int.from_bytes(bytearray(data[10:12]),'big',signed=False)
        res = "light:%d , red:%d , green:%d , blue:%d" % (light,red,green,blue)
    return res

def parse_is_broadcast(byte):
    return (byte & 0x80)

def node_name(byte):
    res ="Unknown"
    if(str(byte) in nodes):
        res = nodes[str(byte)]["name"]
    return res

def int8_signed(byte):
    val = int(byte)
    if(val > 127):
        return (256-val) * (-1)
    else:
        return val

def publish(msg):
    pub = {}
    if(msg["src"] in nodes):
        src_name = nodes[msg["src"]]["name"]
    else:#keep number as identifier
        src_name = msg["src"]
    if(config["mesh"]["whitelist"] != []):
        if(src_name not in config["mesh"]["whitelist"]):
            log.debug(f"{src_name} not whitelisted - discarded")
            return
    if(config["mesh"]["blacklist"] != []):
        if(src_name in config["mesh"]["blacklist"]):
            log.debug(f"{src_name} balcklisted - discarded")
            return
    topic = src_name
    json_payload = {}
    if("rssi" in msg):
        json_payload["rssi"] = int(msg["rssi"])
    if(inv_pid[int(msg["pid"])] == "alive"):
        json_payload["alive_count"] = int(msg["alive"])
        nb_rx = int(msg["nb"])
        for i in range(nb_rx):
            rx_i = "rx"+str(i+1)
            json_payload["path"] = []
            txpow_rssi_nid = msg[rx_i].split(',')
            path_entry = {}
            path_entry["tx_power"]  = int8_signed(txpow_rssi_nid[0])
            path_entry["rssi"]      = txpow_rssi_nid[1]
            last_rssi                       = txpow_rssi_nid[1]
            path_entry["nodeid"]    = txpow_rssi_nid[2]
            json_payload["path"].append(path_entry)
        json_payload["rssi"] = int(last_rssi)
        pub[topic] = json_payload
    elif(inv_pid[int(msg["pid"])] == "bme280"):
        if("temp" in msg):
            json_payload["temperature"] = float(msg["temp"])
            json_payload["humidity"] = float(msg["hum"])
            json_payload["pressure"] = float(msg["press"])
    elif(inv_pid[int(msg["pid"])] == "temperature"):
        if("temp" in msg):
            json_payload["temperature"] = float(msg["temp"])
    elif(inv_pid[int(msg["pid"])] == "light"):
        if("light" in msg):
            json_payload["light"] = float(msg["light"])
    elif(inv_pid[int(msg["pid"])] == "voltage"):
        if("battery" in msg):
            json_payload["voltage"] = float(msg["battery"])
        elif("voltage" in msg):
            json_payload["voltage"] = float(msg["voltage"])
    elif(inv_pid[int(msg["pid"])] == "acceleration"):
        if("accx" in msg):  #check accx is enough as some have size error logs
            json_payload = {}
            json_payload["x"] = float(msg["accx"])
            json_payload["y"] = float(msg["accy"])
            json_payload["z"] = float(msg["accz"])
            pub[topic] = json_payload
    elif(inv_pid[int(msg["pid"])] == "button"):
        if(int(msg["button"]) == 1):
            json_payload["button"] = "down"
        else:
            json_payload["button"] = "up"
    elif(inv_pid[int(msg["pid"])] == "reset"):
        if("reset" in msg):
            json_payload["reset"] = float(msg["reset"])
    pub[topic] = json_payload
    return pub

def line2dict(line):
    res = {}
    entries = line.split(';')
    for entry in entries:
        kv = entry.split(':')
        if(len(kv)==2):
            res[kv[0]] = kv[1]
    return res

def command(cmd,params=[]):
    cmd_list = [exec_cmd[cmd]]+params
    text_cmd = "cmd:0x" + ''.join('%02X' % b for b in cmd_list)+"\r\n"
    ser.send(text_cmd)
    return

def send(payload):
    #print("payload:",payload)
    text_msg = "msg:0x"+''.join('%02X' % b for b in payload)+"\r\n"
    ser.send(text_msg)
    return

def serial_on_line(line):
    log.debug("mesh> uart> "+line)
    ldict = line2dict(line)
    if("ctrl" in ldict):
        if(is_broadcast(ldict["ctrl"])):
            on_broadcast(ldict)
        else:
            if("cmd" in ldict):
                on_cmd_response(ldict,True)
                log.info("mesh> remote cmd resp > "+line)
            else:
                on_message(ldict)
            #log.info("msg > "+line)
    elif("cmd" in ldict):
        on_cmd_response(ldict,False)
        log.info("mesh> cmd resp > "+line)
    return

def run():
    ser.run()
    return

def start(config,mesh_on_broadcast,mesh_on_message,mesh_on_cmd_response):
    global on_broadcast
    global on_message
    global on_cmd_response
    on_broadcast = mesh_on_broadcast
    on_cmd_response = mesh_on_cmd_response
    on_message = mesh_on_message
    ser.serial_start(config,serial_on_line)
    return

def stop():
    ser.serial_stop()
    return


# Fetch arxiv data from S3

from boto3.session import Session
import dotenv
import logging
import os

from ai_papers_with_code import PROJECT_DIR

dotenv.load_dotenv(f"{PROJECT_DIR}/.env")
DATA_PATH = f"{PROJECT_DIR}/inputs/data"

key = os.getenv("aws_access_key_id")
secret_key = os.getenv("aws_secret_access_key")

def main():
    session = Session(aws_access_key_id=key, aws_secret_access_key=secret_key)
    s3 = session.resource("s3")

    pwc_bucket = s3.Bucket("ai-papers-with-code")

    for s3_file in pwc_bucket.objects.all():
        logging.info(f"downloading {s3_file.key}")

        with open(f"{PROJECT_DIR}/inputs/data/{s3_file.key}", "wb") as out:
            pwc_bucket.download_fileobj(s3_file.key, out)

if __name__ == "__main__":
    main()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Sep 28 12:53:52 2021

@author: sb16165
"""
import os
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import xgboost as xgb

import Plots
from typing import Tuple

from scipy.special import gamma, polygamma

### OBJECTIVE FUCNTIONS ###
def MSE(predt: np.ndarray, dtrain: xgb.DMatrix) -> Tuple[np.ndarray, np.ndarray]:
    '''Squared Log Error objective. A simplified version for RMSLE used as
    objective function.
    '''
    def gradient(predt: np.ndarray, dtrain: xgb.DMatrix) -> np.ndarray:
        '''Compute the first derivitive'''
        y = dtrain.get_label()
        return predt - y

    def hessian(predt: np.ndarray, dtrain: xgb.DMatrix) -> np.ndarray:
        '''Compute the second derivitive'''
        y = dtrain.get_label()
        return np.ones(y.shape)

    grad = gradient(predt, dtrain)
    hess = hessian(predt, dtrain)
    return grad, hess

def custom(predt: np.ndarray, dtrain: xgb.DMatrix) -> Tuple[np.ndarray, np.ndarray]:
    '''Squared Log Error objective. A simplified version for RMSLE used as
    objective function.
    '''
    def gradient() -> np.ndarray:
        '''Compute the first derivitive'''
        #return 1 - (y * np.exp(-predt))
        #return (polygamma(0, predt)/gamma(predt)) - predt*np.log(y)
        return (x - y) / x**2

    def hessian() -> np.ndarray:
        '''Compute the second derivitive'''
        y = dtrain.get_label()
        #return y * np.exp(-predt)
        #return (polygamma(1, predt)/gamma(predt)) - (polygamma(0, predt)/gamma(predt))**2 - np.log(y)
        return -(x - 2*y) / x**3

    y = dtrain.get_label()
    x = predt
    #x[x < 0] = 0 + 1e-6
    grad = gradient()
    hess = hessian()
    return grad, hess

def GLM(predt: np.ndarray, dtrain: xgb.DMatrix) -> Tuple[np.ndarray, np.ndarray]:
    '''GLM using gamma devience and log link
    '''
    predt[predt < 0] = 1e-6
    y = dtrain.get_label()
    x = np.log(predt)
    den = (x/y) + (y/x) - 1

    def gradient() -> np.ndarray:
        '''Compute the first derivitive'''
        num = 1/y - (y/x**2)
        return num / den

    def hessian() -> np.ndarray:
        '''Compute the second derivitive'''
        return 2 * y / ( den * x**3)

    grad = gradient()
    hess = hessian() - grad**2
    return grad, hess
### OBJECTIVE FUCNTIONS ###

def MakePlots(prediction, data, training, model, subDirectory="", plot=True, save=False, plot_true_mass = True):

    def InvMass(_range=None, bins=20, name="prediction"):
        pred = prediction
        mass = data["invariant mass (GeV)"]
        true_mass = data["true invariant mass (GeV)"]
        if _range != None:
            pred = prediction[prediction < _range]
            mass = mass[mass < _range]
            true_mass = true_mass[true_mass < _range]

        h_1, h_2, edges = Plots.PlotHistComparison(pred, mass, bins, "invariant mass (GeV)", label_1="predicted", label_2="calculated", alpha=0.5, sf=3)
        if plot_true_mass is True:
            plt.hist(true_mass, edges, label="true Invariant mass", alpha=0.5, density=False, color="C2")
        else:
            plt.axvline(0.13497, ymax=max(h_1), label="$m_{\pi_{0}}$", color="black")
        plt.legend()
        if save is True:
            os.makedirs(subDirectory, exist_ok=True)
            Plots.Save(name, subDirectory, "BDT_regressor")

    ### INVARIANT MASS ###
    plt.rcParams["figure.figsize"] = plt.rcParamsDefault["figure.figsize"]
    InvMass(bins=100)
    InvMass(0.5, name="prediction_zoom")

    ### PREDICTION VS CALCULATED ###
    Plots.PlotHist2D(prediction, data["invariant mass (GeV)"], 100, xlabel="predicted invariant mass (GeV)", ylabel="calculated invariant mass (GeV)")
    if save is True:
        os.makedirs(subDirectory, exist_ok=True)
        Plots.Save("2D", subDirectory, "BDT_regressor")

    residual = prediction - data["invariant mass (GeV)"]
    energy_residual = data["leading energy (GeV)"] + data["secondary energy (GeV)"] - data["total true energy (GeV)"]
    angle_residual = data["opening angle (rad)"] - data["true opening angle (rad)"]

    ### PREDICTION - CALCULATED ###
    Plots.PlotHist(residual, 100, "prediction - calculated (GeV)")
    if save is True:
        Plots.Save("difference", subDirectory, "BDT_regressor")

    ### TRAINING SET INVARIANT MASS ###
    Plots.PlotHist(training["invariant mass (GeV)"], 100, "training set true invariant mass (GeV)")
    if save is True:
        Plots.Save("training", subDirectory, "BDT_regressor")

    ### ENERGY RESIDUAL ###
    Plots.PlotHist(energy_residual, 100, xlabel="energy residual (GeV)")
    if save is True:
        Plots.Save("energy_res", subDirectory, "BDT_regressor")

    ### ANGLE RESIDUAL ###
    Plots.PlotHist(angle_residual, 100, xlabel="opening angle residual (rad)")
    if save is True:
        Plots.Save("angle_res", subDirectory, "BDT_regressor")

    ### RESIDUAL VS ENERGY RESIDUAL ###
    Plots.PlotHist2D(residual, energy_residual, 100, xlabel="prediction - calculated (GeV)", ylabel="energy residual (GeV)")
    if save is True:
        Plots.Save("energy_res_2D", subDirectory, "BDT_regressor")

    ### RESIDUAL VS ANGLE RESIDUAL  ###
    Plots.PlotHist2D(residual, angle_residual, 100, xlabel="prediction - calculated (GeV)", ylabel="opening angle residual (rad)")
    if save is True:
        Plots.Save("angle_res_2D", subDirectory, "BDT_regressor")

    ### INVARIANT MASS RESIDUALS ###
    h_1, h_2, _ = Plots.PlotHistComparison(prediction - data["true invariant mass (GeV)"], data["invariant mass (GeV)"] - data["true invariant mass (GeV)"], 100, "invariant mass residual (GeV)", label_1="predicted", label_2="calculated", alpha=0.5, sf=3)
    plt.legend()
    if save is True:
        os.makedirs(subDirectory, exist_ok=True)
        Plots.Save("residual", subDirectory, "BDT_regressor")

    ### FEATURE IMPORTANCE ###
    plt.rcParams["figure.figsize"] = (8, 5)
    xgb.plot_importance(model, grid=False)
    plt.tight_layout()
    if save is True:
        Plots.Save("importance", subDirectory, "BDT_regressor")
    plt.rcParams["figure.figsize"] = plt.rcParamsDefault["figure.figsize"]

def PlotResiduals(prediction, calculated, true, bins=100, _range=[], subDirectory="", plot=True, save=False):
    l_p = "prediction - true mass (GeV)"
    l_c = "calculated mass - true mass (GeV)"
    res_p = prediction - true
    res_c = calculated - true

    if len(_range) > 1:
        res_p = res_p[res_p > _range[0]]
        res_p = res_p[res_p < _range[1]]
        res_c = res_c[res_c < _range[1]]
        res_c = res_c[res_c > _range[0]]

    Plots.PlotHist(res_p, bins, l_p)
    if save is True: Plots.Save("res_p.png", subDirectory)
    Plots.PlotHist(res_c, bins, l_c)
    if save is True: Plots.Save("res_c.png", subDirectory)
    Plots.PlotHistComparison(res_p, res_c, bins, "residuals (GeV)", label_1=l_p, label_2=l_c, alpha=0.5)
    if save is True: Plots.Save("res_both.png", subDirectory)

def PlotCorrelations(data, df, x_label="x label", subDirectory="", outName="correlations"):
    fig = plt.figure(figsize=(32, 18))
    for i in range(len(df.columns)):
        name = df.columns[i]
        plt.subplot(6, 8, i+1)
        Plots.PlotHist2D(data, df[name], 25, xlabel=None, ylabel=name)
    fig.text(0.5, 0.005, x_label, ha='center')
    out = subDirectory + outName + ".png"
    plt.savefig(out, dpi=500)
    plt.close()

def Run(data, training, n=1000, predict="invariant mass", features=None, param=None, _print=True):

    if features == None:
        features = data.columns[1:12]

    if param == None:
        param = {}

        # Booster parameters
        param['eta']              = 0.3 # learning rate
        param['gamma']            = 0.5
        param['max_depth']        = 5 # maximum depth of a tree
        param['subsample']        = 1 # fraction of events to train tree on
        param['colsample_bytree'] = 0.2 # fraction of features to train tree on

        # Learning task parameters
        param['objective']   = 'reg:gamma' # objective function
        param['eval_metric'] = 'error'           # evaluation metric for cross validation
        param = list(param.items()) + [('eval_metric', 'logloss')] + [('eval_metric', 'rmse')]

    # Assign variable to predict and features
    target_train = training[predict]
    train_DM = xgb.DMatrix(training[features], target_train)

    true = data[predict]
    test_DM = xgb.DMatrix(data[features], true)

    model = xgb.train(param, train_DM, num_boost_round=n) # train model
    prediction = model.predict(test_DM) # use model on data

    # print stats of run
    if _print == True:
        print(model.eval(test_DM))
        print("training mean and st.dev")
        print(np.mean(target_train))
        print(np.std(target_train))
        print("target mean and st.dev")
        print(np.mean(true))
        print(np.std(true))
        print("predicted mean and st.dev")
        print(np.mean(prediction))
        print(np.std(prediction))

    return prediction, true, target_train, model

#xgb.plot_tree(booster, num_trees=booster.best_iteration)


import numpy as np
import pandas as pd
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
from sklearn.ensemble import RandomForestClassifier
from sklearn import tree
# from sklearn.metrics import cr

hr_data = pd.read_csv('data/hr.csv', header=0)
hr_data.head()
print(" Data Set Shape ", hr_data.shape)
print(list(hr_data.columns))
print(" Sample Data", hr_data.head())

data_trnsf = pd.get_dummies(hr_data, columns=['salary', 'sales'])
data_trnsf.columns
X = data_trnsf.drop('left', axis=1)
X.columns

X_train, X_test, Y_train, Y_test = train_test_split(X, data_trnsf.left, test_size=0.3, random_state=42)
print(X_train)

num_trees = 100
max_features = 3
attrition_forest = RandomForestClassifier(n_estimators=num_trees, max_features=max_features)
attrition_forest.fit(X_train, Y_train)

Y_pred = attrition_forest.predict(X_test)
confusionMatrix = confusion_matrix(Y_test, Y_pred)
print(confusionMatrix)

print('Accuracy of Decision Tree classifier on test set:{:.2f}', format(attrition_forest.score(X_test, Y_test)))
print(classification_report(Y_test, Y_pred))

crossval_5_scores = cross_val_score(attrition_forest, X_train, Y_train, cv=5)
print(crossval_5_scores)
print(np.mean(crossval_5_scores))
crossval_10_scores = cross_val_score(attrition_forest, X_train, Y_train, cv=10)
print(crossval_10_scores)
print(np.mean(crossval_10_scores))


#!/usr/bin/env python3
# Fundamentals of Python Network Programming, Third Edition
"""Simple TCP client and server that send and receive 16 octets."""

import argparse
import socket
import sys

def server(interface, port, bytecount):
    """Socket server."""
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    sock.bind((interface, port))
    sock.listen(1)
    print("Server listening at", sock.getsockname())
    try:
        while True:
            conn, addr_info = sock.accept()
            print("Processing up to 1024 bytes at at time from", addr_info)
            n = 0
            while True:
                data = conn.recv(1024)
                if not data:
                    break
                output = data.decode('ascii').upper().encode('ascii')
                conn.sendall(output)  # send back as uppercase
                n += len(data)
                print('\r  {} bytes processed so far'.format(n), end=' ')
                sys.stdout.flush()
            print()
            conn.close()
            print("    Reply sent, socket closed.")
    except KeyboardInterrupt:
        print("Shutting down server.")
        sock.close()
        sys.exit(0)

def client(host, port, bytecount):
    """Socket client."""
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    bytecount = (bytecount + 15) // 16 * 16
    message = b'capitalize this!'  # 16-byte message to repeat over and over
    print('Sending', bytecount, 'bytes of data, in chunks of 16 bytes')
    sock.connect((host, port))
    sent = 0
    while sent < bytecount:
        sock.sendall(message)
        sent += len(message)
        print('\r {} bytes sent'.format(sent), end=' ')
        sys.stdout.flush()

    print()
    sock.shutdown(socket.SHUT_WR)

    print("Receiving all the data the server sends back")

    received = 0
    while True:
        data = sock.recv(42)
        if not received:
            print('    The first data received says', repr(data))
        if not data:
            break
        received += len(data)
        print('\r    {} bytes received'.format(received), end=' ')
    print()
    sock.close()

if __name__ == "__main__":
    choices = {"client": client, "server": server}
    parser = argparse.ArgumentParser(description="Send and receive over TCP")
    parser.add_argument("role", choices=choices, help="which role to play")
    parser.add_argument("host", help="interface the server listens at; host the client send to")
    parser.add_argument("bytecount", type=int, nargs="?", default=16, help="number of bytes for client to send (default 16)")
    parser.add_argument("-p", metavar="PORT", type=int, default=1060, help="TCP port (default 1060)")
    args = parser.parse_args()
    function = choices[args.role]
    function(args.host, args.p, args.bytecount)