"""Command line tool for importing digital objects."""
from __future__ import unicode_literals

import datetime
import fnmatch
import os
import platform
import sys
from uuid import uuid4
import errno

import click
import six

import file_scraper
import premis
from siptools.mdcreator import MetsSectionCreator
from siptools.utils import scrape_file, calc_checksum
from siptools.scripts.premis_event import premis_event, create_premis_event
from siptools.scripts.create_agent import create_agent
from siptools.utils import generate_digest, encode_path

click.disable_unicode_literals_warning = True

ALLOWED_CHARSETS = ['ISO-8859-15', 'UTF-8', 'UTF-16', 'UTF-32']

DEFAULT_VERSIONS = {
    'application/vnd.oasis.opendocument.text': '1.0',
    'application/vnd.oasis.opendocument.spreadsheet': '1.0',
    'application/vnd.oasis.opendocument.presentation': '1.0',
    'application/vnd.oasis.opendocument.graphics': '1.0',
    'application/vnd.oasis.opendocument.formula': '1.0',
}

# For mimetypes that has no version applicable for them.
NO_VERSION = '(:unap)'

# For cases where scraper does not know the version
UNKNOWN_VERSION = '(:unav)'

# Supported bit-level preservation types
SUPPLEMENTARY_TYPES = ["xml_schema"]

@click.command()
@click.argument('filepaths', nargs=-1, type=str)
@click.option(
    '--workspace', type=click.Path(exists=True), default='./workspace/',
    metavar='<WORKSPACE PATH>',
    help="Workspace directory for the metadata files. "
         "Defaults to ./workspace")
@click.option(
    '--base_path', type=click.Path(exists=True), default='.',
    metavar='<BASE PATH>',
    help="Source base path of digital objects. If used, give objects in "
         "relation to this base path.")
@click.option(
    '--skip_wellformed_check', is_flag=True,
    help='Skip file format well-formed check.')
@click.option(
    '--charset', type=str, metavar='<CHARSET>',
    help='Charset encoding of a file.')
@click.option(
    '--original_name', type=str, metavar='<NAME>',
    help='Original name of the file.')
@click.option(
    '--file_format', nargs=2, type=str,
    metavar='<MIMETYPE> <FORMAT VERSION>',
    help='Mimetype and file format version of a file. Use "" for empty '
         'version string.')
@click.option(
    '--format_registry', type=str, nargs=2,
    metavar='<REGISTRY NAME> <REGISTRY KEY>',
    help='The format registry name and key of the digital object.')
@click.option(
    '--identifier', nargs=2, type=str,
    metavar='<IDENTIFIER TYPE> <IDENTIFIER VALUE>',
    help='The identifier type and value of a digital object.')
@click.option(
    '--checksum', nargs=2, type=str,
    metavar='<CHECKSUM ALGORITHM> <CHECKSUM VALUE>',
    help='Checksum algorithm and value of a given file.')
@click.option(
    '--date_created', type=str,
    metavar='<ISO-8601 TIME>',
    help='The actual or approximate date and time the object was created.')
@click.option(
    '--creating_application', nargs=2, type=str,
    metavar='<SOFTWARE NAME> <SOFTWARE VERSION>',
    help='The software and software version that was used to create the '
         'object. Use "" for empty version string.')
@click.option(
    '--order', type=int,
    metavar='<ORDER NUMBER>',
    help='Order number of the digital object.')
@click.option(
    '--event_datetime', type=str,
    metavar='<EVENT DATETIME>',
    help='Timestamp of the event documenting the script actions.')
@click.option(
    '--event_target', type=str,
    metavar='<EVENT TARGET>',
    help='Target for events, if it is not given the package root is used.')
@click.option(
    '--stdout', is_flag=True, help='Print result also to stdout.')
@click.option(
    '--bit_level', is_flag=True,
    help='Mark only for bit-level preservation. If used, then --file_format '
         'is mandatory.')
@click.option(
    '--supplementary', type=click.Choice(SUPPLEMENTARY_TYPES),
    multiple=True, metavar='<SUPPLEMENTARY TYPE>',
    help='Used to mark supplementary files, files that are not part of the '
         'contents per se, but are to be included in the SIP. May be used '
         'multiple times, but currently only "xml_schema" type is supported.')
# pylint: disable=too-many-arguments
def main(**kwargs):
    """Import files to generate digital objects.

    If attributes --charset, --file_format, --identifier, --checksum or
    --date_created are not given, then these are created automatically.

    FILEPATHS: Files or a directory to import, relative path in relation
    to current directory or to --base_path. It depends on your
    attributes whether you may give several files or a single file. For
    example --checksum and --identifier are file dependent metadata, and
    if these are used, then use the script only for one file.
    """
    import_object(**kwargs)
    return 0

def _attribute_values(given_params):
    """
    Give attribute values as a dict for the script.

    :given_params: Arguments as dict.
    :returns: Attribute value dict
    """
    attributes = {
        "filepaths": given_params["filepaths"],
        "workspace": "./workspace/",
        "base_path": ".",
        "skip_wellformed_check": False,
        "charset": None,
        "original_name": None,
        "file_format": (),
        "format_registry": (),
        "identifier": (),
        "checksum": (),
        "date_created": None,
        "creating_application": (),
        "order": None,
        "event_datetime": None,
        "event_target": None,
        "stdout": False,
        "bit_level": None,
        "supplementary": ()
    }
    for key in given_params:
        if given_params[key]:
            attributes[key] = given_params[key]

    if attributes["bit_level"]:
        if not attributes["file_format"]:
            raise ValueError(
                "Argument --file_format is mandatory if --bit_level is given.")
        attributes["skip_wellformed_check"] = True

    return attributes

def import_object(**kwargs):
    """Import files to generate digital objects.

    If attributes charset, file_format, identifier, checksum or
    date_created are not given, then these are created automatically.

    :attributes: Given arguments
                 filepaths: Files or a directory to import
                 workspace: Workspace path
                 base_path: Base path of digital objects
                 skip_wellformed_check: True skips well-formedness
                                        checking
                 charset: Character encoding of a file
                 original_name: Original filename
                 file_format: File format and version (tuple) of a file
                 format_registry: Format registry name and value (tuple)
                 identifier: File identifier type and value (tuple)
                 checksum: Checksum algorithm and value (tuple)
                 date_created: Creation date of a file
                 creating_application: Software and its version that created
                                       the file
                 order: Order number of a file
                 event_datetime: Timestamp of the events. Defaults to
                                 current date YYYY-MM-DD.
                 event_target: The target of the events
                 stdout: True prints output to stdout
                 bit_level: True marks files for bit-level preservation only
                 supplementary: Object type for supplementary files
    """
    attributes = _attribute_values(kwargs)
    date_now = datetime.datetime.utcnow().date().isoformat()

    # Loop files and create premis objects
    files = collect_filepaths(dirs=attributes["filepaths"],
                              base=attributes["base_path"])
    creator = PremisCreator(attributes["workspace"])
    agents = []
    for filepath in files:

        # If the given path is an absolute path and base_path is current
        # path (i.e. not given), relpath will return ../../.. sequences,
        # if current path is not part of the absolute path. In such case
        # we will use the absolute path for filerel and omit base_path
        # relation.
        if attributes["base_path"] not in ['.']:
            filerel = os.path.relpath(filepath, attributes["base_path"])
        else:
            filerel = filepath

        properties = {}
        if attributes["order"] is not None:
            properties['order'] = six.text_type(attributes["order"])
        properties["bit_level"] = attributes["bit_level"]
        properties["supplementary"] = attributes["supplementary"]

        (streams, scraper_info) = creator.add_premis_md(
            filepath, attributes, filerel=filerel, properties=properties)
        for index in scraper_info:
            agents.append(_parse_scraper_tools(scraper_info[index]))

        grade = streams[0]['properties']['grade']

    is_native = grade in (
        file_scraper.defaults.BIT_LEVEL,
        file_scraper.defaults.BIT_LEVEL_WITH_RECOMMENDED
    )

    is_validated = (
        not bool(attributes["skip_wellformed_check"])
        and not is_native
    )

    creator.write(stdout=attributes["stdout"])

    # Resolve event target
    event_target = attributes["event_target"]
    if not event_target:
        event_target = "."

    # Resolve event datetime
    event_datetime = None
    if attributes["event_datetime"] is None:
        event_datetime = date_now
    else:
        event_datetime = attributes["event_datetime"]

    # Create events documenting the technical metadata creation
    _create_events(
        workspace=attributes["workspace"],
        base_path=attributes["base_path"],
        event_datetime=event_datetime,
        event_target=event_target,
        identification_event=not bool(attributes["file_format"]),
        validation_event=is_validated,
        checksum_event=not bool(attributes["checksum"]),
        agents=agents
    )

class PremisCreator(MetsSectionCreator):
    """PREMIS metadata generator for files and streams."""

    def add_premis_md(
            self, filepath, attributes, filerel=None, properties=None):
        """
        Create metadata for PREMIS metadata.

        This method:
        - Scrapes a file
        - Creates PREMIS metadata with amd references for a file
        - Creates PREMIS metadata with amd references for streams in a
          file
        - Returns stream dict from scraper

        :filepath: Full path to file (including base_path)
        :attributes: The following keys
                     skip_wellformed_check: True skips well-formedness
                                            checking
                     charset: Character encoding of a file
                     original_nme: Original filename
                     file_format: File format and version (tuple) of a
                                  file
                     format_registry: Format registry name and value
                                      (tuple)
                     identifier: File identifier type and value (tuple)
                     checksum: Checksum algorithm and value (tuple)
                     date_created: Creation date of a file
        :filerel: Relative path from base_path to file
        :returns: Stream dict and info dict from file-scraper as a tuple
        """
        if not attributes["file_format"]:
            mimetype = "(:unav)"
            version = "(:unav)"
        else:
            mimetype = attributes["file_format"][0]
            version = attributes["file_format"][1]

        (streams, info, grade) = scrape_file(
            filepath=filepath,
            skip_well_check=attributes["skip_wellformed_check"],
            mimetype=mimetype,
            version=version,
            charset=attributes["charset"],
            skip_json=True
        )

        # Add new properties of a file for other script files, e.g.
        # structMap
        streams[0]['properties'] = {'grade': grade}
        if properties:
            streams[0]['properties'].update(properties)

        premis_elem = create_premis_object(filepath, streams, **attributes)
        self.add_md(premis_elem, filerel, given_metadata_dict=streams)
        premis_list = create_streams(streams, premis_elem)

        if premis_list is not None:
            for index, premis_stream in six.iteritems(premis_list):
                self.add_md(
                    premis_stream, filerel, index, given_metadata_dict=streams)

        return (streams, info)

    # pylint: disable=too-many-arguments
    def write(self, mdtype="PREMIS:OBJECT", mdtypeversion="2.3",
              othermdtype=None, section=None, stdout=False,
              file_metadata_dict=None,
              ref_file="import-object-md-references.jsonl"):
        """Write PREMIS metadata."""
        super(PremisCreator, self).write(
            mdtype=mdtype, mdtypeversion=mdtypeversion,
            file_metadata_dict=file_metadata_dict, ref_file=ref_file
        )

def create_streams(streams, premis_file):
    """Create PREMIS objects for streams.

    :streams: Stream dict
    :premis_file: Created PREMIS XML file for the digital object file
    :returns: List of PREMIS etree objects
    """
    if len(streams) < 2:
        return None

    premis_list = {}
    for index, stream in six.iteritems(streams):
        if stream['stream_type'] not in ['video', 'audio']:
            continue

        id_value = six.text_type(uuid4())
        identifier = premis.identifier(
            identifier_type='UUID',
            identifier_value=id_value)
        version = "" if stream['version'] == NO_VERSION else stream['version']
        premis_format_des = premis.format_designation(
            stream['mimetype'], version)
        premis_format = premis.format(child_elements=[premis_format_des])
        premis_objchar = premis.object_characteristics(
            child_elements=[premis_format])
        el_premis_object = premis.object(
            identifier, child_elements=[premis_objchar], bitstream=True)

        premis_list[index] = el_premis_object

        premis_file.append(
            premis.relationship('structural', 'includes', [el_premis_object]))

    return premis_list

def check_metadata(mimetype, version, streams, fname):
    """Validate metadata.

    Check that we will not get None nor (:unav) values to mimetype and
    version. Check that multiple streams are found in
    video containers, and exist only for images or video containers.

    :mimetype: MIME type
    :version: File format version
    :streams: Streams returned from Scraper
    :fname: File name of the digital object
    :raises: ValueError if metadata checking results errors
    """
    if mimetype in [None, "(:unav)"]:
        raise ValueError('MIME type could not be identified for '
                         'file %s' % fname)
    if version in [None, "(:unav)"]:
        raise ValueError('File format version could not be identified for '
                         'file %s' % fname)
    if streams[0]['stream_type'] not in ['videocontainer', 'image'] and \
            len(streams) > 1:
        raise ValueError('The file contains multiple streams which '
                         'is supported only for images and video containers.')
    if streams[0]['stream_type'] in ['videocontainer'] and len(streams) < 2:
        raise ValueError('Video container format without contained streams '
                         'found.')

# pylint: disable=too-many-locals
def create_premis_object(fname, streams, **attributes):
    """
    Create Premis object for given file.

    :fname: File name of the digital object
    :streams: Streams from the Scraper
    :attributes: The following keys:
                 charset: Character encoding of a file
                 original_name: Original filename
                 file_format: File format and version (tuple) of a file
                 format_registry: Format registry name and value (tuple)
                 identifier: File identifier type and value (tuple)
                 checksum: Checksum algorithm and value (tuple)
                 date_created: Creation date of a file
                 creating_application: Software and its version that created
                                       the file
    :returns: PREMIS object as etree
    :raises: ValueError if character set is invalid for text files.
    """
    attributes = _attribute_values(attributes)
    date_created = attributes["date_created"] or creation_date(fname)
    if not attributes["checksum"]:
        attributes["checksum"] = ("MD5", calc_checksum(fname))

    application = None
    application_version = None
    if attributes["creating_application"]:
        (application, application_version) = attributes["creating_application"]

    if streams[0]['stream_type'] == 'text':
        charset = attributes["charset"] or streams[0]['charset']
    else:
        charset = None

    if not attributes["file_format"]:
        if streams[0]["version"] and streams[0]["version"] != UNKNOWN_VERSION:
            format_version = '' if streams[0]["version"] == NO_VERSION else \
                streams[0]["version"]
        else:
            format_version = DEFAULT_VERSIONS.get(streams[0]["mimetype"], None)

        file_format = (streams[0]["mimetype"], format_version)
    else:
        version = "" if attributes["file_format"][1] == NO_VERSION else \
            attributes["file_format"][1]
        file_format = (attributes["file_format"][0], version)

    check_metadata(file_format[0], file_format[1], streams, fname)

    charset_mime = ""
    if charset:
        if charset not in ALLOWED_CHARSETS:
            raise ValueError('Invalid charset.')
        charset_mime = '; charset={}'.format(charset)

    if attributes["identifier"]:
        identifier_type = attributes["identifier"][0]
        identifier_value = attributes["identifier"][1]
    else:
        identifier_type = 'UUID'
        identifier_value = six.text_type(uuid4())

    object_identifier = premis.identifier(
        identifier_type=identifier_type,
        identifier_value=identifier_value
    )

    premis_fixity = premis.fixity(attributes["checksum"][1],
                                  attributes["checksum"][0])
    premis_format_des = premis.format_designation(
        file_format[0] + charset_mime, file_format[1])
    if not attributes["format_registry"]:
        premis_format = premis.format(child_elements=[premis_format_des])
    else:
        premis_registry = premis.format_registry(
            attributes["format_registry"][0],
            attributes["format_registry"][1])
        premis_format = premis.format(child_elements=[premis_format_des,
                                                      premis_registry])
    application_elements = []
    if application:
        application_elements.append(
            premis.creating_application_name(application)
        )
    if application_version:
        application_elements.append(
            premis.creating_application_version(application_version)
        )
    application_elements.append(premis.date_created(date_created))

    premis_create = premis.creating_application(
        child_elements=application_elements
    )
    premis_objchar = premis.object_characteristics(
        child_elements=[premis_fixity, premis_format, premis_create])

    # Create object element
    el_premis_object = premis.object(
        object_identifier,
        original_name=attributes["original_name"],
        child_elements=[premis_objchar])

    return el_premis_object

def collect_filepaths(dirs=None, pattern='*', base='.'):
    """
    Collect file paths recursively from given directory.

    :dirs: Directories from arguments
    :pattern: Filter to match the file names
    :base: Base path (see --base_path)
    :raises: IOError if given path does not exist.
    """
    if dirs is None:
        dirs = ['.']
    files = []

    for directory in dirs:
        directory = os.path.normpath(os.path.join(base, directory))
        if os.path.isdir(directory):
            files += [os.path.join(looproot, filename)
                      for looproot, _, filenames in os.walk(directory)
                      for filename in filenames
                      if fnmatch.fnmatch(filename, pattern)]
        elif os.path.isfile(directory):
            files += [directory]
        else:
            raise IOError

    return files

def creation_date(path_to_file):
    """Return creation date for file.

    Try to get the date that a file was created, falling back to when it
    was last modified if that isn't possible.  See
    http://stackoverflow.com/a/39501288/1709587 for explanation.

    :path_to_file: File path
    :returns: Timestamp for a file
    """
    if platform.system() == 'Windows':
        return datetime.datetime.fromtimestamp(
            os.path.getctime(path_to_file)).isoformat()

    stat = os.stat(path_to_file)
    try:
        return datetime.datetime.fromtimestamp(
            stat.st_birthtime).isoformat()
    except AttributeError:
        # We're probably on Linux. No easy way to get creation dates
        # here, so we'll settle for when its content was last modified.
        return datetime.datetime.fromtimestamp(stat.st_mtime).isoformat()

def _parse_scraper_tools(scraper_info):
    """Helper function to parse the used third party components used
    when identifying files and extracting technical metadata to be used
    as agents when creating events documenting the packaging process.

    :returns: a dict of agent metadata
    """
    agent = {'agent_name': None,
             'detector': False,
             'agent_version': None,
             'tools': None,
             'detail_note': None}

    agent['agent_name'] = scraper_info['class']
    if 'Detector' in scraper_info['class']:
        agent['detector'] = True
    agent['agent_version'] = file_scraper.__version__
    tools = ''
    tools_list = []
    if 'tools' in scraper_info:
        for tool in scraper_info['tools']:
            tools_list.append(tool)
    if tools_list:
        tools = 'Used tools (name-version): ' + ', '.join(tools_list)
        agent['tools'] = tools
    detail_notes = scraper_info['messages'] + scraper_info['errors']
    agent['detail_note'] = ', '.join(detail_notes)

    return agent

def _create_events(
        workspace,
        base_path,
        event_datetime,
        event_target,
        identification_event=False,
        validation_event=False,
        checksum_event=False,
        agents=None):
    """Function to create events documenting the extraction of technical
    metadata as well as the potential message digest calculation and
    the identification and validation of digital objects.

    :workspace: The path to the workspace
    :base_path: Base path (see --base_path)
    :event_datetime: The timestamp for the event
    :event_target: The target of the metadata creation
    :identification_event: Boolean to indicate whether the file formats
                           were identified during the extraction
                           of technical metadata. If True, creates an
                           file format identifiaction event.
    :validation_event: Boolean to indicate whether the digital
                       objects were validated during the extraction
                       of technical metadata. If True, creates a digital
                       object validation event.
    :checksum_event: Boolean to indicate whether the message digest for
                     digital objects were calculated during the
                     extraction of technical metadata. If True, creates
                     a message digest calculation event.
    :agents: A list of the software, as dicts, used to extract the
             metadata
    """
    events = {
        'extraction': {
            'event_type': 'metadata extraction',
            'event_datetime': event_datetime,
            'event_detail': ('Technical metadata extraction as premis '
                             'metadata from digital objects'),
            'event_outcome': 'success',
            'event_outcome_detail': ('Premis metadata successfully created '
                                     'from extracted technical metadata.')},
        'identification': {
            'event_type': 'format identification',
            'event_datetime': event_datetime,
            'event_detail': 'MIME type and version identification',
            'event_outcome': 'success',
            'event_outcome_detail': ('File MIME type and format version '
                                     'successfully identified.')},
        'validation': {
            'event_type': 'validation',
            'event_datetime': event_datetime,
            'event_detail': 'Digital object validation',
            'event_outcome': 'success',
            'event_outcome_detail': ('Digital object(s) evaluated as '
                                     'well-formed and valid.')},
        'checksum': {
            'event_type': 'message digest calculation',
            'event_datetime': event_datetime,
            'event_detail': 'Checksum calculation for digital objects',
            'event_outcome': 'success',
            'event_outcome_detail': ('Checksum(s) successfully calculated '
                                     'for digital object(s).')},
    }

    # Do not create events for steps that haven't been performed by the
    # script
    if not identification_event:
        del events['identification']
    if not validation_event:
        del events['validation']
    if not checksum_event:
        del events['checksum']

    for event_name, event in six.iteritems(events):
        found_event = _find_event(workspace,
                                  event['event_type'],
                                  event['event_datetime'],
                                  event['event_detail'],
                                  event['event_outcome'],
                                  event['event_outcome_detail'])

        if not found_event:

            if event_name == 'checksum':
                create_agent(
                    workspace=workspace,
                    agent_name='file-scraper',
                    agent_version=file_scraper.__version__,
                    agent_type='software',
                    agent_role='executing program',
                    create_agent_file='import-object-%s' % event_name)

            for agent in agents:
                if event_name == 'identification' and not agent['detector']:
                    continue
                if event_name == 'checksum':
                    break
                create_agent(
                    workspace=workspace,
                    agent_name=agent['agent_name'],
                    agent_version=agent['agent_version'],
                    agent_note=agent['tools'],
                    agent_type='software',
                    agent_role='executing program',
                    create_agent_file='import-object-%s' % event_name)

            premis_event(event_type=event['event_type'],
                         event_datetime=event['event_datetime'],
                         event_detail=event['event_detail'],
                         event_outcome=event['event_outcome'],
                         event_outcome_detail=event[
                             'event_outcome_detail'],
                         workspace=workspace,
                         base_path=base_path,
                         event_target=(event_target, ),
                         create_agent_file='import-object-%s' % event_name)

            agent_file = os.path.join(
                workspace, "import-object-%s-AGENTS-amd.json" % event_name)
            try:
                os.remove(agent_file)
            except OSError as exc:  # FileNotFoundError on Python 3
                if exc.errno != errno.ENOENT:
                    raise

def _find_event(workspace,
                event_type,
                event_datetime,
                event_detail,
                event_outcome,
                event_outcome_detail):
    """Helper function to find if a similar event already is created
    by using the digest of the metadata to see if a file already
    exist.
    """

    event = create_premis_event(
        event_type=event_type,
        event_datetime=event_datetime,
        event_detail=event_detail,
        event_outcome=event_outcome,
        event_outcome_detail=event_outcome_detail)

    digest = generate_digest(event)
    expected_filename = encode_path("%s-PREMIS:EVENT-amd.xml" % digest)

    return os.path.exists(os.path.join(workspace, expected_filename))

if __name__ == "__main__":
    RETVAL = main()  # pylint: disable=no-value-for-parameter
    sys.exit(RETVAL)


import numpy as np
import pandas as pd
import os
from sklearn.model_selection import train_test_split, GroupShuffleSplit

class Preprocessing:
    def form_time_sequences(self, data_df, cols, window_size=40, overlap_factor=2):
        """ Produce overlapping time-sequences from our data for use
            in a time-sequencing deep learning model, e.g. LSTM / CNN """
        # keep only desired columns
        lean_df = data_df.loc[:, cols].copy()
        output_data = []
        for i in range(0, lean_df.shape[0] - window_size,
                       int(window_size / overlap_factor)):
            output_data.append(lean_df.iloc[i : (i + window_size)])
        return np.stack(output_data)

    def get_class_trajectories(self, class_name, data_dir, col_names, window_size,
                               overlap_factor=2):
        """ Helper function for creating a dataset of sub-trajectories
            from the chosen class directory and higher-level data
            directory. Each sub-trajectory has a window size equal
            to the given window_size argument.
        """
        flt_filenames = [x for x in os.listdir(os.path.join(data_dir, class_name))
                    if x.endswith('.csv')]

        if 'est_z' in col_names:
            altitude = True
        else:
            altitude = False

        subtrajs = []
        labels = []
        flight_refs = []

        for filename in flt_filenames:
            flt_df = pd.read_csv(os.path.join(data_dir,
                                              class_name,
                                              filename))

            # if window size too large, skip flight
            if flt_df.shape[0] <= window_size + 1:
                continue

            # get sub-trajectories for this flight
            sub_trajs = self.form_time_sequences(flt_df, col_names,
                                            window_size,
                                            overlap_factor)

            # append to all of our sub-trajectories
            subtrajs.append(sub_trajs)

            # get original flight filename for reference
            original_ref = "_".join(filename.split('_')[:-1])

            # give each sub-traj the same filename reference
            flt_refs = np.array([original_ref for _
                                 in range(sub_trajs.shape[0])])
            flight_refs.append(flt_refs)

            # also add correct number of labels to align with sub-trajs
            sub_traj_labels = np.array([class_name for _
                                        in range(sub_trajs.shape[0])])
            labels.append(sub_traj_labels)

        # convert data into numpy arrays
        subtrajs = np.concatenate(subtrajs)
        labels = np.concatenate(labels)
        flight_refs = np.concatenate(flight_refs)
        trajs_std = subtrajs.copy()

        # get first x,y,z co-ordinates of each sub-traj
        n_rows = subtrajs.shape[0]
        x0 = subtrajs[:, 0, 0].reshape((n_rows, 1))
        y0 = subtrajs[:, 0, 1].reshape((n_rows, 1))
        if altitude:
            z0 = subtrajs[:, 0, 2].reshape((n_rows, 1))

        # centralise sub-trajectories to start at (0,0,0)
        trajs_std[:, :, 0] = subtrajs[:, :, 0] - x0
        trajs_std[:, :, 1] = subtrajs[:, :, 1] - y0
        if altitude:
            trajs_std[:, :, 2] = subtrajs[:, :, 2] - z0

        # re-order results into convenient order for processing later
        # (row, features, timestep)
        trajs_std = np.transpose(trajs_std, (0, 2, 1))

        return trajs_std, labels, flight_refs

    def OutliersRemoval(self, UAV_TYPE_IDX, X, y, flight_refs, ANOMALOUS_THRESHOLD = 11000):
        """Remove Outliers from the data in accordance with a user-defined threshold"""

        anomalous_mask = X[:,:UAV_TYPE_IDX,:].max(axis=(1,2))>ANOMALOUS_THRESHOLD
        print(flight_refs[anomalous_mask])

        X = X[~anomalous_mask]
        y = y[~anomalous_mask]
        flight_refs = flight_refs[~anomalous_mask]

        return X, y, flight_refs

    def Train_Val_Test_Split(self, X, y, flight_refs, random_state, n_splits = 1, test_size = 0.10, val_size = 0.20):
        """ Creates training, validation and testing datasets"""

        train_test_splitter = GroupShuffleSplit(n_splits = n_splits, test_size = test_size, random_state = random_state)
        train_test_split = train_test_splitter.split(X, y, groups = flight_refs)
        train_idx, test_idx = next(train_test_split)

        X_temp = X[train_idx]
        y_temp = y[train_idx]
        flight_refs_temp = flight_refs[train_idx]
        X_test = X[test_idx]
        y_test = y[test_idx]

        # split again to get validation data (20% of remaining training data)
        train_val_splitter = GroupShuffleSplit(n_splits = n_splits, test_size = val_size, random_state = random_state)
        train_val_split = train_val_splitter.split(X_temp, y_temp, groups = flight_refs_temp)
        train_idx, val_idx = next(train_val_split)

        X_train = X_temp[train_idx]
        y_train = y_temp[train_idx]
        X_val = X_temp[val_idx]
        y_val = y_temp[val_idx]

        train_flight_refs = flight_refs_temp[train_idx]
        val_flight_refs = flight_refs_temp[val_idx]
        test_flight_refs = flight_refs[test_idx]

        print(X_train.shape, y_train.shape, X_test.shape,
              y_test.shape, X_val.shape, y_val.shape,
             np.unique(train_flight_refs).shape, np.unique(val_flight_refs).shape, np.unique(test_flight_refs).shape)
        return X_train, y_train, X_test, y_test, X_val, y_val, train_flight_refs, val_flight_refs, test_flight_refs

    def time_seqs_with_lookahead(self, data_df, cols, window_size=20,
                                  overlap_factor=10, lookahead=20):
        """ Produce overlapping time-sequences from our data for use
            in a time-sequencing deep learning model, e.g. LSTM / CNN """
        # keep only desired columns
        lean_df = data_df.loc[:, cols].copy()

        max_idx = lean_df.shape[0]
        subtrajs = []
        lookahead_seqs = []

        for i in range(0, max_idx - (lookahead + window_size),
                       int(max(1, window_size / overlap_factor))):
            # append subtrajectory
            subtrajs.append(lean_df.iloc[i : (i + window_size)])

            # also get sequence ahead of current sub-trajectory
            lookahead_seqs.append(lean_df.iloc[i + window_size :
                                               (i + window_size + lookahead)])

        return np.stack(subtrajs), np.stack(lookahead_seqs)

    def get_min_max_differences(self, sub_traj, lookahead_traj):
        """ Determine the minimum and maximum difference between
        the las timestep on the given sub-trajectory and the
        co-ordinates in the lookahead trajectory """
        # get first timestep co-ordinates (x,y,z) for lookahead
        last_timestep = sub_traj[:, -1, :3]

        # get maximum bounds for lookahead seq
        max_lookaheads = lookahead_traj[:, :, :3].max(axis=1)
        min_lookaheads = lookahead_traj[:, :, :3].min(axis=1)

        # find maximum bounds relative to last input timestep
        max_diffs = max_lookaheads - last_timestep
        min_diffs = min_lookaheads - last_timestep

        # combine into one set of labels (xmin,ymin,zmin,xmax,ymax,zmax)
        min_max_diffs = np.hstack([min_diffs, max_diffs])

        return min_max_diffs

    def get_class_reg_trajectories(self, class_name, data_dir,
                                   col_names, window_size=10,
                                   overlap_factor=5, lookahead=30,
                                   sample_times=[10, 20, 30, 40]):
        """ Helper function for creating a dataset of sub-trajectories
            from the chosen class directory and higher-level data
            directory for maximum / minimum bounds regression.

            Labels are created from the lookahead sequences corresponding
            to maximum bounds (in 3D) for each of the given time intervals
            in the argument (sample_times)
        """
        flt_filenames = [x for x in os.listdir(os.path.join(data_dir, class_name))
                         if x.endswith('.csv')]

        all_subtrajs = []
        all_lookaheads = []
        flight_refs = []

        for filename in flt_filenames:
            flt_df = pd.read_csv(os.path.join(data_dir,
                                              class_name,
                                              filename))

            # if window size too large, skip flight
            if flt_df.shape[0] <= window_size + lookahead + 1:
                continue

            # add intent type to data
            flt_df['uav_intent'] = class_name

            # get sub-trajectories for this flight
            sub_trajs, lookaheads = self.time_seqs_with_lookahead(flt_df, col_names,
                                                                  window_size,
                                                                  overlap_factor,
                                                                  lookahead)

            # append to all of our sub-trajectories & lookaheads
            all_subtrajs.append(sub_trajs)
            all_lookaheads.append(lookaheads)

            # get original flight filename for reference
            original_ref = "_".join(filename.split('_')[:-1])

            # give each sub-traj the same filename reference
            flt_refs = np.array([original_ref for _
                                 in range(sub_trajs.shape[0])])
            flight_refs.append(flt_refs)

        # convert data into numpy arrays
        all_subtrajs = np.concatenate(all_subtrajs)
        all_lookaheads = np.concatenate(all_lookaheads)
        flight_refs = np.concatenate(flight_refs)

        # standardise all sub-trajs to start at (0,0,0)
        trajs_std = all_subtrajs.copy()
        lookahead_std = all_lookaheads.copy()

        # get first x,y,z co-ordinates of each sub-traj
        n_rows = all_subtrajs.shape[0]
        x0 = all_subtrajs[:, 0, 0].reshape((n_rows, 1))
        y0 = all_subtrajs[:, 0, 1].reshape((n_rows, 1))
        z0 = all_subtrajs[:, 0, 2].reshape((n_rows, 1))

        # centralise sub-trajectories to start at (0,0,0)
        trajs_std[:, :, 0] = all_subtrajs[:, :, 0] - x0
        trajs_std[:, :, 1] = all_subtrajs[:, :, 1] - y0
        trajs_std[:, :, 2] = all_subtrajs[:, :, 2] - z0

        # also re-centre lookahead trajectories using above translations
        lookahead_std[:, :, 0] = all_lookaheads[:, :, 0] - x0
        lookahead_std[:, :, 1] = all_lookaheads[:, :, 1] - y0
        lookahead_std[:, :, 2] = all_lookaheads[:, :, 2] - z0

        # get min/max bound labels for each sample time
        min_max_labels = []
        for sample_time in sample_times:
            sample_min_maxes = self.get_min_max_differences(trajs_std,
                                                            lookahead_std[:, :sample_time, :])
            min_max_labels.append(sample_min_maxes)

        # horizontally stack labels into array of labels for each input
        min_max_labels = np.hstack(min_max_labels)

        # re-order results into more convenient order:
        # (row, features, timestep)
        trajs_std = np.transpose(trajs_std, (0, 2, 1))
        lookahead_std = np.transpose(lookahead_std, (0, 2, 1))

        return trajs_std, lookahead_std, min_max_labels, flight_refs

    def DataSplitter(self, X, y, y_lookaheads, flight_refs,
                     val_size = 0.20, test_size = 0.10,
                     n_splits = 1, random_state = 18):
        """Creates training, validation, and testing datasets for
        the high-level intent regression models"""
        train_test_splitter = GroupShuffleSplit(test_size = test_size, n_splits = 1, random_state = random_state )
        train_test_split = train_test_splitter.split(X, y, groups=flight_refs)
        train_idx, test_idx = next(train_test_split)

        X_temp = X[train_idx].copy()
        y_temp = y[train_idx].copy()
        y_temp_traj = y_lookaheads[train_idx].copy()
        flight_refs_temp = flight_refs[train_idx].copy()
        X_test = X[test_idx].copy()
        y_test = y[test_idx].copy()
        y_test_traj = y_lookaheads[test_idx].copy()
        y_test_refs = flight_refs[test_idx].copy()

        # split again to get validation data (val_size of remaining training data)
        train_val_splitter = GroupShuffleSplit(test_size = val_size, n_splits = 1, random_state = random_state)
        train_val_split = train_val_splitter.split(X_temp, y_temp, groups=flight_refs_temp)
        train_idx, val_idx = next(train_val_split)

        X_train = X_temp[train_idx].copy()
        y_train = y_temp[train_idx].copy()
        y_train_traj = y_temp_traj[train_idx].copy()
        y_train_refs = flight_refs_temp[train_idx].copy()

        X_val = X_temp[val_idx].copy()
        y_val = y_temp[val_idx].copy()
        y_val_traj = y_temp_traj[val_idx].copy()
        y_val_refs = flight_refs_temp[val_idx].copy()

        print(X_train.shape, y_train.shape, y_train_traj.shape)
        print(X_val.shape, y_val.shape, y_val_traj.shape)
        print(X_test.shape, y_test.shape, y_test_traj.shape)
        return (X_train, y_train, y_train_traj,
                X_val, y_val, y_val_traj,
                X_test, y_test, y_test_traj)


import traceback
import yaml
import gc

from src.neural_analysis import *
from src.wiener_filter import *
from src.filters import *
from src.folder_handler import *
from src.tdt_support import *
from src.decoders import *
from src.plotter import *
from src.phase_decoder_support import *
from sklearn.metrics import r2_score
from sklearn.utils import shuffle
from scipy.signal import resample, find_peaks
from scipy.stats import mode
from sklearn.decomposition import PCA
from sklearn.preprocessing import MinMaxScaler
import scipy.io as sio
import numpy as np
import pandas as pd
import cv2
import copy

class CortProcessor:
    '''
    class that can handle neural + kinematic/EMG data simultaneously
    upon initialization, extracts data from TDT and anipose file
    see cort_processor.md in 'docs' for more information
    '''
    def __init__(self, folder_path):
        if os.path.isdir(folder_path):
            #see docs/data_folder_layout.md for how to structure folder_path
            self.handler = FolderHandler(folder_path)
            self.tdt_data, self.kin_data = self.extract_data()

            self.crop_list = None
            crop = self.parse_config()
            if isinstance(crop, list):
                self.crop_list = crop

            self.data={}
            self.data['rates'] = 'run_process_first'
            self.data['coords'] = 'run_process_first'
            self.data['angles'] = 'run_process_first'
            self.data['toe_height'] = 'run process first, then run process\
                    toehight'

            self.gait_indices = 'run get_gait_indices first'
            self.avg_gait_samples = 'run get_gait_indices first'

        else:
            print('this is filipe data i belive')
            self.handler = sio.loadmat(folder_path)
            self.gait_indices=None
            self.avg_gait_samples=None
            self.extract_filipe()

    def extract_filipe(self):
        self.data={}
        #remoivng 0 rate channels, which
        #throws an error in wiener filter
        temp_rates = self.handler['SelectedSpikeData']
        mask = np.average(temp_rates, 0) > .1
        temp_rates_remove = temp_rates[:, mask]
        self.data['rates'] = [temp_rates_remove]

        self.data['coords'] = [self.handler['SelectedKinematicData']['kindata'][0,0]]
        angles_temp = self.handler['SelectedKinematicData']['kinmeasures'][0,0][0,0]
        self.data['angles'] = [np.squeeze(np.array(angles_temp.tolist())).T]
        angle_names = ['ankle', 'limbfoot', 'hip', 'knee', 'toeheight']
        self.data['angle_names'] = angle_names

        temp_channels  =\
        self.handler['SelectedFieldDatastruct']['FieldCh2Use'][0][0][0].tolist()

        which_channels = copy.deepcopy(temp_channels)
        for idx in range(temp_rates.shape[1]):
            if np.average(temp_rates[:, idx]) < .1:
                baddy = temp_channels[idx]
                which_channels.remove(baddy)

        self.data['which_channels'] = which_channels
#        samples = temp_rates.shape[0]
#        num_channels = 32
#        original_rates = np.zeros((samples, num_channels))
#
#        for idx in range(num_channels):
#            if idx in which_channels:
#                position = np.where(which_channels==idx)[0][0]
#                if np.average(temp_rates[:, position]) > .1:
#                    original_rates[:, idx] = temp_rates[:, position]
#                else:
#                    original_rates[:,idx] = np.nan
#            else:
#                original_rates[:,idx] = np.nan
#
#        self.data['original_rates'] = [original_rates]
#
    def parse_config(self):
        '''
        this loads config.yaml file
        '''
        try:
            path = self.handler.folder_path
            with open(f"{path}/config.yaml", "r") as stream:
                yaml_stuff = yaml.safe_load(stream)
                crop = 'crop'

                if crop in yaml_stuff:
                    return yaml_stuff[crop]
                else:
                    print('no crop')
                    return 0
        except yaml.YAMLError as exc:
            print(exc)

    def extract_data(self):
        '''
        this is called when a CortProcessor is initialized.
        It extracts raw neural,raw angles, and raw coords
        neural data is saved under CortProcessr.tdt_data
        kinematic data is saved under CortProcessor.kin_data

        If you use process cort script, all these attriibutes are overwritten
        since raw data isnt saved after processing.
        '''
        tdt_data_list = []
        raw_ts_list = self.handler.ts_list
        raw_tdt_list = self.handler.tdt_list
        for i in range(len(raw_tdt_list)):
            tdt_data_list.append(extract_tdt(raw_tdt_list[i], raw_ts_list[i]))

        kin_data_list = []

        raw_coords_list = self.handler.coords_list
        raw_angles_list = self.handler.angles_list
        raw_phase_list = self.handler.phase_list
        for i in range(len(raw_coords_list)):
            kin_data_list.append(extract_kin_data(raw_coords_list[i],
                raw_angles_list[i], raw_phase_list[i]))

        return tdt_data_list, kin_data_list

    def process(self, manual_crop_list=None, threshold_multiplier = -3.0, window = 60, binsize = 0.05, clear_storage = True):
        """
        1) takes raw neural data, then bandpass+notch filters, then extracts
        spikes using threshold_multiplier, then bins into firing rates with bin
        size
        2) takes coordinates/angles, then resamples at 20 HZ
        3) saves all under CortHandler.data

        crop is automatically read from config.yaml file, unless manual crop
        list is set.
        """
        if self.crop_list is not None:
            crop_list = self.crop_list
        elif manual_crop_list is not None:
            crop_list = manual_crop_list
        else:
            #TODO
            print('need to make it so that it is uncropped')

        tdt_data_list = self.tdt_data
        kin_data_list = self.kin_data

        self.data = {}

        self.data['bodyparts'] = kin_data_list[0]['bodyparts']
        self.data['angle_names'] = kin_data_list[0]['angles_list']

        self.data['rates']=[]
        self.data['coords']=[]
        self.data['angles']=[]
        self.data['phase'] = []

        for i in range(len(tdt_data_list)):
            #syncs neural data/kinematic data, then crops
            self.tdt_data[i], self.kin_data[i] = self.crop_data(tdt_data_list[i],
                    kin_data_list[i], crop_list[i])

            fs = self.tdt_data[i]['fs'] #quick accessible variable
            neural = self.tdt_data[i]['neural'] #quick accessible variable

            #notch and bandpass filter

            ##common mode rejection assumes equal impedence and is therefore not recommended
            # means = np.mean(neural, axis = 1)
            # commonmode = []
            # for j in range(neural.shape[1]):
            #     commonhold = neural[:,j] - means
            #     commonmode.append(commonhold)
            # commonmode = np.array(commonmode).T

            half_filtered_neural = fresh_filt(neural, 350, 8000, fs, order = 4)
            filtered_neural = notch_filter(half_filtered_neural, fs)
            clean_filtered_neural = remove_artifacts(filtered_neural, fs)

            #extract spikes with rolling window stdv
            spikes = rolling_threshold_crossings(clean_filtered_neural, fs,
                    threshold_multiplier, window)
            firing_rates = spike_binner(spikes, fs, binsize)

            #old processing method
            # spikes = autothreshold_crossings(clean_filtered_neural,
            #         threshold_multiplier)
            # firing_rates = spike_binner(spikes, fs, binsize)

            self.data['rates'].append(firing_rates)

            temp_angles = self.kin_data[i]['angles'] #quick accessible variable
            temp_coords = self.kin_data[i]['coords']
            temp_phase = self.kin_data[i]['phase']

            #resample at same frequency of binned spikes
            resampled_angles = resample(temp_angles, firing_rates.shape[0],
                    axis=0)
            resampled_coords = resample(temp_coords, firing_rates.shape[0],
                    axis=0)
            resampled_phase = self.resample_mode(temp_phase, firing_rates.shape[0])

            self.data['angles'].append(resampled_angles)
            self.data['coords'].append(resampled_coords)
            self.data['phase'].append(resampled_phase)

            #remove raw data to save memory
            if clear_storage == True:
                self.tdt_data[i] = 0
                self.kin_data[i] = 0
            gc.collect()

        #returning stitched rates --- we don't directly use this for anything.
        return np.vstack(self.data['rates']), np.vstack(self.data['angles']), np.hstack(self.data['phase'])

    def resample_mode(self, phase, shape):
        re = resample(phase, shape, axis=0)
        for i in range(re.shape[0]):
            re[i] = int(np.round(re[i]))
        return re

    def process_toe_height(self):
        """
        extracts toe height from data['coords'], then scales such that lowest
        toe height is 0.

        toe_num is which bodypart in data['coords'] is the toe. default is
        bodypart 0
        """
        try:
            toe_num = self.bodypart_helper('toe')
            temp_list = []
            self.data['toe_height'] = []
            minimum_toe = 1000 #arbitrary high number to be beat
            for i in range(len(self.data['coords'])):
                #find y-value of toe
                toe_height = self.data['coords'][i][:, toe_num, 1]
                if np.min(toe_height) < minimum_toe:
                    minimum_toe = np.min(toe_height)
                temp_list.append(toe_height)
            for i in range(len(temp_list)):
                norm_toe_height = temp_list[i] - minimum_toe
                self.data['toe_height'].append(norm_toe_height)

            return np.hstack(self.data['toe_height'])
        except Exception as e:
            print('failed!! did you run process first')
            print(e)

    def marker_position(self, bodypart, dimension):
        """
        updated version of process_toe_height that
        a) takes any bodypart
        b) accounts for reformatting splices
        """
        mark_num = self.bodypart_helper(bodypart)
        mark_stack = []
        for i in range(len(self.data['coords'])):
            mark_stack.append(self.data['coords'][i][9:, mark_num, dimension])
        mark_y = np.hstack(mark_stack)
        mark_min = np.min(mark_y)
        mark_y_norm = mark_y - mark_min
        return mark_y_norm.T

    def crop_data(self, tdt_data, kin_data, crop):
        '''
        helper function that both syncs and crops neural/kinematic data
        see docs/cam_something.odg for how it syncs em.
        '''

        crop_tdt_datafile=tdt_data
        crop_kin_datafile=kin_data

        start_time = crop[0]
        end_time = crop[1]

        kin_start = start_time*200
        kin_end = end_time * 200

        init_start_sample = get_sync_sample(tdt_data)

        start_sample = round(init_start_sample + (start_time * tdt_data['fs']))
        end_sample = round(init_start_sample + (end_time * tdt_data['fs']))

        temp_neural = tdt_data['neural'] #going to slice variable
        temp_ts = tdt_data['ts']

        crop_tdt_datafile['neural'] = temp_neural[start_sample:end_sample,:]
        crop_tdt_datafile['ts'] = temp_ts[start_sample:end_sample]

        del(temp_neural)
        del(temp_ts)

        gc.collect()

        temp_coords = kin_data['coords']
        temp_angles = kin_data['angles']
        temp_phase = kin_data['phase']
        temp_frames = kin_data['fnum']

        crop_kin_datafile['coords'] = temp_coords[kin_start:kin_end,:,:]
        crop_kin_datafile['angles'] = temp_angles[kin_start:kin_end,:]
        crop_kin_datafile['phase'] = temp_phase[kin_start:kin_end]
        crop_kin_datafile['fnum'] = temp_frames[kin_start:kin_end]

        #maybe need edge-case if only single angle/bodypart

        return crop_tdt_datafile, crop_kin_datafile

    def stitch_and_format(self, firing_rates_list=None,
            resampled_angles_list=None, N = 10):
        """
        takes list of rates, list of angles, then converts them into lags of 10
        using format rate in wiener_filter.py, and then stitches them into one
        big array

        both lists must have same # of elements, and each array inside list
        must have the same size as the corresponding array in the other list.
        """
        if firing_rates_list == None and resampled_angles_list == None:
            firing_rates_list = self.data['rates']
            resampled_angles_list = self.data['angles']

        assert isinstance(firing_rates_list, list), 'rates must be list'
        assert isinstance(resampled_angles_list, list), 'angles must be list'
        formatted_rates = []
        formatted_angles = []

        for i in range(len(firing_rates_list)):
            f_rate, f_angle = format_data(firing_rates_list[i],
                    resampled_angles_list[i], N)
            formatted_rates.append(f_rate)
            formatted_angles.append(f_angle)

        if len(formatted_rates)==1: #check if just single array in list
            rates = np.array(formatted_rates)
        else: #if multiple, stitch into single array
            rates = np.vstack(formatted_rates)

        if len(formatted_angles)==1: #check if single array
            kin = np.array(formatted_angles)
        elif formatted_angles[0].ndim > 1: #check if multiple angles
            kin = np.vstack(formatted_angles)
        else:
            kin = np.hstack(formatted_angles)

        return np.squeeze(rates), np.squeeze(kin)

    def stitch_data(self, firing_rates_list, resampled_angles_list):
        '''
        deprecated. you can just use np.vstack instead of this
        WARNING: VSTACK WILL NOT BE INDEXED AT PARITY TO STITCH AND FORMAT UNLESS THE LAST 10 ENTRIES OF EACH LIST IS OMITTED
        '''
        rates = np.vstack(firing_rates_list)
        kin = np.vstack(resampled_angles_list)

        return rates, kin

    def subsample(self, percent, X=None, Y=None):
        '''
        function to subsample RAW data (not gait aligned).
        we keep it in order since we go by folds.
        '''
        if X is None:
            X = self.data['rates']
        if Y is None:
            Y = self.data['angles']

        if percent==1.0:
            return X, Y

        new_x=[]
        new_y=[]

        for i in range(len(X)):
            subsample = int(np.round(percent * X[i].shape[0]))
            new_x.append(X[i][:subsample,:])
            new_y.append(Y[i][:subsample,:])

        return new_x, new_y

    def decode_angles(self, X=None, Y=None, metric_angle='limbfoot', scale=False):
        """
        takes list of rates, angles, then using a wiener filter to decode.
        if no parameters are passed, uses data['rates'] and data['angles']

        returns best_h, vaf (array of all angles and all folds), test_x (the
        x test set that had best performance, and test_y (the y test set that
        had best formance)
        """
        try:
            if X is None:
                X = self.data['rates']
            if Y is None:
                Y = self.data['angles']

            if scale is True:
                X = self.apply_scaler(X)
            X, Y = self.stitch_and_format(X,Y)

            metric = self.angle_name_helper(metric_angle)

            h_angle, vaf_array, final_test_x, final_test_y, _ = decode_kfolds(X,Y,
                    metric_angle=metric)

            self.h_angle = h_angle
            return h_angle, vaf_array, final_test_x, final_test_y
        except Exception as e:
            print(e)
            print('did you run process() first.')

    def decode_toe_height(self):
        '''
        a PIPELINE code, which you can you only after you run
        process_toe_height. You cannot pass any custom parameters in this. it
        uses a wiener filter to create a decoder just for toe height
        '''
        try:
            X,Y = self.stitch_and_format(self.data['rates'],
                    self.data['toe_height'])
            h_toe, vaf_array, final_test_x, final_test_y = decode_kfolds_single(X, Y)
            self.h_toe = h_toe
            return h_toe, vaf_array, final_test_x, final_test_y
        except:
            print('did you run process_toe_height() yet?????')

#     def decode_phase(self, rates=None, angles=None, metric_angle='limbfoot'):
#         '''
#         this has a non-zero chance of returning overfit cos test cases
#         this probability is assumed to be less than the 1% predicted by random selection of kfolds
#         as the quality of the data along equally divded splits is assumed to be variable
#         '''
#         if rates is None and angles is None:
#             full_rates, full_angles = self.stitch_and_format(self.data['rates'],
#                         self.data['angles'])

#         elif isinstance(rates, list):
#             full_rates, full_angles = self.stitch_and_format(rates, angles)
#         else:
#             full_rates = rates
#             full_angles = angles

#         angle_number = self.angle_name_helper(metric_angle)
#         phase_list = []

#         for i in range(full_angles.shape[1]):
#             peak_list = tailored_peaks(full_angles, i, self.data['angle_names'][i])
#             phase_list_tmp = to_phasex(peak_list, full_angles[:,i])
#             phase_list.append(phase_list_tmp)
#         phase_list = np.array(phase_list).T
#         sin_array, cos_array = sine_and_cosine(phase_list)

#         h_sin, r2_sin, test_rates, test_sin, sin_test_index = decode_kfolds(X=full_rates, Y=sin_array, metric_angle=angle_number, vaf_scoring=False)
#         h_cos, r2_cos, _, test_cos, _ = decode_kfolds(X=full_rates, Y=cos_array, metric_angle=angle_number, vaf_scoring=False, forced_test_index = sin_test_index)
#         predicted_sin = predicted_lines(test_rates, h_sin)
#         predicted_cos = predicted_lines(test_rates, h_cos)
#         # order_test = predicted_lines(test_rates, h_atantest)
#         predicted_arctans = arctan_fn(predicted_sin, predicted_cos)
#         test_arctans = phase_list[sin_test_index, :]

#         self.h_sin = h_sin
#         self.h_cos = h_cos
#         self.phase_list = phase_list

#         return h_sin, h_cos, np.mean((r2_sin,r2_cos), axis=0), predicted_arctans, test_arctans, test_rates, phase_list

    def phase_train(self):
        rect_phase_list = []
        for i in range(len(self.data['phase'])):
            scrub_phase = ss_cleaner(self.data['phase'][i], 3)
            rect_phase_list.append(scrub_phase)
        form_rates, form_phase = self.stitch_and_format(self.data['rates'], rect_phase_list)
        phase_12 = form_phase[np.nonzero(form_phase)]
        rates_12 = form_rates[np.nonzero(form_phase),:][0]
        phase_angles, swing_mean = get_phase_angles(phase_12)
        sin_arr, cos_arr = sine_and_cosine(phase_angles)
        h_sin_nl, lsq_sin, r2_sin_nl, test_rates, test_sin, sin_test_index = decode_kfolds_single_nonlinear(X=rates_12, Y=sin_arr, scoring='R2')
        h_cos_nl, lsq_cos, r2_cos_nl, _, test_cos, _ = decode_kfolds_single_nonlinear(X=rates_12, Y=cos_arr, scoring='R2', forced_test_index = sin_test_index)
        predicted_sin_nl = test_nonlinear_wiener_filter(test_rates, h_sin_nl, lsq_sin)
        predicted_cos_nl = test_nonlinear_wiener_filter(test_rates, h_cos_nl, lsq_cos)
        predicted_arctans_nl = arctan_fn(predicted_sin_nl, predicted_cos_nl)
        test_arctans = phase_angles[sin_test_index]

        self.predicted_arctans_nl = predicted_arctans_nl
        self.test_arctans = test_arctans
        self.swing_mean = swing_mean
        self.h_sin_nl = h_sin_nl
        self.lsq_sin = lsq_sin
        self.h_cos_nl = h_cos_nl
        self.lsq_cos = lsq_cos

        return h_sin_nl, lsq_sin, h_cos_nl, lsq_cos, predicted_arctans_nl, test_arctans, swing_mean

    def phase_evaluate(self, manual_threshold = None, refractory_tics = 0, bounds = [-4, 4], plotting = False):
        if manual_threshold == None:
            threshold = self.swing_mean
        else:
            threshold = manual_threshold
        crossingsA = np.diff(self.test_arctans >threshold, prepend=0)
        np.put(crossingsA, np.where(crossingsA==-1), 0)
        crossingsB = np.diff(self.predicted_arctans_nl >threshold, prepend=0)
        np.put(crossingsB, np.where(crossingsB==-1), 0)
        crossingsC = stim_cooldown(crossingsB, refractory_tics)
        true_indicies, new_indicies, delay_array_list, spacing_mean = phase_sychrony(crossingsA, crossingsC)
        true_score, effective_score = phase_diagnositc(self.predicted_arctans_nl, self.test_arctans, threshold, true_indicies, new_indicies, delay_array_list, spacing_mean, bounds, plotting)
        return true_score, effective_score

    def get_H(self, H):
        if H == 'toe':
            H_mat = self.h_toe
        if H == 'angle':
            H_mat = self.h_angle
        if H == 'cos':
            H_mat = self.h_cos
        if H == 'sin':
            H_mat = self.h_sin
        return H_mat

    def impulse_response(self, AOI, H = None, plotting = True):
        phase_list = self.phase_list
        if H == None:
            H = 'sin'
        h_mat = self.get_H(H)
        response = impulse_response(AOI, h_mat, phase_list, plotting)
        return response

    def neuron_tuning(self, rates_gait=None):
        '''
        this takes in neural data that is divided into gaits, and sorts each
        channel by where in the gait cycle each channel is most active

        feed the output of this into plot_raster!
        '''
        try:
            if rates_gait is None:
                rates_gait = self.rates_gait
            rates_gait = np.vstack(rates_gait)
            gait_array_avg = np.average(rates_gait, axis=0)
            df = pd.DataFrame(gait_array_avg)
            temp = df.iloc[:, df.idxmax(axis=0).argsort()]
            self.norm_sorted_neurons=((temp-temp.min())/(temp.max()-temp.min()))
            return self.norm_sorted_neurons

        except Exception as e:
            print(e)
            print('make sure you run divide into gaits first')

    def apply_PCA(self, dims=None, X=None, transformer=None):
        '''
        this works now?
        '''
        if X is None:
            X=self.data['rates']
        if dims is None:
            dims=.95
        X_full = np.vstack(X)
        if transformer is None:
            pca_object = PCA(n_components = dims)
            pca_object.fit(X_full)
        else:
            pca_object = transformer

        x_pca = []

        for X_recording in X:
            x_pca.append(pca_object.transform(X_recording))

        self.num_components = x_pca[0].shape[1]
        self.pca_object = pca_object
        return x_pca

    def apply_scaler(self, X=None, scaler=None):
        if X is None:
            X=self.data['rates']
        if scaler is None:
            my_scaler=StandardScaler()
            X_full = np.vstack(X)
            my_scaler.fit(X_full)
        else:
            my_scaler = scaler

        scaled_X = []
        for X_recording in X:
            scaled_X.append(my_scaler.transform(X_recording))

        self.scaler = my_scaler
        return scaled_X

    def display_frame(self, video_path, recording_number, sample_number):
        cap = cv2. VideoCapture(video_path)
        crop_times = self.crop_list[recording_number]
        first_frame = crop_times[0] * 200
        my_frame = first_frame + (sample_number * 4)
        cap.set(1, my_frame)
        ret, frame = cap.read()

        video_time = my_frame/200
        print(f'video_time is: {video_time}')

        return frame

    def predicted_lines_malleable(self, h_sin, h_cos):
        try:
            full_rates, full_angles = self.stitch_and_format(self.data['rates'],
                            self.data['angles'])
            phase_list = self.phase_list
            sin_array, cos_array = sine_and_cosine(phase_list)
            predicted_sin = predicted_lines(full_rates, h_sin)
            predicted_cos = predicted_lines(full_rates, h_cos)
            arctans = arctan_fn(predicted_sin, predicted_cos)
            r2_array = []
            for i in range(sin_array.shape[1]):
                r2_sin = r2_score(sin_array[:,i], predicted_sin[:,i])
                r2_cos = r2_score(cos_array[:,i], predicted_cos[:,i])
                r2_array.append(np.mean((r2_sin,r2_cos)))
            return arctans, phase_list, r2_array
        except:
            print('error lol')
            print('Feed in sin and cos H matricies from another session to test gernealizability')

    def angle_name_helper(self, angle_name):
        return self.data['angle_names'].index(angle_name)

    def bodypart_helper(self, bodypart):
        return self.data['bodyparts'].index(bodypart)


from matplotlib import pyplot as plt
import numpy as np
import csv
from collections import Counter
import ast
import pandas as pd
from textblob import TextBlob
import re
from textblob.sentiments import NaiveBayesAnalyzer

def clean_tweet(tweet):
    return ' '.join(re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)", " ", tweet).split()) #removing hyperlinks and special charaters

def analyse_tweet(tweet):
    analysis = TextBlob(tweet)
    if analysis.sentiment.polarity > 0: # positive
        return "positive"
    elif analysis.sentiment.polarity == 0: # neutral / we don't know
        pass
    else: # negative
        return "negative"

with open('twitterData_public_tweets.csv') as file:
    csv_reader =  csv.DictReader(file)

    sentiments = []
    num = []

    for row in csv_reader:
        sentiments.append(analyse_tweet(row['content']))
        num.append(TextBlob(row['content']).sentiment.polarity)

data = {'Sentiment':sentiments,'Value':num}
data_pd = pd.DataFrame(data)

data_pd.boxplot(by='Sentiment', column='Value')
plt.xlabel('Sentiment of tweets')
plt.ylabel('Polarity value')
plt.show()


from fastapi import FastAPI, Request
from fastapi.responses import HTMLResponse
from fastapi.staticfiles import StaticFiles
from fastapi.templating import Jinja2Templates

from databse.db import init_db

app = FastAPI()
app.mount("/static", StaticFiles(directory="static"), name="static")
templates = Jinja2Templates(directory="templates")

@app.on_event("startup")
async def start_db():
    await init_db()

@app.get("/")
async def root():
    return {"message": "Hello World"}

@app.get("/student", response_class=HTMLResponse)
async def first(request: Request):
    return templates.TemplateResponse("add_student.html", {"request": request})


from unified.core.auth import Auth
import requests
import json
from datetime import datetime, timezone

def get_access_token(context):
    """ Returns new access token """

    headers = context['headers']
    auth_info = {
        "client_id": headers["client_id"],
        "client_secret": headers["client_secret"],
        "token": headers["token"],
        "refresh_url": headers["refresh_url"],
    }
    token = Auth().get_oauth2_token(auth_info)

    return (json.loads(token))['access_token']

def rest(method, url, access_token, org_id, body=None, files=None):
    """ Send REST request and returns response  """

    headers = {
        "Accept": "*/*",
        "Content-Type": "application/json; charset=utf-8",
        "Authorization": f"Zoho-oauthtoken {access_token}"
    }

    if org_id:
        headers['orgId'] = org_id

    if files:
        del headers['Content-Type']
        response = requests.post(url, files=files, headers=headers)
        return response

    response = requests.request(method, url, headers=headers, json=body, files=files)

    return response

def epoch_to_format(epoch, format='%Y-%m-%dT%H:%M:%SZ'):
    """ Convert  epoch to given format """

    return datetime.fromtimestamp(int(epoch[:10]), tz=timezone.utc).strftime(format)

def get_url():
    """ Utility function for getting url """

    return "https://desk.zoho.in/api/v1/"


from rl_framework.agent import Agent
from rl_framework.environment import Environment
from stable_baselines3.common.base_class import BaseAlgorithm
from stable_baselines3.common.env_util import make_vec_env
from stable_baselines3.common.vec_env import DummyVecEnv
from stable_baselines3 import A2C, DDPG, DQN, HER, PPO, SAC, TD3
from typing import List, Text, Optional, Dict
from huggingface_sb3 import package_to_hub, load_from_hub
from functools import partial
from enum import Enum

class StableBaselinesAlgorithm(Enum):
    A2C = A2C
    DDPG = DDPG
    DQN = DQN
    HER = HER
    PPO = PPO
    SAC = SAC
    TD3 = TD3

class StableBaselinesAgent(Agent):
    @property
    def model(self):
        return self._model

    @model.setter
    def model(self, value):
        self._model = value

    def __init__(
        self,
        rl_algorithm: StableBaselinesAlgorithm = StableBaselinesAlgorithm.PPO,
        rl_algorithm_parameters: Dict = None,
        pretrained_model: Optional[BaseAlgorithm] = None,
    ):
        """
        Initialize an agent which will trained on one of Stable-Baselines3 algorithms.

        Args:
            rl_algorithm (StableBaselinesAlgorithm): Enum with values being SB3 RL Algorithm classes (Types).
                Specifies the algorithm for RL training.
                Defaults to PPO.
            rl_algorithm_parameters (Dict): Parameters / keyword arguments for the specified SB3 RL Algorithm class.
                See https://stable-baselines3.readthedocs.io/en/master/modules/base.html for details on common params.
                See individual docs (e.g., https://stable-baselines3.readthedocs.io/en/master/modules/ppo.html)
                for algorithm-specific params.
            pretrained_model (BaseAlgorithm): Pretrained SB3 model.
                This variable is mainly used for loading previously saved models.
        """

        self.sb3_algorithm = rl_algorithm.value
        self.model = pretrained_model

        if rl_algorithm_parameters is None:
            rl_algorithm_parameters = {
                "policy": "MlpPolicy",
                "learning_rate": 0.001,
            }

        self._model_builder = partial(
            self.sb3_algorithm,
            **rl_algorithm_parameters
        )

    def train(self, training_environments: List[Environment], total_timesteps: int = 100000, *args, **kwargs):
        """
        Train the instantiated agent on the environment.

        This training is done by using the agent-on-environment training method provided by Stable-baselines3.

        The model is changed in place, therefore the updated model can be accessed in the `.model` attribute
        after the agent has been trained.

        Args:
            training_environments (List[Environment): Environment on which the agent should be trained on.
                If n_environments is set above 1, multiple environments enables parallel training of an agent.
            total_timesteps (int): Amount of individual steps the agent should take before terminating the training.
        """

        environment_iterator = iter(training_environments)
        training_env = make_vec_env(
            lambda: next(environment_iterator), n_envs=len(training_environments)
        )

        self.model = self._model_builder(env=training_env)

        self.model.learn(total_timesteps=total_timesteps)

    def choose_action(self, observation: object, *args, **kwargs):
        """
        Chooses action which the agent will perform next, according to the observed environment.

        Args:
            observation (object): Observation of the environment

        Returns: action (int): Action to take according to policy.

        """

        action, _ = self.model.predict(observation)
        return action

    def save(self, file_path: Text):
        """
        Save the model of the agent to a zipped file.

        Args:
            file_path (Text): Path where the model should be saved to.
        """
        self.model.save(file_path)

    def upload_to_huggingface_hub(
        self,
        evaluation_environment: Environment,
        model_name: Text,
        model_architecture: Text,
        environment_name: Text,
        repository_id: Text,
        commit_message: Text
    ) -> None:
        """

        Args:
            evaluation_environment (Environment): Environment used for final evaluation and clip creation before upload.
            model_name (Text): Name of the model (uploaded model .zip will be named accordingly).
            model_architecture (Text): Name of the used model architecture (only used for model card and metadata).
            environment_name (Text): Name of the environment (only used for model card and metadata).
            repository_id (Text): Id of the model repository from the Hugging Face Hub.
            commit_message (Text): Commit message for the HuggingFace repository commit.

        NOTE: If after running the package_to_hub function, and it gives an issue of rebasing, please run the following code
            `cd <path_to_repo> && git add . && git commit -m "Add message" && git pull`
            And don't forget to do a `git push` at the end to push the change to the hub.

        """
        # Create a Stable-baselines3 vector environment (required for HuggingFace upload function)
        vectorized_evaluation_environment = DummyVecEnv(
            [lambda: evaluation_environment]
        )

        model = self.model
        package_to_hub(
            model=model,
            model_name=model_name,
            model_architecture=model_architecture,
            env_id=environment_name,
            eval_env=vectorized_evaluation_environment,
            repo_id=repository_id,
            commit_message=commit_message,
        )

    def download_from_huggingface_hub(
        self, repository_id: Text, filename: Text
    ):
        """
        Download a reinforcement learning model from the HuggingFace Hub and update the agent policy in-place.

        Args:
            repository_id (Text): Repository ID of the reinforcement learning model we want to download.
            filename (Text): The model filename (file ending with .zip) located in the hugging face repository.

        """

        # When the model was trained on Python 3.8 the pickle protocol is 5
        # But Python 3.6, 3.7 use protocol 4
        # In order to get compatibility we need to:
        # 1. Install pickle5
        # 2. Create a custom empty object we pass as parameter to PPO.load()
        custom_objects = {
            "learning_rate": 0.0,
            "lr_schedule": lambda _: 0.0,
            "clip_range": lambda _: 0.0,
        }

        checkpoint = load_from_hub(repository_id, filename)
        model = self.sb3_algorithm.load(
            checkpoint, custom_objects=custom_objects, print_system_info=True
        )
        self.model = model


import pyautogui as pya
import pyperclip
import time
import keyboard
import webbrowser

lst = []

def copy_clipboard():
    pyperclip.copy("")
    pya.hotkey('ctrl','c')
    time.sleep(.1)
    return pyperclip.paste()

def double_click_copy():
    pya.doubleClick(pya.position())
    pya.click(pya.position())
    var=copy_clipboard()
    lst.append(var)
    print(lst)
    webbrowser.open('https://stackoverflow.com/')
    time.sleep(3)
    pya.hotkey('ctrl','v')
    pya.press('enter')

keyboard.add_hotkey('ctrl+f9',double_click_copy)
keyboard.wait()


import configparser

from cpe_help import util

def get_configuration():
    config = configparser.RawConfigParser()
    config.read(util.path.CONFIG_PATH)
    return config

def get_acs_variables():
    """
    Return a dictionary mapping variable names to be queried into how
    these variables should be named locally.
    """
    config = get_configuration()
    default_keys = list(config['DEFAULT'])
    result = {k.upper(): v
              for k, v in config['ACS Variables'].items()
              if k not in default_keys}
    return result


#! /usr/bin/env python

import argparse
import re
import sys
import webbrowser
import requests
import logging

from gen_html import init_html, story_html, end_html, comment_html
from http.server import BaseHTTPRequestHandler, HTTPServer

logger = logging.getLogger(__name__)

class HandlerClass(BaseHTTPRequestHandler):
    def do_GET(self):
        self.send_response(200)

        logger.info("received new request")
        logger.info("path: " + self.path)

        if self.path == "/":
            logger.info("true")
            self.end_headers()
            out = get_top(1)
            self.wfile.write(bytes(out, "utf8"))

        if self.path == "/style.css":
            self.send_header('Content-type', "text/css")
            self.end_headers()
            with open("./style.css", "r") as file:
                self.wfile.write(bytes(file.read(), "utf8"))

def run(server: str, port: int, num_stories):
    httpd = HTTPServer((server, port), HandlerClass)
    httpd.serve_forever()

def open_web_browser(server: str, port: int):
    server_address = 'https://' + server + ':' + str(port)
    webbrowser.open_new(server_address)

def get_top(num_stories: int):
    i = 0
    res = [init_html()]
    top_story_list = requests.get(
        "https://hacker-news.firebaseio.com/v0/topstories.json?print=pretty"
    )
    stories = top_story_list.text.split(", ")

    # TODO: use queue of threads for this
    # see https://docs.python.org/3/library/queue.html
    while i < num_stories:
        url = get_api_url(stories[i])
        story = requests.get(url).json()
        title = story.get("title")

        # TODO: create new css class, make title a link to post on HN
        #       HN url: https://news.ycombinator.com/item?id=<id>
        try:
            res.append(story_html(story.get("url"), story.get("title")))
        except TypeError:
            pass
        print_comments(story)
        # time.sleep(0.01) # to stay under rate limit
        i += 1

    res.append(end_html())
    # TODO lets not use print() here
    print("".join(res))
    return "".join(res)

def get_api_url(item):
    base_url = "https://hacker-news.firebaseio.com/v0/item/"
    postfix = ".json?print=pretty"
    item_number = trim_id(str(item))
    return base_url + item_number + postfix

def get_hn_url(item):
    base_url = "https://news.ycombinator.com/item?id="
    return base_url + item

def print_comments(story):
    res = []

    comments = story.get("kids")

    print(comments)

    try:
        for comment in comments:
            comment = requests.get(get_api_url(comment)).json()
            res.append(comment_html(comment.get("text")))
    except TypeError:
        pass

    print("".join(res))
    return "".join(res)

def trim_id(n: str):
    return re.sub(r"\D", "", n)

def process_args():
    """process arguments
# default server address
SERVER_ADDRESS = ("127.0.0.1", 8080)
    """
    parser = argparse.ArgumentParser()

    parser.add_argument('--num_stories',
                        type=int,
                        default=20,
                        help="number of stories to return")
    parser.add_argument('--server',
                        type=str,
                        default='127.0.0.1',
                        help="Address to serve page on.")
    parser.add_argument('--port',
                        type=int,
                        default=8080,
                        help="Port to use.")
    parser.add_argument('--verbose',
                        action='store_true',
                        help="Verbose logging")

    return parser.parse_args()

def init_logging(verbose):
    log_level = logging.ERROR if args.verbose else logging.INFO
    _log_handler = logging.StreamHandler(stream=sys.stdout)
    logger.addHandler(_log_handler)
    logger.setLevel(log_level)

if __name__ == "__main__":
    args = process_args()
    init_logging(args.verbose)
    open_web_browser(args.server, args.port)
    run(args.server, args.port, args.num_stories)


from django.urls import path

from django.conf import settings
from django.conf.urls.static import static

from . import views

urlpatterns = [
    path('',views.home,name='home'),
    path('all-news',views.all_news,name='all-news'),
    path('all-teachers',views.all_teachers,name='all-teachers'),
    path('all-students',views.all_students,name='all-students'),
    path('faoliyat',views.faoliyat,name='faoliyat'),
    path('kafedra',views.kafedra,name='kafedra'),
    path('detail/<int:id>',views.detail,name="detail"),
    path('contact',views.contact,name = "contact"),
    # path('category/<int:id>',views.category,name='category'),
] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)


from django.db import models
from authen.models import UserProfile, Content

# Create your models here.

class Forum(models.Model):
    creator = models.ForeignKey(UserProfile, on_delete=models.CASCADE) # Perlu kah?
    title = models.CharField(max_length = 255, unique=True)
    description = models.TextField()
    contents = models.ManyToManyField(Content)
    date_created = models.DateTimeField(auto_now_add=True)

class Members(models.Model):
    forum = models.OneToOneField(Forum, on_delete=models.CASCADE)
    subscriptor = models.ManyToManyField(UserProfile) # Mau make ginian?


import logging

from flask import request
from flask_restplus import Resource, fields
from structlog import wrap_logger

from ras_backstage import party_api
from ras_backstage.controllers import party_controller

logger = wrap_logger(logging.getLogger(__name__))

respondent_details = party_api.model('RespondentDetails', {
    'first_name': fields.String(required=True),
    'last_name': fields.String(required=True),
    'telephone': fields.String(required=True),
    'email_address': fields.String(required=True),
    'new_email_address': fields.String(required=True)
})

@party_api.route('/update-respondent-details/<respondent_id>')
class UpdateRespondentDetails(Resource):
    @staticmethod
    @party_api.expect(respondent_details, validate=True)
    def put(respondent_id):
        logger.info('Retrieving updated respondent details', respondent_id=respondent_id)
        respondent_contact_details = request.get_json()
        party_controller.update_respondent_details(respondent_id, respondent_contact_details)
        logger.info('Successfully updated user details', respondent_id=respondent_id)
        return 200


#! /usr/bin/env python
# coding: utf-8

import requests
import config

class Geocode:
    """ This class search the geolocation informations of a place """

    def __init__(self, place=""):
        self.api_key = config.GOOGLE_API_KEY
        self.url = "https://maps.googleapis.com/maps/api/geocode/json"
        self.geocode_params = {
            "address": place,
            "key": self.api_key,
        }
        self.get_place_infos()

    def get_place_infos(self):
        self.placeInfos = {}
        req = requests.get(self.url, self.geocode_params)
        if req.status_code >= 200 and req.status_code <= 400:
            data = req.json()
            self.placeInfos["status"] = data["status"]
            if self.placeInfos["status"] == "OK":
                self.placeInfos["address"] = data["results"][0]["formatted_address"]
                self.placeInfos["location"] = data["results"][0]["geometry"]["location"]
                self.placeInfos["place_id"] = data["results"][0]["place_id"]
            else:
                try:
                    self.placeInfos['error_message'] = data["error_message"]
                except KeyError:
                    pass
            return self.placeInfos
        else:
            self.placeInfos['status'] = "Erreur 404"
            self.placeInfos['error_message'] = "Page not found or error in URL"
            return self.placeInfos

    @property
    def infos(self):
        return self.placeInfos

    @property
    def status(self):
        return self.placeInfos["status"]

    @property
    def address(self):
        try:
            if self.placeInfos["address"]:
                return self.placeInfos["address"]
        except KeyError:
            return "'address' not found for this request"

    @property
    def location(self):
        try:
            if self.placeInfos["location"]:
                return self.placeInfos["location"]
        except KeyError:
            return "'location' not found for this request"

    @property
    def place_id(self):
        try:
            if self.placeInfos["place_id"]:
                return self.placeInfos["place_id"]
        except KeyError:
            return "'place_id' not found for this request"

    @property
    def message_error(self):
        try:
            if self.placeInfos["error_message"]:
                return self.placeInfos["error_message"]
        except KeyError:
            return "No message Error"

# if __name__=="__main__":
#     geo = Geocode("OpenClassrooms Paris")
#     print(geo.status)
#     print(geo.address)
#     print(geo.location)
#     print(geo.location['lat'])
#     print(geo.location['lng'])
#     print(geo.place_id)
#     print(geo.message_error)


# Exploring the Bitcoin market

import time, requests, json
import datetime as dt

# API access key
CMC_PRO_API_KEY = 'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx'

# URL query and formatting
url = 'https://sandbox-api.coinmarketcap.com/v1/cryptocurrency/listings/latest'
parameters = {
  'start':'1',
  'limit':'5000',
  'convert':'USD'
}

headers = {
  'Accepts': 'application/json',
  'X-CMC_PRO_API_KEY': CMC_PRO_API_KEY,
}

# fetch query
r = requests.get(url, headers=headers)

# checking for errors when fetching
if r.status_code == 200:
  response = json.loads(r.text)

# dump json file
with open('BTC_data.json','w') as f:
  json.dump(response, f)


from setuptools import setup, find_packages

VERSION = (0, 6, 1)

# Dynamically calculate the version based on VERSION tuple
if len(VERSION)>2 and VERSION[2] is not None:
    str_version = "%d.%d_%s" % VERSION[:3]
else:
    str_version = "%d.%d" % VERSION[:2]

version= str_version

setup(
    name = 'django-colissimo',
    version = version,
    description = "Django app to get shipping prices from La Poste Colissimo",
    long_description = """django-colissimo provides the ability to get prices from the La Poste Colissimo postal service.""",
    author = 'Mathias Monnerville',
    author_email = 'mathias@monnerville.com',
	url = 'https://github.com/matm/django-colissimo',
    license = 'GPL 3',
    platforms = ['any'],
    classifiers = [
		'Development Status :: 5 - Production/Stable',
		'Environment :: Web Environment',
		'Intended Audience :: Developers',
		'License :: OSI Approved :: GNU General Public License (GPL)',
		'Operating System :: OS Independent',
		'Programming Language :: Python',
		'Framework :: Django'],
    packages = find_packages(),
    setup_requires=["setuptools_hg"],
    include_package_data = True,
)


from flask_wtf import FlaskForm
from wtforms import StringField, SubmitField, SelectField, IntegerField
from wtforms.validators import DataRequired

from app.models import Menu

class OrderForm(FlaskForm):
    order_name = StringField('order_name', validators=[DataRequired()])
    phone = StringField('phone', validators=[DataRequired()])
    adress = StringField('adress', validators=[DataRequired()])
    dish1 = SelectField(default=1)
    dish2 = SelectField(default=1)
    dish3 = SelectField(default=1)
    dish4 = SelectField(default=1)
    dish5 = SelectField(default=1)
    amount = IntegerField()
    submit1 = SubmitField('Сделать заказ', validators=[DataRequired()])

    def __init__(self, *args, **kwargs):
        super(OrderForm, self).__init__(*args, **kwargs)
        self.dish1.choices = [(c.id, c.name) for c in Menu.query.all()]
        self.dish2.choices = [(c.id, c.name) for c in Menu.query.all()]
        self.dish3.choices = [(c.id, c.name) for c in Menu.query.all()]
        self.dish4.choices = [(c.id, c.name) for c in Menu.query.all()]
        self.dish5.choices = [(c.id, c.name) for c in Menu.query.all()]

class OrderSatus(FlaskForm):
    search = StringField('search')
    submit = SubmitField('Поиск', validators=[DataRequired()])

    def __init__(self, *args, **kwargs):
        super(OrderSatus, self).__init__(*args, **kwargs)


from project_management.clickup import util
from project_management.clickup.entities.clickup_folder import ClickupFolder
from project_management.clickup.entities.clikup_tasklist import ClickupTasklist
from project_management.clickup.entities.clickup_task import ClickupTask
import json

class ClickupApi:
    def folder(self, context, params):
        '''gets a folder'''

        url = f"https://api.clickup.com/api/v2/folder/{params['folder_id']}"

        response = util.rest("GET", url, context=context)

        folder_obj = json.loads(response.text)

        folder = ClickupFolder(
            space_id=folder_obj['space']['id'],
            folder_name=folder_obj['name'],
            folder_id=folder_obj['id'],
            is_archived=folder_obj['archived'],
            task_count=folder_obj['task_count'],
        )

        return folder.__dict__

    def tasklist(self, context, params):
        '''gets a tasklist'''

        url = f"https://api.clickup.com/api/v2/list/{params['tasklist_id']}"
        response = util.rest("GET", url, context=context)

        tasklist_obj = json.loads(response.text)

        tasklist = ClickupTasklist(
            folder_id=tasklist_obj['folder']['id'],
            priority=tasklist_obj['priority'],
            assignee_to=tasklist_obj['assignee'],
            due_date=tasklist_obj['due_date'],
            id=tasklist_obj['id'],
            name=tasklist_obj['name'],
        )

        return tasklist.__dict__

    def task(self, context, params):
        '''get a task'''

        url = f"https://api.clickup.com/api/v2/task/{params['task_id']}"
        response = util.rest("GET", url, context=context)

        task_obj = json.loads(response.text)
        task = ClickupTask(
            due_date=task_obj['due_date'],
            list_id=task_obj['list']['id'],
            tag=task_obj['tags'],
            id=task_obj['id'],
            name=task_obj['name'],
            description=task_obj['description'],
            task_id=task_obj['id'],
            assigned_to=task_obj['assignees'],
        )

        return task.__dict__

    def tasks(self, context, params):
        '''get list of task in list or team '''

        if params.get('list_id'):
            url = f"https://api.clickup.com/api/v2/list/{params['list_id']}/task"
        else:
            url = f"https://api.clickup.com/api/v2/team/{params['team_id']}/task"

        response = util.rest("GET", url, context=context)

        task_list = json.loads(response.text)['tasks']

        tasks = []

        for task_obj in task_list:
            task = ClickupTask(
                due_date=task_obj['due_date'],
                list_id=task_obj['list']['id'],
                tag=task_obj['tags'],
                id=task_obj['id'],
                name=task_obj['name'],
                description=task_obj['description'],
                task_id=task_obj['id'],
                assigned_to=task_obj['assignees'],
            )
            tasks.append(task.__dict__)

        return json.dumps(tasks)

    def profile(self, context, payload):
        url = "https://api.clickup.com/api/v2/user"
        response_data = util.rest("GET", url, context=context)
        response = response_data.json()['user']
        profile = {
            'id':response['id'],
            'email':response['email'],
            'name':response['username']
        }
        return profile, response_data.status_code

    def verify(self, context, payload):
        url = "https://api.clickup.com/api/v2/team"
        response_data = util.rest("GET", url, context=context)
        response = response_data.json()
        return response, response_data.status_code