"""Managing common operations found in ETL jobs developed using AWS Glue.

This module aims to put together everything it's needed to improve and enhance
the development journey of Glue jobs on AWS.

___
"""

# Importing libraries
import sys
from time import sleep
from sparksnake.utils.log import log_config

from awsglue.utils import getResolvedOptions
from awsglue.context import GlueContext
from awsglue.job import Job
from awsglue.dynamicframe import DynamicFrame

from pyspark.context import SparkContext
from pyspark.sql import DataFrame

# Setting up a logger object
logger = log_config(logger_name=__file__)

class GlueJobManager():
    """Enables users to use AWS Glue features in their Spark applications.

    This class provides an enhanced experience on developing Glue jobs using
    Apache Spark. The core idea behind it is related complex code encapsulation
    that is mandatory to build and run Glue jobs. By that, it's possible to
    say that this `GlueJobManager` class has attributes and methods that uses
    the `awsglue` library to handle almost everything that would be handled
    individually by users if they are not using sparksnake.

    By the end, it's important to mention that all attributes and methods of
    this class are inherited by `SparkETLManager` class in the `manager` module
    when users initialize it with `mode="glue"`. So, when doing that, users
    need to pass some additional arguments in order to make things work
    properly in the Glue specific world.

    Args:
        argv_list (list):
            List with all user defined arguments for the job

        data_dict (dict):
            Dictionary with all source data references to be read from the
            catalog and used in the ETL job.

    This class can get some other attributes along the execution of its
    methods. Those are:

    Attributes:
        args (dict):
            Dictionary with all arguments of the job. This is a joint
            between user defined arguments and system arguments (`sys.argv`)

        sc (SparkContext):
            A Spark context object used for creating a Glue context object

        glueContext (GlueContext):
            A Glue context object used for creating a Spark session object

        spark (SparkSession):
            A Spark session object used as a central point for job operations

    Tip: About setting up the data_dict class attribute
        The data_dict dictionary passed as a required attribute for the
        `GlueJobManager` class must be defined following some rules. Then main
        purpose of such attribute is to provide a single variable to handle
        all data sources to be read by the Glue job application.

        With that in mind, it's important to say that the data_dict attribute
        can be defined using everything that is available and acceptable in
        the Glue DynamicFrameReader class. Users can take a look at the AWS
        official docs about the DynamicFrameReader class to see more details
        about reading DynamicFrame objects in Glue jobs.

        On this class scope, the data_dict dictionary can also have additional
        keys that can used to guide reading proccesses and apply some special
        conditions. The additional keys that can be put on data_dict class
        attribute are:

        - "source_method": str -> Defines if users want to read data
        from catalog ('from_catalog') or from other options ('from_options').
        Under the hood, the 'source_method' dictionary key defines which method
        will be used along the `glueContext.create_dynamic_frame` method.
        The default value is 'from_catalog'.

        - "create_temp_view": bool -> Sets the creation of a Spark temporary
        table (view) after reading the data source as a DynamicFrame. If this
        additional key is set as True, then the `DataFrame.createTempView()`
        method is executed in order to create temporary tables using the
        table name as the main reference for the temp view.

        By the end, an example on how to define the data_dict class attribute
        dictionary can be find right below:

        ```python
        # Defining elements of all data sources used on the job
        ```python
        {
            "orders": {
                "database": "ra8",
                "table_name": "orders",
                "transformation_ctx": "dyf_orders"
            },
            "customers": {
                "database": "ra8",
                "table_name": "customers",
                "transformation_ctx": "dyf_customers",
                "push_down_predicate": "anomesdia=20221201",
                "create_temp_view": True,
                "additional_options": {
                    "compressionType": "lzo"
                }
            },
            "payments": {
                "source_method": "from_options",
                "connection_type": "s3",
                "connection_options": {
                    "paths": [
                        "s3://some-bucket-name/some-prefix/file.csv"
                    ],
                    "recurse": True
                },
                "format": "csv",
                "format_options": {
                    "withHeader": True,
                    "separator": ",",
                    "quoteChar": '"'
                },
                "transformation_ctx": "dyf_payments"
            }
        }
        ```

        As you can see, the DATA_DICT variable defined in the example above
        uses mixed data sources, each one with special configurations
        accepted by the DynamicFrameReader Glue methods. But don't worry,
        you will find more examples along this documentation page.
    """

    def __init__(self, argv_list: list, data_dict: dict) -> None:
        self.argv_list = argv_list
        self.data_dict = data_dict

        # Getting job arguments
        self.args = getResolvedOptions(sys.argv, self.argv_list)

    def job_initial_log_message(self) -> None:
        """Preparing a detailed log message for job start up.

        This method is responsible for composing an initial log message to be
        logged in CloudWatch after the user starts a Glue Job. The message aims
        to clarify some job details, such as the data sources mapped and its
        push down predicate values (if used). This can be a good practice in
        order to develop more organized Glue jobs.
        """

        # Defining a initial string for composing the message
        welcome_msg = f"Successfully started Glue job {self.args['JOB_NAME']}"\
                      ". Data sources in this ETL process:\n\n"
        initial_msg = ""

        # Iterating over the data_dict dicionary for extracting some info
        for _, params in self.data_dict.items():
            # Getting table name and start building the message
            tbl_ref = f"{params['database']}.{params['table_name']}"
            table_msg = f"Table {tbl_ref} "

            # Looking for a push_down_predicate information in the dictionary
            if "push_down_predicate" in params:
                table_msg += "with the following push down predicate info: "\
                             f"{params['push_down_predicate']}\n"
            else:
                table_msg += "without push down predicate\n"

            # Concat final message
            initial_msg += table_msg

        # Logging the message to CloudWatch
        logger.info(welcome_msg + initial_msg)

    def print_args(self) -> None:
        """Getting and logging job arguments in CloudWatch.

        This method is responsible to show users all the arguments used in the
        job. It achieves its goal by iterating over all args in self.args class
        attribut for composing a detailed log message.
        """

        # Formatting and composing a log message with job arguments
        args_formatted = "".join([f'--{k}="{v}"\n'
                                  for k, v in self.args.items()])
        logger.info(f"Job arguments:\n\n{args_formatted}")
        sleep(0.01)

    def get_context_and_session(self) -> None:
        """Getting context and session elements for Glue job application.

        This method is a central point for creating the following elements:
        SparkContext, GlueContext and SparkSession.
        """
        logger.info("Creating SparkContext, GlueContext and SparkSession")
        self.sc = SparkContext.getOrCreate()
        self.glueContext = GlueContext(self.sc)
        self.spark = self.glueContext.spark_session

    def init_job(self):
        """Starting a Glue job.

        This method consolidates all the necessary steps required for a Glue
        job initialization process. In its definition, it calls the following
        methods:

        * `self.job_initial_log_message()`
        * `self.print_args()`
        * `self.get_context_and_session()`

        After that, the class has all the attributes required for calling the
        Job class used by Glue to initialize a job object.
        """

        # Getting job arguments and logging useful infos for users
        self.job_initial_log_message()
        self.print_args()

        # Getting context and session elements
        self.get_context_and_session()

        # Initializing a Glue job based on a GlueContext element
        self.job = Job(self.glueContext)
        self.job.init(self.args['JOB_NAME'], self.args)

    def generate_dynamicframes_dict(self) -> dict:
        """Getting a dictionary of DynamicFrames objects read from the catalog.

        This method uses the data_dict class attribute for reading and getting
        DynamicFrame objects mapped as data sources on the mentioned
        dictionary. The main advantage of using this method is explained by
        having the possibility to read multiple data sources with a single
        method call. The result of this method is a dictionary containing all
        DynamicFrames objects as values of keys mapped on data_dict class
        attribute. The user will be able to access those DynamicFrames in
        a easy way through indexing.

        Examples:
            ```python
            # Getting a dictionary of Glue DynamicFrames
            dyfs_dict = spark_manager.generate_dynamicframes_dict()

            # Inexing and getting individual DynamicFrames
            dyf_orders = dyfs_dict["orders"]
            dyf_customers = dyfs_dict["customers"]
            ```

        Returns:
            Python dictionary with keys representing the identification of the\
            data source put into the data_dict class attributes and values\
            representing DynamicFrame objects read from catalog.

        Tip: Details about the returned dictionary from the method
            In order to provide a clear view of the return of this method,
            consider the following definition example of `self.data_dict` as a
            class attribute:

            ```python
            {
                "orders": {
                    "database": "ra8",
                    "table_name": "orders",
                    "transformation_ctx": "dyf_orders"
                },
                "customers": {
                    "database": "ra8",
                    "table_name": "customers",
                    "transformation_ctx": "dyf_customers",
                    "push_down_predicate": "anomesdia=20221201",
                    "create_temp_view": True,
                    "additional_options": {
                        "compressionType": "lzo"
                    }
                }
            }
            ```

            As stated before, all elements on Glue DynamicFrameReader class
            methods can be used as dictionary keys on `self.data_dict`
            definition. Moreover, some additional keys can be defined by the
            user for some special purposes, such as:

            - "source_method": str -> Defines if users want to read data
            from catalog ('from_catalog') or from other options
            ('from_options'). Under the hood, the 'source_method' dictionary
            key defines which method will be used along the
            `glueContext.create_dynamic_frame` method. The default value is
            'from_catalog'.

            - "create_temp_view": bool -> Sets the creation of a Spark
                temporary table (view) after reading the data source as a
                DynamicFrame

            The return of the method `generate_dynamicframes_dict()` will be
            presented as the following format:

            ```python
            {
                "orders": <DynamicFrame>
                "customers": <DynamicFrame>
            }
            ```

            where the tags <DynamicFrame> means objects of DynamicFrame type
            read for each data source.
        """

        logger.info("Reading all data sources as Glue DynamicFrame objects")
        try:
            dynamic_frames = []
            for t in self.data_dict.keys():
                # Defining the source method to be called to read the dyf
                source_method = self.data_dict[t]["source"].lower().strip() \
                    if "source" in self.data_dict[t].keys()\
                    else "from_catalog"

                # Entering a condition where the data is read from catalog
                if source_method == "from_catalog":
                    # Getting some required args: database and table_name
                    database = self.data_dict[t]["database"]
                    table_name = self.data_dict[t]["table_name"]

                    # Getting non required args: redshift_tmp_dir
                    redshift_tmp_dir = self.data_dict[t]["redshift_tmp_dir"]\
                        if "redshift_tmp_dir" in self.data_dict[t].keys()\
                        else ""

                    # Getting non required args: transformation_ctx
                    transformation_ctx = \
                        self.data_dict[t]["transformation_ctx"]\
                        if "transformation_ctx" in self.data_dict[t].keys()\
                        else ""

                    # Getting non required args: push_down_predicate
                    push_down_predicate = \
                        self.data_dict[t]["push_down_predicate"]\
                        if "push_down_predicate" in self.data_dict[t].keys()\
                        else ""

                    # Getting non required args: additional_options
                    additional_options = \
                        self.data_dict[t]["additional_options"] \
                        if "additional_options" in self.data_dict[t].keys()\
                        else {}

                    # Getting non required args: catalog_id
                    catalog_id = self.data_dict[t]["catalog_id"] \
                        if "catalog_id" in self.data_dict[t].keys()\
                        else None

                    # Reads a DynamicFrame from catalog
                    dyf = self.glueContext.create_dynamic_frame.from_catalog(
                        database=database,
                        table_name=table_name,
                        redshift_tmp_dir=redshift_tmp_dir,
                        transformation_ctx=transformation_ctx,
                        push_down_predicate=push_down_predicate,
                        additional_options=additional_options,
                        catalog_id=catalog_id
                    )

                # Entering a condition where the data is read from other option
                elif source_method == "from_options":
                    # Getting some required args: connection_type
                    connection_type = self.data_dict[t]["connection_type"]

                    # Getting non required args: connection_options
                    conn_options = self.data_dict[t]["connection_options"] \
                        if "connection_options" in self.data_dict[t].keys()\
                        else {}

                    # Getting non required args: catalog_id
                    format = self.data_dict[t]["format"] \
                        if "format" in self.data_dict[t].keys()\
                        else None

                    # Getting non required args: format_options
                    format_options = self.data_dict[t]["format_options"] \
                        if "format_options" in self.data_dict[t].keys()\
                        else {}

                    # Reads a DynamicFrame from options
                    dyf = self.glueContext.create_dynamic_frame.from_options(
                        connection_type=connection_type,
                        connection_options=conn_options,
                        format=format,
                        format_options=format_options,
                        transformation_ctx=transformation_ctx
                    )

                else:
                    # Raising an error if source_method is not expected
                    raise TypeError("Invalid value for 'source_method' in "
                                    f"DATA_DICT dictionary ({source_method}). "
                                    "Acceptable values are 'from_catalog' or "
                                    "'from_options'.")

                # Appending the DynamicFrame obnject to DynamicFrames list
                dynamic_frames.append(dyf)

        except Exception as e:
            logger.error("Failed to read data sources mapped on DATA_DICT "
                         f"dictionary as Glue DynamicFrames. Exception: {e}")
            raise e

        # Creating a DynamicFrames dictionary
        dynamic_dict = {
            k: dyf for k, dyf in zip(self.data_dict.keys(), dynamic_frames)
        }

        # Preparing a final message with all data sources read
        print_dict = {k: type(v) for k, v in dynamic_dict.items()}
        logger.info("Sucessfully read all data sources. There are "
                    f"{len(dynamic_dict.values())} DynamicFrames available "
                    "in a Python dictionary presented as following: \n"
                    f"{print_dict}")

        # Returning the dictionary
        sleep(0.01)
        return dynamic_dict

    def generate_dataframes_dict(self) -> dict:
        """Getting a dictionary of DataFrames objects read from the catalog.

        This method uses the data_dict class attribute for reading and getting
        DataFrame objects mapped as data sources on the mentioned
        dictionary. The main advantage of using this method is explained by
        having the possibility to read multiple data sources with a single
        method call. The result of this method is a dictionary containing all
        DataFrames objects as values of keys mapped on data_dict class
        attribute. The user will be able to access those DataFrames in
        a easy way through indexing.

        In its behalf, this method calls `generate_dynamicframes_dict()` method
        for getting a dictionary of DynamicFrames after applying the method
        `toDF()` for transforming all objects into Spark DataFrames.

        Examples:
            ```python
            # Getting a dictionary of Spark DataFrames
            dyfs_dict = spark_manager.generate_dataframes_dict()

            # Inexing and getting individual DataFrames
            dyf_orders = dyfs_dict["orders"]
            dyf_customers = dyfs_dict["customers"]
            ```

        Returns:
            Python dictionary with keys representing the identification of the\
            data source put into the data_dict class attributes and values\
            representing DataFrame objects read from catalog.

        Tip: Details about the returned dictionary from the method
            In order to provide a clear view of the return of this method,
            consider the following definition example of `self.data_dict` as a
            class attribute:

            ```python
            {
                "orders": {
                    "database": "ra8",
                    "table_name": "orders",
                    "transformation_ctx": "dyf_orders"
                },
                "customers": {
                    "database": "ra8",
                    "table_name": "customers",
                    "transformation_ctx": "dyf_customers",
                    "push_down_predicate": "anomesdia=20221201",
                    "create_temp_view": True,
                    "additional_options": {
                        "compressionType": "lzo"
                    }
                }
            }
            ```

            As stated before, all
            `glueContext.create_dynamic_frame.from_catalog()` can be used as
            dictionary keys on `self.data_dict` definition. Moreover, some
            additional keys can be defined by the user for some special
            purposes, such as:

            - "create_temp_view": bool -> Sets the creation of a Spark
                temporary table (view) after reading the data source as a
                DataFrame

            The return of the method `generate_dataframes_dict()` will be
            presented as the following format:

            ```python
            {
                "orders": <DataFrame>
                "customers": <DataFrame>
            }
            ```

            where the tags <DataFrame> means objects of DataFrame type
            read for each data source.
        """

        # Getting a DynamicFrame objects dictionary
        dyf_dict = self.generate_dynamicframes_dict()

        # Transforming DynamicFrames into DataFrames
        logger.info(f"Transforming {len(dyf_dict.keys())} "
                    "Glue DynamicFrames into Spark DataFrames")
        try:
            # Transforming objects and mapping the result into a python dict
            df_dict = {k: dyf.toDF() for k, dyf in dyf_dict.items()}

            # Preparing a log message
            print_dict = {k: type(v) for k, v in df_dict.items()}
            logger.info("Sucessfully generated Spark DataFrames from Glue "
                        "DynamicFrames previously read. There are "
                        f"{len(df_dict.values())} Spark DataFrames available "
                        "in a Python dictionary presented as following: "
                        f"{print_dict}")
            sleep(0.01)

        except Exception as e:
            logger.error("Failed to transform Glue DynamicFrames into Spark "
                         f"DataFrames. Exception: {e}")
            raise e

        # Creating spark temporary tables if applicable
        for table_key, params in self.data_dict.items():
            try:
                # Extracting useful variables from the dict
                df = df_dict[table_key]
                table_name = params["table_name"]

                # Creating temp tables (if the flag for it was set as True)
                if "create_temp_view" in params\
                        and bool(params["create_temp_view"]):
                    df.createOrReplaceTempView(table_name)

                    logger.info("Successfully created a Spark temporary table "
                                f"(view) named {table_name}")

            except Exception as e:
                logger.error("Failed to create a Spark temporary table for "
                             f"data source {table_name}. Exception: {e}")
                raise e

        # Returning the DataFrames dict
        sleep(0.01)
        return df_dict

    def drop_partition(self, s3_partition_uri: str,
                       retention_period: int = 0) -> None:
        """Deleting (purging) a physical table partition directly on S3.

        This methods is responsable for excluding physical partitions on S3
        through purge method from a GlueContext. The users can use this feature
        for ensuring that new writing processes in a specific partitions will
        only be done after deleting existing references for the given
        partition.

        Examples:
            ```python
            # Dropping a physycal partition on S3
            partition_uri = "s3://some-bucket/some-table/partition=value/"
            spark_manager.drop_partition(partition_uri)

            # The result is the elimination of everything under the prefix
            # partition=name (including the prefix itself and all table files)
            ```

        Args:
            s3_partition_uri (str): Partition URI on S3
            retention_period (int): Hours to data retention

        Raises:
            Exception: Generic exception raises when a failed attempt of
            executing the `glueContext.purge_s3_path()` method.
        """

        logger.info(f"Searching for partition {s3_partition_uri} and "
                    "dropping it (if exists)")
        try:
            self.glueContext.purge_s3_path(
                s3_path=s3_partition_uri,
                options={"retentionPeriod": retention_period}
            )
        except Exception as e:
            logger.error(f"Failed to purge partition {s3_partition_uri}. "
                         f"Exception: {e}")
            raise e

    def write_and_catalog_data(
            self,
            df: DataFrame or DynamicFrame,
            s3_table_uri: str,
            output_database_name: str,
            output_table_name: str,
            partition_name: str or list = None,
            connection_type: str = "s3",
            update_behavior: str = "UPDATE_IN_DATABASE",
            compression: str = "snappy",
            enable_update_catalog: bool = True,
            output_data_format: str = "parquet"
    ) -> None:
        """Writing data on S3 anda cataloging on Data Catalog.

        This methods is responsible to put together all the steps needed to
        write a Spark DataFrame or a Glue DynamicFrame into S3 and catalog
        its metadata on Glue Data Catalog. In essence, this methods include
        the following steps:

        1. Check if the data object type (df argument) is a Glue DynamicFrame
        (if it's not, it converts it)
        2. Make a sink with data catalog
        3. Write data in S3 and update the data catalog with the behavior
        chosen by the user

        Examples:
            ```python
            # Writing and cataloging data
            spark_manager.write_and_catalog_data(
                df=df_orders,
                s3_table_uri="s3://some-bucket/some-table",
                output_database_name="db_corp_business_inteligence",
                output_table_name="tbl_orders_prep",
                partition_name="anomesdia"
            )
            ```

        Args:
            df (DataFrame or DynamicFrame):
                A data object that can be a Spark DataFrame
                (`pyspark.sql.DataFrame`) or a Glue DynamicFrame
                (`awsglue.dynamicframe.DynamicFrame`) considered as a target
                element for writing and cataloging processes. This information
                is used on `glueContext.getSink().writeFrame()` method.

            s3_table_uri (str):
                Table URI in the format `s3://bucket-name/table-prefix/`. This
                information is used on parameter "path" from
                `glueContext.getSink()` method.

            output_database_name (str):
                Reference for the database used on the catalog process for the
                table. This information is used on parameter "catalogDatabase"
                from `glueContext.getSink().setCatalogInfo()` method.

            output_table_name (str):
                Reference for the table used on the catalog process. This
                information is used on parameter "catalogTableName"
                from `glueContext.getSink().setCatalogInfo()` method.

            partition_name (str or list or None):
                Partition coumn name chosen for the table. This information is
                used on parameter "partitionKeys" from `glueContext.getSink()`
                method. In case of partitioning by multiple columns, users can
                pass a list with partition names.

            connection_type (str):
                Connection type used in storage. This information is used on
                parameter "connection_type" from `glueContext.getSink()` method

            update_behavior (str):
                Defines the update behavior for the target table. This
                information is used on parameter "updateBehavior" from
                `glueContext.getSink()` method.

            compression (str):
                Defines the compression method used on data storage process.
                This information is used on parameter "compression" from
                `glueContext.getSink()` method.

            enable_update_catalog (bool):
                Enables the update of data catalog with data storage by this
                method. This information is used on parameter
                "enableUpdateCatalog" from  `glueContext.getSink()` method.

            output_data_format (str):
                Defines the data format for the data to be stored on S3. This
                information is used on parameter "output_data_format" from
                `glueContext.getSink().setFormat()` method.
        """

        # Converting DataFrame into DynamicFrame
        if type(df) == DataFrame:
            logger.info("Converting the Spark DataFrame as Glue DynamicFrame")
            try:
                dyf = DynamicFrame.fromDF(df, self.glueContext, "dyf")
            except Exception as e:
                logger.error("Failed to transform the Spark DataFrame object "
                             f"in a Glue DynamicFrame object. Exception: {e}")
                raise e
        else:
            dyf = df

        # Handling partition information
        if isinstance(partition_name, str):
            partition_keys = [partition_name]
        elif isinstance(partition_name, list):
            partition_keys = partition_name
        else:
            raise ValueError("Invalid type for argument partition_name. "
                             "Acceptable values are str or list")

        # Creating a sink with S3
        logger.info("Creating a sink with all configurations provided")
        try:
            # Criando relação de escrita de dados
            data_sink = self.glueContext.getSink(
                path=s3_table_uri,
                connection_type=connection_type,
                updateBehavior=update_behavior,
                partitionKeys=partition_keys,
                compression=compression,
                enableUpdateCatalog=enable_update_catalog,
                transformation_ctx="data_sink",
            )
        except Exception as e:
            logger.error("Failed to create a sink for the output table. "
                         f"Exception: {e}")
            raise e

        # Setting up data catalog info
        logger.info("Setting up information on the Data Catalog for the table")
        try:
            data_sink.setCatalogInfo(
                catalogDatabase=output_database_name,
                catalogTableName=output_table_name
            )
            data_sink.setFormat(output_data_format,
                                useGlueParquetWriter=True)
            data_sink.writeFrame(dyf)

            logger.info(f"Successfully updated table {output_database_name}"
                        f".{output_table_name} in Glue Data Catalog. The "
                        "table data are stored in the following S3 path: "
                        f"{s3_table_uri}")
        except Exception as e:
            logger.error("Failed to write and catalog data for table  "
                         f"{output_database_name}.{output_table_name}. "
                         f"Exception: {e}")
            raise e


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
compute gm with load_data.py and test them.
Created on Wed Sep 19 16:12:13 2018

@author: ljia
"""

"""Shortest-Path graph kernel.
Python implementation based on: "Shortest-path kernels on graphs", by
Borgwardt, K.M.; Kriegel, H.-P., in Data Mining, Fifth IEEE
International Conference on , vol., no., pp.8 pp.-, 27-30 Nov. 2005
doi: 10.1109/ICDM.2005.132
Author : Sandro Vega-Pons, Emanuele Olivetti
"""

import sys
sys.path.insert(0, "../../")
import numpy as np
import networkx as nx
from gklearn.utils.graphfiles import loadDataset
import matplotlib.pyplot as plt
from numpy.linalg import eig

class GK_SP:
    """
    Shorthest path graph kernel.
    """

    def compare(self, g_1, g_2, verbose=False):
        """Compute the kernel value (similarity) between two graphs.
        Parameters
        ----------
        g1 : networkx.Graph
            First graph.
        g2 : networkx.Graph
            Second graph.
        Returns
        -------
        k : The similarity value between g1 and g2.
        """
        # Diagonal superior matrix of the floyd warshall shortest
        # paths:
        fwm1 = np.array(nx.floyd_warshall_numpy(g_1))
        fwm1 = np.where(fwm1 == np.inf, 0, fwm1)
        fwm1 = np.where(fwm1 == np.nan, 0, fwm1)
        fwm1 = np.triu(fwm1, k=1)
        bc1 = np.bincount(fwm1.reshape(-1).astype(int))

        fwm2 = np.array(nx.floyd_warshall_numpy(g_2))
        fwm2 = np.where(fwm2 == np.inf, 0, fwm2)
        fwm2 = np.where(fwm2 == np.nan, 0, fwm2)
        fwm2 = np.triu(fwm2, k=1)
        bc2 = np.bincount(fwm2.reshape(-1).astype(int))

        # Copy into arrays with the same length the non-zero shortests
        # paths:
        v1 = np.zeros(max(len(bc1), len(bc2)) - 1)
        v1[range(0, len(bc1)-1)] = bc1[1:]

        v2 = np.zeros(max(len(bc1), len(bc2)) - 1)
        v2[range(0, len(bc2)-1)] = bc2[1:]

        return np.sum(v1 * v2)

    def compare_normalized(self, g_1, g_2, verbose=False):
        """Compute the normalized kernel value between two graphs.
        A normalized version of the kernel is given by the equation:
        k_norm(g1, g2) = k(g1, g2) / sqrt(k(g1,g1) * k(g2,g2))
        Parameters
        ----------
        g1 : networkx.Graph
            First graph.
        g2 : networkx.Graph
            Second graph.
        Returns
        -------
        k : The similarity value between g1 and g2.
        """
        return self.compare(g_1, g_2) / (np.sqrt(self.compare(g_1, g_1) *
                                                 self.compare(g_2, g_2)))

    def compare_list(self, graph_list, verbose=False):
        """Compute the all-pairs kernel values for a list of graphs.
        This function can be used to directly compute the kernel
        matrix for a list of graphs. The direct computation of the
        kernel matrix is faster than the computation of all individual
        pairwise kernel values.
        Parameters
        ----------
        graph_list: list
            A list of graphs (list of networkx graphs)
        Return
        ------
        K: numpy.array, shape = (len(graph_list), len(graph_list))
        The similarity matrix of all graphs in graph_list.
        """
        n = len(graph_list)
        k = np.zeros((n, n))
        for i in range(n):
            for j in range(i, n):
                k[i, j] = self.compare(graph_list[i], graph_list[j])
                k[j, i] = k[i, j]

        k_norm = np.zeros(k.shape)
        for i in range(k.shape[0]):
            for j in range(k.shape[1]):
                k_norm[i, j] = k[i, j] / np.sqrt(k[i, i] * k[j, j])

        return k_norm

ds_name = 'PAH'
datafile = '../../datasets/PAH/dataset.ds'
dataset, y = loadDataset(datafile, filename_y=None, extra_params=None)
gk_sp = GK_SP()
x = gk_sp.compare_list(dataset)
np.savez('../check_gm/' + ds_name + '.gm.jstsp', gms=x)

plt.imshow(x)
plt.colorbar()
plt.savefig('../check_gm/' + ds_name + '.gm.jstsp.eps', format='eps', dpi=300)
# print(np.transpose(x))
print('if symmetric: ', np.array_equal(x, np.transpose(x)))

print('diag: ', np.diag(x))
print('sum diag < 0.1: ', np.sum(np.diag(x) < 0.1))
print('min, max diag: ', min(np.diag(x)), max(np.diag(x)))
print('mean x: ', np.mean(np.mean(x)))

[lamnda, v] = eig(x)
print('min, max lambda: ', min(lamnda), max(lamnda))


from sqlalchemy import create_engine
from sqlalchemy import select

import argparse
import pandas as pd

__author__ = 'jaquinoa'

db_engine = create_engine('mysql+pymysql://user:password@localhost/pydb')

parser = argparse.ArgumentParser(description='Copiar Nexus en Excel a DB')
parser.add_argument('-e', '--excel', help='Archivo Excel', required=True)
parser.add_argument('-s', '--sheet', help='Hoja Excel data Nexus', required=False)
args = parser.parse_args()

print("-----------------------------------------------------")
print("")
print("*******  Copiar Excel Nexus a una Tabla MySQL *******")

print("Leyendo archivo Excel...")
xlsfile = args.excel
sheet   = args.sheet
df =  pd.read_excel(xlsfile, skiprows = 4, dtype = str)

# Set columns names
df.columns = ['region', 'distrito', 'ugel', 'provincia', 'distrito', 'tipo_ie', 'gestion', 'zona', 'codmod_ie', 'clave8', 'niveleducativo', 'nombre_ie', 'codigo_plaza', 'tipo_trabajador', 'subtipo_trabajador', 'cargo', 'situacion_laboral', 'motivo_vacante', 'paterno', 'materno', 'nombres', 'categoria_remunerativa', 'jornada_laboral', 'estado', 'codigo_modular', 'fecha_nacimiento', 'dni', 'fecha_inicio', 'fecha_termino', 'tipo_registro', 'ley', 'preventiva', 'referencia_preventiva']

print(df.info())

print("Copiar a Base de Datos...")
df.to_sql(con=db_engine, name='_nexus2018', if_exists='replace', index_label = 'id')

print('Done...')


import copy
import logging
import pprint

from .diffutils import contains, findby, treefindby, get_descendants
from .presets import diff_presets

logger = logging.getLogger('treediffs')
logger.setLevel(logging.DEBUG)

# EXTERNAL API
################################################################################

def treediff(treeA, treeB, preset=None, format="simplified", sort_order_changes=False,
             attrs=None, exclude_attrs=[], mapA={}, mapB={},
             assessment_items_key='assessment_items', setlike_attrs=['tags']):
    """
    Compute the diff between `treeA` (old tree) and `treeB` (new tree).
    """
    # 0. load diff preset
    if preset is not None:
        if preset in diff_presets:
            kwargs = diff_presets[preset]
            attrs = kwargs['attrs'] if 'attrs' in kwargs else attrs
            exclude_attrs = kwargs['exclude_attrs'] if 'exclude_attrs' in kwargs else exclude_attrs
            mapA = kwargs['mapA'] if 'mapA' in kwargs else mapA
            mapB = kwargs['mapB'] if 'mapB' in kwargs else mapB
            assessment_items_key = kwargs['assessment_items_key'] if 'assessment_items_key' in kwargs else assessment_items_key
            setlike_attrs = kwargs['setlike_attrs'] if 'setlike_attrs' in kwargs else setlike_attrs

            # print(preset, format, sort_order_changes, attrs, exclude_attrs, mapA, mapB, assessment_items_key, setlike_attrs)

    # 1. compute the tree diff
    # special handling of tree root nodes??? (might not have the same IDs)
    raw_diff = diff_subtree(None, treeA, None, treeB, root=True,
                            attrs=attrs, exclude_attrs=exclude_attrs,
                            mapA=mapA, mapB=mapB,
                            assessment_items_key=assessment_items_key,
                            setlike_attrs=setlike_attrs)

    # 2. detect node moves
    nodes_moved = detect_moves(raw_diff['nodes_deleted'], raw_diff['nodes_added'])
    raw_diff['nodes_moved'] = nodes_moved

    if format == "raw" and sort_order_changes:
        # keep sort_order changes and count them as moves
        return raw_diff
    elif format == "raw" and not sort_order_changes:
        # filter out nodes for which only sort_order has changed (local moves)
        new_nodes_moved = [nm for nm in nodes_moved if not nm['sort_order_change']]
        raw_diff['nodes_moved'] = new_nodes_moved
        return raw_diff

    # 3. simplify (remove nodes moved from nodes added/deleted lists)
    simplified_diff = simplify_diff(raw_diff)
    if format == "simplified" and sort_order_changes:
        return simplified_diff
    elif format == "simplified" and not sort_order_changes:
        # filter out nodes for which only sort_order has changed (local moves)
        nodes_moved = simplified_diff['nodes_moved']
        new_nodes_moved = [nm for nm in nodes_moved if not nm['sort_order_change']]
        simplified_diff['nodes_moved'] = new_nodes_moved
        return simplified_diff

    # 4. restructure (un-flatten)
    restructured_diff = restructure_diff(simplified_diff, treeA, treeB, mapA=mapA, mapB=mapB)
    if not sort_order_changes:
        # filter out nodes for which only sort_order has changed (local moves)
        nodes_moved = restructured_diff['nodes_moved']
        new_nodes_moved = [nm for nm in nodes_moved if not nm['sort_order_change']]
        restructured_diff['nodes_moved'] = new_nodes_moved
    if format == "restructured":
        return restructured_diff

# INTERNAL API
################################################################################

def diff_subtree(parent_idA, nodeA, parent_idB, nodeB, root=False,
    attrs=None, exclude_attrs=[], mapA={}, mapB={},
    assessment_items_key='assessment_items', setlike_attrs=['tags']):
    """
    Compute the changes between the node `nodeA` in the old tree and the
    corresponding `nodeB` in the new tree. This is the main workhorse call and
    includes diff of attributes, and recusive diff of node's children.
    """
    if root:
        # special handling for root node of the tree
        node_id_keyA = mapA.get('root.node_id', 'node_id')   # a.k.a. channel_id
        node_id_keyB = mapB.get('root.node_id', 'node_id')   # a.k.a. channel_id
        content_id_keyB = mapB.get('root.content_id', 'content_id')
    else:
        # regular nodes
        node_id_keyA = mapA.get('node_id', 'node_id')
        node_id_keyB = mapB.get('node_id', 'node_id')
        content_id_keyB = mapB.get('content_id', 'content_id')

    # Get IDs for nodeA and nodeB
    node_idA = nodeA[node_id_keyA]
    node_idB, content_idB = nodeB[node_id_keyB], nodeB[content_id_keyB]

    attrs_diff = diff_attributes(nodeA, nodeB, root=root,
                                 attrs=attrs, exclude_attrs=exclude_attrs,
                                 mapA=mapA, mapB=mapB,
                                 assessment_items_key=assessment_items_key,
                                 setlike_attrs=setlike_attrs)
    nodes_modified = []
    if attrs_diff['added'] or attrs_diff['deleted'] or attrs_diff['modified']:
        node = dict(
            node_id=node_idB,
            parent_id=parent_idB,
            content_id=content_idB,
            attributes=attrs_diff['attributes'],
        )
        if attrs_diff['added']:
            node['added'] = attrs_diff['added']
        if attrs_diff['deleted']:
            node['deleted'] = attrs_diff['deleted']
        if attrs_diff['modified']:
            node['modified'] = attrs_diff['modified']
        nodes_modified.append(node)

    nodes_deleted, nodes_added = [], []
    if 'children' in nodeA and 'children' in nodeB:
        children_diff = diff_children(node_idA, nodeA['children'],
                                      node_idB, nodeB['children'],
                                      attrs=attrs, exclude_attrs=exclude_attrs,
                                      mapA=mapA, mapB=mapB,
                                      assessment_items_key=assessment_items_key,
                                      setlike_attrs=setlike_attrs)
        nodes_added.extend(children_diff['nodes_added'])
        nodes_deleted.extend(children_diff['nodes_deleted'])
        nodes_modified.extend(children_diff['nodes_modified'])

    return {
        'nodes_deleted': nodes_deleted,
        'nodes_added': nodes_added,
        'nodes_modified': nodes_modified,
    }

# NODE ATTRIBUTES
################################################################################

def diff_attributes(nodeA, nodeB, root=False,
    attrs=None, exclude_attrs=[], mapA={}, mapB={},
    assessment_items_key='assessment_items', setlike_attrs=['tags']):
    """
    Compute the diff between the attributes of `nodeA` and `nodeB`.
    Returns a dict { added=[], deleted=[], modifeid=[], attributes={} }
    """
    attributes = {}
    added, deleted, modified = [], [], []

    if attrs is None:
        # Calculate "all attrs" based on the node attrs and the given attr-maps
        attrs = set()
        attrs.update( set(mapA.keys()).union(set(mapB.keys())) )
        #
        mapA_vals = set(mapA.values())
        mapB_vals = set(mapB.values())
        for keyA in nodeA.keys():
            if keyA not in mapA_vals:
                attrs.add(keyA)
        for keyB in nodeB.keys():
            if keyB not in mapB_vals:
                attrs.add(keyB)
        attrs = sorted(attrs)

    # 1. Regular attributes
    for attr in attrs:
        if attr in exclude_attrs or attr in setlike_attrs \
            or attr == assessment_items_key or attr == 'files' or attr == 'children':
            continue

        attrA = mapA.get(attr, attr)
        attrB = mapB.get(attr, attr)

        if attrA not in nodeA and attrB not in nodeB:
            logger.warning("requested diff for missing attr " + attr)
            continue
        elif attrA not in nodeA:
            attributes[attr] = {'value': nodeB[attrB]}
            added.append(attr)
        elif attrB not in nodeB:
            attributes[attr] = {'old_value': nodeA[attrA]}
            deleted.append(attr)
        elif nodeA[attrA] == nodeB[attrB]:
            # no difference
            attributes[attr] = {'value': nodeB[attrB]}
        else:
            # both exist and there is a difference
            attributes[attr] = {'old_value': nodeA[attrA], 'value': nodeB[attrB]}
            modified.append(attr)

    # 2. Set-like attributes
    for attr in setlike_attrs:
        attrA = mapA.get(attr, attr)
        attrB = mapB.get(attr, attr)

        if attrA not in nodeA and attrB not in nodeB:
            continue
        elif attrA not in nodeA:
            attributes[attr] = {'value': nodeB[attrB]}
            added.append(attr)
        elif attrB not in nodeB:
            attributes[attr] = {'old_value': nodeA[attrA]}
            deleted.append(attr)
        else:
            # both attributes exist
            valueA_set = set(nodeA.get(attrA, []))
            valueB_set = set(nodeB.get(attrB, []))
            if valueA_set == valueB_set:
                attributes[attr] = {'value': nodeB[attrB]}
            else:
                attributes[attr] = {
                    'old_value': nodeA[attrA],
                    'value': nodeB[attrB],
                    'added': sorted(valueB_set - valueA_set),
                    'deleted': sorted(valueA_set - valueB_set),
                }
                modified.append(attr)

    # 3. Files
    if 'files' not in exclude_attrs and 'files' in nodeA and 'files' in nodeB:
        files_diff = diff_files(nodeA['files'], nodeB['files'], exclude_attrs=exclude_attrs, mapA=mapA, mapB=mapA)
        if files_diff['added'] or files_diff['deleted']:
            modified.append('files')
            attributes['files'] = {
                'old_value': nodeA['files'],
                'value': nodeB['files'],
                'added': files_diff['added'],
                'deleted': files_diff['deleted'],
            }
        else:
            attributes['files'] = {
                'value': nodeB['files'],
            }

    # 4. assessment items
    if assessment_items_key and assessment_items_key in nodeA and assessment_items_key in nodeB:
        listA = nodeA[assessment_items_key]
        listB = nodeB[assessment_items_key]
        ais_diff = diff_assessment_items(listA, listB, mapA=mapA, mapB=mapB, exclude_attrs=exclude_attrs)
        if ais_diff['added'] or ais_diff['deleted'] or ais_diff['moved'] or ais_diff['modified']:
            modified.append(assessment_items_key)
            attributes[assessment_items_key] = {
                'old_value': nodeA[assessment_items_key],
                'value': nodeB[assessment_items_key],
                'added': ais_diff['added'],
                'deleted': ais_diff['deleted'],
                'moved': ais_diff['moved'],
                'modified': ais_diff['modified'],
            }
        else:
            attributes[assessment_items_key] = {
                'value': nodeB[assessment_items_key],
            }

    # return attrs_diff
    return {
        'added': added,
        'deleted': deleted,
        'modified': modified,
        'attributes': attributes,
    }

def diff_files(listA, listB, exclude_attrs=[], mapA={}, mapB={}):
    """
    Compute the diff of two lists for files, treating them as set-like.
    """
    added = []
    deleted = []

    cleanlistA = []
    for fileA in listA:
        cleanfileA = fileA.copy()
        for exclude_attr in exclude_attrs:
            if exclude_attr.startswith('files.'):
                file_attr = exclude_attr.replace('files.', '')
                cleanfileA.pop(file_attr)
        cleanlistA.append(cleanfileA)

    cleanlistB = []
    for fileB in listB:
        cleanfileB = fileB.copy()
        for exclude_attr in exclude_attrs:
            if exclude_attr.startswith('files.'):
                file_attr = exclude_attr.replace('files.', '')
                cleanfileB.pop(file_attr)
        cleanlistB.append(cleanfileB)

    for cleanfileA in cleanlistA:
        if cleanfileA not in cleanlistB:
            deleted.append(cleanfileA)
    for cleanfileB in cleanlistB:
        if cleanfileB not in cleanlistA:
            added.append(cleanfileB)
    return {
        'added': added,
        'deleted': deleted,
    }

def diff_assessment_items(listA, listB, exclude_attrs=[], mapA={}, mapB={}):
    """
    Compute the diff between the lists of assessment items `listA` and `listB`,
    using the key `assessment_id` to detect modifications and reorderings.
    Note: the code assumes there are no duplicate `assessment_id`s in a list.
    """
    # 1A. prepropocess listA to extract order and assessment_id
    itemsA = []
    assessment_idsA = set()
    for i, aiA in enumerate(listA):
        assessment_id_keyA = mapA.get('assessment_id', 'assessment_id')
        assessment_idA = aiA[assessment_id_keyA]
        order_keyA = mapA.get('order', 'order')
        orderA = aiA.get(order_keyA, None)
        if orderA is None:
            orderA = float(i + 1)  # 1-based indexitng
        itemA = dict(
            order=orderA,
            assessment_id=assessment_idA,
            assessment_item=aiA,
        )
        itemsA.append(itemA)
        assessment_idsA.add(assessment_idA)

    # 1B. prepropocess listB to extract order and assessment_id
    itemsB = []
    assessment_idsB = set()
    for j, aiB in enumerate(listB):
        assessment_id_keyB = mapB.get('assessment_id', 'assessment_id')
        assessment_idB = aiB[assessment_id_keyB]
        order_keyB = mapB.get('order', 'order')
        orderB = aiB.get(order_keyB, None)
        if orderB is None:
            orderB = float(j + 1)  # 1-based indexitng
        itemB = dict(
            order=orderB,
            assessment_id=assessment_idB,
            assessment_item=aiB,
        )
        itemsB.append(itemB)
        assessment_idsB.add(assessment_idB)

    # 2. compare lists
    added_items = [itB for itB in itemsB if not contains(itemsA, itB, by=('assessment_id'))]
    deleted_items = [itA for itA in itemsA if not contains(itemsB, itA, by=('assessment_id'))]

    # 3. check for moves and modifications in common items
    common_item_tuples = []
    for itA in itemsA:
        itB = findby(itemsB, itA, by=('assessment_id'))
        if itB:
            common_item_tuples.append((itA, itB))
    moved, modified = [], []
    for itA, itB in common_item_tuples:

        # remove order attribute of the assessment_item
        aiA, aiB = itA['assessment_item'], itB['assessment_item']
        aiAattrs, aiBattrs = copy.deepcopy(aiA), copy.deepcopy(aiB)
        order_keyA = mapA.get('order', 'order')
        if order_keyA in aiAattrs:
            aiAattrs.pop(order_keyA)
        order_keyB = mapB.get('order', 'order')
        if order_keyB in aiBattrs:
            aiBattrs.pop(order_keyB)

        # remove attribute from the exclude_attrs dict
        for exclude_attr in exclude_attrs:
            if exclude_attr.startswith('assessment_items.'):
                ai_attr = exclude_attr.replace('assessment_items.', '')
                aiAattrs.pop(ai_attr)
                aiBattrs.pop(ai_attr)

        files_changed = False
        if 'files' in aiA and 'files' in aiB:
            aiAattrs.pop('files')
            aiBattrs.pop('files')
            files_diff = diff_files(aiA['files'], aiB['files'], exclude_attrs=exclude_attrs, mapA=mapA, mapB=mapA)
            if files_diff['added'] or files_diff['deleted']:
                files_changed = True

        if aiAattrs == aiBattrs and itA['order'] == itB['order'] and not files_changed:
            continue  # same attrs and same sort order, so not modified or moved
        elif aiAattrs == aiBattrs and not files_changed and itA['order'] != itB['order']:
            moved.append(aiB)
        else:
            modified.append(aiB)

    # format for diff output
    added = [item['assessment_item'] for item in added_items]
    deleted = [item['assessment_item'] for item in deleted_items]

    return {
        'added': added,
        'deleted': deleted,
        'moved': moved,
        'modified': modified,
    }

# TREE STUCTURE
################################################################################

def flatten_subtree(parent_id, sort_order, subtree, kind, map={}):
    """
    Recusively flatten the `subtree` of nodes and return the info a a flat list.
    The diff format depends on what `kind` of diff it is: `added` or `deleted`.
    """
    subtree =  copy.deepcopy(subtree)  # to avoid modifying original tree
    flatlist = []
    node_id_key = map.get('node_id', 'node_id')
    content_id_key = map.get('content_id', 'content_id')
    sort_order_key = map.get('sort_order', 'sort_order')

    if 'children' in subtree:
        children = subtree.pop('children')
    else:
        children = []

    # first add yourself...
    if kind == "deleted":
        node = dict(
            old_node_id=subtree[node_id_key],
            old_parent_id=parent_id,
            old_sort_order=sort_order,
            content_id=subtree[content_id_key],
            attributes=dict(
                (attr, {'value': val}) for attr, val in subtree.items()
            )
        )
        flatlist.append(node)
    elif kind == "added":
        node = dict(
            node_id=subtree[node_id_key],
            parent_id=parent_id,
            sort_order=sort_order,
            content_id=subtree[content_id_key],
            attributes=dict(
                (attr, {'value': val}) for attr, val in subtree.items()
            )
        )
        flatlist.append(node)
    else:
        raise ValueError('Unexpected flatlist kind ' + str(kind) + ' found.')

    # ... then add your children
    for i, child in enumerate(children):
        sort_order = child.get(sort_order_key, None)
        if sort_order is None:
            sort_order = float(i + 1)  # 1-based indexitng
        childflatlist = flatten_subtree(subtree[node_id_key], sort_order, child, kind=kind, map=map)
        flatlist.extend(childflatlist)

    return flatlist

def diff_children(parent_idA, childrenA, parent_idB, childrenB,
    attrs=None, exclude_attrs=[], mapA={}, mapB={},
    assessment_items_key='assessment_items', setlike_attrs=['tags']):
    """
    Compute the diff between the nodes in `childrenA` and the nodes in `childrenB`.
    Args:
      - attrs (list(str)): what attributes to check in comparison
      - mapA: map of diff attribues to childrenA node attributes
      - mapB: map of diff attribues to childrenB node attributes
    """
    # 1A. prepropocess childrenA nodes
    itemsA = []
    node_idsA = set()
    for i, nodeA in enumerate(childrenA):
        node_id_keyA = mapA.get('node_id', 'node_id')
        content_id_keyA = mapA.get('content_id', 'content_id')
        node_idA, content_idA = nodeA[node_id_keyA], nodeA[content_id_keyA]
        sort_order_keyA = mapA.get('sort_order', 'sort_order')
        sort_orderA = nodeA.get(sort_order_keyA, None)
        if sort_orderA is None:
            sort_orderA = float(i + 1)  # 1-based indexitng
        itemA = dict(
            parent_id=parent_idA,
            node_id=node_idA,
            sort_order=sort_orderA,
            content_id=content_idA,
            node=nodeA,
        )
        itemsA.append(itemA)
        node_idsA.add(node_idA)

    # 1B. prepropocess childrenB nodes
    itemsB = []
    node_idsB = set()
    for j, nodeB in enumerate(childrenB):
        node_id_keyB = mapB.get('node_id', 'node_id')
        content_id_keyB = mapB.get('content_id', 'content_id')
        node_idB, content_idB = nodeB[node_id_keyB], nodeB[content_id_keyB]
        sort_order_keyB = mapB.get('sort_order', 'sort_order')
        sort_orderB = nodeB.get(sort_order_keyB, None)
        if sort_orderB is None:
            sort_orderB = float(j + 1)  # 1-based indexitng
        itemB = dict(
            parent_id=parent_idB,
            node_id=node_idB,
            sort_order=sort_orderB,
            content_id=content_idB,
            node=nodeB,
        )
        itemsB.append(itemB)
        node_idsB.add(node_idB)

    # 2. nodes deleted
    nodes_deleted = []
    deleted_items = [itA for itA in itemsA if not contains(itemsB, itA, by=('node_id', 'sort_order'))]
    for item in deleted_items:
        flatlist = flatten_subtree(parent_idA, item['sort_order'], item['node'], kind="deleted", map=mapA)
        nodes_deleted.extend(flatlist)

    # 3. nodes added
    nodes_added = []
    added_items = [itB for itB in itemsB if not contains(itemsA, itB, by=('node_id', 'sort_order'))]
    for item in added_items:
        flatlist = flatten_subtree(parent_idB, item['sort_order'], item['node'], kind="added", map=mapB)
        nodes_added.extend(flatlist)

    # 4. recursely diff common nodes
    nodes_modified = []
    common_item_tuples = []
    for itA in itemsA:
        itB = findby(itemsB, itA, by=('node_id'))
        if itB:
            common_item_tuples.append((itA, itB))
    for itA, itB in common_item_tuples:
        diff = diff_subtree(parent_idA, itA['node'], parent_idB, itB['node'],
                            attrs=attrs, exclude_attrs=exclude_attrs,
                            mapA=mapA, mapB=mapB,
                            assessment_items_key=assessment_items_key,
                            setlike_attrs=setlike_attrs)
        nodes_deleted.extend(diff['nodes_deleted'])
        nodes_added.extend(diff['nodes_added'])
        nodes_modified.extend(diff['nodes_modified'])

    # return combined info (note: node moves will be detected at a later stage)
    return {
        'nodes_deleted': nodes_deleted,
        'nodes_added': nodes_added,
        'nodes_modified': nodes_modified,
    }

# PHASE 2
################################################################################

def detect_moves(nodes_deleted, nodes_added):
    """
    Look for nodes with the same `content_id` that appear in both lists, and
    interpret those nodes as having moved. Returns `nodes_moved` (list).
    """
    nodes_deleted_old_node_id = {}
    for node in nodes_deleted:
        nodes_deleted_old_node_id[node['old_node_id']] = node

    nodes_added_by_new_node_id = {}
    for node in nodes_added:
        nodes_added_by_new_node_id[node['node_id']] = node

    nodes_moved_by_new_node_id = {}
    for old_node_id, nd in nodes_deleted_old_node_id.items():
        for new_node_id, na in nodes_added_by_new_node_id.items():
            if nd['content_id'] == na['content_id']:
                if new_node_id not in nodes_moved_by_new_node_id.keys():
                    nm = copy.deepcopy(na)
                    nm['old_node_id'] = old_node_id
                    nm['old_parent_id'] = nd['old_parent_id']
                    nm['old_sort_order'] = nd['old_sort_order']
                    nodes_moved_by_new_node_id[new_node_id] = nm
                else:
                    logger.info('A node move with content_id=' + nd['content_id'] + ' already exists.')

    nodes_moved = []
    for nm in nodes_moved_by_new_node_id.values():
        if nm['old_parent_id'] == nm['parent_id'] and nm['old_node_id'] == nm['node_id']:
            nm['sort_order_change'] = True
        else:
            nm['sort_order_change'] = False
        nodes_moved.append(nm)

    return nodes_moved

# PHASE 3: simplify to avoid nodes moved showing up in added and deleted
################################################################################

def simplify_diff(raw_diff):
    """
    For presentation purposes, nodes that were recognized as "moved" don't need
    to show up in the `nodes_deleted` and `nodes_added` lists.
    """
    nodes_moved = raw_diff['nodes_moved']
    old_node_ids_moved = set(nm['old_node_id'] for nm in nodes_moved)
    new_node_ids_moved = set(nm['node_id'] for nm in nodes_moved)

    nodes_deleted = raw_diff['nodes_deleted']
    new_nodes_deleted = [nd for nd in nodes_deleted if nd['old_node_id'] not in old_node_ids_moved]

    nodes_added = raw_diff['nodes_added']
    new_nodes_added = [na for na in nodes_added if na['node_id'] not in new_node_ids_moved]

    simplified_diff = {
        'nodes_deleted': new_nodes_deleted,
        'nodes_added': new_nodes_added,
        'nodes_moved': nodes_moved,
        'nodes_modified': raw_diff['nodes_modified'],
    }
    return simplified_diff

# PHASE 4: restructure for displaying diffs in tree form
################################################################################

def restructure_diff(simplified_diff, treeA, treeB, mapA={}, mapB={}):
    """
    Go thorugh flatlists of nodes deleted, added, and moved and organize them
    into subtrees to make get a more compact representation for display purposes.
    """
    node_id_keyA = mapA.get('node_id', 'node_id')
    node_id_keyB = mapB.get('node_id', 'node_id')

    def restructure_list(difflist, parent_id_key="parent_id", diff_node_id_key="node_id",
                         tree=None, by="node_id"):
        """
        Restucture list of diff nodes `difflist` by identifing complere subtrees
        of changes. The logic used is to loop over all diff nodes in `difflist`,
        get all their descendants the tree `tree` and do the restructuing if all
        the descendants are also in `difflist`.
        Returns a list of individual nodes and restructured node subtrees.
        """
        # 1. store for the restructured nodes:
        nodes_by_id = dict((node[diff_node_id_key], node) for node in difflist)
        # 2. a shallow copy of 1. to use as pointers to all nodes for lookups
        pointers_to_nodes = nodes_by_id.copy()
        all_node_ids = set(nodes_by_id.keys())
        #
        node_ids_restructured = set()
        for diff_node_id in all_node_ids:
            if diff_node_id in node_ids_restructured:
                continue  # skip nodes thave have already been restrucutred
            assert diff_node_id in nodes_by_id, 'must be still in list'
            #
            # now let's lookup node_id in the tree and get its descendants
            tree_node = treefindby(tree, diff_node_id, by=by)
            descendants = get_descendants(tree_node, include_self=False)
            descendants_ids = set(dnode[by] for dnode in descendants)
            if descendants_ids and descendants_ids.issubset(all_node_ids):
                # full subtree rooted at tree_node in list, so let's restrucutre
                for dtree_node in descendants:
                    dnode_id = dtree_node[by]
                    if dnode_id in nodes_by_id:
                        # this descendant hasn't been restructured yet, do it!
                        ddiff_node = nodes_by_id.pop(dnode_id)
                        parent_id = ddiff_node[parent_id_key]
                        parent_diff_node = pointers_to_nodes.get(parent_id)
                        if parent_diff_node:
                            if 'children' in parent_diff_node:
                                parent_diff_node['children'].append(ddiff_node)
                            else:
                                parent_diff_node['children'] = [ddiff_node]
                        else:
                            logger.warning('did not find parent ' + parent_id \
                                + 'in pointers_to_nodes for node id ' + dnode_id)
                        node_ids_restructured.add(dnode_id)
                    else:
                        logger.debug('node ' + dnode_id + ' has already been restructured')
            else:
                pass  # no restructuring: leave diff_node in nodes_by_id for now
        return list(nodes_by_id.values())

    nodes_deleted = simplified_diff['nodes_deleted']
    new_nodes_deleted = restructure_list(nodes_deleted,
        parent_id_key="old_parent_id", diff_node_id_key="old_node_id",
        tree=treeA, by=node_id_keyA)

    nodes_added = simplified_diff['nodes_added']
    new_nodes_added = restructure_list(nodes_added,
        parent_id_key="parent_id", diff_node_id_key="node_id",
        tree=treeB, by=node_id_keyB)

    nodes_moved = simplified_diff['nodes_moved']
    new_nodes_moved = restructure_list(nodes_moved,
        parent_id_key="parent_id", diff_node_id_key="node_id",
        tree=treeB, by=node_id_keyB)

    restructured_diff = {
        'nodes_deleted': new_nodes_deleted,
        'nodes_added': new_nodes_added,
        'nodes_moved': new_nodes_moved,
        'nodes_modified': simplified_diff['nodes_modified'],
    }
    return restructured_diff


# -*- coding: utf-8 -*-
from copy import deepcopy
from typing import List

import numpy as np

from nata.containers import ParticleDataset
from nata.containers import ParticleQuantity
from nata.plugins.register import register_container_plugin

@register_container_plugin(ParticleDataset, name="filter")
def filter_particle_dataset(
    dataset: ParticleDataset,
    mask: List[bool] = None,
    quantities: List[str] = None,
    slicing: slice = None,
) -> ParticleDataset:
    """Filters a :class:`nata.containers.ParticleDataset` according to a\
       selection of quantities.

        Parameters
        ----------
        mask: ``np.ndarray``, optional
            Array of booleans indicating the particles to be filtered. Particles
            with ``True`` (``False``) mask entries are selected (hidden). The
            shape of ``mask`` must match that of each particle quantity.

        slicing: ``slice``, optional
            Slice of particles to be filtered. Acts only on particle indices
            and not on time, as time slicing should be done on the dataset.
            When provided together with the ``mask argument``, slicing is done
            on the masked array.

        quantities: ``list``, optional
            List of quantities to be filtered, ordered by the way they should be
            sorted in the returned dataset.

        Returns
        ------
        :class:`nata.containers.ParticleDataset`:
            Filtered dataset with only the quantities selected in
            ``quantities``.

        Examples
        --------
        The filter plugin is used to get dataset with only a selection, say
        ``'x1'`` and ``'x2'``, of its quantities.

        >>> from nata.containers import ParticleDataset
        >>> ds = ParticleDataset("path/to"file")
        >>> ds_flt = ds.filter(quantities=["x1","p1"])

    """

    dataset_c = deepcopy(dataset)

    if quantities is not None:
        if not isinstance(quantities, list) or not all(
            isinstance(quant, str) for quant in quantities
        ):
            raise TypeError("'quantities' must be a list of strings.")

        quants = {
            quant: dataset_c.quantities[quant]
            for quant in quantities
            if quant in dataset_c.quantities.keys()
        }

        if not bool(quants):
            raise ValueError(
                "None of the quantities in 'quantities' were found in the dataset."
            )

    else:
        quants = dataset_c.quantities

    if mask is not None:
        if not isinstance(mask, np.ndarray) or not mask.dtype == bool:
            raise TypeError("'mask' must be an 'np.ndarray' of dtype 'bool'.")

        quant = next(iter(quants.values()))
        if quant.data.shape != mask.shape:
            raise ValueError(
                f"'mask.shape' ({mask.shape}) does not match the shape of "
                + "each quantity ({quant.data.shape})."
            )

        for name, quant in quants.items():
            quants[name].data[~mask] = np.ma.masked

    if slicing is not None:
        if not isinstance(slicing, slice):
            raise TypeError("'slicing' must be of type 'slice'.")

        if len(dataset_c) > 1:
            slicing = (slice(None), slicing)

        quants = {
            quant.name: ParticleQuantity(
                data=quant.data[slicing],
                name=quant.name,
                label=quant.label,
                unit=quant.unit,
                dtype=quant.data.dtype,
            )
            for quant in quants.values()
        }

    return ParticleDataset(
        iteration=dataset_c.axes["iteration"],
        time=dataset_c.axes["time"],
        name=dataset_c.name,
        quantities=quants,
    )


# encoding:utf-8

from starlette.datastructures import UploadFile

def test():
    f = open('./.gitignore', 'rb')
    upload_file = UploadFile('gitignorefile', f, content_type='text')

    data = upload_file.read()
    print(data.send("你好吗"))

if __name__ == '__main__':
    test()


# Day 18. Boiling Boulders
# https://adventofcode.com/2022/day/18

from collections import deque
from util.input_util import read_input_file

def parse_lines():
    lines = read_input_file(18)
    cubes = []
    for l in lines:
        # (x, y, z) location of cube
        cubes.append(tuple([int(x) for x in l.split(',')]))

    return set(cubes)

def get_sides(cubes):
    sides = []
    for x, y, z in cubes:
        # check all 6 sides
        for d in (-1, 1):
            if (x+d, y, z) not in cubes:
                sides.append((x+d, y, z))
            if (x, y+d, z) not in cubes:
                sides.append((x, y+d, z))
            if (x, y, z+d) not in cubes:
                sides.append((x, y, z+d))

    return sides

def solution1():
    cubes = parse_lines()
    sides = get_sides(cubes)
    return len(sides)

def get_limits(cubes):
    # Get maxes. Mins known to be 0
    mx, my, mz = 0, 0, 0
    for x, y, z in cubes:
        mx = max(mx, x)
        my = max(my, y)
        mz = max(mz, z)
    return mx+1, my+1 , mz+1

def build_outer_shell(cubes, X, Y, Z):
    shell = set()
    #BFS
    q = deque([(0, 0, 0)])
    while q:
        x, y, z = q.popleft()
        if (x, y, z) in shell:
            continue

        shell.add((x, y, z))
        # Look in all 6 directions and look one box beyond the outer boundaries
        for d in (-1, 1):
            if -1 <= x+d < X+1 and (x+d, y, z) not in shell and (x+d, y, z) not in cubes:
                q.append((x+d, y, z))
            if -1 <= y+d < Y+1 and (x, y+d, z) not in shell and (x, y+d, z) not in cubes:
                q.append((x, y+d, z))
            if -1 <= z+d < Z+1 and (x, y, z+d) not in shell and (x, y, z+d) not in cubes:
                q.append((x, y, z+d))

    return shell

def solution2():
    cubes = parse_lines()
    X, Y, Z = get_limits(cubes)

    shell = build_outer_shell(cubes, X, Y, Z)
    sides = get_sides(shell)
    # Filter out sides that are on the outer part of the shell.
    # Only count a side if there's a lava cube on the other side.
    count = 0
    for x, y, z in sides:
        if (x, y, z) in cubes:
            count += 1

    return count

if __name__ == '__main__':
    print(solution1())

    print('--------------')

    print(solution2())


from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import sys
import os

rnd=sys.argv[1].upper()

PATH = "C:\\selenium\\geckodriver.exe"
driver = webdriver.Firefox(executable_path=PATH)
driver.get("https://codechef.com/"+rnd)

urls=[]

for problem in driver.find_elements_by_class_name("cc-problem-name"):
	print(problem)
	urls.append(problem.find_element_by_tag_name('div').find_element_by_tag_name('span').find_element_by_tag_name('a').get_attribute('href'))

for url in urls:
	driver.get(url)
	path=rnd+'/'+url.split('/')[-1]
	try:
		os.makedirs(path)
	except:
		pass
	WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "problem-statement")))
	driver.find_element_by_id("problem-statement").screenshot(path+"/problem.png")
	for test in driver.find_elements_by_tag_name("pre"):
		print(test)
		print(path+'/'+test.find_element_by_xpath("preceding-sibling::h3[1]").get_attribute('id')[:42]+'.txt')
		print(test.find_element_by_tag_name("code").get_attribute("innerText"))
		open(path+'/'+test.find_element_by_xpath("preceding-sibling::h3[1]").get_attribute('id')[:42]+'.txt','w').write(test.find_element_by_tag_name("code").get_attribute("innerText"))

driver.quit()


import pandas as pd
import matplotlib.pyplot as plt
import folium
import numpy as np
import seaborn as sns
from sklearn import linear_model
from scipy import stats
import statsmodels.api as sm

# Question 1 visualization (map)
# Load the cleaned up dataframe
earthquake_df = pd.read_csv ('final_df.csv')
#earthquake_df.info()

# Create blank map
world_map = folium.Map(location = (0, 0), zoom_start = 2.1)

# Create a color dictionary for severity of earthquake magnitude
colordict = {0: 'green', 1: 'yellow', 2: 'orange', 3: 'red'}
earthquake_df['Magnitude Quartile'] = pd.qcut(earthquake_df['Magnitude'], 4, labels = False) # quartiles of magnitude

# Title
title_map = 'Incidence of Seismic Events Over the Past 30 Days (1/13 - 2/11)'
title_html = '''

{}

             '''.format(title_map)

world_map.get_root().html.add_child(folium.Element(title_html))

# We iterate through the dataframe and plot the locations on the map
for i in range(len(earthquake_df)):
    folium.Circle(
        location = [earthquake_df.iloc[i]['Latitude'], earthquake_df.iloc[i]['Longitude']],
        radius = 60000,
        color = 'b',
        key_on = earthquake_df.iloc[i]['Magnitude Quartile'],
        threshold_scale = [0, 1, 2, 3],
        fill_color = colordict[earthquake_df.iloc[i]['Magnitude Quartile']],
        fill = True,
        fill_opacity = 0.7,
        zoom_control = False,
        scrollWheelZoom = False,
        dragging = False

    ).add_to(world_map)

# Adding the tectonic plates over the map
url = 'https://raw.githubusercontent.com/fraxen/tectonicplates/master/GeoJSON/PB2002_boundaries.json'

folium.GeoJson(
    url,
    name = 'geojson'
).add_to(world_map)

folium.LayerControl().add_to(world_map)

# Save to file
world_map.save('earthquakes.html')

# Question 2 visualization of the multilinear regression model
# Prepare the dataframe with only the variables that'll be used
multi_df = earthquake_df[['Magnitude', 'Intensity', 'Depth']]
multi_df['Intensity'].replace('', np.nan, inplace = True)
multi_df = multi_df.dropna(subset=['Intensity'])
#print(multi_df)

# Visualize the data using scatter plot and histogram pairplots
sns.set_palette('colorblind')
pair_plots = sns.pairplot(data = multi_df, height = 2)
plt.subplots_adjust(top = 0.9)
plt.suptitle('Matrix Plots of Earthquake Magnitude, Intensity and Depth', fontsize = 10)
pair_plots.savefig('pair_plots.png', dpi = 200)

# Set independent and dependent variables
X = multi_df[['Intensity', 'Depth']]
y = multi_df['Magnitude']

# Initialize model and fit it to data
regr = linear_model.LinearRegression()
model = regr.fit(X, y)

# Set independent and dependent variables
X = multi_df[['Intensity', 'Depth']]
y = multi_df['Magnitude']

# Initialize model and fit it to data
regr = linear_model.LinearRegression()
model = regr.fit(X, y)

# y-hat = 4.401 + 0.125*X_1 + 0.003*X_2
print('Intercept:', model.intercept_)
print('Coefficients:', model.coef_)

# Validation of Model
#X = multi_df[['Intensity', 'Depth']]
X = sm.add_constant(X) # adding a constant
olsmod = sm.OLS(y, X).fit()
print(olsmod.summary())
print('R2 score:', olsmod.rsquared)
print(olsmod.pvalues)

# Question 3: Frequency of seismic occurrence
# Plotting hours and frequency of seismic events
sort_time = earthquake_df['Time']
times = []
bins_sort = np.arange(0, 24, 1)

# Extracting the hour
for elem in sort_time:
    hour = int(elem[0:2])
    times.append(hour)

# Sorting times for histogram
times.sort()

# plotting times
plt.hist(times, bins = bins_sort, align='mid')
plt.xticks(np.arange(0, 24, 1))
plt.title("Number of Seismic Events by Hour")
plt.xlabel("Hour (UTC)")
plt.ylabel("Seismic Events")
plt.savefig('hoursplt.png')
plt.close()

# Plotting the past 30 days and seismic events
days = []
for elem in earthquake_df['Date']:
    days.append(int(elem[-2:]))

# plotting times
plt.hist(days, bins = 30, align='mid')
plt.xticks(np.arange(0, 31, 2))
plt.title("Frequency of Seismic Events Over the Past Thirty Days (1/13 - 2/11)")
plt.xlabel("Day")
plt.ylabel("Seismic Events")
plt.savefig('times1.png')
plt.close()


import sys
import pandas as pd
from sqlalchemy import create_engine

import re
import nltk
from nltk.corpus import stopwords
from nltk.stem.wordnet import WordNetLemmatizer
from nltk.tokenize import word_tokenize

nltk.download(['punkt', 'stopwords', 'wordnet'])

from sklearn.multioutput import MultiOutputClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer

from sklearn.metrics import classification_report

from sklearn.model_selection import train_test_split, GridSearchCV

import pickle

def load_data(database_filepath):
    """
    This function loads data from an sql database and transform it into a pandas dataframe

    Input:
        database_filepath - String with the name of the database

    Output:
        X - DataFrame with the messages to be classified
        Y - Dataframe with the labels
    """

    engine = create_engine('sqlite:///' + database_filepath)
    df = pd.read_sql_table('messages', engine)
    X = df['message']
    Y = df.drop(labels = ['id', 'original', 'genre', 'message'], axis = 1)

    return X, Y, Y.columns

def tokenize(text):
    """
    Function that tokenize and lemmatize the given text

    Input:
        String with the text to be transformed

    Output:
        String with the lemmas of the string given as input
    """

    # Loading the stop words and Lemmatizer
    stop_words = stopwords.words("english")
    lemmatizer = WordNetLemmatizer()

    # Transform to lowercase and remove punctuation
    text = re.sub(r"[^a-zA-Z0-9]", " ", text.lower())

    # Tokenize the resulting text
    tokens = word_tokenize(text)

    # lemmatize and remove stop words
    tokens = [lemmatizer.lemmatize(word) for word in tokens if word not in stop_words]

    return tokens

def build_model():
    """
    This functions makes the model to classify messages, it has a countvectorizer, a tfidftransformer
    for the data processing step and a random forest for the model step

    Output:
        Pipeline with the steps of the pipeline
    """

    pipeline = Pipeline(steps = [
        ('text_pipeline', Pipeline([
            ('vect', CountVectorizer(tokenizer=tokenize)),
            ('tfidf', TfidfTransformer())
        ])),
        ('model', MultiOutputClassifier(RandomForestClassifier(random_state = 17)))
    ])

    return pipeline

def evaluate_model(model, X_test, y_test, category_names):
    """
    This functions prints the classification report for each category name

    Input:
        model - The model to be tested
        X_test - Dataframe with the input variables to be predicted and tested for model performance
        Y_test - Dataframe containing the true values of the classification
        category_names - Array with the name of each category
    """

    y_pred = model.predict(X_test)
    y_pred = pd.DataFrame(y_pred, columns = y_test.columns)

    for i in range(y_test.shape[1]):
        print('Classification report for label: ' + category_names[i])
        print(classification_report(y_test.iloc[:,i], y_pred.iloc[:,i], labels = [0, 1]))

def save_model(model, model_filepath):
    """
    This function saves the model as a pickle file

    Input:
        model - The model to save
        model_filepath - String with the filepath in which the model will be saved
    """
    with open(model_filepath, 'wb') as f:
        pickle.dump(model, f)

def main():
    if len(sys.argv) == 3:
        database_filepath, model_filepath = sys.argv[1:]
        print('Loading data...\n    DATABASE: {}'.format(database_filepath))
        X, Y, category_names = load_data(database_filepath)
        X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2)

        print('Building model...')
        model = build_model()

        print('Training model...')
        model.fit(X_train, Y_train)

        print('Evaluating model...')
        evaluate_model(model, X_test, Y_test, category_names)

        print('Saving model...\n    MODEL: {}'.format(model_filepath))
        save_model(model, model_filepath)

        print('Trained model saved!')

    else:
        print('Please provide the filepath of the disaster messages database '\
              'as the first argument and the filepath of the pickle file to '\
              'save the model to as the second argument. \n\nExample: python '\
              'train_classifier.py ../data/DisasterResponse.db classifier.pkl')

if __name__ == '__main__':
    main()


# Contains user class which has methods to analyse a user's trajectories

# u = user(randUsr('/Users/JBremner/Desktop/Beijing Trajectories/Geolife Trajectories 1.3/Data'))

import csv, os, sys
from datetime import datetime
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.cluster import DBSCAN
from sklearn import metrics
from sklearn.neighbors import NearestNeighbors
from sklearn.neighbors import radius_neighbors_graph
from Scripts.mathsFuncs import geometric_median
from Scripts.chooseUsr import randUsr
from nolds import corr_dim

class user:
	'''
	Contains methods to analyse a user's trajectories

	'''

	def __init__(self, path):
		self.folder = path
		self.path = path + '/Trajectory'
		self.dirs = [name for name in os.listdir(self.path) if os.path.isdir(os.path.join(self.path, name))]
		self.len = self.trajNo()

		# Initial values to be filled on request but then saved to cut processing time.
		self.epsilon = False
		self.clusters = False
		self.points = False

	# ~~~~~~~~~~~~~~~~~~~~ SPATIAL ROUTINES ~~~~~~~~~~~~~~~~~~~~

	def startLocs(self):
		'''
		Returns array of start locations for the user's trajectories

		'''
		locs = []
		for direc in self.dirs:
			trajs = os.listdir(self.path + '/' + direc)
			for traj in trajs:
				trajPath = self.path + '/' + direc + '/' + traj
				df = pd.read_csv(trajPath)
				locs.append([df['x'].iloc[0], df['y'].iloc[0]])

		return np.array(locs)

	def endLocs(self):
		'''
		Returns array of end locations for the user's trajectories

		'''
		locs = []
		for direc in self.dirs:
			trajs = os.listdir(self.path + '/' + direc)
			for traj in trajs:
				trajPath = self.path + '/' + direc + '/' + traj
				df = pd.read_csv(trajPath)
				locs.append([df['x'].iloc[-1], df['y'].iloc[-1]])

		return np.array(locs)

	def startEndLocs(self):
		'''
		Returns commbined array of start and end locations for the user's trajectories

		'''

		locs = np.concatenate([self.startLocs(),self.endLocs()], axis=0)
		return locs

	def locs(self):
		'''
		Returns array with every location point from every trajectory - could be used to create heatmap.

		'''

		locs = []
		for direc in self.dirs:
			trajs = os.listdir(self.path + '/' + direc)
			for traj in trajs:
				trajPath = self.path + '/' + direc + '/' + traj
				df = pd.read_csv(trajPath)
				s = np.array([df['x'], df['y']]).T
				for pos in s:
					locs.append(pos)

		self.points = np.array(locs)

		return np.array(locs)

	def corrDim(self):
		'''
		Returns the correlation dimension of the start/end locations

		'''
		return corr_dim(self.startEndLocs(), emb_dim=5)
	# ~~~~~~~~~~~~~~~~~~~~ CLUSTERING ROUTINES ~~~~~~~~~~~~~~~~~~~~

	def nnDist(self, theLocs):
		'''
		Takes an array of coordinates and returns an array of the n-n distances

		'''
		nbrs = NearestNeighbors(n_neighbors=2, algorithm='ball_tree').fit(theLocs)
		distances, indices = nbrs.kneighbors(theLocs)

		return distances[:,1]

	def nnDistDisp(self, theLocs):
		'''
		Takes an array of coordinate pairs and produces a histogram of the nearest neighbour distance distribution

		'''
		distances = self.nnDist(theLocs)

		fig = plt.figure()
		ax = fig.add_subplot(111)
		ax.grid(True,which='both')
		ax.hist(distances,100)
		ax.set_ylabel('Frequency')
		plt.show()

	def findEps(self, theLocs):
		'''
		Returns the appropriate value of epsilon (95th percentile of n-n distances) to use with DBSCAN from given locations - there are more sophisticated ways to do this.

		'''
		self.epsilon = np.percentile(self.nnDist(theLocs),95)
		return self.epsilon

	def clusterLabels(self):
		'''
		Uses the start/end locations to find clusters of popular destinations or start locations.
		Returns the cluster labels, the array of original start/end coordinates and the core_samples_mask

		'''
		s = self.startEndLocs()
		if not self.epsilon:
			self.findEps(self.startEndLocs())

		eps = self.epsilon

		# e = 60
		# if eps < e:
		# 	eps = e

		db = DBSCAN(eps=eps, min_samples=10).fit(s)
		core_samples_mask = np.zeros_like(db.labels_, dtype=bool)
		core_samples_mask[db.core_sample_indices_] = True
		labels = db.labels_

		self.n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0)

		return labels, s, core_samples_mask

	def clusterLocs(self):
		'''
		Returns an array of cluster positions (calulated from the mean position of all sites belonging to the cluster in question)

		'''
		labels, s = self.clusterLabels()[:2]
		labels = np.array(labels)

		unique_labels = set(labels)
		unique_labels = [lbl for lbl in unique_labels if lbl != -1]

		centres = []

		for lbl in unique_labels:
			indices = np.where(labels==lbl)[0]
			locs = []
			for i in indices:
				locs.append(s[i])
			centre = geometric_median(np.array(locs))
			centres.append(centre)

		self.clusters = np.array(centres)

		return np.array(centres)

	def plotClusters(self):
		'''
		Plots the clusters and noise points found from the DBSCAN algorithm

		'''
		labels, s, core_samples_mask = self.clusterLabels()

		# Black removed and is used for noise instead.
		unique_labels = set(labels)
		colors = [plt.cm.Spectral(each)
		          for each in np.linspace(0, 1, len(unique_labels))]
		for k, col in zip(unique_labels, colors):
		    if k == -1:
		        # Black used for noise.
		        col = [0, 0, 0, 1]

		    class_member_mask = (labels == k)

		    xy = s[class_member_mask & core_samples_mask]
		    plt.plot(xy[:, 0], xy[:, 1], 'o', markerfacecolor=tuple(col),
		             markeredgecolor='k', markersize=10)

		    xy = s[class_member_mask & ~core_samples_mask]
		    plt.plot(xy[:, 0], xy[:, 1], 'o', markerfacecolor=tuple(col),
		             markeredgecolor='k', markersize=2)

		plt.title('Estimated number of clusters: %d' % self.n_clusters_)
		plt.show()

	# ~~~~~~~~~~~~~~~~~~~~ TEMPORAL ROUTINES ~~~~~~~~~~~~~~~~~~~~

	def startTs(self):
		'''
		Returns an array of start datetimes from a user in datetime format

		'''
		Ts = []

		for direc in self.dirs:
			trajs = os.listdir(self.path + '/' + direc)
			for traj in trajs:
				trajPath = self.path + '/' + direc + '/' + traj
				df = pd.read_csv(trajPath)
				Ts.append(datetime.strptime(df['datetime'].iloc[0], '%Y-%m-%d %H:%M:%S'))

		return np.array(Ts)

	def endTs(self):
		'''
		Returns an array of end datetimes from a user in datetime format

		'''
		Ts = []

		for direc in self.dirs:
			trajs = os.listdir(self.path + '/' + direc)
			for traj in trajs:
				trajPath = self.path + '/' + direc + '/' + traj
				df = pd.read_csv(trajPath)
				Ts.append(datetime.strptime(df['datetime'].iloc[-1], '%Y-%m-%d %H:%M:%S'))

		return np.array(Ts)

	def times_hist(self):
		'''
		Plots a histogram of the start times

		'''
		# Ts = np.concatenate((self.startTs(),self.endTs()), axis=0)
		Ts = self.startTs()
		Ts = np.array([t.time() for t in Ts])

		fig = plt.figure()
		ax = fig.add_subplot(111)
		ax.grid(True,which='both')
		ax.hist(Ts,48)
		ax.set_ylabel('Frequency')
		# plt.show()

	def dates_hist(self):
		'''
		Plots a histogram of the start dates

		'''
		Ts = self.startTs()

		fig = plt.figure()
		ax = fig.add_subplot(111)
		ax.grid(True,which='both')
		ax.hist(Ts,50)
		ax.set_ylabel('Frequency')
		ax.set_xlabel('Date')
		# plt.show()

	# ~~~~~~~~~~~~~~~~~~~~ HEATMAP ROUTINES ~~~~~~~~~~~~~~~~~~~~

	def plotHeatmap(self, coords):
		'''
		Takes an array of coordinate pairs and returns a heatmap with cluster locations

		'''
		x = coords[:,0]
		y = coords[:,1]

		heatmap, xedges, yedges = np.histogram2d(x, y, bins=(100,100))
		extent = [xedges[0], xedges[-1], yedges[0], yedges[-1]]

		plt.clf()
		plt.imshow(heatmap.T, cmap='hot', extent=extent, origin='lower', interpolation='bilinear')

		if type(self.clusters) == bool:
			clusters = self.clusterLocs()
		else:
			clusters = self.clusters

		if len(clusters) != 0:
			clx = clusters[:,0]
			cly = clusters[:,1]
			plt.plot(clx, cly,'g^', fillstyle='none')
			plt.title('Estimated number of clusters: %d' % self.n_clusters_)

		# plt.show()

	def plotAllTrajs(self):
		'''
		Plots all of the user's trajectories with cluster locations

		'''
		plt.clf()

		if type(self.points) == bool:
			locs = self.locs()
		else:
			locs = self.points

		locx = locs[:,0]
		locy = locs[:,1]

		fig = plt.figure()
		ax = fig.add_subplot(111)
		ax.plot(locx, locy, 'wo', markersize=0.1)

		if type(self.clusters) == bool:
			clusters = self.clusterLocs()
		else:
			clusters = self.clusters

		if len(clusters) != 0:
			clx = clusters[:,0]
			cly = clusters[:,1]
			ax.plot(clx, cly,'r^', fillstyle='none')

		ax.set_title('Estimated number of clusters: ' + str(self.n_clusters_))
		ax.set_ylabel('y')
		ax.set_xlabel('x')
		ax.set_facecolor('black')
		# plt.show()

	def heatmap(self):
		'''
		Produces heatmap from all user's trajectories

		'''
		if type(self.points) == bool:
			locs = self.locs()
		else:
			locs = self.points

		self.plotHeatmap(locs)

	def start_heatmap(self):
		'''
		Produces heatmap from the start positions of the user's trajectories

		'''
		self.plotHeatmap(self.startLocs())

	def end_heatmap(self):
		'''
		Produces heatmap from the end positions of the user's trajectories

		'''
		self.plotHeatmap(self.endLocs())

	def startEnd_heatmap(self):
		'''
		Produces heatmap from the start and end positions of the user's trajectories

		'''
		self.plotHeatmap(self.startEndLocs())

	# ~~~~~~~~~~~~~~~~~~~~ MISC ROUTINES ~~~~~~~~~~~~~~~~~~~~

	def trajNo(self):
		'''
		Returns the number of trajectories a user has

		'''
		tot = 0

		for direc in self.dirs:
			trajs = os.listdir(self.path + '/' + direc)
			tot += len(trajs)

		return tot

	def modeInfo(self):
		'''
		Produces info on the common mode of transport for a user

		'''
		files = os.listdir(self.folder)

		# WARNING - Not strictly correct since labels.csv contains labels which don't refer to any trajectory

		if 'labels.csv' in files:
			df = pd.read_csv(self.folder + '/labels.csv')
			print('Number of Labelled Trajectories:', str(len(df)) + '/' + str(self.len))
			return df['Transportation Mode']\
											.groupby(df['Transportation Mode'])\
											.count()\
											.sort_values(ascending=False)

		else:
			print('No labels provided')


from django.urls import path
from users import views

urlpatterns = [
    path('user_login',views.user_login,name="user_login"),
    path('user_logout',views.user_logout,name="user_logout"),
    path('user_home', views.user_home, name="user_home"),
    path('user_contribute', views.user_contribute, name="user_contribute"),

]