import ciso8601
import datetime
from dateutil import parser
import json
import logging
import os

import requests
from cryptography.fernet import Fernet
from django.core.exceptions import FieldError
from django.db.utils import DataError
from django.db import transaction, IntegrityError
from .models import (
    Contact,
    Phone,
    EmailAddress,
    ConstantContactList,
    Note,
    Address,
    UserStatusOnCCList,
    RequiringRemediation
)
from profilestats import profile

from .settings import BASE_DIR
from .utils import updt

API_KEY = None
AUTH_KEY = None

# Optional load of secret_settings.py, if it is in package
try:
    from .secret_settings import API_KEY, AUTH_KEY
except ImportError:
    pass

BASE_URI = 'https://api.constantcontact.com'
HTTP_FAIL_THRESHOLD = 400
HERE = os.path.join(BASE_DIR, "datacombine")

class CombineException(BaseException):
    def __init__(self, message=""):
        super()
        self.message = message

class DataCombine():
    def __init__(self, api_key=API_KEY, auth_key=AUTH_KEY,
                 loglvl=logging.ERROR, logger_name=__name__,
                 logfile='dcombine.log'):
        """Manages relationship between the local DB and ConstantContact API

        Uses the optional file `secret_settings.py` to set the default API_KEY,
        AUTH_KEY. Some methods require the postgres password, which can also be
        in this file. The `secret_settings.py` is not maintained in the github
        repo, for obvious security reasons, and must be in the same package
        as `data_combine.py`.

        :param api_key: (str) ConstantContact developer API key
        :param auth_key: (str) ConstantContact account authorization key
        :param loglvl: (int, default: logging.ERROR == 40) Log level severity
            threshold
        :param logger_name: (str) Name of the logger used
        :param logfile: (str) Name of the logfile
        """
        self.api_key = api_key
        self.token = auth_key
        # Check for log directory
        logdir = os.path.join(HERE, "logs")
        if not os.path.isdir(logdir):
            os.mkdir(logdir)
        self._setup_logger(loglvl, logger_name, logfile)
        self.headers = {'Authorization': f'Bearer {self.token}'}
        self.contacts = []
        self.cclists = []
        self.highrise_contacts_json = dict()
        self.bad_phone_nums = dict()
        self.bad_m2m = dict()

    def _setup_logger(self, lvl, logger, logfile="dcombine.log",
                      max_bytes=1000000, backup_count=5):
        self.logger = logging.getLogger(logger)
        self.logger.setLevel(lvl)
        path_to_logfile = os.path.join(HERE, "logs", logfile)
        rtfh = logging.handlers.RotatingFileHandler(
            path_to_logfile,
            maxBytes=max_bytes,
            backupCount=backup_count,
            encoding='utf-8'
        )
        formatter = logging.Formatter(
            '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
        )
        rtfh.setFormatter(formatter)
        self.logger.addHandler(rtfh)

    def _report_cc_api_request_fail(self, r):
        # Poop pants. Die.
        self.logger.error(
            f"Attempt to harvest encountered {r.status_code}: "
            f"{r.reason}: {r.content}"
        )
        return None

    def _harvest_contact_page(self, params, api_url):
        url = f"{BASE_URI}{api_url}"

        # Parameters should only be required on the initial GET request
        params = {
            'api_key': self.api_key
        } if api_url.__contains__('next') else params

        limit = params.get('limit')
        if limit:
            self._limit = limit

        # GET them contacts
        r = requests.get(url, params=params, headers=self.headers)
        self.logger.debug(f"Getting '{self._limit}' contacts from {url}.")

        # In case of UH-OH!
        if r.status_code >= HTTP_FAIL_THRESHOLD:
            return self._report_cc_api_request_fail(r)

        # Pick them tasty contacts
        rjson = r.json()
        self.contacts.extend(rjson['results'])

        # Get next_link or signal end
        if "next_link" in rjson['meta']['pagination']:
            next_link = rjson['meta']['pagination']['next_link']
            self.logger.debug(f"Found next link to harvest: '{next_link}'")
            return next_link
        else: # DONE!
            self.logger.debug("No more contacts to harvest.")
            return False

    def harvest_contacts(self, status='ALL', limit='500', modified_since=None,
                         api_uri='/v2/contacts', delete_contacts=True):
        """Downloads contacts from ConstantContact account

        :param status: (str): From one of UNCONFIRMED, ACTIVE, OPTOUT, REMOVED,
            NON_SUBSCRIBER or for all of the above, ALL. See:
            https://developer.constantcontact.com/docs/contacts-api/
                contacts-index.html
            for more informaton on these statuses.
        :param limit: (str / int) Number of contacts to download per page.
            Maximum value allowed by ConstantContact is 500.
        :param modified_since: (datetime) Only download contacts modified after
            this date and time
        :param api_uri: (str) The API endpoint for ConstantContact contacts
        :param delete_contacts: (bool) Delete any values currently in
            `self.contacts`
        :return: None, contacts are saved in json formatted list in
            `self.contacts`
        """
        if delete_contacts:
            self.contacts = []
        else:
            raise CombineException(
                "Contacts already exist and 'delete_contacts' parameter "
                "is False"
            )

        if limit and not type(limit) == str:
            limit = str(limit)

        params = {
            'status': status,
            'limit': limit,
            'api_key': self.api_key,
        }
        if modified_since:
            if not self._check_for_iso_8601_format(modified_since):
                raise TypeError(f"'{modified_since}' is not in iso8601 format")
            else:
                params['modified_since'] = modified_since
        next_page = api_uri
        while next_page:
            next_page = self._harvest_contact_page(params, next_page)

    def _check_for_iso_8601_format(self, dt):
        return bool(ciso8601.parse_datetime(dt))

    def harvest_lists(self, modified_since=None,
                      api_uri='/v2/lists', delete_cclists=True):
        """Downloads lists from ConstantContact account

        :param modified_since: (datetime) Only download lists modified after
            this date and time
        :param api_uri: (str) The API endpoint for ConstantContact lists
        :param delete_cclists: (bool) Delete any values current in
            `self.cclists`
        :return: None, lists are saved in json formatted list in `self.cclists`
        """
        if delete_cclists:
            self.cclists = []
        else:
            raise CombineException(
                "cclists already exist and 'delete_cclists' parameter "
                "is False"
            )

        params = {
            'api_key': self.api_key,
        }
        url = f"{BASE_URI}{api_uri}"

        if modified_since:
            if not self._check_for_iso_8601_format(modified_since):
                raise TypeError(f"'{modified_since}' is not in iso8601 format")
            else:
                params['modified_since'] = modified_since

        # GET dem lists!
        r = requests.get(url, params=params, headers=self.headers)
        self.logger.debug(f"Making list request: {r}")

        # In case of UH-OH!
        if r.status_code >= HTTP_FAIL_THRESHOLD:
            return self._report_cc_api_request_fail(r)

        # GET'ten dem lists
        self.cclists = r.json()
        self.logger.debug(
            f"Successfully downloaded '{len(self.cclists)}' lists"
        )

    def _get_most_recent_datetime(self, cls_obj, param):
        # Order in decreasing order so first is now last, and last is now first
        by_most_recent = cls_obj.objects.order_by('-'+param)
        if not by_most_recent:
            return None
        mrdt = getattr(by_most_recent.first(), param)
        if mrdt:
            return mrdt.isoformat()
        else:
            return None

    def update_local_db_caches(self):
        """Finds the most recent lists and contacts, and downloads any new ones

        Works primarly through the `harvest` functions, and will delete any
        contacts or lists in their respective variables, and overwrite them
        with new lists and contacts.

        :return: None. Overwrites `self.contacts` and `self.cclists` with
            new contacts and lists, respectively.
        """
        most_recent_list_dt = self._get_most_recent_datetime(
            ConstantContactList, 'modified_date'
        )
        most_recent_contact_dt = self._get_most_recent_datetime(
            Contact, 'cc_modified_date'
        )

        # Begin the harvesting!!!
        self.harvest_lists(modified_since=most_recent_list_dt)
        self.harvest_contacts(modified_since=most_recent_contact_dt)

        self.logger.info(
            "Harvested Constant Contact lists from "
            f"{most_recent_list_dt} and harvested Constant Contact"
            f" Contacts from {most_recent_contact_dt}."
        )

    def combine_and_update_new_entries(self):
        """Updates local db caches and combines them into the db

        Note: "db caches" are defined as `self.cclists` and `self.contacts`

        :return: None
        """
        # Get all updated lists and contacts / HARVEST
        self.update_local_db_caches()

        # Update databases / COMBINE
        for cclist in self.cclists:
            self.combine_cclist_json_into_db(cclist)
        self.combine_contacts_into_db()

    def read_from_highrise_contact_stash(self, jfname="yaya.json"):
        """Reads HighRise contacts that have been converted to json

        :param jfname: (str) Filename of HighRise stash
        :return: Populates `self.highrise_contacts_json` with HighRise contacts
        """
        with open(jfname, 'r') as f:
            self.highrise_contacts_json = json.loads(f.read())

    def read_constantcontact_objects_from_json(
            self,
            jfname=os.path.join(HERE, "yaya_cc.json"),
            override_contacts=True,
            override_lists=True
    ):
        """Read previously collected ConstantContact objects from JSON file

        :param jfname: (str) Path to json file with ConstantContact data
        :param override_contacts: (bool) Replace `self.contact` with contacts
            in json file
        :param override_lists: (bool) Replace `self.cclist` with lists from
            json file
        :return: None
        """
        with open(jfname, 'r') as jf:
            data = json.loads(jf.read())
            if hasattr(self, 'contacts'):
                if override_contacts:
                    self.logger.debug("Overriding contacts...")
                    self.contacts = data['contacts']
                else:
                    self.logger.info(f"'{len(self.contacts)}' already found, "
                                     f"not overriding. Merging with contacts.")
                    self._update_ccobj("contacts", data['contacts'])
            if hasattr(self, 'cclists'):
                if override_lists:
                    self.logger.debug("Overriding lists...")
                    self.cclists = data['cclists']
                else:
                    self.logger.info(f"'{len(self.cclists)}' already found, "
                                     f"not overriding. Merging with lists.")
                    self._update_ccobj("cclists", data['cclists'])
            remed = data.get('to_remidate')
            if remed:
                self.bad_phone_nums = remed.get('bad_phone_nums')
                self.bad_m2m = remed.get('bad_m2m')

    def _prep_objects_to_dump(self, data):
        add_contacts = []
        add_lists = []
        fields = [
            (add_contacts, 'contacts'),
            (add_lists, 'cclists'),
        ]

        # Check if record exists in contacts and cclists
        # If so, don't re-add it
        for dumps, field in fields:
            for self_field in getattr(self, field):
                if any([x['id'] == self_field.get('id')\
                        for x in data.get(field)]):
                    continue
                else:
                    dumps.append(self_field)
        return (add_contacts, add_lists)

    def _update_ccobj(self, field, new_ccobj):
        obj_ids = [obj.get('id') for obj in getattr(self, field)]
        if hasattr(self, field):
            # Grab that field in the ccobj
            ccobj = getattr(self, field) #<- updates (adds to) this field
            for obj in new_ccobj:
                if obj.get('id') in obj_ids:
                    continue
                else:
                    ccobj.append(obj)

    def dump_constantcontact_objects_from_json(
            self,
            jfname=os.path.join(HERE, "yaya_cc.json"),
            override_json=True,
            encrypt=True
    ):
        """Dump ConstantContact objects into a JSON formatted file

        :param jfname: (str) Path to JSON file
        :param override_json: (bool) If file exists at `jfname`, delete?
        :return: None, but there should be data in the file at `jfname`
        """
        mode = 'w' if not override_json else 'w+'
        data = dict()
        with open(jfname, mode) as jf:
            if mode == 'w+': # Read and write
                try:
                    self.logger.debug("Preparing to dump objects...")
                    data = json.load(jf)
                    contacts, cclists = self._prep_objects_to_dump(data)
                except json.JSONDecodeError as jde:
                    fileinfo = os.stat(jfname)
                    if fileinfo.st_size == 0:
                        msg = f"File '{jfname}' is 0 bytes...writing to it"
                        if logging.INFO >= self.logger.level:
                            print(msg)
                        self.logger.info(msg)
                    else:
                        self.logger.exception(
                            f"Could not read JSON file {jfname}. "
                            f"No objects dumped"
                        )
                        raise jde
                    contacts, cclists = (dict(), dict())
                self._update_ccobj('contacts', contacts)
                self._update_ccobj('cclists', cclists)
            data = {
                'contacts': self.contacts,
                'cclists': self.cclists,
                'to_remidate': {
                    'bad_phone_numbers': dict(),
                    'bad_m2m': dict()
                }
            }
            if hasattr(self, 'bad_phone_numbers'):
                data['to_remidate']['bad_phone_nums'] = self.bad_phone_nums
            if hasattr(self, 'bad_m2m'):
                data['to_remidate']['bad_m2m'] = self.bad_m2m
            json.dump(data, jf)
            self.logger.debug("Objects dumped successfully")

    @classmethod
    def get_init_values_for_model(_, cls):
        """Returns all fields that are not relations or auto-incremented id's

        :param cls: The class model to get the fields of
        :return: (`list` of `str`) A list of the names of each non-relation and
            non-auto-incremented `django.db.models.fields`
        """
        return [f.name for f in cls._meta.get_fields() if not f.is_relation
                and f.name != 'id']

    @staticmethod
    def convert_choice_to_field(choice):
        """Makes choice a field by changing the text

        :param choice: Entry to make a field
        :return: (str) `choice` made uppercase and replacing spaces with '_'
        """
        return choice.upper().replace(' ', '_')

    @classmethod
    def _setup_model_object(_, cls, attrs):
        kwargs = {}
        for fld in cls._meta.get_fields():
            if fld.name == 'id' or fld.many_to_many:
                continue
            elif fld.name.startswith("cc_"):
                if fld.name == "cc_id":
                    # If object already exists in DB, return None
                    if cls.objects.filter(cc_id=attrs[fld.name[3:]]):
                        return
                kwargs[fld.name] = attrs[fld.name[3:]]
            elif hasattr(fld, 'choices') and fld.choices:
                # Map uppercase field names to DB 2-letter codes
                chmap = {
                    DataCombine.convert_choice_to_field(ch[1]): ch[0]\
                    for ch in fld.choices
                }
                try:
                    kwargs[fld.name] = chmap[attrs[fld.name]]
                except KeyError:
                    kwargs[fld.name] = None
            else:
                try:
                    kwargs[fld.name] = attrs[fld.name]
                except KeyError:
                    kwargs[fld.name] = None
        # Return a new model object
        return cls(**kwargs)

    @transaction.atomic
    def combine_cclist_json_into_db(self, cclist_json):
        """Creates a `model.ConstantContactList` and saves it to the local DB

        :param cclist_json: (dict) JSON with ConstantContact list information
        :return: None
        """
        md = cclist_json.get('modified_date')
        cd = cclist_json.get('created_date')
        if not self._check_for_iso_8601_format(md):
            raise TypeError("Modified date is not in ISO-8601 format")
        if not self._check_for_iso_8601_format(cd):
            raise TypeError("Created date is not in ISO-8601 format")
        ccid = cclist_json.get('id')
        id_in_db = ConstantContactList.objects.filter(cc_id=ccid)

        ccl = ConstantContactList(
            cc_id=ccid,
            status="AC" if cclist_json.get('status').startswith('A')\
                        else "HI",
            name=cclist_json.get('name'),
            created_date=cd,
            modified_date=md
        )
        if id_in_db:
            l = id_in_db.first()
            if ccl == l:
                self.logger.debug(
                    f"List named '{l.name}' already found in db skipping..."
                )
                return
            else:
                raise IntegrityError(
                    f"Lists with same ID do not match: {ccl.name} and {l.name}"
                    f", {ccl.cc_id} and {l.cc_id}."
                )
        ccl.save()
        return None

    @transaction.atomic
    def _initial_contact_setup_from_json(self, contact):
        try:
            cf = DataCombine.get_init_values_for_model(Contact)
            fields = dict()
            # Set up fields for Contact
            for x in cf:
                try:
                    if not x.startswith('cc_'):
                        fields[x] = contact.get(x)
                    else:
                        fields[x] = contact.get(x[3:])
                except KeyError as ke:
                    key = ke.args[0]
                    if Contact._meta.get_field(key).null:
                        fields[x] = ""
                    else:
                        raise ke
            # Initialize new Contact object
            newContact = Contact(**fields)
            newContact.status = Contact.convert_status_str_to_code(
                newContact.status
            )
            newContact.save()
            return newContact
        except KeyError as ke:
            key = ke.args[0]
            self.logger.error(
                f"Encountered a KeyError on setting "
                f"'_initial_contact_setup_from_json', there is no key "
                f"'{key}' in contact: {contact}"
            )
            raise

    @transaction.atomic
    def combine_phone_number_into_db(self, phone_num, newContact, phfld):
        """Initialize a `models.Phone` from a sting and save to local DB

        :param phone_num: (str) A sting representing a phone number
        :param newContact: (`models.Contact`) The new Contact to add the phone
            object to.
        :param phfld: (`django.db.models.fields.related.ManyToManyField`) The
            field (home_phone, cell_phone, work_phone, or fax) that the
            `phone_num` string will be added to in `newContact`
        :return: None, but a new Phone object should be added to the local DB
        """
        if not phone_num:
            return
        ph = Phone()
        try:
            ph.create_from_str(phone_num)
        except FieldError:
            raise FieldError(phone_num, newContact, phfld)

        if ph == None:
            self.logger.info(f"phone_num='{phone_num}' produces None")
            return

        ph_in_db = Phone.is_phone_in_db(ph)
        phobj = getattr(newContact, phfld)
        if ph_in_db:
            self.logger.debug(
                f"Phone number '{ph}' is already in database"
            )
            phobj.add(ph_in_db.first())
        else:
            ph.save()
            phobj.add(ph)

    @transaction.atomic
    def _combine_m2m_field_into_db(self, cls_obj, m2mattrs, newContact, m2m):
        m2mobj = DataCombine._setup_model_object(
            cls_obj, m2mattrs
        )
        if m2mobj:
            try:
                m2mobj.save()
            except DataError as de:
                too_long = "value too long for type character varying"
                if de.args[0].__contains__(too_long):
                    for key, val in m2mattrs.items():
                        cf = cls_obj._meta.get_field(key)
                        if not val or not cf.max_length:
                            continue
                        if len(val) > cf.max_length:
                            de.args = (cls_obj, key, val)
                            raise de
            ncontact_m2mfield = getattr(newContact, m2m)
            ncontact_m2mfield.add(m2mobj)
        else:
            self.logger.debug(
                f"{m2m.capitalize()} {m2mobj} already in database"
                f"...skipping"
            )
        return newContact

    @transaction.atomic
    def _combine_notes_into_db(self, notes, newContact):
        for note in notes:
            newNote = DataCombine._setup_model_object(Note, note)
            if newNote:
                newNote.contact = newContact
                newNote.save()

    @transaction.atomic
    def _save_ustat_object(self, ustat_object):
        ustat_object.save()

    @transaction.atomic
    def save_for_remediation(self, contact, json_entry):
        """Make a new `models.RequiringRemediation` object and save to local DB

        :param contact: (`models.Contact`) Entry with bad field
        :param json_entry: (dict) Fields that are incorrect matched to bad data
        :return: None, local DB is updated
        """
        rr = RequiringRemediation(contact_pk=contact, fields=json_entry)
        rr.save()

    def _save_ustat_objects(self, contact, newContact, updating):
        for xcclist in contact.get('lists'):
            listobj = ConstantContactList.objects.filter(
                cc_id=xcclist['id']
            )
            liststat = "HI" if xcclist.get('status').startswith('H') \
                else "AC"
            ustat_obj = UserStatusOnCCList(
                cclist=listobj.first(), user=newContact, status=liststat
            )
            if updating:
                con = newContact.cc_lists.filter(cc_id=xcclist['id']).first()
                if con.status != liststat:
                    con.status = liststat
                    con.update()
                return
            self._save_ustat_object(ustat_obj)

    def _continue_combine(self, count):
        # Update progress bar
        updt(len(self.contacts), count)
        return count+1

    #@profile(print_stats=10, dump_stats=True, profile_filename="p3.out")
    def combine_contacts_into_db(self, update_web_interface=False):
        """Adds all contacts and lists available to `self` to local DB

        This is the core of the "combining" process. After lists and contacts
        have been "harvested" (ie. downloaded or read from JSON file) from
        ConstantContact and added to `self.cclists` and `self.contacts`,
        respectively; this function will make the relevant Django ORM models
        and then save each to the local DB

        :param update_web_interface: (str) If true, this function will generate
            a dictonary object with 'processed' and 'total' keys, matched to
            the number of Contact iterations completed and the number of
            Contact objects yet to be added (there usually are far fewer lists
            than contacts, so these aren't included)

            If not true, calls `utils.updt` to fulfill the same purpose as
            updating the web interface, but to the console instead
        :return: None, but should update local DB
        """
        begin_time = datetime.datetime.now()
        processed = 0

        if not hasattr(self, 'contacts'):
            raise AttributeError("No contacts found")
        elif not hasattr(self, 'cclists'):
            raise AttributeError("No constant contact list found")

        # These are all the UserStatusOnCCList objects which
        # Need to be saved when the user has been entered into the database
        ustat_objects = []
        for cclist in self.cclists:
            self.combine_cclist_json_into_db(cclist)
        for c_i, contact in enumerate(self.contacts):
            try:
                newContact = None
                bad_m2m_entry = None
                bad_phone_entry = None
                updatingContact = False

                # Check if Contact is already in DB, and act appropriately
                contact_in_db = Contact.objects.filter(cc_id=contact.get("id"))
                if contact_in_db:
                    contact_in_db = contact_in_db.first()
                    contact_date = parser.parse(contact.get("modified_date"))
                    if contact_in_db.cc_modified_date == contact_date:
                        continue
                    else:
                        newContact = contact_in_db
                        del contact_in_db
                else:
                    newContact = self._initial_contact_setup_from_json(contact)

                non_phone_or_cclist_m2m = [
                    (Address, "addresses"),
                    (EmailAddress, "email_addresses"),
                ]

                # Setup and save connections from this contact to various lists
                # (ie. `models.UserStatusOnCCList` objects)
                self._save_ustat_objects(contact, newContact, updatingContact)

                # Setup and combine phone numbers for contact
                phone_fields = [
                    'home_phone',
                    'work_phone',
                    'cell_phone',
                    'fax'
                ]
                for phfld in phone_fields:
                    try:
                        self.combine_phone_number_into_db(
                            contact.get(phfld), newContact, phfld
                        )
                    except FieldError as fe:
                        bad_phone_entry = {phfld:contact.get(phfld)}

                # Setup and combine many to many fields for contact
                for cls_obj, m2m in non_phone_or_cclist_m2m:
                    for m2mattrs in contact.get(m2m):
                        try:
                            newContact = self._combine_m2m_field_into_db(
                                cls_obj, m2mattrs, newContact, m2m
                            )
                        except DataError:
                            bad_m2m_entry = {newContact.cc_id: m2mattrs}

                # Set up and save notes about contact
                self._combine_notes_into_db(contact.get('notes'), newContact)

                # Save new contact to database
                newContact.save()

                # Setup and save any entries which will need to be remediated
                # by a human operator
                if bad_m2m_entry:
                    self.bad_m2m.setdefault(newContact.cc_id, [])\
                        .append(bad_m2m_entry)
                    self.save_for_remediation(newContact, bad_m2m_entry)

                if bad_phone_entry:
                    self.bad_phone_nums.setdefault(newContact.cc_id, [])\
                        .append(bad_phone_entry)
                    self.save_for_remediation(newContact, bad_phone_entry)

            except KeyboardInterrupt:
                self.logger.info("Interrupt signal received...quitting.")
                return
            except DataError as de:
                if len(de.args) == 3:
                    self.logger.error(
                        "For class object "
                        f"{de.args[0]}.{de.args[1]}={de.args[2]}. Is too long."
                    )
                else:
                    self.logger.error(
                        f"Data error on contact #{c_i} {de.args[0]}"
                    )
            except FieldError:
                 self.logger.warning(
                    f"Field error on contact #{c_i} {contact} for {phfld}"
                    "...skipping..."
                )
            except: # Keep calm, fuck this, and carry on
                self.logger.exception(f"Exception on contact #{c_i}...skipping...")
            finally: # Update dem progress trackers
                if update_web_interface:
                    yield {'processed': processed, 'total': len(self.contacts)}
                    processed += 1
                else:
                    processed = self._continue_combine(processed)

        # Note time taken to complete, for log
        end_time = datetime.datetime.now()
        total_time = (end_time - begin_time).total_seconds()
        if update_web_interface:
            yield {'processed': len(self.contacts), 'total': len(self.contacts)}
        else:
            updt(len(self.contacts), len(self.contacts))
        self.logger.info(
            f"Combined time to process '{processed}' contacts: "
            f"{total_time // 60} minutes and {total_time % 60} seconds."
         )

if __name__ == '__main__':
    dc = DataCombine()
    dc.read_constantcontact_objects_from_json()
    dc.combine_contacts_into_db()


import scipy.misc
import numpy as np
import matplotlib.pyplot as plt
import cv2

def getIntersetion(l1, l2, toPrint = False):
	def lineParams(p1, p2):
		if(p1[0] == p2[0]):
			raise("You can't take lines parallel to the image plane")
		m = (p1[1] - p2[1])/(p1[0] - p2[0])
		c = p1[1] - m * p1[0]
		return [m , c]
	def lines(params):
		return "y = " + str(params[0]) + "*x + " + str(params[1])
	def intersectionPoints(par1, par2):
		if(par1[0] == par2[0]):
			raise("the lines shouldn't be parallel!!!")
		x = (par1[1] - par2[1])/(par2[0] - par1[0])
		y = par1[0]*x + par1[1]
		return (x, y)
	params1 = lineParams(l1[0], l1[1])
	params2 = lineParams(l2[0], l2[1])
	if(toPrint):
		print("The first line equation is " + lines(params1))
		print("The second line equation is " + lines(params2))
		print("------------------------------------------------------------------------------------------")
	return intersectionPoints(params1, params2)

def getVanishingPoints(nLines = 4):
	plt.title("Click the start and end cordinates for "+str(nLines//2)+" set of parallel lines(Total " +str(nLines)+" lines).")
	points = plt.ginput(nLines*2)
	vanishingPoints = []
	for i in range(0, len(points), nLines):
		A1 = points[i]
		A2 = points[i+1]
		B1 = points[i+2]
		B2 = points[i+3]
		plt.plot([A1[0], A2[0]], [A1[1], A2[1]], marker = 'o')
		plt.plot([B1[0], B2[0]], [B1[1], B2[1]], marker = 'o')
		vanishingPoints.append(getIntersetion([A1, A2], [B1, B2], True))
	return vanishingPoints

def getPointsForObj():
	points = plt.ginput(n = 2, timeout = 0)
	top = points[0]
	bottom = points[1]
	plt.plot([top[0], bottom[0]], [top[1], bottom[1]], marker = 'o')
	return top, bottom

def getHeight(height, refTop, refBottom, heightPlt):
	return height * (np.linalg.norm(np.array(heightPlt)-np.array(refBottom))*1.0/np.linalg.norm(np.array(refTop)-np.array(refBottom)))

image = scipy.misc.imread("img.jpg")
N = int(input("Enter the number of objects for which you want to get the height: "))
plt.imshow(image)
vanishingPoints = getVanishingPoints()
plt.imshow(image)
plt.title("Click the coordinates for the pole(ref obj) (Top - Bottom)")
pole_top, pole_bottom = getPointsForObj()
poleHeight = 1.65
heights = ""
for i in range(1, N+1):
	plt.title("Click the coordinates of the obj-"+str(i)+" for which height is to be calculated (Top - Bottom)")
	obj_top, obj_bottom = getPointsForObj()

	vlPoint = getIntersetion([pole_bottom, obj_bottom], [vanishingPoints[0], vanishingPoints[1]])
	toGetHeightPlt = getIntersetion([pole_top, pole_bottom], [obj_top, vlPoint])

	plt.plot([vlPoint[0], toGetHeightPlt[0]], [vlPoint[1], toGetHeightPlt[1]], marker = 'o')
	plt.plot([obj_bottom[0], vlPoint[0]], [obj_bottom[1], vlPoint[1]], marker = 'o')
	plt.plot([obj_top[0], vlPoint[0]], [obj_top[1], vlPoint[1]], marker = 'o')

	objHeight = getHeight(poleHeight, pole_top, pole_bottom, toGetHeightPlt)
	print("Height of the obj"+str(i)+" : " + str(objHeight))
	heights += "Height of obj-" + str(i) + ": " + str(objHeight) + "\n"

print("\n----------------------------------------------------\n")
forCameraHeight = getIntersetion([pole_bottom, pole_top], [vanishingPoints[0], vanishingPoints[1]])
cameraHeight = getHeight(poleHeight, pole_top, pole_bottom, forCameraHeight)
print("Height of camera : "+ str(cameraHeight))
heights += "Height of camera"+ ": " + str(cameraHeight) + "\nThe dotted line is the vanishing line"
plt.title("")
plt.text(0, 0, heights)
plt.plot([vanishingPoints[0][0], vanishingPoints[1][0]], [vanishingPoints[0][1], vanishingPoints[1][1]], color='red', marker = 'o', linestyle='dashed')
plt.show()


import argparse
import os
import numpy as np
import torch
import torch.nn as nn
from torch.nn import functional as F
import matplotlib.pyplot as plt
from sklearn import metrics
from statistics import mean
import pickle
import math
import sys
import torchvision.models as models

sys.path.insert(0, 'utils/')
sys.path.insert(0, 'data/')
sys.path.insert(0, 'models/')
sys.path.insert(0, 'layers/')

from vgg_utils import _nanargmin, set_parameter_requires_grad
from data_vgg import get_data_loaders

#%%

EPOCHS = 100

loss_fn = F.cross_entropy

LEARNING_RATE = 1e-04
REG = 1e-04

def fit(feature_extractor, task_classifier, adv_classifier, data, device, model_path, output):
    # train and validation loaders
    train_loader, valid_loader = data
    print("Train/Val batches: {}/{}".format(len(train_loader),
                                            len(valid_loader)))

    # Set the optimizer
    task_opt = torch.optim.Adam(list(feature_extractor.parameters()) +
                                list(task_classifier.parameters()),
                                lr=LEARNING_RATE,
                                weight_decay=REG)

    adv_opt = torch.optim.Adam(list(adv_classifier.parameters()),
                               lr=LEARNING_RATE,
                               weight_decay=REG)

    # Start training
    train_history = {'train_loss': [], 'train_acc': [], 'train_apcer': [], 'train_bpcer': [], 'train_eer': [], 'train_bpcer_apcer1': [], 'train_bpcer_apcer5': [], 'train_bpcer_apcer10': [], 'train_apcer1': [], 'train_apcer5': [], 'train_apcer10': [],
                     'val_loss': [], 'val_acc': [], 'val_apcer': [], 'val_bpcer': [], 'val_eer': [], 'val_bpcer_apcer1': [], 'val_bpcer_apcer5': [], 'val_bpcer_apcer10': [], 'val_apcer1': [], 'val_apcer5': [], 'val_apcer10': []}

    # Best validation params
    best_val = -float('inf')
    best_epoch = 0

    for epoch in range(EPOCHS):
        print('\nEPOCH {}/{}\n'.format(epoch + 1, EPOCHS))

        # TRAINING
        # set model to train
        feature_extractor.train()
        task_classifier.train()
        adv_classifier.train()
        for i, (x, y, f, f_norm) in enumerate(train_loader):  # iterations loop
            # send mini-batch to gpu
            x = x.to(device)

            y = y.type(torch.LongTensor)
            y = y.to(device)

            f = f.type(torch.LongTensor)
            f = f.to(device)

            f_norm = f_norm.type(torch.LongTensor)
            f_norm = f_norm.to(device)

            # split real and fake observations
            x_real = x[f == -1]
            x_fake = x[f != -1]
            x_concat = torch.cat([x_real, x_fake], dim=0)

            y_real = y[f == -1]
            y_fake = y[f != -1]
            y_concat = torch.cat([y_real, y_fake], dim=0)

            f_norm_fake = f_norm[f != -1]

            if x_real.shape[0] == 0 or x_fake.shape[0] == 0  :
                continue

            # forward pass
            h_conv = feature_extractor(x_concat)
            h_fake = feature_extractor(x_fake)

            h_conv_flat = h_conv.view(h_conv.size(0), -1)
            h_fake_flat = h_fake.view(h_fake.size(0), -1)

            y_task = task_classifier(h_conv_flat)
            y_adv = adv_classifier(h_fake_flat)

            # Compute task-specific loss
            task_loss = loss_fn(y_task, y_concat)

            # Compute adversial loss
            adv_loss = loss_fn(y_adv, f_norm_fake)

            task_opt.zero_grad()
            task_loss.backward(retain_graph=True)
            task_opt.step()

            adv_opt.zero_grad()
            adv_loss.backward()
            adv_opt.step()

            # display the mini-batch loss
            sys.stdout.write("\r" + '........{}-th mini-batch loss: {:.3f}'.format(i, task_loss.item()))
            sys.stdout.flush()

        # Validation
        tr_loss, tr_acc, tr_apcer, tr_bpcer, tr_eer, tr_bpcer_apcer1, tr_bpcer_apcer5, tr_bpcer_apcer10, tr_apcer1, tr_apcer5, tr_apcer10 = eval_model(feature_extractor, task_classifier, adv_classifier, train_loader, device)
        train_history['train_loss'].append(tr_loss.item())
        train_history['train_acc'].append(tr_acc)
        train_history['train_apcer'].append(tr_apcer)
        train_history['train_bpcer'].append(tr_bpcer)
        train_history['train_eer'].append(tr_eer)
        train_history['train_bpcer_apcer1'].append(tr_bpcer_apcer1)
        train_history['train_bpcer_apcer5'].append(tr_bpcer_apcer5)
        train_history['train_bpcer_apcer10'].append(tr_bpcer_apcer10)
        train_history['train_apcer1'].append(tr_apcer1)
        train_history['train_apcer5'].append(tr_apcer5)
        train_history['train_apcer10'].append(tr_apcer10)

        val_loss, val_acc, val_apcer, val_bpcer, val_eer, val_bpcer_apcer1, val_bpcer_apcer5, val_bpcer_apcer10, val_apcer1, val_apcer5, val_apcer10 = eval_model(feature_extractor, task_classifier, adv_classifier, valid_loader, device)
        train_history['val_loss'].append(val_loss.item())
        train_history['val_acc'].append(val_acc)
        train_history['val_apcer'].append(val_apcer)
        train_history['val_bpcer'].append(val_bpcer)
        train_history['val_eer'].append(val_eer)
        train_history['val_bpcer_apcer1'].append(val_bpcer_apcer1)
        train_history['val_bpcer_apcer5'].append(val_bpcer_apcer5)
        train_history['val_bpcer_apcer10'].append(val_bpcer_apcer10)
        train_history['val_apcer1'].append(val_apcer1)
        train_history['val_apcer5'].append(val_apcer5)
        train_history['val_apcer10'].append(val_apcer10)

        # save best validation model
        if best_val < val_acc:
            #torch.save(model.state_dict(), model_path + 'vgg_fpad.pth')
            best_val = val_acc
            best_epoch = epoch

        # display the training loss
        print()
        print('\n>> Train loss: {:.3f}  |'.format(tr_loss.item()) + ' Train Acc: {:.3f}'.format(tr_acc) + '\n   Train APCER: {:.3f} |'.format(tr_apcer) + ' Train BPCER: {:.3f}'.format(tr_bpcer) + '\n   Train EER: {:.3f}'.format(tr_eer))

        print('\n>> Valid loss: {:.3f}  |'.format(val_loss.item()) + ' Valid Acc: {:.3f}'.format(val_acc) + '\n   Valid APCER: {:.3f} |'.format(val_apcer) + ' Valid BPCER: {:.3f}'.format(val_bpcer) + '\n   Valid EER: {:.3f}'.format(val_eer))

        print('\n>> Best model: {} / Acc={:.3f}'.format(best_epoch+1, best_val))
        print()

    # save train/valid history
    plot_fn = output + 'vgg_fpad_history.png'
    plot_train_history(train_history, plot_fn=plot_fn)

    # return best validation model
    #model.load_state_dict(torch.load(model_path + 'vgg_fpad.pth'))

    #return model, train_history, valid_loader, best_epoch+1
    return train_history, valid_loader, best_epoch+1

def plot_train_history(train_history, plot_fn=None):
    plt.switch_backend('agg')

    best_val_epoch = np.argmin(train_history['val_loss'])
    best_val_acc = train_history['val_acc'][best_val_epoch]
    best_val_loss = train_history['val_loss'][best_val_epoch]
    plt.figure(figsize=(7, 5))
    epochs = len(train_history['train_loss'])
    x = range(epochs)
    plt.subplot(211)
    plt.plot(x, train_history['train_loss'], 'r-')
    plt.plot(x, train_history['val_loss'], 'g-')
    plt.plot(best_val_epoch, best_val_loss, 'bx')
    plt.xlabel('Epoch')
    plt.ylabel('Train/Val loss')
    plt.legend(['train_loss', 'val_loss'])
    plt.axis([0, epochs, 0, max(train_history['train_loss'])])
    plt.subplot(212)
    plt.plot(x, train_history['train_acc'], 'r-')
    plt.plot(x, train_history['val_acc'], 'g-')
    plt.plot(best_val_epoch, best_val_acc, 'bx')
    plt.xlabel('Epoch')
    plt.ylabel('Train/Val acc')
    plt.legend(['train_acc', 'val_acc'])
    plt.axis([0, epochs, 0, 1])
    if plot_fn:
        #plt.show()
        plt.savefig(plot_fn)
        plt.close()
    else:
        plt.show()

def eval_model(feature_extractor, task_classifier, adv_classifier, data_loader, device, debug=False):
    with torch.no_grad():

        feature_extractor.eval()
        task_classifier.eval()
        adv_classifier.eval()

        loss_eval = 0
        N = 0
        n_correct = 0

        TP = 0
        TN = 0
        FP = 0
        FN = 0

        PA = 0
        BF = 0

        eer_list = []

        BPCER_APCER1_list = []
        BPCER_APCER5_list = []
        BPCER_APCER10_list = []

        APCER1_list = []
        APCER5_list = []
        APCER10_list = []

        for i, (x, y, f, f_norm) in enumerate(data_loader):
            # send mini-batch to gpu
            x = x.to(device)

            y = y.type(torch.LongTensor)
            y = y.to(device)

            f = f.type(torch.LongTensor)
            f = f.to(device)

            f_norm = f_norm.type(torch.LongTensor)
            f_norm = f_norm.to(device)

            # split real and fake observations
            x_real = x[f == -1]
            x_fake = x[f != -1]
            x_concat = torch.cat([x_real, x_fake], dim=0)

            y_real = y[f == -1]
            y_fake = y[f != -1]
            y_concat = torch.cat([y_real, y_fake], dim=0)

            f_norm_fake = f_norm[f != -1]

            if x_real.shape[0] == 0 or x_fake.shape[0] == 0  :
                continue

            # forward pass
            h_conv = feature_extractor(x_concat)
            h_fake = feature_extractor(x_fake)

            h_conv_flat = h_conv.view(h_conv.size(0), -1)
            h_fake_flat = h_fake.view(h_fake.size(0), -1)

            y_task = task_classifier(h_conv_flat)
            y_adv = adv_classifier(h_fake_flat)

            # Compute task-specific loss
            task_loss = loss_fn(y_task, y_concat)

            # Compute adversial loss
            adv_loss = loss_fn(y_adv, f_norm_fake)

            loss_eval += task_loss * x.shape[0]

            y_pred = y_task

            # Compute Acc
            N += x.shape[0]
            ypred_ = torch.argmax(y_pred, dim=1)
            n_correct += torch.sum(1.*(ypred_ == y)).item()

            y = y.cpu().numpy()
            ypred_ = ypred_.cpu().numpy()

            # Biometric metrics

            TP += np.sum(np.logical_and(ypred_, y))
            TN += np.sum(np.logical_and(1-ypred_, 1-y))

            FP += np.sum(np.logical_and(ypred_, 1-y))
            FN += np.sum(np.logical_and(1-ypred_, y))

            PA += np.sum(y == 0)
            BF += np.sum(y == 1)

            probs = torch.nn.functional.softmax(y_pred, dim=0)

            probs = probs.cpu().numpy()

            probs = probs[:, 1]

            fpr, tpr, threshold = metrics.roc_curve(y, probs)
            fnr = 1 - tpr

            BPCER_APCER1_list.append(fpr[(np.abs(fnr - 0.01)).argmin()])
            BPCER_APCER5_list.append(fpr[(np.abs(fnr - 0.05)).argmin()])
            BPCER_APCER10_list.append(fpr[(np.abs(fnr - 0.1)).argmin()])

            APCER1_list.append(fnr[(np.abs(fnr - 0.01)).argmin()])
            APCER5_list.append(fnr[(np.abs(fnr - 0.05)).argmin()])
            APCER10_list.append(fnr[(np.abs(fnr - 0.1)).argmin()])

            index = _nanargmin(np.absolute((fnr - fpr)))
            if math.isnan(index) == False:
                eer_list.append(fpr[index])

        loss_eval = loss_eval / N
        acc = n_correct / N
        APCER = (FP * 1.) / (FP + TN)
        BPCER = (FN * 1.) / (FN + TP)

        BPCER_APCER1=mean(BPCER_APCER1_list)
        BPCER_APCER5=mean(BPCER_APCER5_list)
        BPCER_APCER10=mean(BPCER_APCER10_list)

        APCER1=mean(APCER1_list)
        APCER5=mean(APCER5_list)
        APCER10=mean(APCER10_list)

        if eer_list != []:
            EER = mean(eer_list)
        else:
            EER = -1000000000

        return loss_eval, acc, APCER, BPCER, EER, BPCER_APCER1, BPCER_APCER5, BPCER_APCER10, APCER1, APCER5, APCER10

def main():

    IMG_PATH = "/ctm-hdd-pool01/DB/LivDet2015/train/"

    print()
    if torch.cuda.is_available():
        DEVICE = torch.device("cuda:1")  # you can continue going on here, like cuda:1 cuda:2....etc.
        print("Running on the GPU")
    else:
        DEVICE = torch.device("cpu")
        print("Running on the CPU")

    mode = input("Enter the mode [train/test]: ")
    #DATASET = input("Dataset [CrossMatch/Digital_Persona/GreenBit/Hi_Scan/Time_Series]: ")
    unseen_ = input("Unseen attack? [y/n]: ")

    if unseen_ == "y":
        unseen = True
        NUM_ITERATIONS = 1
        attack_txt = "UA"
    elif unseen_ == "n":
        unseen = False
        NUM_ITERATIONS = 3
        attack_txt = "OA"
    else:
        sys.exit("Error ('Unseen attack?'): incorrect input!")

    sensors = ["CrossMatch", "Digital_Persona", "GreenBit", "Hi_Scan", "Time_Series"]

    for DATASET in sensors:

        print("\nRunning in " + DATASET + "...\n")

        if DATASET == "CrossMatch" or DATASET=="Time_Series":
            NUM_MATERIALS = 3
        else:
            NUM_MATERIALS = 4

        # For LOOP - Test splits
        train_results_ = []
        results = []
        best_epochs = []

        for iteration in range(NUM_ITERATIONS):

            print("\n-- ITERATION {}/{} --".format(iteration+1, NUM_ITERATIONS))

            for test_material in range(NUM_MATERIALS):

                output_fn = "results/" + DATASET + "/" + DATASET + "_" + str(test_material) + "_"
                model_path = "/ctm-hdd-pool01/afpstudents/jaf/VGG19bn_" + DATASET + "_" + str(test_material) + "_"

                model = models.vgg19_bn(pretrained=True)
                set_parameter_requires_grad(model, feature_extracting=False)
                num_ftrs = model.classifier[6].in_features
                model.classifier[6] = nn.Linear(num_ftrs,2)

                feature_extractor = torch.nn.Sequential(*list(model.children())[0:2])

                task_classifier = torch.nn.Sequential(*list(model.children())[2])

                adv_classifier = torch.nn.Sequential(*list(model.children())[2])
                num_ftrs = model.classifier[6].in_features
                adv_classifier[6] = nn.Linear(num_ftrs,3)

                feature_extractor = feature_extractor.to(DEVICE)
                task_classifier = task_classifier.to(DEVICE)
                adv_classifier = adv_classifier.to(DEVICE)

                # Train or test
                if mode == 'train':

                    (train_loader, valid_loader, test_loader) = get_data_loaders(IMG_PATH, DATASET, test_material, croped=True, unseen_attack=unseen)

                    # Fit model
                    train_history, _, best_epoch = fit(feature_extractor=feature_extractor,
                                                  task_classifier = task_classifier,
                                                  adv_classifier = adv_classifier,
                                                  data=(train_loader, valid_loader),
                                                  device=DEVICE,
                                                  model_path = model_path,
                                                  output=output_fn)

                    # save train history
                    train_res_fn = output_fn + "history.pckl"
                    pickle.dump(train_history, open(train_res_fn, "wb"))

                elif mode == 'test':
                    sys.exit("Error: in construction yet!")
                    '''
                    model.load_state_dict(torch.load(
                                          os.path.join(*(output_fn, 'mlp_fpad.pth'))))

                    # load train history
                    res_fn = os.path.join(*(output_fn, '_history.pckl'))
                    train_history = pickle.load(open(res_fn, "rb"))
                    plot_fn = os.path.join(*(output_fn, 'mlp_fpad_history.png'))
                    plot_train_history(train_history, plot_fn=plot_fn)
                    '''
                else:
                    sys.exit("Error: incorrect mode!")

                #Train results
                train_results = pickle.load(open(train_res_fn, "rb"))
                train_results_.append([train_results['train_acc'][EPOCHS-1], train_results['train_apcer'][EPOCHS-1], train_results['train_bpcer'][EPOCHS-1], train_results['train_eer'][EPOCHS-1], train_results['train_bpcer_apcer1'][EPOCHS-1], train_results['train_bpcer_apcer5'][EPOCHS-1], train_results['train_bpcer_apcer10'][EPOCHS-1], train_results['train_apcer1'][EPOCHS-1], train_results['train_apcer5'][EPOCHS-1], train_results['train_apcer10'][EPOCHS-1]])

                # Test results
                test_loss, test_acc, test_apcer, test_bpcer, test_eer, test_bpcer_apcer1, test_bpcer_apcer5, test_bpcer_apcer10, test_apcer1, test_apcer5, test_apcer10 = eval_model(feature_extractor, task_classifier, adv_classifier, test_loader, DEVICE)
                print('\nTest loss: {:.3f}            |'.format(test_loss.item()) + ' Test Acc: {:.3f}'.format(test_acc) + '\nTest APCER: {:.3f}           |'.format(test_apcer) + ' Test BPCER: {:.3f}'.format(test_bpcer))
                print('Test BPCER@APCER=1%: {:.3f}  | Test APCER1: {:.3f}'.format(test_bpcer_apcer1, test_apcer1))
                print('Test BPCER@APCER=5%: {:.3f}  | Test APCER5: {:.3f}'.format(test_bpcer_apcer5, test_apcer5))
                print('Test BPCER@APCER=10%: {:.3f} | Test APCER10: {:.3f}'.format(test_bpcer_apcer10, test_apcer10))
                print('Test EER: {:.3f}'.format(test_eer))
                results.append((test_loss.item(), test_acc, test_apcer, test_bpcer, test_eer, test_bpcer_apcer1, test_bpcer_apcer5, test_bpcer_apcer10, test_apcer1, test_apcer5, test_apcer10))

                best_epochs.append(best_epoch)

                # save results
                res_fn = output_fn + 'results.pckl'
                pickle.dump(results, open(res_fn, "wb"))
                results = pickle.load(open(res_fn, "rb"))

            # Compute average and std
            acc_array = np.array([i[1] for i in results])
            apcer_array = np.array([i[2] for i in results])
            bpcer_array = np.array([i[3] for i in results])
            eer_array = np.array([i[4] for i in results])
            bpcer_apcer1_array = np.array([i[5] for i in results])
            bpcer_apcer5_array = np.array([i[6] for i in results])
            bpcer_apcer10_array = np.array([i[7] for i in results])
            apcer1_array = np.array([i[8] for i in results])
            apcer5_array = np.array([i[9] for i in results])
            apcer10_array = np.array([i[10] for i in results])

            print('\n\n\n---------------------------------\n-------------------------------------------')
            print('Average Acc: {:.3f}             |'.format(np.mean(acc_array)) + ' Std: {:.3f}'.format(np.std(acc_array)))
            print('Average APCER: {:.3f}           |'.format(np.mean(apcer_array)) + ' Std: {:.3f}'.format(np.std(apcer_array)))
            print('Average BPCER: {:.3f}           |'.format(np.mean(bpcer_array)) + ' Std: {:.3f}'.format(np.std(bpcer_array)))
            print('Average EER: {:.3f}             |'.format(np.mean(eer_array)) + ' Std: {:.3f}'.format(np.std(eer_array)))
            print('Average BPCER@APCER=1%: {:.3f}  |'.format(np.mean(bpcer_apcer1_array)) + ' Std: {:.3f}'.format(np.std(bpcer_apcer1_array)))
            print('Average BPCER@APCER=5%: {:.3f}  |'.format(np.mean(bpcer_apcer5_array)) + ' Std: {:.3f}'.format(np.std(bpcer_apcer5_array)))
            print('Average BPCER@APCER=10%: {:.3f} |'.format(np.mean(bpcer_apcer10_array)) + ' Std: {:.3f}'.format(np.std(bpcer_apcer10_array)))
            print('Average APCER1: {:.3f}          |'.format(np.mean(apcer1_array)) + ' Std: {:.3f}'.format(np.std(apcer1_array)))
            print('Average APCER5: {:.3f}          |'.format(np.mean(apcer5_array)) + ' Std: {:.3f}'.format(np.std(apcer5_array)))
            print('Average APCER10: {:.3f}         |'.format(np.mean(apcer10_array)) + ' Std: {:.3f}'.format(np.std(apcer10_array)))

            #Best epochs
            print('\nBest epochs:', end=" ")
            for epoch in best_epochs:
                print(epoch, end="   ")

            #Results of all loops (train and test)
            np.set_printoptions(formatter={'float': '{: 0.3f}'.format})
            print()
            print()
            print("*** " + DATASET + " RESULTS ***")
            print()
            print(">>Train results [Acc, APCER, BPCER, EER, BPCER@APCER=1%, BPCER@APCER=5%, BPCER@APCER=10%, APCER1, APCER5, APCER10]")
            print()
            for k in range(NUM_MATERIALS):
                print(*train_results_[k], sep = ", ")

            print()
            print()
            print(">>Test results")

            results_test = []

            for j in range(len(list(acc_array))):
                res = []
                res.append(acc_array[j])
                res.append(apcer_array[j])
                res.append(bpcer_array[j])
                res.append(eer_array[j])
                res.append(bpcer_apcer1_array[j])
                res.append(bpcer_apcer5_array[j])
                res.append(bpcer_apcer10_array[j])
                res.append(apcer1_array[j])
                res.append(apcer1_array[j])
                res.append(apcer10_array[j])

                print(*res, sep = ", ")

                results_test.append(res)

            if iteration == NUM_ITERATIONS-1:
                np.savetxt(DATASET + '_' + attack_txt + '_test.txt', results_test, fmt='%.3f', delimiter=',')

            '''
            print()
            print()
            print(">>Test results")
            print()
            print("Acc:             {}".format(acc_array))
            print("APCER:           {}".format(apcer_array))
            print("BPCER:           {}".format(bpcer_array))
            print("EER:             {}".format(eer_array))
            print("BPCER@APCER=1%:  {}".format(bpcer_apcer1_array))
            print("BPCER@APCER=5%:  {}".format(bpcer_apcer5_array))
            print("BPCER@APCER=10%: {}".format(bpcer_apcer10_array))
            print("APCER1:          {}".format(apcer1_array))
            print("APCER5:          {}".format(apcer5_array))
            print("APCER10:         {}".format(apcer10_array))
            '''

    print("\n\nDONE!")

if __name__ == '__main__':
    main()


import glob
import os

from lxml import etree
from lxml.etree import LxmlError

from SpiffWorkflow.bpmn.specs.bpmn_process_spec import BpmnProcessSpec
from SpiffWorkflow.bpmn.specs.defaults import (
    UserTask,
    ManualTask,
    NoneTask,
    ScriptTask,
    ServiceTask,
    CallActivity,
    SubWorkflowTask,
    TransactionSubprocess,
    InclusiveGateway,
    ExclusiveGateway,
    ParallelGateway,
    StartEvent,
    EndEvent,
    IntermediateCatchEvent,
    IntermediateThrowEvent,
    SendTask,
    ReceiveTask,
    BoundaryEvent,
    EventBasedGateway
)
from SpiffWorkflow.bpmn.specs.event_definitions.simple import NoneEventDefinition
from SpiffWorkflow.bpmn.specs.event_definitions.timer import TimerEventDefinition
from SpiffWorkflow.bpmn.specs.mixins.subworkflow_task import SubWorkflowTask as SubWorkflowTaskMixin
from SpiffWorkflow.bpmn.specs.mixins.events.start_event import StartEvent as StartEventMixin

from .ValidationException import ValidationException
from .ProcessParser import ProcessParser
from .node_parser import DEFAULT_NSMAP
from .spec_description import SPEC_DESCRIPTIONS
from .util import full_tag, xpath_eval, first
from .TaskParser import TaskParser
from .task_parsers import (
    GatewayParser,
    ConditionalGatewayParser,
    CallActivityParser,
    ScriptTaskParser,
    SubWorkflowParser,
)
from .event_parsers import (
    EventBasedGatewayParser,
    StartEventParser, EndEventParser,
    BoundaryEventParser,
    IntermediateCatchEventParser,
    IntermediateThrowEventParser,
    SendTaskParser,
    ReceiveTaskParser
)

XSD_PATH = os.path.join(os.path.dirname(__file__), 'schema', 'BPMN20.xsd')

class BpmnValidator:

    def __init__(self, xsd_path=XSD_PATH, imports=None):
        with open(xsd_path) as xsd:
            schema = etree.parse(xsd)
        if imports is not None:
            for ns, fn in imports.items():
                elem = etree.Element(
                    '{http://www.w3.org/2001/XMLSchema}import',
                    namespace=ns,
                    schemaLocation=fn
                )
                schema.getroot().insert(0, elem)
        self.validator = etree.XMLSchema(schema)

    def validate(self, bpmn, filename=None):
        try:
            self.validator.assertValid(bpmn)
        except ValidationException as ve:
            ve.file_name = filename
            ve.line_number = self.validator.error_log.last_error.line
        except LxmlError:
            last_error = self.validator.error_log.last_error
            raise ValidationException(last_error.message, file_name=filename,
                                      line_number=last_error.line)

class BpmnParser(object):
    """
    The BpmnParser class is a pluggable base class that manages the parsing of
    a set of BPMN files. It is intended that this class will be overriden by an
    application that implements a BPMN engine.

    Extension points: OVERRIDE_PARSER_CLASSES provides a map from full BPMN tag
    name to a TaskParser and Task class. PROCESS_PARSER_CLASS provides a
    subclass of ProcessParser. DATA_STORE_CLASSES provides a mapping of names to
    subclasses of BpmnDataStoreSpecification that provide a data store
    implementation.
    """

    PARSER_CLASSES = {
        full_tag('startEvent'): (StartEventParser, StartEvent),
        full_tag('endEvent'): (EndEventParser, EndEvent),
        full_tag('userTask'): (TaskParser, UserTask),
        full_tag('task'): (TaskParser, NoneTask),
        full_tag('subProcess'): (SubWorkflowParser, SubWorkflowTask),
        full_tag('manualTask'): (TaskParser, ManualTask),
        full_tag('exclusiveGateway'): (ConditionalGatewayParser, ExclusiveGateway),
        full_tag('parallelGateway'): (GatewayParser, ParallelGateway),
        full_tag('inclusiveGateway'): (ConditionalGatewayParser, InclusiveGateway),
        full_tag('callActivity'): (CallActivityParser, CallActivity),
        full_tag('transaction'): (SubWorkflowParser, TransactionSubprocess),
        full_tag('scriptTask'): (ScriptTaskParser, ScriptTask),
        full_tag('serviceTask'): (TaskParser, ServiceTask),
        full_tag('intermediateCatchEvent'): (IntermediateCatchEventParser, IntermediateCatchEvent),
        full_tag('intermediateThrowEvent'): (IntermediateThrowEventParser, IntermediateThrowEvent),
        full_tag('boundaryEvent'): (BoundaryEventParser, BoundaryEvent),
        full_tag('receiveTask'): (ReceiveTaskParser, ReceiveTask),
        full_tag('sendTask'): (SendTaskParser, SendTask),
        full_tag('eventBasedGateway'): (EventBasedGatewayParser, EventBasedGateway),
    }

    OVERRIDE_PARSER_CLASSES = {}

    PROCESS_PARSER_CLASS = ProcessParser

    DATA_STORE_CLASSES = {}

    def __init__(self, namespaces=None, validator=None, spec_descriptions=SPEC_DESCRIPTIONS):
        """
        Constructor.
        """
        self.namespaces = namespaces or DEFAULT_NSMAP
        self.validator = validator
        self.spec_descriptions = spec_descriptions
        self.process_parsers = {}
        self.collaborations = {}
        self.process_dependencies = set()
        self.messages = {}
        self.correlations = {}
        self.data_stores = {}

    def _get_parser_class(self, tag):
        if tag in self.OVERRIDE_PARSER_CLASSES:
            return self.OVERRIDE_PARSER_CLASSES[tag]
        elif tag in self.PARSER_CLASSES:
            return self.PARSER_CLASSES[tag]
        return None, None

    def get_process_parser(self, process_id):
        """
        Returns the ProcessParser for the given process ID or name. It matches
        by name first.
        """
        if process_id in self.process_parsers:
            return self.process_parsers[process_id]

    def get_process_ids(self):
        """Returns a list of process IDs"""
        return list(self.process_parsers.keys())

    def add_bpmn_file(self, filename):
        """
        Add the given BPMN filename to the parser's set.
        """
        self.add_bpmn_files([filename])

    def add_bpmn_files_by_glob(self, g):
        """
        Add all filenames matching the provided pattern (e.g. *.bpmn) to the
        parser's set.
        """
        self.add_bpmn_files(glob.glob(g))

    def add_bpmn_files(self, filenames):
        """
        Add all filenames in the given list to the parser's set.
        """
        for filename in filenames:
            with open(filename, 'r') as f:
                self.add_bpmn_io(f, filename)

    def add_bpmn_io(self, file_like_object, filename=None):
        """
        Add the given BPMN file like object to the parser's set.
        """
        self.add_bpmn_xml(etree.parse(file_like_object), filename)

    def add_bpmn_str(self, bpmn_str, filename=None):
        """
        Add the given BPMN string to the parser's set.
        """
        self.add_bpmn_xml(etree.fromstring(bpmn_str), filename)

    def add_bpmn_xml(self, bpmn, filename=None):
        """
        Add the given lxml representation of the BPMN file to the parser's set.

        :param svg: Optionally, provide the text data for the SVG of the BPMN
          file
        :param filename: Optionally, provide the source filename.
        """
        if self.validator:
            self.validator.validate(bpmn, filename)

        # we need to parse the data stores before _add_process since it creates
        # the parser instances, which need to know about the data stores to
        # resolve data references.
        self._add_data_stores(bpmn)

        self._add_processes(bpmn, filename)
        self._add_collaborations(bpmn)
        self._add_messages(bpmn)
        self._add_correlations(bpmn)

    def _add_processes(self, bpmn, filename=None):
        for process in bpmn.xpath('.//bpmn:process', namespaces=self.namespaces):
            self._find_dependencies(process)
            self.create_parser(process, filename)

    def _add_collaborations(self, bpmn):
        collaboration = first(bpmn.xpath('.//bpmn:collaboration', namespaces=self.namespaces))
        if collaboration is not None:
            collaboration_xpath = xpath_eval(collaboration)
            name = collaboration.get('id')
            self.collaborations[name] = [ participant.get('processRef') for participant in collaboration_xpath('.//bpmn:participant') ]

    def _add_messages(self, bpmn):
        for message in bpmn.xpath('.//bpmn:message', namespaces=self.namespaces):
            if message.attrib.get("id") is None:
                raise ValidationException(
                    "Message identifier is missing from bpmn xml"
                )
            self.messages[message.attrib.get("id")] = message.attrib.get("name")

    def _add_correlations(self, bpmn):
        for correlation in bpmn.xpath('.//bpmn:correlationProperty', namespaces=self.namespaces):
            correlation_identifier = correlation.attrib.get("id")
            if correlation_identifier is None:
                raise ValidationException("Correlation identifier is missing from bpmn xml")
            correlation_property_retrieval_expressions = correlation.xpath(
                ".//bpmn:correlationPropertyRetrievalExpression", namespaces = self.namespaces)
            if not correlation_property_retrieval_expressions:
                raise ValidationException(
                    f"Correlation is missing correlation property retrieval expressions: {correlation_identifier}"
                )
            retrieval_expressions = []
            for cpre in correlation_property_retrieval_expressions:
                message_model_identifier = cpre.attrib.get("messageRef")
                if message_model_identifier is None:
                    raise ValidationException(
                        f"Message identifier is missing from correlation property: {correlation_identifier}"
                    )
                children = cpre.getchildren()
                expression = children[0].text if len(children) > 0 else None
                retrieval_expressions.append({"messageRef": message_model_identifier,
                                             "expression": expression})
            self.correlations[correlation_identifier] = {
                "name": correlation.attrib.get("name"),
                "retrieval_expressions": retrieval_expressions
            }

    def _add_data_stores(self, bpmn):
        for data_store in bpmn.xpath('.//bpmn:dataStore', namespaces=self.namespaces):
            data_store_id = data_store.attrib.get("id")
            if data_store_id is None:
                raise ValidationException(
                    "Data Store identifier is missing from bpmn xml"
                )
            data_store_name = data_store.attrib.get("name")
            if data_store_name is None:
                raise ValidationException(
                    "Data Store name is missing from bpmn xml"
                )
            if data_store_name not in self.DATA_STORE_CLASSES:
                raise ValidationException(
                    f"Data Store with name {data_store_name} has no implementation"
                )
            data_store_spec = self.DATA_STORE_CLASSES[data_store_name](
                data_store_id,
                data_store_name,
                data_store.attrib.get('capacity'),
                data_store.attrib.get('isUnlimited'))
            self.data_stores[data_store_id] = data_store_spec

    def _find_dependencies(self, process):
        """Locate all calls to external BPMN, and store their ids in our list of dependencies"""
        for call_activity in process.xpath('.//bpmn:callActivity', namespaces=self.namespaces):
            self.process_dependencies.add(call_activity.get('calledElement'))

    def create_parser(self, node, filename=None, lane=None):
        parser = self.PROCESS_PARSER_CLASS(self, node, self.namespaces, self.data_stores, filename=filename, lane=lane)
        if parser.bpmn_id in self.process_parsers:
            raise ValidationException(f'Duplicate process ID: {parser.bpmn_id}', node=node, file_name=filename)
        self.process_parsers[parser.bpmn_id] = parser

    def get_process_dependencies(self):
        return self.process_dependencies

    def get_spec(self, process_id, required=True):
        """
        Parses the required subset of the BPMN files, in order to provide an
        instance of BpmnProcessSpec (i.e. WorkflowSpec)
        for the given process ID or name. The Name is matched first.
        """
        parser = self.get_process_parser(process_id)
        if required and parser is None:
            raise ValidationException(
                f"The process '{process_id}' was not found. "
                f"Did you mean one of the following: "
                f"{', '.join(self.get_process_ids())}?")
        elif parser is not None:
            return parser.get_spec()

    def get_subprocess_specs(self, name, specs=None, require_call_activity_specs=True):
        used = specs or {}
        wf_spec = self.get_spec(name)
        for task_spec in wf_spec.task_specs.values():
            if isinstance(task_spec, SubWorkflowTaskMixin) and task_spec.spec not in used:
                subprocess_spec = self.get_spec(task_spec.spec, required=require_call_activity_specs)
                used[task_spec.spec] = subprocess_spec
                if subprocess_spec is not None:
                    self.get_subprocess_specs(task_spec.spec, used)
        return used

    def find_all_specs(self):
        # This is a little convoluted, but we might add more processes as we generate
        # the dictionary if something refers to another subprocess that we haven't seen.
        processes = dict((id, self.get_spec(id)) for id in self.get_process_ids())
        while sorted(processes.keys()) != sorted(self.process_parsers.keys()):
            for process_id in tuple(self.process_parsers.keys()):
                processes[process_id] = self.get_spec(process_id)
        return processes

    def get_collaboration(self, name):
        self.find_all_specs()
        spec = BpmnProcessSpec(name)
        subprocesses = {}
        participant_type = self._get_parser_class(full_tag('callActivity'))[1]
        start_type = self._get_parser_class(full_tag('startEvent'))[1]
        end_type = self._get_parser_class(full_tag('endEvent'))[1]
        start = start_type(spec, 'Start Collaboration', NoneEventDefinition())
        spec.start.connect(start)
        end = end_type(spec, 'End Collaboration', NoneEventDefinition())
        end.connect(spec.end)
        for process in self.collaborations[name]:
            process_parser = self.get_process_parser(process)
            if process_parser and process_parser.process_executable:
                sp_spec = self.get_spec(process)
                subprocesses[process] = sp_spec
                subprocesses.update(self.get_subprocess_specs(process))
                if len([s for s in sp_spec.task_specs.values() if
                        isinstance(s, StartEventMixin) and
                        isinstance(s.event_definition, (NoneEventDefinition, TimerEventDefinition))
                ]):
                    participant = participant_type(spec, process, process)
                    start.connect(participant)
                    participant.connect(end)
        return spec, subprocesses


from __future__ import annotations

import logging
import os
import re
import shutil
from typing import Any, Callable, Sequence, Sized

import torch
from accelerate import Accelerator
from torch.optim import Optimizer
from torch.utils.data import DataLoader
from tqdm.auto import tqdm

try:
    from torch.optim.lr_scheduler import LRScheduler
except ImportError:
    from torch.optim.lr_scheduler import _LRScheduler as LRScheduler

logger = logging.getLogger(__name__)

class Trainer:
    def __init__(
        self,
        *,
        model: torch.nn.Module,
        train_dataloader: DataLoader,
        optimizer: Optimizer,
        accelerator: Accelerator,
        validation_dataloader: DataLoader | None = None,
        epochs: int = 3,
        lr_scheduler: LRScheduler | None = None,
        log_interval: int = 50,
        save_on_epoch_end: bool = True,
        epoch_end_callbacks: Sequence[Callable[['Trainer'], None]] | None = None,
    ):
        self.model = model
        self.optimizer = optimizer
        self.train_dataloader = train_dataloader
        self.validation_dataloader = validation_dataloader
        self.lr_scheduler = lr_scheduler
        self.accelerator = accelerator
        self.epochs = epochs
        self.log_interval = log_interval
        self.save_on_epoch_end = save_on_epoch_end

        self.train_loss_tracker = LossTracker()
        self.validation_loss_tracker = LossTracker()
        if isinstance(self.train_dataloader.dataset, Sized):
            num_steps_per_epoch = len(self.train_dataloader)
        else:
            num_steps_per_epoch = None
        self.progress_bar = DistributedTqdmProgressBar(self.epochs, num_steps_per_epoch=num_steps_per_epoch)
        self.epoch_end_callbacks = epoch_end_callbacks or []
        self.current_step = 0

    def train(self):
        for current_epoch in range(1, self.epochs + 1):
            self.model.train()
            self.progress_bar.on_epoch_start()

            for batch_index, batch in enumerate(self.train_dataloader):
                with self.accelerator.accumulate(self.model):
                    self.optimizer.zero_grad()
                    batch_output = self.model(**batch)
                    loss = batch_output['loss']
                    self.accelerator.backward(loss)
                    self.optimizer.step()
                    if self.lr_scheduler is not None:
                        self.lr_scheduler.step()
                    self.train_loss_tracker.update(loss)

                self.progress_bar.update()
                self.current_step += 1
                if batch_index % self.log_interval == 0:
                    self.log_metrics(
                        {'loss': self.train_loss_tracker.loss},
                        step=self.current_step,
                    )

            train_metrics = self.add_prefix({'loss': self.train_loss_tracker.loss}, 'train')
            self.accelerator.log(train_metrics, step=current_epoch)
            self.train_loss_tracker.on_epoch_end()
            self.progress_bar.on_epoch_end()

            if self.validation_dataloader:
                validation_loss = evaluate(
                    self.model,
                    self.validation_dataloader,
                    self.validation_loss_tracker,
                )
                validation_metrics = self.add_prefix({'loss': validation_loss}, 'validation')
                self.accelerator.print(f'Epoch {current_epoch} Validation loss: {validation_loss:.4f}')
                self.accelerator.log(validation_metrics, step=current_epoch)

            if self.save_on_epoch_end:
                self.accelerator.save_state(self.get_checkpoint_dir())

            if self.epoch_end_callbacks:
                for callback in self.epoch_end_callbacks:
                    callback(self)

        self.accelerator.end_training()

    def log_metrics(self, metrics: dict[str, float], step: int):
        self.accelerator.log(metrics, step=step)
        self.progress_bar.show_metrics(metrics)

    @staticmethod
    def add_prefix(values: dict[str, Any], prefix: str):
        return {f'{prefix}/{k}': v for k, v in values.items()}

    def get_checkpoint_dir(self):
        # COPY FROM accelerator to fix Checkpoint bug
        self.accelerator.project_configuration.automatic_checkpoint_naming = False
        output_dir = os.path.join(self.accelerator.project_dir, 'checkpoints')
        if self.accelerator.is_local_main_process:
            os.makedirs(output_dir, exist_ok=True)
            folders = [os.path.join(output_dir, folder) for folder in os.listdir(output_dir)]
            if self.accelerator.project_configuration.total_limit is not None and (
                len(folders) + 1 > self.accelerator.project_configuration.total_limit
            ):

                def _inner(folder):
                    return list(map(int, re.findall(r'[\/]?([0-9]+)(?=[^\/]*$)', folder)))[0]

                folders.sort(key=_inner)
                logger.warning(
                    f'Deleting {len(folders) + 1 - self.accelerator.project_configuration.total_limit}'
                    'checkpoints to make room for new checkpoint.'
                )
                for folder in folders[: len(folders) + 1 - self.accelerator.project_configuration.total_limit]:
                    shutil.rmtree(folder)

        output_dir = os.path.join(output_dir, f'checkpoint_{self.accelerator.save_iteration}')
        if self.accelerator.is_local_main_process:
            os.makedirs(output_dir, exist_ok=True)
        logger.info(f'Saving current state to {output_dir}')
        return output_dir

def evaluate(
    model: torch.nn.Module,
    dataloader: DataLoader,
    loss_tracker: LossTracker | None = None,
):
    model = model.eval()
    loss_tracker = loss_tracker or LossTracker()
    for batch in dataloader:
        with torch.inference_mode():
            batch_output = model(**batch)
            loss_tracker.update(batch_output['loss'])
    loss = loss_tracker.loss
    loss_tracker.on_epoch_end()
    return loss

class DummyProgressBar:
    def update(self, n: int = 1) -> None:
        pass

    def close(self) -> None:
        pass

    def set_description(self, description: str) -> None:
        pass

class DistributedTqdmProgressBar:
    def __init__(self, epochs: int, num_steps_per_epoch: int | None, **kwargs) -> None:
        self.accelerator = Accelerator()
        self.epochs = epochs
        self.current_epoch = 1
        self.num_steps_per_epoch = num_steps_per_epoch
        self.tqdm_kwargs = kwargs

    def on_epoch_start(self):
        if self.accelerator.is_main_process:
            self.progress_bar = tqdm(total=self.num_steps_per_epoch, **self.tqdm_kwargs)
        else:
            self.progress_bar = DummyProgressBar()

    def update(self, n: int = 1) -> None:
        self.progress_bar.update(n)

    def close(self) -> None:
        self.progress_bar.close()

    def on_epoch_end(self) -> None:
        self.current_epoch += 1
        self.progress_bar.close()

    def show_metrics(self, metrics: dict[str, float]) -> None:
        description = f'Epoch {self.current_epoch}/{self.epochs}'
        for name, score in metrics.items():
            description += f' - {name}: {score:.4f}'
        self.progress_bar.set_description(description)

class LossTracker:
    def __init__(
        self,
        ndigits=4,
    ) -> None:
        self.ndigits = ndigits
        self._loss: float = 0.0
        self.loss_count: int = 0
        self.history: list[float] = []

    def update(self, loss_tensor: torch.Tensor):
        loss = loss_tensor.item()
        self._loss = (self._loss * self.loss_count + loss) / (self.loss_count + 1)
        self.loss_count += 1

    def reset(self):
        self._loss = 0
        self.loss_count = 0

    def on_epoch_end(self, reset: bool = True):
        self.history.append(self.loss)
        if reset:
            self.reset()

    @property
    def loss(self) -> float:
        return round(float(self._loss), self.ndigits)


import cv2
import numpy as np
import pandas as pd
from tqdm import tqdm
from scipy.spatial.distance import directed_hausdorff
from itertools import combinations
from .utils import (is_black_image, create_overlapping_image,
                    scale_down, draw_hausdorff_lines, read_binary,
                    make_hyperlink, create_overlapping_images,
                    _max_dim_diff)
import pyximport
pyximport.install()
from .distance import get_distance_matrix

def get_canny_edge(img:np.ndarray, threshold1=0, threshold2=255,
                **kwargs)->np.ndarray:
    """
    Extract line by Canny Edge Method.
    """
    cnt = cv2.Canny(img, threshold1, threshold2)
    return np.argwhere(cnt>0)

def get_contour(img:np.ndarray, retrieve_mode:int=cv2.RETR_TREE,
               approximation:int=cv2.CHAIN_APPROX_SIMPLE, **kwargs)->np.ndarray:

    """
    Extract line by cv2.findContours().
    """
    cnt, _ = cv2.findContours(img1, retrieve_mode, approximation)
    r, _, c = cnt[0].shape
    cnt_line = cnt[0].reshape(r,c)

    return cnt_line

def get_raw_points(img:np.ndarray, **kwargs)->np.ndarray:
    """
    Extract every point from binary image.
    """
    return np.argwhere(img>0)

def get_thin_line(img:np.ndarray, **kwargs)->np.ndarray:
    """
    Extract line by thinning method.
    """
    return np.argwhere(cv2.ximgproc.thinning(img) >0)

def average_hausdorff(segmentation: np.ndarray,
                      ground_truth: np.ndarray,
                      balanced:bool=False) -> float:

    """
    Implementation of O.U. Aydin, A.A. Taha, A. Hilbert, A.A Khalil,
    I. Galinovic, J.B. Fiebach, D. Frey, V.I. Madai. (2021).
    On the usage of average Hausdorff distance for segmentation
    performance assessment: hidden error when used for ranking.

    """

    S = segmentation.shape[0]
    G = ground_truth.shape[0]
    segmentation=np.asarray(segmentation, dtype=np.int32, order='c')
    ground_truth=np.asarray(ground_truth, dtype=np.int32, order='c')
    distance_matrix = get_distance_matrix(segmentation, ground_truth)
    StoG = distance_matrix.min(axis=0).sum()
    GtoS = distance_matrix.min(axis=1).sum()

    if balanced:
        return ((StoG / G) + (GtoS / G)) / 2

    return ((StoG / S) + (GtoS) / G) /2

def hausdorff_distance(img1:np.ndarray, img2: np.ndarray,
                      extraction_method:str='canny',
                      retrieve_mode:int=cv2.RETR_EXTERNAL,
                      approximation:int=cv2.CHAIN_APPROX_TC89_KCOS,
                      threshold1=0,
                      threshold2=255,
                      **kwargs):

    extraction_methods = {
        "thin" : get_thin_line,
        "canny" : get_canny_edge,
        "contour" : get_contour,
        "raw_points" : get_raw_points
    }

    na = (-1.0, (-1.0,(0,0),(0,0)), (-1.0, (0,0), (0,0)))
    ignore_error = kwargs.get('ignore_error')
    tolerance = kwargs.get('tolerance')
    index = kwargs.get('index')
    raters = kwargs.get('raters')
    point_threshold = kwargs.get('point_threshold')
    average= kwargs.get('average')
    balanced = kwargs.get('balanced') if kwargs.get('balanced') else False

    if ignore_error:
        if ((is_black_image(img1))
        or (is_black_image(img2))):
            return na

        if img1.shape != img2.shape:
            if tolerance is None:
                return na

            tol_pixel, dimension = tolerance
            pixel_difference = _max_dim_diff(img1, img2, dimension)

            if pixel_difference > tol_pixel:
                return na

    else:
        assert img1.shape == img2.shape, \
        f"""Unequal image size at {index}:\n
        ``{'Image 1' if not raters else raters[0]}`'s image has size {img1.shape}\n
        while `{'Image 2' if not raters else raters[1]}`'s image has size {img2.shape}\n"""

    line1 = extraction_methods[extraction_method](img1,
                                                  retrieve_mode=retrieve_mode,
                                                  approximation=approximation,
                                                  threshold1=threshold1,
                                                  threshold2=threshold2)
    line2 = extraction_methods[extraction_method](img2,
                                                  retrieve_mode=retrieve_mode,
                                                  approximation=approximation,
                                                  threshold1=threshold1,
                                                  threshold2=threshold2)

    if ignore_error:
        if not (line1).all() or not (line2).all():
            return na

    if point_threshold:
        if (len(line1.ravel()) < point_threshold
         or len(line2.ravel()) < point_threshold):
            return na

    hAB, hBA = directed_hausdorff(line1,line2), directed_hausdorff(line2,line1)

    if average:
        hd, AtoB, BtoA = (average_hausdorff(line1, line2, balanced),
                          hAB[1:], hBA[1:])
    else:
        hd, AtoB, BtoA = max(hAB[0], hBA[0]), hAB[1:], hBA[1:]

    return (hd,
            (hAB[0], tuple(line1[AtoB[0]][::-1]), tuple(line2[AtoB[1]][::-1])),
            (hBA[0], tuple(line2[BtoA[0]][::-1]), tuple(line1[BtoA[1]][::-1])))
    # Deprecated
    # return hd

def hausdorff_distances(df:pd.DataFrame, **kwargs)->pd.DataFrame:

    """
    Get Hausdorff Distance from all images in a DataFrame with image path.
    """

    assert len(df.columns) > 1, \
    "DataFrame must contain more than 1 rater."

    if kwargs.get('ignore_inconsistent_name') == False:
        assert not inconsistent_name(df), \
        "DataFrame contains inconsistent names. Please check file names again."

    repeated_image = kwargs.get('repeated_image')
    ignore_error = kwargs.get('ignore_error')
    point_threshold = kwargs.get('point_threshold')
    tolerance = kwargs.get('tolerance')
    output_dir = kwargs.get('output_dir')
    create_overlapping_image = kwargs.get('create_overlapping_image')
    average = kwargs.get('average')
    balanced = kwargs.get('balanced')
    extraction_method = kwargs.get('extraction_method')

    hd_df = pd.DataFrame()
    temp_dfs = []
    tqdm.pandas()
    hyperlink_df = None

    if repeated_image:
        indices = df.index.values
    else:
        indices = np.vectorize(lambda x: x.name)(df[df.columns[0]].values)

    for comb in combinations(df.columns, 2):
        carrier_df = pd.DataFrame()
        rater_a, rater_b = comb
        print(f'Comparing {rater_a} and {rater_b} ...')
        temp = df.progress_apply(
        lambda x : hausdorff_distance(read_binary(x[rater_a]),
                                      read_binary(x[rater_b]),
                                      index=x.name,
                                      extraction_method=extraction_method,
                                      raters=(rater_a, rater_b),
                                      ignore_error=ignore_error,
                                      point_threshold=point_threshold,
                                      tolerance=tolerance,
                                      average=average,
                                      balanced=balanced),
                                      axis = 1).apply(pd.Series)
        carrier_df[['hd', 'hAB', 'hBA']] = temp
        hd_df[f"Hausdorff_Distance-{rater_a}-{rater_b}"] = carrier_df['hd']

        if create_overlapping_image:
            if not kwargs.get('output_dir'): continue
            print('creating overlapped images....')
            distances = carrier_df[['hAB', 'hBA']].apply(tuple, axis=1)
            paths = create_overlapping_images(df[rater_a],
                                              df[rater_b],
                                              output_dir=output_dir,
                                              indices=indices,
                                              raters=(rater_a, rater_b),
                                              distances=distances,
                                              channels='rgb')

            temp_df = pd.DataFrame(hd_df[f"Hausdorff_Distance-{rater_a}-{rater_b}"]
                                    .astype('string') + "#" + paths)
            temp_df.fillna('-1.0#0', inplace=True)

            temp_df = temp_df.applymap(lambda x : make_hyperlink(x))
            temp_dfs.append(temp_df)

    if create_overlapping_image:
        hyperlink_df = pd.concat(temp_dfs, axis=1)
        hyperlink_df = hyperlink_df.set_index(indices)

    return hd_df.set_index(indices), hyperlink_df


import os
import json
import random

#os.system('clear')
directorios = [
    './categorias/ciencias',
    './categorias/entretenimiento',
    './categorias/finanzas',
    './categorias/salud',
    './categorias/tecnologia'
]

def obtener_carpetas(directorio):
    try:
        elementos = os.listdir(directorio)
        carpetas = [os.path.join(directorio, elemento).replace('.', '') for elemento in elementos if os.path.isdir(os.path.join(directorio, elemento)) and elemento != '__pycache__']
        return carpetas
    except OSError as e:
        print(f"[Barlow (ERROR); Error al acceder a la ruta {directorio}. {e}]")
        return []

diccionario_carpetas = {}
for i, directorio in enumerate(directorios, start=1):
    carpetas = obtener_carpetas(directorio)
    clave = f'cat{i}'
    carpetas_aleatorias = random.sample(carpetas, len(carpetas))
    diccionario_carpetas[clave] = carpetas_aleatorias

with open('./categorias/logs.json', 'w', encoding='utf-8') as archivo:
    json.dump(diccionario_carpetas, archivo, ensure_ascii=False)
print("[Barlow; Fichero 'logs.json' creado correctamente].")


from rest_framework import serializers

from SocialNetwork.models import Post

class PostSerializer(serializers.ModelSerializer):
    url = serializers.SerializerMethodField(read_only=True)

    class Meta:
        model = Post
        fields = [
            'url',
            'pk',
            'user',
            'title',
            'content',
            'timestamp',
            'likes'
        ]
        read_only_fields = ['pk', 'user']

    def get_url(self, obj):
        request = self.context.get("request")
        return obj.get_api_url(request=request)

    def validate_title(self, value):
        query = Post.objects.filter(title__iexact=value)
        if self.instance:
            query = query.exclude(pk=self.instance.pk)
        if query.exists():
            raise serializers.ValidationError("This title has already been used")
        return value


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models
import django.db.models.deletion
from django.conf import settings

class Migration(migrations.Migration):

    dependencies = [
        ('base', '0010_add_rabbitmq_fields'),
    ]

    operations = [
        migrations.AlterField(
            model_name='controller',
            name='organization',
            field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, blank=True, to='organizations.Organization', null=True),
        ),
        migrations.AlterField(
            model_name='controller',
            name='owner',
            field=models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=django.db.models.deletion.PROTECT),
        ),
    ]


from django.contrib import admin
from .models import TextField

# Register your models here.

class TextFieldAdmin(admin.ModelAdmin):
    list_display=(
        'name',
        'about_title',
        'about_blurb'
    )

    ordering = (
        'name',
    )

admin.site.register(TextField, TextFieldAdmin)


import geopandas as gpd

if __name__ == "__main__":
    DRIVE_FILE_ID = "1kdqgi94qhdEVvOJAvlpHvurYcurh_5d9"
    url = f"https://drive.google.com/uc?export=download&id={DRIVE_FILE_ID}"

    df_kgiop_objects = gpd.read_file(url)
    print(df_kgiop_objects.head())

    DRIVE_FILE_ID = "107DtdB8wUehAFoASn-rtQ1jokVam04FN"
    url = f"https://drive.google.com/uc?export=download&id={DRIVE_FILE_ID}"

    df_roads = gpd.read_file(url)
    df_roads = df_roads.set_index("id")
    print(df_roads.head())

    K_MEAN_POINTS = 5
    mean_distance = df_roads.geometry.apply(
        lambda road: df_kgiop_objects.distance(road).nsmallest(K_MEAN_POINTS).mean())

    mean_distance.to_json("k_mean_points_distance.json")


import sys,pygame

# from pygame. locals import *
pygame.init()
screen_info = pygame.display.Info()
# set the width and height to the size of screen
size = (width, height) = ((int(screen_info.current_w), int(screen_info.current_h)))
screen = pygame.display.set_mode(size)
clock = pygame.time.Clock()
color = (110, 170, 255)

fish_image = pygame.image.load("fish.png")
fish_image = pygame.transform.smoothscale(fish_image,(280, 160))

fish_rect = fish_image.get_rect()

fish_rect.center = (width//2, height//2)

speed=pygame.math.Vector2(0,0)

def movefish():
    global color
    global fish_image
    size = (width, hight) = (int(screen_info.current_w), int(screen_info.current_h))
    fish_rect.move_ip(speed)

    fish_rect.left < 0
    fish_rect.right
    fish_rect.top

    if fish_rect.bottom > height:
        speed[1] *= -1

        color = (255,125, 20)
    if fish_rect.top < (0):
        speed[1] *= -1
        color = (255, 120, 120)
    if fish_rect.right > width:
        speed[0] *= -1
        color = (0,255, 120)
        fish_image = pygame.transform.flip(fish_image, True, False)
    if fish_rect.left < (0):
        speed[0] *= -1
        color = (110, 170, 255)
        fish_image = pygame.transform.flip(fish_image, True, False)
def main():

    while True:

        global speed
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                pygame.quit(); sys.exit()
                main = fales

            if event.type == pygame.KEYDOWN:
                if event.key == pygame.K_LEFT or event.key == ord('a'):
                    print ('left')
                    speed[0] =(-2)
                if event.key == pygame.K_RIGHT or event.key == ord('d'):
                    print ('right')
                    speed[0] =(2)
                if event.key == pygame.K_UP or event.key == ord('w'):
                    print ('jump')
                    speed[1] =(-2)
                if event.key == pygame.K_DOWN or event.key == ord('s'):
                    print ('down')
                    speed[1] =(2)
            if event.type == pygame.KEYUP:
                if event.key == pygame.K_LEFT or event.key == ord('a'):
                    print('left stop')
                    speed[0] = (0)
                if event.key == pygame.K_RIGHT or event.key == ord('d'):
                    print('right stop')
                    speed[0] = (0)
                if event.key == pygame.K_UP or event.key == ord('w'):
                    print('up stop')
                    speed[1] = (0)
                if event.key == pygame.K_DOWN or event.key == ord('s'):
                    print('down stop')
                    speed[1] = (0)

        clock.tick(60)
        movefish()
        screen.fill(color)
        screen.blit(fish_image,fish_rect)
        pygame.display .flip()

if __name__ == '__main__':
    main()


import numpy as np
import torch
from scipy import io as sio
import binvox_rw
import matplotlib.pyplot as plt
from PIL import Image
import projtransform
import numpy
from stl import mesh

def cuboid3D_to_unitbox3D(cub3D):
    device = cub3D.device
    dst = torch.tensor(
        [[-1.0, -1.0], [1.0, -1.0], [-1.0, 1.0], [1.0, 1.0]], device=device, dtype=torch.float32
    )
    dst = dst.view(1, 4, 2).expand(cub3D.shape[0], -1, -1)
    # for (x,z) plane
    txz = projtransform.ProjectiveTransform()
    src = torch.stack(
        [
            torch.stack([cub3D[:, 0, 0], cub3D[:, 4, 0]], dim=1),
            torch.stack([cub3D[:, 0, 1], cub3D[:, 4, 0]], dim=1),
            torch.stack([cub3D[:, 2, 0], cub3D[:, 4, 1]], dim=1),
            torch.stack([cub3D[:, 2, 1], cub3D[:, 4, 1]], dim=1),
        ],
        dim=1,
    )
    if not txz.estimate(src, dst):
        raise ValueError("Estimate failed")
    # for (y,z) plane
    tyz = projtransform.ProjectiveTransform()
    src = torch.stack(
        [
            torch.stack([cub3D[:, 1, 0], cub3D[:, 4, 0]], dim=1),
            torch.stack([cub3D[:, 1, 1], cub3D[:, 4, 0]], dim=1),
            torch.stack([cub3D[:, 3, 0], cub3D[:, 4, 1]], dim=1),
            torch.stack([cub3D[:, 3, 1], cub3D[:, 4, 1]], dim=1),
        ],
        dim=1,
    )
    if not tyz.estimate(src, dst):
        raise ValueError("Estimate failed")
    return txz, tyz

def box2D_to_cuboid3D(zranges, Ks, boxes, im_sizes):
    device = boxes.device

    if boxes.shape[0] != Ks.shape[0] != zranges.shape[0]:
        raise ValueError("Ks, boxes and zranges must have the same batch dimension")
    if zranges.shape[1] != 2:
        raise ValueError("zrange must have two entries per example")
    w, h = im_sizes.t()
    sx, px, py = Ks.t()
    sy = sx
    x1, y1, x2, y2 = boxes.t()
    # transform 2d box from image coordinates to world coordinates
    x1 = w - 1 - x1 - px
    y1 = h - 1 - y1 - py
    x2 = w - 1 - x2 - px
    y2 = h - 1 - y2 - py

    cub3D = torch.zeros((boxes.shape[0], 5, 2), device=device, dtype=torch.float32)
    for i in range(2):
        z = zranges[:, i]
        x3D_min = x2 * z / sx
        x3D_max = x1 * z / sx
        y3D_min = y2 * z / sy
        y3D_max = y1 * z / sy
        cub3D[:, i * 2 + 0, 0] = x3D_min
        cub3D[:, i * 2 + 0, 1] = x3D_max
        cub3D[:, i * 2 + 1, 0] = y3D_min
        cub3D[:, i * 2 + 1, 1] = y3D_max
    cub3D[:, 4, 0] = zranges[:, 0]
    cub3D[:, 4, 1] = zranges[:, 1]
    return cub3D

def transform_verts(verts, R, t):
    """
    Transforms verts with rotation R and translation t
    Inputs:
        - verts (tensor): of shape (N, 3)
        - R (tensor): of shape (3, 3) or None
        - t (tensor): of shape (3,) or None
    Outputs:
        - rotated_verts (tensor): of shape (N, 3)
    """

    rot_verts = verts.clone().t()

    if R is not None:
       # R[0][2] *= -1
       # R[1][2] *= -1
       # R[2][2] *= -1

        assert R.dim() == 2
        assert R.size(0) == 3 and R.size(1) == 3
        rot_verts = torch.mm(R, rot_verts)

    if t is not None:
        assert t.dim() == 1
        assert t.size(0) == 3
        rot_verts = rot_verts + t.unsqueeze(1)

    rot_verts = rot_verts.t()

    return rot_verts

def read_voxel(voxelfile):
    """
    Reads voxel and transforms it in the form of verts
    """
    #voxel = sio.loadmat(voxelfile)["voxel"]

    #print("DEBUGGING READ_VOXEL FROM SHAPE.PY")

    with open(voxelfile, 'rb') as f:
            voxel = binvox_rw.read_as_3d_array(f)

    #print("voxel dimensions: ", voxel.dims)

    voxel = voxel.data
    voxel = np.array( voxel > 0, dtype=int)
    #print("voxel dimensions after np.array(voxel > 0): ", voxel.shape)

    #voxel = np.rot90(voxel, k=3, axes=(1, 2))
    #print("voxel dimensions after np.rot90: ", voxel.shape)

    verts = np.argwhere(voxel > 0).astype(np.float32, copy=False)
    #print("verts after np.argwhere(voxel > 0: ",verts.shape)

    # centering and normalization

    #normalize pixel values
    min_x = np.min(verts[:, 0])
    max_x = np.max(verts[:, 0])
    min_y = np.min(verts[:, 1])
    max_y = np.max(verts[:, 1])
    min_z = np.min(verts[:, 2])
    max_z = np.max(verts[:, 2])

    #centralize pixel values
    verts[:, 0] = verts[:, 0] - (max_x + min_x) / 2
    verts[:, 1] = verts[:, 1] - (max_y + min_y) / 2
    verts[:, 2] = verts[:, 2] - (max_z + min_z) / 2
    scale = np.sqrt(np.max(np.sum(verts ** 2, axis=1))) * 2
    verts /= scale

    verts = torch.tensor(verts, dtype=torch.float32)
    #print("verts final: ", verts.size())

    #print("-- END -- DEBUGGING READ_VOXEL FROM SHAPE.PY")

    return verts