import discord
import json
import properties
import random
import urllib.request
import bookService
import gameService

client = discord.Client()
giphyApiBase = "http://api.giphy.com/v1/gifs/"
giphyApiKey = properties.giphyToken

memberToNotify = "snacks#7277"
fillScreenImageUrl = "https://i.imgur.com/jlFXbsF.png"
aw_yeahUrl = "https://media1.giphy.com/media/3nozJyPYl195u/giphy.gif"

botName = "neatbot"

#Triggers
class Triggers:
    gifTriggerString = "gif me"
    notifyTriggerString = "pubg"
    rollTriggerString = "!roll"
    topTrendingGifTrigger = "top trending gif"
    randomTrendingGifTrigger = "random trending gif"

#Files - These are used to track state
memberDataFile = "memberData.json"
bookDataFile = "data.json"

def getActiveMembers():
    activeMembers = []
    for member in client.get_all_members():
        if member.status == discord.Status.online and not member.bot:
            activeMembers.append(member)
    return activeMembers

def getJson(url):
    response = urllib.request.urlopen(url)
    data = json.load(response)
    return data

def getGif(searchTerm):
    params = urllib.parse.urlencode({"tag":searchTerm,"rating":"G"})
    gifData = getJson(giphyApiBase + "random?api_key=" + giphyApiKey + "&" + params)
    gifUrl = gifData["data"]["url"]
    return gifUrl

def getTrendingGif(gifNumber, maxRecords):
    params = urllib.parse.urlencode({"limit":maxRecords,"rating":"G"})
    gifData = getJson(giphyApiBase + "trending?api_key=" + giphyApiKey + "&" + params)
    gifUrl = gifData["data"][gifNumber]["url"]
    return gifUrl

def getRandomTrendingGif():
    trendingGifNumber = random.randint(1, 100) - 1
    return getTrendingGif(trendingGifNumber, 100)

def roll(maxRoll):
    return random.randint(1, maxRoll)

def help():
    keys = [i for i in Triggers.__dict__.keys() if i[:1] != '_']
    return [Triggers.__dict__[x] for x in keys]

@client.event
async def on_ready():
    print('Logged in as')
    print(client.user.name)
    print(client.user.id)
    print('------')

@client.event
async def on_message(message):
    if message.author.bot:
        return

    userName = message.author.name
    response = ""
    cmd = message.content.lower()

    if cmd == 'hey ' + botName:
        response = getGif("hello")
    elif cmd == 'aw_yeah':
        response = aw_yeahUrl
    elif cmd.startswith(Triggers.gifTriggerString):
        searchTerm = ""
        if len(message.content) > len(Triggers.gifTriggerString):
            searchTerm = message.content[len(Triggers.gifTriggerString):].lstrip()
        response = getGif(searchTerm)
    elif cmd.startswith(Triggers.topTrendingGifTrigger):
        response = getTrendingGif(0, "1")
    elif cmd.startswith(Triggers.randomTrendingGifTrigger):
        response = getRandomTrendingGif()
    elif cmd.startswith(Triggers.rollTriggerString):
        maxRoll = 100
        if (len(message.content) > len(Triggers.rollTriggerString)):
            maxRoll = int(cmd[len(Triggers.rollTriggerString):].lstrip())

        response = roll(maxRoll)
    elif cmd == 'randomwinner':
        highRoll = 0
        winner = ""
        for member in getActiveMembers():
            currRoll = roll(100)
            if currRoll > highRoll:
                highRoll = currRoll
                winner = member.name

        response = winner + " wins with a roll of " + str(highRoll)
    elif cmd == 'fillscreen':
        response = fillScreenImageUrl
    elif Triggers.notifyTriggerString in cmd:
        channel = message.channel
        member = channel.server.get_member_named(memberToNotify)
        # Only notify if the member is offline
        if member not in getActiveMembers():
            await message.channel.send(member, 'PUBG was mentioned in ' + channel.name)
    elif cmd.startswith(bookService.Triggers.addBookTrigger):
        if len(message.content) > len(bookService.Triggers.addBookTrigger):
            bookName = message.content[len(bookService.Triggers.addBookTrigger):].lstrip()
            response = bookService.addBookToList(bookName)
    elif cmd == bookService.Triggers.listBooksTrigger:
        response = bookService.listBooks()
    elif cmd.startswith(bookService.Triggers.removeBookTrigger):
        if len(message.content) > len(bookService.Triggers.removeBookTrigger):
            bookName = message.content[len(bookService.Triggers.removeBookTrigger):].lstrip()
            response = bookService.removeBookFromList(bookName)
    elif cmd.startswith(bookService.Triggers.assignBookTrigger):
        if len(message.content) > len(bookService.Triggers.assignBookTrigger):
            bookName = message.content[len(bookService.Triggers.assignBookTrigger):].lstrip()
            response = bookService.assignBook(bookName, userName)
    elif cmd == bookService.Triggers.assignRandomBookTrigger:
        response = bookService.Triggers.bookService.assignRandomBook(userName)
    elif cmd == bookService.Triggers.unassignBookTrigger:
        response = bookService.unassignBook(userName)
    elif cmd == bookService.Triggers.getMyBookTrigger:
        response = bookService.getAssignedBook(userName)
    elif cmd == bookService.Triggers.getAllBooksTrigger:
        response = bookService.getAllAssignedBooks()
    elif "help" in cmd and "book" in cmd:
        response = bookService.bookHelp()
    elif cmd.startswith(botName) and "help" in cmd:
        response = bookService.bookHelp() + help() + gameService.gameHelp()
    #gameService
    elif cmd.startswith(gameService.Triggers.addGameTrigger):
        if len(message.content) > len(gameService.Triggers.addGameTrigger):
            gameName = message.content[len(gameService.Triggers.addGameTrigger):].lstrip()
            response = gameService.addGameToList(gameName)
    elif cmd.startswith(gameService.Triggers.removeGameTrigger):
        if len(message.content) > len(gameService.Triggers.removeGameTrigger):
            gameName = message.content[len(gameService.Triggers.removeGameTrigger):].lstrip()
            response = gameService.removeGameFromList(gameName)
    elif cmd.startswith(gameService.Triggers.listGamesTrigger):
            response = gameService.listGames()
    elif cmd.startswith(gameService.Triggers.gameHelp):
        response = gameService.gameHelp()
    if response != "":
        await message.channel.send(response)

@client.event
async def on_member_update(before, after):
    response = ""

    # The member started or changed games
    # if before.game is None and after.game is not None:
        # Disabling the response for now to avoid annoyance
        #response = "Hey " + after.name + ", have fun playing " + after.game.name + ". Thanks for inviting the rest of us."

    if response != "":
        #this doesn't work with discord > 1 - I'm not sure it's used, kinda assuming not due to the server name? funny, haha?
        await client.send_message(
            discord.utils.get(client.get_all_channels(), server__name='Steamy Pile', name='general'), response)

client.run(properties.neatbotToken)


'''
Given a non-empty array of integers, every element appears twice except for one. Find that single one.

Note:

Your algorithm should have a linear runtime complexity. Could you implement it without using extra memory?

Example 1:

Input: [2,2,1]
Output: 1
Example 2:

Input: [4,1,2,1,2]
Output: 4
'''

from unittest import TestCase
from typing import List

def singleNumber(nums: List[int]) -> int:
    ret = 0
    for n in nums:
        ret ^= n

    return ret

if __name__ == "__main__":
    test_case = TestCase()
    test_case.assertEqual(23^23^8, 8)
    test_case.assertEqual(singleNumber([3,4,4,4,4]), 3)


import re
import sys

from http import HTTPStatus

def eprint(*args, **kwargs):
    print(*args, file=sys.stderr, **kwargs)

def die(message, code=1):
    eprint(message)
    sys.exit(code)

class HTTPHeader:
    req_line_pat = re.compile(r"([A-Z]+) (\S+) (HTTP\/\d+\.\d+)")
    res_line_pat = re.compile(r"(HTTP/\d+\.\d+) ((\d{3}) (.+))")  # e.g. HTTP/1.1 404 Not Found
    url_pat = re.compile(r"https?:\/\/(\S+?)?(\/(\S+)?)")
    tunn_pat = re.compile(r"(\S+):\d+")

    @classmethod
    def parse(cls, header_str, is_response=False):
        request = {}
        response = {}
        headers = {}

        lines = header_str.split("\r\n")
        for line in lines:
            if not len(line.strip()):
                continue
            match_req = cls.req_line_pat.match(line)
            match_res = cls.res_line_pat.match(line)
            if match_req:
                request["method"] = match_req[1]
                request["url"] = match_req[2]
                request["version"] = match_req[3]

                if request["url"].startswith("http://") or request["url"].startswith("https://"):
                    url_match = cls.url_pat.match(request["url"])
                    request["hostname"] = url_match[1]
                    request["path"] = url_match[2]
                else:
                    tunn_match = cls.tunn_pat.match(request["url"])
                    request["hostname"] = tunn_match[1]
            elif match_res:
                response["version"] = match_res[1]
                response["status"] = {
                    "code": int(match_res[3]),  # 404
                    "phrase": match_res[4],     # "Not Found"
                    "message": match_res[2],    # "404 Not Found"
                }
            else:
                [key, value] = line.split(": ")
                if key == "Proxy-Connection":
                    key = "Connection"
                headers[key] = value

        # print("[HTTPHeader.parse]", request, headers)
        if is_response:
            return response, headers
        else:
            return request, headers

    @classmethod
    def generate(cls, request, headers):
        lines = []
        if request["method"] == "CONNECT":
            return
            # lines.append("%s %s:443 %s".format(request["method"], request["hostname"], request["version"]))
        else:
            lines.append("{} {} {}".format(request["method"], request["path"], request["version"]))
        if not "Host" in headers:
            lines.append("Host: {}".format(request["hostname"]))
        for key in headers:
            lines.append(f"{key}: {headers[key]}")
        return "\r\n".join(lines)

def get_parts(data: bytes):
    if TERMB not in data:
        return None
    return data.split(TERMB)

def get_headers(data: bytes):
    parts = get_parts(data)
    if parts is None:
        return None
    return HTTPHeader.parse(parts[0])

def get_body(data: bytes):
    parts = get_parts(data)
    if parts is None:
        return None
    return parts[1]

def get_status_message(code: int):
    return str(code) + " " + HTTPStatus(code).phrase

def dict_get_insensitive(d: dict, key, default=None):
    for k in d:
        if k.lower() == key.lower():
            return d[k]
    return default


from contextlib import contextmanager
from datetime import datetime
import logging
import os
import threading
from typing import Optional, Tuple
from urllib.parse import urlparse, urlunparse

import cachetools
from pympler.asizeof import asizeof
from sqlalchemy import and_, bindparam, Column, create_engine, DateTime, String, VARCHAR
from sqlalchemy.ext import baked
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
from sqlalchemy_utils import create_database, database_exists

from lookout.core.analyzer import AnalyzerModel
from lookout.core.metrics import record_event
from lookout.core.model_repository import ModelRepository
from lookout.core.ports import Type

Base = declarative_base()

class Model(Base):
    """The only object stored in the database - trained model metadata."""

    __tablename__ = "models"
    analyzer = Column(String(40), primary_key=True)
    repository = Column(String(40 + 100), primary_key=True)
    path = Column(VARCHAR)
    updated = Column(DateTime(timezone=True), default=datetime.utcnow)

class ContextSessionMaker:
    """
    Adds the `__enter__()`/`__exit__()` to an SQLAlchemy session and thus automatically closes it.
    """

    def __init__(self, factory):  # noqa: 102
        self.factory = factory

    @contextmanager
    def __call__(self):  # noqa: D102
        session = self.factory()
        try:
            yield session
        finally:
            session.close()

class SQLAlchemyModelRepository(ModelRepository):
    """Stores models in file system and their metadata in any database supported by SQLAlchemy."""

    MAX_SUBDIRS = 1024
    _log = logging.getLogger("SQLAlchemyModelRepository")

    def __init__(self, db_endpoint: str, fs_root: str, max_cache_mem: int, ttl: int,
                 engine_kwargs: dict=None):
        """
        Initialize a new instance of SQLAlchemyModelRepository.

        :param db_endpoint: SQLAlchemy connection string.
        :param fs_root: Root directory where to store the models.
        :param max_cache_mem: Maximum memory size to use for model cache (in bytes).
        :param ttl: Time-to-live for each model in the cache (in seconds).
        :param engine_kwargs: Passed directly to SQLAlchemy's `create_engine()`.
        """
        self.fs_root = fs_root
        # A version of db_endpoint that never contain password is needed for logging
        db_endpoint_components = urlparse(db_endpoint)
        if db_endpoint_components.password is not None:
            password, netloc = db_endpoint_components.password, db_endpoint_components.netloc
            password_index = netloc.rindex(password)
            safe_netloc = "%s%s%s" % (
                db_endpoint_components.netloc[:password_index],
                "<PASSWORD>",
                db_endpoint_components.netloc[password_index + len(password):])
            safe_db_endpoint_components = list(db_endpoint_components)
            safe_db_endpoint_components[1] = safe_netloc
            self._safe_db_endpoint = urlunparse(safe_db_endpoint_components)
        else:
            self._safe_db_endpoint = db_endpoint
        must_initialize = not database_exists(db_endpoint)
        if must_initialize:
            self._log.debug("%s does not exist, creating", self._safe_db_endpoint)
            create_database(db_endpoint)
            self._log.warning("created a new database at %s", self._safe_db_endpoint)
        self._engine = create_engine(
            db_endpoint, **(engine_kwargs if engine_kwargs is not None else {}))
        must_initialize |= not self._engine.has_table(Model.__tablename__)
        if must_initialize:
            Model.metadata.create_all(self._engine)
        self._sessionmaker = ContextSessionMaker(sessionmaker(bind=self._engine))
        bakery = baked.bakery()
        self._get_query = bakery(lambda session: session.query(Model))
        self._get_query += lambda query: query.filter(
            and_(Model.analyzer == bindparam("analyzer"),
                 Model.repository == bindparam("repository")))
        self._cache = cachetools.TTLCache(maxsize=max_cache_mem, ttl=ttl, getsizeof=asizeof)
        self._cache_lock = threading.Lock()

    def __repr__(self) -> str:
        """Represent the model repository as a eval()-able string."""
        return "SQLAlchemyModelRepository(db_endpoint=%r, fs_root=%r, max_cache_mem=%r, " \
               "ttl=%r)" % (self._safe_db_endpoint, self.fs_root, self._cache.maxsize,
                            self._cache.ttl)

    def __str__(self) -> str:
        """Summarize the model repository as a string."""
        return "SQLAlchemyModelRepository(db=%s, fs=%s)" % (self._safe_db_endpoint, self.fs_root)

    def get(self, model_id: str, model_type: Type[AnalyzerModel],
            url: str) -> Tuple[Optional[AnalyzerModel], bool]:  # noqa: D102
        cache_key = self.cache_key(model_id, model_type, url)
        record_event("SQLAlchemyModelRepository.cache.length", len(self._cache))
        record_event("SQLAlchemyModelRepository.cache.size", self._cache.currsize)
        with self._cache_lock:
            model = self._cache.get(cache_key)
        if model is not None:
            self._log.debug("used cache for %s with %s", model_id, url)
            record_event("SQLAlchemyModelRepository.cache.hit", 1)
            return model, False
        record_event("SQLAlchemyModelRepository.cache.miss", 1)
        with self._sessionmaker() as session:
            models = self._get_query(session).params(analyzer=model_id, repository=url).all()
        if len(models) == 0:
            self._log.debug("no models found for %s with %s", model_id, url)
            return None, True
        model = model_type().load(models[0].path)
        with self._cache_lock:
            self._cache[cache_key] = model
        self._log.debug("loaded %s with %s from %s", model_id, url, models[0].path)
        return model, True

    def set(self, model_id: str, url: str, model: AnalyzerModel):  # noqa: D102
        path = self.store_model(model, model_id, url)
        with self._sessionmaker() as session:
            session.merge(Model(analyzer=model_id, repository=url, path=path))
            session.commit()
        self._log.debug("set %s with %s", model_id, url)

    def init(self):  # noqa: D102
        self._log.info("initializing")
        Model.metadata.drop_all(self._engine)
        Model.metadata.create_all(self._engine)
        os.makedirs(self.fs_root, exist_ok=True)

    def shutdown(self):  # noqa: D102
        self._log.debug("shutting down")
        self._cache.clear()
        self._engine.dispose()

    @staticmethod
    def split_url(url: str):
        """Explode a Git remote URL into FS-friendly pieces."""
        if url.endswith(".git"):
            url = url[:-4]
        return url[url.find("://") + 3:].split("/")

    @staticmethod
    def cache_key(model_id: str, model_type: Type[AnalyzerModel], url: str) -> str:
        """Compose the cache key for the given model and Git remote."""
        return model_id + "_" + model_type.__name__ + "_" + url

    def store_model(self, model: AnalyzerModel, model_id: str, url: str) -> str:
        """
        Save the model on disk.

        :param model: Instance of the model to save.
        :param model_id: The key of the model (based on the bound analyzer name and version).
        :param url: Git repository remote.
        :return: The path to the saved model.
        """
        url_parts = self.split_url(url)
        if url_parts[0] == "github" or url_parts[0] == "bitbucket":
            url_parts = url_parts[:2] + [url_parts[2][:2]] + url_parts[2:]
        path = os.path.join(self.fs_root, *url_parts, "%s.asdf" % model_id.replace("/", "_"))
        model.save(path)
        return path


"""Document store."""

from collections import defaultdict
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Sequence

from dataclasses_json import DataClassJsonMixin

from gpt_index.constants import TYPE_KEY
from gpt_index.data_structs.node_v2 import ImageNode, IndexNode, Node
from gpt_index.readers.schema.base import Document
from gpt_index.schema import BaseDocument

@dataclass
class DocumentStore(DataClassJsonMixin):
    """Document store."""

    docs: Dict[str, BaseDocument] = field(default_factory=dict)
    ref_doc_info: Dict[str, Dict[str, Any]] = field(
        default_factory=lambda: defaultdict(dict)
    )

    def serialize_to_dict(self) -> Dict[str, Any]:
        """Serialize to dict."""
        docs_dict = {}
        for doc_id, doc in self.docs.items():
            doc_dict = doc.to_dict()
            doc_dict[TYPE_KEY] = doc.get_type()
            docs_dict[doc_id] = doc_dict
        return {"docs": docs_dict, "ref_doc_info": self.ref_doc_info}

    @classmethod
    def load_from_dict(
        cls,
        docs_dict: Dict[str, Any],
    ) -> "DocumentStore":
        """Load from dict."""
        docs_obj_dict = {}
        for doc_id, doc_dict in docs_dict["docs"].items():
            doc_type = doc_dict.pop(TYPE_KEY, None)
            doc: BaseDocument
            if doc_type == "Document" or doc_type is None:
                doc = Document.from_dict(doc_dict)
            elif doc_type == Node.get_type():
                doc = Node.from_dict(doc_dict)
            elif doc_type == ImageNode.get_type():
                doc = ImageNode.from_dict(doc_dict)
            elif doc_type == IndexNode.get_type():
                doc = IndexNode.from_dict(doc_dict)
            else:
                raise ValueError(f"Unknown doc type: {doc_type}")

            docs_obj_dict[doc_id] = doc
        return cls(
            docs=docs_obj_dict,
            ref_doc_info=defaultdict(dict, **docs_dict.get("ref_doc_info", {})),
        )

    @classmethod
    def from_documents(cls, docs: Sequence[BaseDocument]) -> "DocumentStore":
        """Create from documents."""
        obj = cls()
        obj.add_documents(docs)
        return obj

    @classmethod
    def merge(cls, docstores: Sequence["DocumentStore"]) -> "DocumentStore":
        merged_docstore = cls()
        for docstore in docstores:
            merged_docstore.update_docstore(docstore)
        return merged_docstore

    def update_docstore(self, other: "DocumentStore") -> None:
        """Update docstore."""
        self.docs.update(other.docs)

    def add_documents(
        self, docs: Sequence[BaseDocument], allow_update: bool = False
    ) -> None:
        """Add a document to the store."""
        for doc in docs:
            if doc.is_doc_id_none:
                raise ValueError("doc_id not set")

            # NOTE: doc could already exist in the store, but we overwrite it
            if not allow_update and self.document_exists(doc.get_doc_id()):
                raise ValueError(
                    f"doc_id {doc.get_doc_id()} already exists. "
                    "Set allow_update to True to overwrite."
                )
            self.docs[doc.get_doc_id()] = doc
            self.ref_doc_info[doc.get_doc_id()]["doc_hash"] = doc.get_doc_hash()

    def get_document(
        self, doc_id: str, raise_error: bool = True
    ) -> Optional[BaseDocument]:
        """Get a document from the store."""
        doc = self.docs.get(doc_id, None)
        if doc is None and raise_error:
            raise ValueError(f"doc_id {doc_id} not found.")
        return doc

    def set_document_hash(self, doc_id: str, doc_hash: str) -> None:
        """Set the hash for a given doc_id."""
        self.ref_doc_info[doc_id]["doc_hash"] = doc_hash

    def get_document_hash(self, doc_id: str) -> Optional[str]:
        """Get the stored hash for a document, if it exists."""
        return self.ref_doc_info[doc_id].get("doc_hash", None)

    def document_exists(self, doc_id: str) -> bool:
        """Check if document exists."""
        return doc_id in self.docs

    def delete_document(
        self, doc_id: str, raise_error: bool = True
    ) -> Optional[BaseDocument]:
        """Delete a document from the store."""
        doc = self.docs.pop(doc_id, None)
        self.ref_doc_info.pop(doc_id, None)
        if doc is None and raise_error:
            raise ValueError(f"doc_id {doc_id} not found.")
        return doc

    def get_nodes(self, node_ids: List[str], raise_error: bool = True) -> List[Node]:
        """Get nodes from docstore."""
        return [self.get_node(node_id, raise_error=raise_error) for node_id in node_ids]

    def get_node(self, node_id: str, raise_error: bool = True) -> Node:
        """Get node from docstore."""
        doc = self.get_document(node_id, raise_error=raise_error)
        if not isinstance(doc, Node):
            raise ValueError(f"Document {node_id} is not a Node.")
        return doc

    def get_node_dict(self, node_id_dict: Dict[int, str]) -> Dict[int, Node]:
        """Get node dict from docstore given a mapping of index to node ids."""
        return {
            index: self.get_node(node_id) for index, node_id in node_id_dict.items()
        }


#!/usr/bin/env python3
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import os.path
import json
from time import sleep

import RPi.GPIO as GPIO

import google.oauth2.credentials
from google.assistant.library import Assistant
from google.assistant.library.event import EventType
from google.assistant.library.file_helpers import existing_file

PIN_BTN = 17
PIN_LED = 22
ASSISTANT = None

def callback(channel):
    global CREDENTIALS
    if not CREDENTIALS:
        return
    print("callback")
    assistant = Assistant(CREDENTIALS)
    ret = assistant.start_conversation()
    print(ret)

GPIO.setmode(GPIO.BCM)
GPIO.setup(PIN_BTN, GPIO.IN, GPIO.PUD_UP)
GPIO.setup(PIN_LED, GPIO.OUT)
GPIO.output(PIN_LED, GPIO.LOW)

def callback_start_conversation(channel):
    global ASSISTANT
    if not ASSISTANT:
        return
    ASSISTANT.start_conversation()

def process_event(event):
    if event.type == EventType.ON_CONVERSATION_TURN_STARTED:
        print('Google Assistant is listening to you now.')
        GPIO.output(PIN_LED, GPIO.HIGH)
    if event.type == EventType.ON_RECOGNIZING_SPEECH_FINISHED:
        print("You:", event.args['text'])
    if event.type == EventType.ON_ALERT_STARTED:
        print('An alert has started sounding.')
    if event.type == EventType.ON_ALERT_FINISHED:
        print('The alert has finished sounding.')
    if (event.type == EventType.ON_CONVERSATION_TURN_FINISHED and
            event.args and event.args['with_follow_on_turn'] == True):
        #print('Google Assistant is listening to your response.')
        pass
    if event.type == EventType.ON_END_OF_UTTERANCE:
        print('Google Assistant may not have understood what you has said')
    if (event.type == EventType.ON_CONVERSATION_TURN_FINISHED and
            event.args and event.args['with_follow_on_turn'] == False):
        print('Conversation finished.')
        GPIO.output(PIN_LED, GPIO.LOW)

def main():
    global ASSISTANT

    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument('--credentials', type=existing_file,
                        metavar='OAUTH2_CREDENTIALS_FILE',
                        default=os.path.join(
                            os.path.expanduser('~/.config'),
                            'google-oauthlib-tool',
                            'credentials.json'
                        ),
                        help='Path to store and read OAuth2 credentials')
    args = parser.parse_args()
    with open(args.credentials, 'r') as f:
        credentials = google.oauth2.credentials.Credentials(token=None,
                                                            **json.load(f))
    ASSISTANT = Assistant(credentials)
    GPIO.add_event_detect(PIN_BTN, GPIO.FALLING,
                callback=callback_start_conversation, bouncetime=300)
    for event in ASSISTANT.start():
        process_event(event)

    while True:
        pass

if __name__ == '__main__':
    try:
        main()
    except KeyboardInterrupt:
        print("Google Assistant finised.")
        GPIO.cleanup()


# To run this code, you must download the data folder
# or first run figure3.m.

import warnings
warnings.filterwarnings("ignore")
import scipy.io as sio
import matplotlib.pyplot as plt
import numpy as np
from matplotlib import cm
import matplotlib.patches as mpatches
from set_style import set_style

set_style('article', w=1, h=2.5)

filename = np.array([['../data/figure4/bias_d', '../data/figure5/std_d', '../data/figure6/rmse_d'], ['../data/figure4/bias_e', '../data/figure5/std_e', '../data/figure6/rmse_e'], ['../data/figure4/bias_f', '../data/figure5/std_f', '../data/figure6/rmse_f'], ['../data/figure3/bias', '../data/figure3/std', '../data/figure3/rmse']])

panel = np.array([['A', 'B', 'C'], ['D', 'E', 'F'], ['G', 'H', 'I'], ['J', 'K', 'L']])

fig, axarr = plt.subplots(4,3)

# BIAS
for i in range(0,4):
    j = 0

    data = sio.loadmat(filename[i,j])
    x = data['H_est'][0]
    y = data['H_est'][0]
    X, Y = np.meshgrid(x, y)
    Z = data['scaled_bias']
    cmap = cm.bwr

    cset1 = axarr[i,j].imshow(Z, extent=[min(x),max(x),min(y),max(y)], origin='lower', cmap=cm.get_cmap(cmap))
    if i == 0:
        cbar = fig.colorbar(cset1, ax=axarr[i,j], ticks=np.arange(-10, 11, 5), extend='both')
        cbar.ax.set_yticklabels(['-10', '-5', '0', '5', '10'])
        cset1.set_clim(-10, 10)
    else:
        cbar = fig.colorbar(cset1, ax=axarr[i,j], ticks=np.arange(-100, 101, 50), extend='both')
        cbar.ax.set_yticklabels(['-100', '-50', '0', '50', '100'])
        cset1.set_clim(-100, 100)
    if i == 0:
        axarr[i,0].set_ylabel('$\hat{d}\mathrm{_q}=%d$' % data['d_q'])
    else:
        axarr[i,0].set_ylabel('$\hat{d}\mathrm{_q}=%.2f$' % data['d_q'])

# STD
for i in range(0,4):
    j = 1

    data = sio.loadmat(filename[i,j])
    x = data['H_est'][0]
    y = data['H_est'][0]
    X, Y = np.meshgrid(x, y)
    Z = data['scaled_std']
    cmap = cm.Reds

    cset1 = axarr[i,j].imshow(Z, extent=[min(x),max(x),min(y),max(y)], origin='lower', cmap=cm.get_cmap(cmap))

    if i == 0:
        cset1.set_clim(1e3,1e5)
        cbar = fig.colorbar(cset1, ax=axarr[i,j], ticks=[1e3, 1e5], extend='both')
        cbar.ax.set_yticklabels(['$10^3$', '$10^5$'])
    elif i == 1:
        cset1.set_clim(20,50)
        fig.colorbar(cset1, ax=axarr[i,j], ticks=np.arange(20, 50.1, 10), extend='both')
    elif i == 2:
        cset1.set_clim(2,5)
        fig.colorbar(cset1, ax=axarr[i,j], ticks=np.arange(2, 5.1, 1), extend='both')
    elif i == 3:
        cset1.set_clim(0,2)
        fig.colorbar(cset1, ax=axarr[i,j], ticks=np.arange(0, 2.1, 0.5), extend='max')

# RMSE
for i in range(0,4):
    j = 2

    a = sio.loadmat(filename[i,j])
    x = a['H_est'][0]
    y = a['H_est'][0]
    X, Y = np.meshgrid(x, y)
    Z = a['scaled_rmse']
    cmap = cm.Reds

    cset1 = axarr[i,j].imshow(Z, extent=[min(x),max(x),min(y),max(y)], origin='lower', cmap=cm.get_cmap(cmap))

    if i == 0:
        cset1.set_clim(1e3,1e5)
        cbar = fig.colorbar(cset1, ax=axarr[i,j], ticks=[1e3, 1e5], extend='both')
        cbar.ax.set_yticklabels(['$10^3$', '$10^5$'])
    else:
        cset1.set_clim(0, 100)
        fig.colorbar(cset1, ax=axarr[i,j], ticks=np.arange(0, 100.1, 25), extend='max')

for i in range(0,4):
    for j in range(0,3):
        # axes
        axarr[i][j].get_xaxis().tick_bottom()
        axarr[i][j].get_yaxis().tick_left()
        axarr[i,j].set_aspect('equal', 'box')
        # mark vessel
        vessel = mpatches.Circle((max(x)/2,max(y)/2), 6./141, facecolor='black')
        axarr[i,j].add_patch(vessel)
        # ABC
        if j == 0:
            axarr[i,j].text(-0.2, 1.1, panel[i,j], transform=axarr[i,j].transAxes, fontsize=16, fontweight='bold', va='top', ha='right')
        else:
            axarr[i,j].text(-0.1, 1.1, panel[i,j], transform=axarr[i,j].transAxes, fontsize=16, fontweight='bold', va='top', ha='right')

# axes
for ax in axarr.flat:
    ax.set(xticks=np.arange(max(x)/2-1, max(x)/2+1.01, 1), yticks=np.arange(max(y)/2-1, max(y)/2+1.01, 1))
    ax.set(xticklabels=['-1', '0', '1'])
    ax.set(yticklabels=['-1', '0', '1'])
for i in range(0,3):
    for j in range(0,3):
        plt.setp(axarr[i,j].get_xticklabels(), visible=False)
for i in range(0,4):
    for j in range(1,3):
        plt.setp(axarr[i,j].get_yticklabels(), visible=False)

axarr[0,0].set_title('BIAS $[\%]$')
axarr[0,1].set_title('SD $[\%]$')
axarr[0,2].set_title('RMSE $[\%]$')

plt.tight_layout()
plt.savefig('figures_pdf/figure3.pdf', dpi=300)


from datetime import datetime
from datetime import timedelta
import json
import urllib
import boto3
import logging
import time
import os
from aws_xray_sdk.core import xray_recorder
from aws_xray_sdk.core import patch_all

patch_all()

AthQueryLambdaName = os.getenv('AthQueryLambdaName')

logger = logging.getLogger()
logger.setLevel(logging.INFO)

print('Loading function')

client = boto3.client('lambda')

payload = {}

def handler(event, context):
    try:
        xray_recorder.begin_subsegment('handler')
        xray_recorder.current_subsegment().put_annotation('Lambda', context.function_name)
        rekEmotions = ["HAPPY","SAD","ANGRY","CONFUSED","DISGUSTED","SURPRISED","CALM","FEAR"]
        for emotion in rekEmotions:
            query_str = """
            with selfie_emotions as (
              select face_id, image_url, first_name, last_name, (agerange.low + agerange.high)/2 as age,
                gender.value as gender_value, emotion.type as etype,
                round(emotion.confidence,3) as confidence,
                bbox_left, bbox_top, bbox_width, bbox_height,
                imgWidth, imgHeight, updated_at
              from default.selfie_reports TABLESAMPLE BERNOULLI (50)
              cross JOIN UNNEST(emotions) as t(emotion)
              where emotion.type = '%s'
              order by confidence desc
              LIMIT 1
            )
            select * from selfie_emotions
            """ % (emotion)

            payload["type"] = emotion
            payload["query"] = query_str

            response = client.invoke(
                FunctionName=AthQueryLambdaName,
                InvocationType='Event',
                Payload=json.dumps(payload)
            )

            logger.info(response)

        return {'result': True }

    except Exception as e:
        logger.error('Something went wrong: ' + str(e))
        return {'result': False, 'msg': str(e)}


from collections import defaultdict

from math import cos, sin, atan, pi
from operator import itemgetter
from typing import Optional, Tuple, Iterator

import numpy as np
from kivy.uix.widget import Widget
from kivy.core.text import Label
from kivy.graphics.fbo import Fbo
from kivy.graphics import (Translate, Rectangle, Quad, Color, InstructionGroup)
from kivy.properties import (NumericProperty, ListProperty, ObjectProperty)
from kivy.clock import Clock, mainthread

try:
	from kivy.garden.collider import Collide2DPoly
except (KeyError, ImportError):
	from ..collide import Collide2DPoly
from ..util import get_thin_rect_vertices, fortyfive, \
 DEFAULT_ARROW_LABEL_KWARGS

cos45 = cos(fortyfive)
sin45 = sin(fortyfive)

def up_and_down(orig, dest, taillen):
	if orig.center_y == dest.center_y:
		raise ValueError("Can't draw an arrow at a point")
	flipped = orig.center_y > dest.center_y
	if flipped:
		orig, dest = dest, orig
	x = int(orig.center_x)
	dy = int(dest.y)
	oy = int(orig.top)
	if flipped:
		oy, dy = dy, oy
	off1 = cos45 * taillen
	off2 = sin45 * taillen
	x0 = x
	y0 = oy
	endx = x
	endy = dy
	x1 = endx - off1
	x2 = endx + off1
	y1 = y2 = endy - off2 if oy < dy else endy + off2
	return [x0, y0, endx, endy], [x1, y1, endx, endy, x2, y2]

def left_and_right(orig, dest, taillen):
	if orig.center_x == dest.center_x:
		raise ValueError("Can't draw an arrow at a point")
	flipped = orig.center_x > dest.center_x
	if flipped:
		orig, dest = dest, orig
	y = int(orig.center_y)
	dx = int(dest.x)
	ox = int(orig.right)
	if flipped:
		ox, dx = dx, ox
	off1 = cos45 * taillen
	off2 = sin45 * taillen
	x0 = ox
	y0 = y
	endx = dx
	endy = y
	y1 = endy - off1
	y2 = endy + off1
	x1 = x2 = endx - off2 if ox < dx else endx + off2
	return [x0, y0, endx, endy], [x1, y1, endx, endy, x2, y2]

def _get_points_first_part(orig, dest, taillen):
	ox, oy = orig.center
	ow, oh = orig.size
	dx, dy = dest.center
	dw, dh = dest.size
	# Flip the endpoints around so that the arrow faces up and right.
	# We'll flip it back at the end as needed.
	xco = 1 if ox < dx else -1
	yco = 1 if oy < dy else -1
	ox *= xco
	dx *= xco
	oy *= yco
	dy *= yco
	# Nudge my endpoints so that I start and end at the edge of
	# a Spot.
	if dy - oy > dx - ox:
		topy = dy - dh / 2
		boty = oy + oh / 2
		leftx = ox
		rightx = dx
	else:
		leftx = ox + ow / 2
		rightx = dx - dw / 2
		topy = dy
		boty = oy
	# Degenerate cases.
	# Also, these need to be handled specially to avoid
	# division by zero later on.
	if rightx == leftx:
		return up_and_down(orig, dest, taillen)
	if topy == boty:
		return left_and_right(orig, dest, taillen)
	return ow, oh, dw, dh, xco, leftx, rightx, yco, topy, boty

def get_points_multi(args):
	"""Return a dictionary mapping (orig, dest) to pairs of point lists for arrows

	Takes an iterable of (orig, dest, taillen) where orig and dest are Spot instances

	taillen is an integer specifying how long the arrowhead should be.

	"""
	ret = {}
	keys = []
	topys = []
	botys = []
	leftxs = []
	rightxs = []
	taillens = []
	xcos = []
	ycos = []
	for (orig, dest, taillen) in args:
		try:
			p1 = _get_points_first_part(orig, dest, taillen)
		except ValueError:
			p1 = 2, 2, 2, 2, 1, 0, 1, 1, 0, 1
		if len(p1) == 2:
			ret[orig.name, dest.name] = p1
			continue
		ow, oh, dw, dh, xco, leftx, rightx, yco, topy, boty = p1
		keys.append((orig.name, dest.name))
		leftxs.append(leftx)
		rightxs.append(rightx)
		topys.append(topy)
		botys.append(boty)
		taillens.append(taillen)
		xcos.append(xco)
		ycos.append(yco)
	topys = np.array(topys)
	botys = np.array(botys)
	rightxs = np.array(rightxs)
	leftxs = np.array(leftxs)
	rises = np.subtract(topys, botys)
	runs = np.subtract(rightxs, leftxs)
	slopes = np.divide(rises, runs)
	unslopes = np.divide(runs, rises)
	start_thetas = np.arctan(slopes)
	end_thetas = np.arctan(unslopes)
	top_thetas = np.subtract(start_thetas, fortyfive)
	bot_thetas = np.subtract(pi - fortyfive, end_thetas)
	topsins = np.sin(top_thetas)
	topcoss = np.cos(top_thetas)
	botsins = np.sin(bot_thetas)
	botcoss = np.cos(bot_thetas)
	taillens = np.array(taillens)
	xoff1s = np.multiply(topcoss, taillens)
	yoff1s = np.multiply(topsins, taillens)
	xoff2s = np.multiply(botcoss, taillens)
	yoff2s = np.multiply(botsins, taillens)
	xcos = np.array(xcos)
	ycos = np.array(ycos)
	x1s = np.multiply(np.subtract(rightxs, xoff1s), xcos)
	x2s = np.multiply(np.subtract(rightxs, xoff2s), xcos)
	y1s = np.multiply(np.subtract(topys, yoff1s), ycos)
	y2s = np.multiply(np.subtract(topys, yoff2s), ycos)
	startxs = np.multiply(leftxs, xcos)
	startys = np.multiply(botys, ycos)
	endxs = np.multiply(rightxs, xcos)
	endys = np.multiply(topys, ycos)
	for key, startx, starty, endx, endy, x1, y1, endx, endy, x2, y2 in zip(
		keys, startxs, startys, endxs, endys, x1s, y1s, endxs, endys, x2s,
		y2s):
		ret[key] = ([startx, starty, endx, endy], [x1, y1, endx, endy, x2, y2])
	return ret

def get_points(orig, dest, taillen):
	"""Return a pair of lists of points for use making an arrow.

	The first list is the beginning and end point of the trunk of the arrow.

	The second list is the arrowhead.

	"""
	p1 = _get_points_first_part(orig, dest, taillen)
	if len(p1) == 2:
		return p1
	ow, oh, dw, dh, xco, leftx, rightx, yco, topy, boty = p1
	rise = topy - boty
	run = rightx - leftx
	slope = rise / run
	unslope = run / rise
	start_theta = atan(slope)
	end_theta = atan(unslope)

	# make the little wedge at the end so you can tell which way the
	# arrow's pointing, and flip it all back around to the way it was
	top_theta = start_theta - fortyfive
	bot_theta = pi - fortyfive - end_theta
	xoff1 = cos(top_theta) * taillen
	yoff1 = sin(top_theta) * taillen
	xoff2 = cos(bot_theta) * taillen
	yoff2 = sin(bot_theta) * taillen
	x1 = (rightx - xoff1) * xco
	x2 = (rightx - xoff2) * xco
	y1 = (topy - yoff1) * yco
	y2 = (topy - yoff2) * yco
	startx = leftx * xco
	starty = boty * yco
	endx = rightx * xco
	endy = topy * yco
	return [startx, starty, endx, endy], [x1, y1, endx, endy, x2, y2]

def get_quad_vertices(ox,
						oy,
						dx,
						dy,
						x1,
						y1,
						endx,
						endy,
						x2,
						y2,
						bgr,
						fgr,
						label_w=100,
						label_h=100):
	if dx > ox:
		if dy > oy:
			label_pos = x2 - label_w, dy - label_h
		elif dy < oy:
			label_pos = x1 - label_w, dy
		else:
			label_pos = x2 - label_w, y2 - label_h
	elif dx < ox:
		if dy < oy:
			label_pos = x2, y2
		elif dy > oy:
			label_pos = x1, y1 - label_h
		else:
			label_pos = x2, dy - label_h
	else:
		if dy > oy:
			label_pos = x2 - label_w, dy - label_h
		elif dy < oy:
			label_pos = x2, y2
		else:
			label_pos = dx, dy
	return {
		'shaft_bg': get_thin_rect_vertices(ox, oy, dx, dy, bgr),
		'shaft_fg': get_thin_rect_vertices(ox, oy, dx, dy, fgr),
		'left_head_bg': get_thin_rect_vertices(x1, y1, endx, endy, bgr),
		'right_head_bg': get_thin_rect_vertices(x2, y2, endx, endy, bgr),
		'left_head_fg': get_thin_rect_vertices(x1, y1, endx, endy, fgr),
		'right_head_fg': get_thin_rect_vertices(x2, y2, endx, endy, fgr),
		'label_pos': label_pos
	}

eight0s = tuple([0] * 8)

class GraphArrow:

	@property
	def slope(self) -> float:
		"""Return a float of the increase in y divided by the increase in x,
		both from left to right.

		Returns ``None`` when vertical.

		"""
		orig = self.origin
		dest = self.destination
		ox = orig.x
		oy = orig.y
		dx = dest.x
		dy = dest.y
		if oy == dy:
			return 0.
		elif ox == dx:
			return None
		else:
			rise = dy - oy
			run = dx - ox
			return rise / run

	@property
	def y_intercept(self) -> Optional[float]:
		"""Return my Y-intercept.

		I probably don't really hit the left edge of the window, but
		this is where I would, if I were long enough.

		"""
		orig = self.origin
		dest = self.destination
		(ox, oy) = orig.pos
		(dx, dy) = dest.pos
		denominator = dx - ox
		x_numerator = (dy - oy) * ox
		y_numerator = denominator * oy
		return (y_numerator - x_numerator), denominator

	@property
	def reciprocal(self) -> Optional["GraphArrow"]:
		"""The arrow connecting the same spots in the opposite direction"""
		if self.destination.name not in self.board.pred_arrow:
			return
		if self.origin.name not in self.board.pred_arrow[
			self.destination.name]:
			return
		return self.board.pred_arrow[self.destination.name][self.origin.name]

	@property
	def selected(self) -> bool:
		return self is self.board.app.selection

	@selected.setter
	def selected(self, b: bool):
		self.board.app.selection = self
		self.repoint(b)

	def __init__(self, *, board, origin, destination):
		self.board = board
		self.origin = origin
		self.destination = destination

	def collide_point(self, x: float, y: float) -> bool:
		return self.board.arrow_plane._colliders_map[
			self.origin.name, self.destination.name].collide_point(x, y)

	def pos_along(self, pct: float) -> Tuple[float, float]:
		"""Return coordinates for where a Pawn should be if it has travelled
		along ``pct`` of my length (between 0 and 1).

		Might get complex when I switch over to using beziers for
		arrows, but for now this is quite simple, using distance along
		a line segment.

		"""
		if pct < 0 or pct > 1:
			raise ValueError("Invalid portion")
		(ox, oy) = self.origin.center
		(dx, dy) = self.destination.center
		xdist = (dx - ox) * pct
		ydist = (dy - oy) * pct
		return ox + xdist, oy + ydist

	@mainthread
	def repoint(self, selected: bool = None) -> None:
		arrow_plane = self.board.arrow_plane
		fbo = arrow_plane._fbo
		fbo.bind()
		fbo.clear_buffer()
		shaft_points, head_points = get_points(self.origin, self.destination,
												arrow_plane.arrowhead_size)
		r = arrow_plane.arrow_width / 2
		if selected or self.selected:
			bg_scale = arrow_plane.bg_scale_selected
			bg_color = arrow_plane.bg_color_selected
			fg_color = arrow_plane.fg_color_selected
		else:
			bg_scale = arrow_plane.bg_scale_unselected
			bg_color = arrow_plane.bg_color_unselected
			fg_color = arrow_plane.fg_color_unselected
		plane = self.board.arrow_plane
		portal = self.board.character.portal[self.origin.name][
			self.destination.name]
		portal_text = str(portal.get(portal.get('_label_stat', None), ''))
		label_kwargs = dict(portal.get("label_kwargs", ()))
		if portal_text is not None:
			label_kwargs['text'] = portal_text
		try:
			label = self.board.arrow_plane._labels[self.origin.name][
				self.destination.name]
			label.text = portal_text
		except KeyError:
			label = self.board.arrow_plane.labels[self.origin.name][
				self.destination.name] = Label(**DEFAULT_ARROW_LABEL_KWARGS,
												**label_kwargs)
		if (self.origin.name,
			self.destination.name) in plane._instructions_map:
			verts = get_quad_vertices(*shaft_points, *head_points,
										r * bg_scale, r, *label.render())
			insts = self.board.arrow_plane._instructions_map[
				self.origin.name, self.destination.name]
			insts['color0'].rgba = bg_color
			insts['color1'].rgba = fg_color
			insts['shaft_bg'].points = verts['shaft_bg']
			insts['left_head_bg'].points = verts['left_head_bg']
			insts['right_head_bg'].points = verts['right_head_bg']
			insts['shaft_fg'].points = verts['shaft_fg']
			insts['left_head_fg'].points = verts['left_head_fg']
			insts['right_head_fg'].points = verts['right_head_fg']
			insts['label_rect'].pos = verts['label_pos']
			insts['label_rect'].size = label.render()
			label.refresh()
			insts['label_rect'].texture = label.texture
			plane._colliders_map[self.origin.name,
									self.destination.name] = Collide2DPoly(
										points=verts['shaft_bg'])
		else:
			plane._instructions_map[
				self.origin.name,
				self.destination.name] = insts = get_instructions(
					*shaft_points, *head_points, bg_color, fg_color,
					*label.render(), label)
			plane._colliders_map[
				self.origin.name,
				self.destination.name].points = Collide2DPoly(
					points=insts['shaft_bg'].points)
		myidx = plane._port_index[self.origin.name, self.destination.name]
		(ox, oy, dx, dy) = shaft_points
		plane._bot_left_corner_xs[myidx] = min((ox, dx))
		plane._bot_left_corner_ys[myidx] = min((oy, dy))
		plane._top_right_corner_xs[myidx] = max((ox, dx))
		plane._top_right_corner_ys[myidx] = max((oy, dy))
		fbo.release()
		fbo.ask_update()
		arrow_plane.canvas.ask_update()

def get_instructions(ox: float,
						oy: float,
						dx: float,
						dy: float,
						x1: float,
						y1: float,
						endx: float,
						endy: float,
						x2: float,
						y2: float,
						bgr: float,
						r: float,
						bg_color: Tuple[float, float, float, float],
						fg_color: Tuple[float, float, float, float],
						label_kwargs: dict = None,
						label: Label = None) -> dict:
	if label_kwargs is None:
		label_kwargs = {'text': '', **DEFAULT_ARROW_LABEL_KWARGS}
	else:
		label_kwargs = dict(label_kwargs)
		for k, v in DEFAULT_ARROW_LABEL_KWARGS.items():
			if k not in label_kwargs:
				label_kwargs[k] = v
	if label is None:
		label = Label(**label_kwargs)
	else:
		label.text = label_kwargs['text']
	text_size = label.render()
	quadverts = get_quad_vertices(ox, oy, dx, dy, x1, y1, endx, endy, x2, y2,
									bgr, r, *text_size)
	label.refresh()
	return {
		'color0':
		Color(rgba=bg_color),
		'shaft_bg':
		Quad(points=quadverts['shaft_bg']),
		'left_head_bg':
		Quad(points=quadverts['left_head_bg']),
		'right_head_bg':
		Quad(points=quadverts['right_head_bg']),
		'color1':
		Color(rgba=fg_color),
		'shaft_fg':
		Quad(points=quadverts['shaft_fg']),
		'left_head_fg':
		Quad(points=quadverts['left_head_fg']),
		'right_head_fg':
		Quad(points=quadverts['right_head_fg']),
		'label_rect':
		Rectangle(pos=quadverts['label_pos'],
					size=text_size,
					texture=label.texture),
		'label':
		label
	}

class GraphArrowWidget(Widget, GraphArrow):
	arrowhead_size = NumericProperty(10)
	arrow_width = NumericProperty(2)
	bg_scale = NumericProperty(5)
	bg_color = ListProperty([0.5, 0.5, 0.5, 0.5])
	fg_color = ListProperty([1.0, 1.0, 1.0, 1.0])

	def __init__(self, **kwargs):
		self._trigger_repoint = Clock.create_trigger(self.repoint)
		super().__init__(**kwargs)

	def on_origin(self, *args):
		if hasattr(self, '_origin_bind_uid'):
			self.unbind_uid(self._origin_bind_uid)
			del self._origin_bind_uid
		if not self.origin:
			return
		self._origin_bind_uid = self.origin.fbind('pos', self._trigger_repoint)

	def on_destination(self, *args):
		if hasattr(self, '_destination_bind_uid'):
			self.unbind_uid(self._destination_bind_uid)
			del self._destination_bind_uid
		if not self.destination:
			return
		self._destination_bind_uid = self.destination.fbind(
			'pos', self._trigger_repoint)

	def on_parent(self, *args):
		if not self.origin or not self.destination or not self.canvas:
			Clock.schedule_once(self.on_parent, 0)
			return
		self.canvas.clear()
		if self.parent is None:
			return
		shaft_points, head_points = get_points(self.origin, self.destination,
												self.arrowhead_size)
		with self.canvas:
			r = self.arrow_width / 2
			self._instructions = get_instructions(*shaft_points, *head_points,
													r * self.bg_scale, r,
													self.bg_color,
													self.fg_color)

	def repoint(self, *args):
		shaft_points, head_points = get_points(self.origin, self.destination,
												self.arrowhead_size)
		r = self.arrow_width / 2
		try:
			portal = self.board.character.portal[self.origin.name][
				self.destination.name]
			portal_text = str(portal.get(portal.get('_label_stat', None), ''))
		except (KeyError, AttributeError):
			portal_text = ''
		if hasattr(self, '_label'):
			label = self._label
			label.text = portal_text
		else:
			label = self._label = Label(text=portal_text)
		label_size = label.render()
		verts = get_quad_vertices(*shaft_points, *head_points,
									r * self.bg_scale, r, *label_size)
		insts = self._instructions
		insts['color0'].rgba = self.bg_color
		insts['color1'].rgba = self.fg_color
		insts['shaft_bg'].points = verts['shaft_bg']
		insts['left_head_bg'].points = verts['left_head_bg']
		insts['right_head_bg'].points = verts['right_head_bg']
		insts['shaft_fg'].points = verts['shaft_fg']
		insts['left_head_fg'].points = verts['left_head_fg']
		insts['right_head_fg'].points = verts['right_head_fg']
		insts['label_rect'].pos = verts['label_pos']
		insts['label_rect'].size = label_size
		label.refresh()
		insts['label_rect'].texture = label.texture

class ArrowPlane(Widget):
	data = ListProperty()
	arrowhead_size = NumericProperty(10)
	arrow_width = NumericProperty(2)
	bg_scale_unselected = NumericProperty(4)
	bg_scale_selected = NumericProperty(5)
	bg_color_selected = ListProperty([0.0, 1.0, 1.0, 1.0])
	bg_color_unselected = ListProperty([0.5, 0.5, 0.5, 0.5])
	fg_color_unselected = ListProperty([1.0, 1.0, 1.0, 1.0])
	fg_color_selected = ListProperty([1.0, 1.0, 1.0, 1.0])

	def __init__(self, **kwargs):
		self._labels = defaultdict(dict)
		self._trigger_redraw = Clock.create_trigger(self.redraw)
		self._redraw_bind_uid = self.fbind('data', self._trigger_redraw)
		self.bind(arrowhead_size=self._trigger_redraw)
		self._colliders_map = {}
		self._instructions_map = {}
		self._port_index = {}
		self._port_l = []
		self._bot_left_corner_ys = []
		self._bot_left_corner_xs = []
		self._top_right_corner_ys = []
		self._top_right_corner_xs = []
		super().__init__(**kwargs)

	def on_parent(self, *args):
		if not self.canvas:
			Clock.schedule_once(self.on_parent, 0)
			return
		with self.canvas:
			self._fbo = Fbo(size=self.size)
			self._translate = Translate(x=self.x, y=self.y)
			self._rectangle = Rectangle(size=self.size,
										texture=self._fbo.texture)
		self._trigger_redraw()

	def have_arrow(self, orig, dest):
		return (orig, dest) in self._port_index

	def redraw(self, *args):
		if not hasattr(self, '_rectangle'):
			self._trigger_redraw()
			return
		fbo = self._fbo
		fbo.bind()
		fbo.clear()
		fbo.clear_buffer()
		add = fbo.add
		r = self.arrow_width // 2
		bg_scale_selected = self.bg_scale_selected
		bg_color_unselected = self.bg_color_unselected
		fg_color_unselected = self.fg_color_unselected
		taillen = self.arrowhead_size
		points_map = get_points_multi(
			(datum['origspot'], datum['destspot'], taillen)
			for datum in self.data)
		port_l = []
		bot_left_corner_xs = []
		bot_left_corner_ys = []
		top_right_corner_xs = []
		top_right_corner_ys = []
		port_index = self._port_index
		colliders_map = self._colliders_map
		oxs = []
		oys = []
		dxs = []
		dys = []
		for ((ox, oy, dx, dy), _) in points_map.values():
			oxs.append(ox)
			oys.append(oy)
			dxs.append(dx)
			dys.append(dy)
		widths = np.abs(np.array(dxs) - np.array(oxs))
		heights = np.abs(np.array(dys) - np.array(oys))
		lengths = np.sqrt(np.square(widths) + np.square(heights))
		for (length, (port, ((ox, oy, dx, dy), (x1, y1, endx, endy, x2, y2))),
				datum) in zip(lengths, points_map.items(), self.data):
			if length < r:
				continue
			bgr = r * bg_scale_selected  # change for selectedness pls
			instructions = get_instructions(ox, oy, dx, dy, x1, y1, endx, endy,
											x2, y2, bgr, r,
											bg_color_unselected,
											fg_color_unselected,
											datum.get('label_kwargs', {}),
											self._labels[port[0]].get(port[1]))
			instructions['group'] = grp = InstructionGroup()
			grp.add(instructions['color0'])
			grp.add(instructions['shaft_bg'])
			grp.add(instructions['left_head_bg'])
			grp.add(instructions['right_head_bg'])
			grp.add(instructions['color1'])
			grp.add(instructions['shaft_fg'])
			grp.add(instructions['left_head_fg'])
			grp.add(instructions['right_head_fg'])
			grp.add(instructions['label_rect'])
			add(grp)
			self._instructions_map[port] = instructions
			self._labels[port[0]][port[1]] = instructions['label']
			if ox < dx:
				leftx = ox
				rightx = dx
			else:
				rightx = ox
				leftx = dx
			if oy < dy:
				boty = oy
				topy = dy
			else:
				boty = dy
				topy = oy
			port_index[port] = len(port_l)
			port_l.append(port)
			bot_left_corner_xs.append(leftx - bgr)
			bot_left_corner_ys.append(boty - bgr)
			top_right_corner_xs.append(rightx + bgr)
			top_right_corner_ys.append(topy + bgr)
			colliders_map[port] = Collide2DPoly(
				points=instructions['shaft_bg'].points)
		fbo.release()
		self.canvas.ask_update()
		self._port_l = port_l
		self._bot_left_corner_xs = np.array(bot_left_corner_xs)
		self._bot_left_corner_ys = np.array(bot_left_corner_ys)
		self._top_right_corner_xs = np.array(top_right_corner_xs)
		self._top_right_corner_ys = np.array(top_right_corner_ys)

	def add_new_portal(self, datum: dict) -> None:
		orig_spot = datum['origspot']
		dest_spot = datum['destspot']
		shaft_points, head_points = get_points(orig_spot, dest_spot,
												self.arrowhead_size)
		self.unbind_uid('data', self._redraw_bind_uid)
		self.data.append(datum)
		r = self.arrow_width / 2
		bgr = r * self.bg_scale_unselected
		instructions = self._instructions_map[
			orig_spot.name, dest_spot.name] = get_instructions(
				*shaft_points, *head_points, bgr, r,
				self.bg_color_unselected, self.fg_color_unselected,
				datum.get('label_kwargs',
							{}), self._labels[orig_spot.name][dest_spot.name])
		instructions['group'] = grp = InstructionGroup()
		grp.add(instructions['color0'])
		grp.add(instructions['shaft_bg'])
		grp.add(instructions['left_head_bg'])
		grp.add(instructions['right_head_bg'])
		grp.add(instructions['color1'])
		grp.add(instructions['shaft_fg'])
		grp.add(instructions['left_head_fg'])
		grp.add(instructions['right_head_fg'])
		grp.add(instructions['label'])
		self._labels[orig_spot.name][dest_spot.name] = instructions['label']
		self._fbo.add(grp)
		ox, oy, dx, dy = shaft_points
		if ox < dx:
			left_x = ox
			right_x = dx
		else:
			right_x = ox
			left_x = dx
		if oy < dy:
			bot_y = oy
			top_y = dy
		else:
			bot_y = dy
			top_y = oy
		self._port_index[orig_spot.name, dest_spot.name] = len(self._port_l)
		self._port_l.append((orig_spot.name, dest_spot.name))
		self._bot_left_corner_xs = np.array(
			list(self._bot_left_corner_xs) + [left_x - bgr])
		self._bot_left_corner_ys = np.array(
			list(self._bot_left_corner_ys) + [bot_y - bgr])
		self._top_right_corner_xs = np.array(
			list(self._top_right_corner_xs) + [right_x + bgr])
		self._top_right_corner_ys = np.array(
			list(self._top_right_corner_ys) + [top_y + bgr])
		self._colliders_map[orig_spot.name, dest_spot.name] = Collide2DPoly(
			points=instructions['shaft_bg'].points)
		self.canvas.ask_update()
		self._redraw_bind_uid = self.fbind('data', self._trigger_redraw)

	def remove_edge(self, orig, dest):
		self._fbo.bind()
		self._fbo.clear_buffer()
		self._fbo.remove(self._instructions_map[orig, dest]['group'])
		index = self._port_index[orig, dest]
		for port in self._port_l[index:]:
			self._port_index[port] -= 1
		del self._port_index[orig, dest]
		del self._instructions_map[orig, dest]
		del self._colliders_map[orig, dest]
		del self._port_l[index]
		for arr in ('_bot_left_corner_ys', '_bot_left_corner_xs',
					'_top_right_corner_ys', '_top_right_corner_xs'):
			dat = list(getattr(self, arr))
			del dat[index]
			setattr(self, arr, np.array(dat))
		self._fbo.release()
		self.canvas.ask_update()

	def update_portal_label(self, orig, dest, text):
		rect = self._instructions_map[orig, dest]['label']
		label = self._labels[orig][dest]
		label.text = text
		label.refresh()
		rect.texture = label.texture

	def iter_collided_edges(self, x: float, y: float) -> Iterator:
		x, y = map(float, (x, y))
		collider_map = self._colliders_map
		hits = (self._bot_left_corner_xs <= x) & (
			self._bot_left_corner_ys <= y
		) & (x <= self._top_right_corner_xs) & (y <= self._top_right_corner_ys)
		for port in map(itemgetter(0),
						filter(itemgetter(1), zip(self._port_l, hits))):
			if collider_map[port].collide_point(x, y):
				yield port

	def on_pos(self, *args):
		if not hasattr(self, '_translate'):
			return
		self._translate.x, self._translate.y = self.pos
		self.canvas.ask_update()

	def on_size(self, *args):
		if not hasattr(self, '_rectangle') or not hasattr(self, '_fbo'):
			return
		self._rectangle.size = self._fbo.size = self.size
		self.redraw()


# Databricks notebook source
# MAGIC %md
# MAGIC Author: milos.colic@databricks.com

# COMMAND ----------

# DBTITLE 1,Lib Installation
# MAGIC %pip install geopandas h3 s2 shapely folium

# COMMAND ----------

# MAGIC %md
# MAGIC ## Point Data Preparation (Trips Data)

# COMMAND ----------

# MAGIC %md
# MAGIC For our point data we will use New York City Taxis data. </br>
# MAGIC This data provides us a volume of 1.6B records and in addition it contains two different Point Features - pick up and drop off. </br>
# MAGIC Our Polygon features dataset will remain the New York City Neighbourhoods.

# COMMAND ----------

# DBTITLE 1,Read NYC Taxis Data
taxi_trips = spark.read.format("delta").load("/databricks-datasets/nyctaxi/tables/nyctaxi_yellow/")

# COMMAND ----------

# DBTITLE 1,Visualise a Sample of Data
display(taxi_trips)

# COMMAND ----------

# DBTITLE 1,Verify Row Count
taxi_trips.count()

# COMMAND ----------

# MAGIC %md
# MAGIC In order to perform Point In Polygon joins we will make sure our point data contains H3 indices at resolution 10. </br>
# MAGIC We will create an User Defined Function that will perform this for us and we will apply it to the data.

# COMMAND ----------

# DBTITLE 1,UDF for H3 encoding
import h3
from pyspark.sql import functions as F

@udf("string")
def lat_lng_2_h3(lat, lng, res):
  import h3
  try:
    result = h3.geo_to_h3(lat, lng, res)
    return result
  except:
    return None # invalid coordinates will result in null index value.

taxi_trips = taxi_trips.withColumn(
  "h3_pickup", lat_lng_2_h3(F.col("pickup_latitude"), F.col("pickup_longitude"), F.lit(10))
).withColumn(
  "h3_dropoff", lat_lng_2_h3(F.col("dropoff_latitude"), F.col("dropoff_longitude"), F.lit(10))
)

# COMMAND ----------

# DBTITLE 1,View that H3 columns
display(taxi_trips)

# COMMAND ----------

# DBTITLE 1,Define location for data storage (silver layer)
username = "milos.colic" #please update with a correct user
silver_data_location = f"Users/{username}/geospatial/workshop/data/silver"

# COMMAND ----------

# DBTITLE 1,Write data out to a delta table
dbutils.fs.mkdirs(f"dbfs:/{silver_data_location}/h3/trips")
taxi_trips.write.format("delta").save(f"/{silver_data_location}/h3/trips/")

# COMMAND ----------

# MAGIC %md
# MAGIC For convenience purpose we will generate a managed database in the metastore. </br>
# MAGIC We will register the tables with our data against this database. </br>
# MAGIC This simplifies the way we are handling the data at consumption time.

# COMMAND ----------

# DBTITLE 1,Create a database for our workshop
# MAGIC %sql
# MAGIC Create DATABASE if Not EXISTS GeospatialWorkshop

# COMMAND ----------

# DBTITLE 1,Use the database
# MAGIC %sql
# MAGIC use GeospatialWorkshop

# COMMAND ----------

# DBTITLE 1,Create the table that will contain trips with h3 indices
# MAGIC %sql
# MAGIC CREATE TABLE IF NOT EXISTS silver_h3_trips
# MAGIC using delta
# MAGIC LOCATION '/Users/milos.colic/geospatial/workshop/data/silver/h3/trips' -- please updated the username value in the path

# COMMAND ----------

# MAGIC %md
# MAGIC Delta is able to physically colocate the similar data based on z-order. </br>
# MAGIC Since we know we will be serving PIP joins using h3 columns we will z-order data by both h3 columns we have created.

# COMMAND ----------

# DBTITLE 1,ZOrder Data
# MAGIC %sql
# MAGIC optimize silver_h3_trips ZORDER by (h3_pickup, h3_dropoff)

# COMMAND ----------

# DBTITLE 1,View Trips Data
taxi_trips = spark.read.table("GeospatialWorkshop.silver_h3_trips")
display(taxi_trips)

# COMMAND ----------

# MAGIC %md
# MAGIC ## Polygon Data Preparation - Neighbourhoods Data

# COMMAND ----------

# DBTITLE 1,Fetch Neighbours Data
# MAGIC %sh wget -O /tmp/neighbourhoods.zip https://www1.nyc.gov/assets/planning/download/zip/data-maps/open-data/nynta2020_21d.zip

# COMMAND ----------

# DBTITLE 1,Unzip the Data
# MAGIC %sh
# MAGIC rm -R /tmp/neighbourhoods/
# MAGIC mkdir /tmp/neighbourhoods/
# MAGIC unzip /tmp/neighbourhoods.zip -d /tmp/neighbourhoods

# COMMAND ----------

# DBTITLE 1,Move the Data to dbfs
username = "milos.colic" #please update with a correct user
raw_data_location = f"Users/{username}/geospatial/workshop/data/raw/"
dbutils.fs.mkdirs(f"dbfs:/{raw_data_location}/")
dbutils.fs.cp("file:/tmp/neighbourhoods/nynta2020_21d/nynta2020.prj", f"dbfs:/{raw_data_location}/neighbourhoods.prj")
dbutils.fs.cp("file:/tmp/neighbourhoods/nynta2020_21d/nynta2020.shx", f"dbfs:/{raw_data_location}/neighbourhoods.shx")
dbutils.fs.cp("file:/tmp/neighbourhoods/nynta2020_21d/nynta2020.shp", f"dbfs:/{raw_data_location}/neighbourhoods.shp")

# COMMAND ----------

# DBTITLE 1,Load Data to geopandas
import geopandas as gpd

neighborhoods_pd = gpd.read_file(f"/dbfs/{raw_data_location}/neighbourhoods.shp")
display(neighborhoods_pd.head())

# COMMAND ----------

# MAGIC %md
# MAGIC In order to prepare our polygon data we will be using two set representation via h3 decompositon. </br>
# MAGIC The only difference to what we discussed in the notebook 01 is that we will be using WKB representation for the chips. </br>
# MAGIC Binary representation of polygons will result in more optimal runtime handling of chips. </br>
# MAGIC In addition we will be returning a single array of structs instead of separate collections for core and border chips. </br>
# MAGIC This will simplify our join logic.

# COMMAND ----------

# DBTITLE 1,H3 Two Set Representation
import shapely
import math
import h3

@udf("array<struct<index: string, is_border: boolean, chip: binary>>")
def h3_2set_polyfill(geometry, resolution):
  # we need to account for possibility of MultiPolygon shapes
  def to_h3(geometry):
    if 'MultiPolygon' in geometry.geom_type:
      geometry = list(geometry.geoms)
    else:
      geometry = [geometry]

    hex_set = set()
    for p in geometry:
      p_geojson = shapely.geometry.mapping(p)
      hex_set = hex_set.union(h3.polyfill_geojson(p_geojson, resolution))

    return hex_set

  # convenience method for converting an index to a chip of a polygon
  def get_result_struct(hex_i, polygon, dirty_set):
    if hex_i in dirty_set:
      hex_polygon = shapely.geometry.Polygon(h3.h3_to_geo_boundary(hex_i, geo_json=True))
      intersection = polygon.intersection(hex_polygon)
      if intersection.is_empty:
        return None
      elif intersection.equals(hex_polygon):
        return (hex_i, False, None)
      else:
        return (hex_i, True, intersection.wkb)
    else:
      return (hex_i, False, None)

  polygon = shapely.wkb.loads(bytes(geometry))

  # compute the buffer radius
  # we cannot use the hexagon side lenght due to curvatrure
  # we use minimum rotated rectange - we assume that the rectangle is near rectangle in shape
  # the alternative would be to iterate through boundary vertices and take the longest side
  cent = polygon.centroid.xy
  centroid_hex = h3.geo_to_h3(cent[1][0], cent[0][0], resolution)
  centroid_geom = shapely.geometry.Polygon(h3.h3_to_geo_boundary(centroid_hex, geo_json=True))
  radius = math.sqrt(centroid_geom.minimum_rotated_rectangle.area)/2

  # any index that may touch the boundary
  dirty = polygon.boundary.buffer(radius)

  original_set = to_h3(polygon)
  dirty_set = to_h3(dirty)

  result = [get_result_struct(hex_i, polygon, dirty_set) for hex_i in list(original_set.union(dirty_set))]

  return result

# COMMAND ----------

# MAGIC %md
# MAGIC Our methods expect wkb representation. When we have loaded geopandas dataframe into spark, all our geometries were loaded as wkt. </br>
# MAGIC We will create an udf to perform the necessary conversions.

# COMMAND ----------

# DBTITLE 1,Convert WKT to WKB in the Input Data
from pyspark.sql import functions as F

@udf("binary")
def wkt_to_wkb(wkt_poly):
  from shapely import wkt as wkt_loader
  from shapely import wkb as wkb_writer

  geom = wkt_loader.loads(wkt_poly)
  wkb_poly = wkb_writer.dumps(geom, hex=False)

  return wkb_poly

neighborhoods_pd = neighborhoods_pd.to_crs("EPSG:4326")
neighborhoods_pd["wkt_polygon"] = neighborhoods_pd["geometry"].apply(lambda x: str(x.wkt))
neighborhoods_df = spark.createDataFrame(neighborhoods_pd[["wkt_polygon"]])
neighborhoods_df = neighborhoods_df.withColumn("wkb_polygon", wkt_to_wkb(F.col("wkt_polygon")))

# COMMAND ----------

# DBTITLE 1,View Conversions between WKT and WKB
display(neighborhoods_df)

# COMMAND ----------

# DBTITLE 1,Represent Geometries via H3 Two Set Decomposition
neighborhoods_df = neighborhoods_df.withColumn("chips", h3_2set_polyfill(F.col("wkb_polygon"), F.lit(10)))

# COMMAND ----------

# DBTITLE 1,View Decompositions
display(neighborhoods_df)

# COMMAND ----------

# MAGIC %md
# MAGIC We will perform 2 different write operations with Neighbourhoods data. </br>
# MAGIC One option is to keep the data in compact format with the array data kept at storage but exploded at join time. </br>
# MAGIC The other option is to pre-explode the arrays, store the data in flat way and z order by the index values. </br>

# COMMAND ----------

# DBTITLE 1,Write out Neighbourhoods Data
dbutils.fs.mkdirs(f"dbfs:/{silver_data_location}/h3/neighbourhoods")

neighborhoods_df = neighborhoods_df.select(
  F.monotonically_increasing_id().alias("id"),
  F.col("wkb_polygon"),
  F.col("chips")
).cache()

neighborhoods_df.write.format("delta").mode("overwrite").option("overwriteSchema", "true").save(f"/{silver_data_location}/h3/neighbourhoods/compact/")
neighborhoods_df.select(
  F.col("id"),
  F.col("wkb_polygon"),
  F.explode("chips").alias("chip")
).write.format("delta").mode("overwrite").option("overwriteSchema", "true").save(f"/{silver_data_location}/h3/neighbourhoods/flat/")

# COMMAND ----------

# DBTITLE 1,Create Compact Table
# MAGIC %sql
# MAGIC CREATE TABLE IF NOT EXISTS silver_h3_neighbourhoods_compact
# MAGIC using delta
# MAGIC LOCATION '/Users/milos.colic/geospatial/workshop/data/silver/h3/neighbourhoods/compact' -- please updated the username value in the path

# COMMAND ----------

# DBTITLE 1,Create Flat Table
# MAGIC %sql
# MAGIC CREATE TABLE IF NOT EXISTS silver_h3_neighbourhoods_flat
# MAGIC using delta
# MAGIC LOCATION '/Users/milos.colic/geospatial/workshop/data/silver/h3/neighbourhoods/flat' -- please updated the username value in the path

# COMMAND ----------

# DBTITLE 1,Optimize Flat Table
# MAGIC %sql
# MAGIC OPTIMIZE silver_h3_neighbourhoods_flat ZORDER BY (chip.index)

# COMMAND ----------

# DBTITLE 1,View Compact Data
neighborhoods_compact = spark.read.table("GeospatialWorkshop.silver_h3_neighbourhoods_compact")
display(neighborhoods_compact)

# COMMAND ----------

# DBTITLE 1,View Flat Data
neighborhoods_flat = spark.read.table("GeospatialWorkshop.silver_h3_neighbourhoods_flat")
display(neighborhoods_flat)

# COMMAND ----------

# MAGIC %md
# MAGIC ## Polygon Data Preparation - Ranomized Polygons

# COMMAND ----------

neighborhoods_df = spark.read.table("GeospatialWorkshop.silver_h3_neighbourhoods_compact").select("id", "wkb_polygon")

# COMMAND ----------

# MAGIC %md
# MAGIC We will generate 2 polygon datasets of ~2.5M polygons to demonstrate polygon intersection joins. </br>
# MAGIC We will randomly translate existing polygons many times in order to generate a dataset with many to many replationships.

# COMMAND ----------

from shapely import wkb
from shapely import affinity
from pyspark.sql import functions as F

@udf("binary")
def translate_polygon(in_wkb):
  from shapely import wkb
  from shapely import affinity
  import random

  polygon = wkb.loads(bytes(in_wkb))
  x_d = random.uniform(-1, 1)
  y_d = random.uniform(-1.5, 1.5)
  translated = affinity.translate(polygon, xoff=x_d, yoff=y_d)
  return translated.wkb

@udf("array<int>")
def gerate_simple_array(size):
  return [i for i in range(size)]

# COMMAND ----------

translated = neighborhoods_df.withColumn(
  "id2", F.explode(gerate_simple_array(F.lit(100))) # generate 100 numers that we will use to explode the data
).withColumn(
  "id3", F.explode(gerate_simple_array(F.lit(100))) # generate 100 numers that we will use to explode the data
).repartition(
  320, F.rand() # repartition the data
).withColumn(
  "wkb_polygon",
  translate_polygon(F.col("wkb_polygon")) # randomly translate the data
).withColumn(
  "id", F.concat_ws("-", F.col("id"), F.col("id2"), F.col("id3")) # generate a single ID by combining the original ID with two new IDs
).drop("id2", "id3")

display(translated)

# COMMAND ----------

# MAGIC %md
# MAGIC We will run the "same" run operation twice, both data outputs will be different. </br>
# MAGIC This is due to radom translations and the fact we have not fixed the random seed.

# COMMAND ----------

translated.write.format("delta").mode("overwrite").option("overwriteSchema", "true").save(f"/{silver_data_location}/h3/neighbourhoods/random/dataset_1")

# COMMAND ----------

translated.write.format("delta").mode("overwrite").option("overwriteSchema", "true").save(f"/{silver_data_location}/h3/neighbourhoods/random/dataset_2")

# COMMAND ----------

# MAGIC %md
# MAGIC We will run the decomposition operation on both of these randomized datasets and store the data.

# COMMAND ----------

from pyspark.sql import functions as F

polygons1 = spark.read.format("delta").load(f"/{silver_data_location}/h3/neighbourhoods/random/dataset_1")
polygons1 = polygons1.withColumn("chips", h3_2set_polyfill(F.col("wkb_polygon"), F.lit(8)))
polygons1.write.format("delta").mode("overwrite").option("overwriteSchema", "true").save(f"/{silver_data_location}/h3/neighbourhoods/random/dataset_1_decomposed")

# COMMAND ----------

from pyspark.sql import functions as F

polygons2 = spark.read.format("delta").load(f"/{silver_data_location}/h3/neighbourhoods/random/dataset_2")
polygons2 = polygons2.withColumn("chips", h3_2set_polyfill(F.col("wkb_polygon"), F.lit(8)))
polygons2.write.format("delta").mode("overwrite").option("overwriteSchema", "true").save(f"/{silver_data_location}/h3/neighbourhoods/random/dataset_2_decomposed")

# COMMAND ----------

sessions/2021-12-14 | Geospatial Analytics and AI at Scale/notebooks/mosaic-notebooks/02-DataPreparation.py


from rdkit import Chem
from rdkit.Chem import rdchem, rdmolfiles, rdmolops

# pop_size, mut_rate, gen_gap, query, sim treshold

def attach_capping(mol1, mol2):
    """it is connecting all Nterminals with the desired capping

    Arguments:
        mol1 {rdKit mol object} -- first molecule to be connected
        mol2 {rdKit mol object} -- second molecule to be connected - chosen N-capping

    Returns:
        rdKit mol object -- mol1 updated (connected with mol2, one or more)
    """

    count = 0

    # detects all the N terminals in mol1
    for atom in mol1.GetAtoms():
        atom.SetProp('Cterm', 'False')
        if atom.GetSmarts() == '[N:2]' or atom.GetSmarts() == '[NH2:2]' or atom.GetSmarts() == '[NH:2]':
            count += 1
            atom.SetProp('Nterm', 'True')
        else:
            atom.SetProp('Nterm', 'False')

    # detects all the C terminals in mol2 (it should be one)
    for atom in mol2.GetAtoms():
        atom.SetProp('Nterm', 'False')
        if atom.GetSmarts() == '[C:1]' or atom.GetSmarts() == '[CH:1]':
            atom.SetProp('Cterm', 'True')
        else:
            atom.SetProp('Cterm', 'False')

    # mol2 is addes to all the N terminal of mol1
    for i in range(count):
        combo = rdmolops.CombineMols(mol1, mol2)
        Nterm = []
        Cterm = []

        # saves in two different lists the index of the atoms which has to be connected
        for atom in combo.GetAtoms():
            if atom.GetProp('Nterm') == 'True':
                Nterm.append(atom.GetIdx())
            if atom.GetProp('Cterm') == 'True':
                Cterm.append(atom.GetIdx())

        # creates the amide bond
        edcombo = rdchem.EditableMol(combo)
        edcombo.AddBond(Nterm[0], Cterm[0], order=Chem.rdchem.BondType.SINGLE)
        clippedMol = edcombo.GetMol()

        # removes tags and lables form the atoms which reacted
        clippedMol.GetAtomWithIdx(Nterm[0]).SetProp('Nterm', 'False')
        clippedMol.GetAtomWithIdx(Cterm[0]).SetProp('Cterm', 'False')
        clippedMol.GetAtomWithIdx(Nterm[0]).SetAtomMapNum(0)
        clippedMol.GetAtomWithIdx(Cterm[0]).SetAtomMapNum(0)
        # uptades the 'core' molecule
        mol1 = clippedMol

    return mol1

class PDGA:

    # list of possible aminoacids
    AA = ['R', 'H', 'K', 'E', 'S', 'T', 'N', 'Q', 'G', 'P', 'A', 'V', 'I', 'L', 'F', 'Y', 'W', 'C', 'D', 'M', 'Z', 'O',
          '!', '?', '=', '%', '$', '@', '#']
    # list of possible branching units (1=Dap, 2=Dab, 3=Orn, 4=Lys)
    B = ['1', '2', '3', '4']
    # list of possible C-terminals
    CT = ['+']
    # list of possible N-capping
    NT = ['&']

    # variables for SMILES generation
    B_SMILES = {'1': '[N:2][C@@H](C[N:2])[C:1](O)=O', '2': '[N:2][C@@H](CC[N:2])[C:1](O)=O',
                '3': '[N:2][C@@H](CCC[N:2])[C:1](O)=O', '4': '[N:2][C@@H](CCCC[N:2])[C:1](O)=O',
                '5': '[N:2][C@H](C[N:2])[C:1](O)=O', '6': '[N:2][C@H](CC[N:2])[C:1](O)=O',
                '7': '[N:2][C@H](CCC[N:2])[C:1](O)=O', '8': '[N:2][C@H](CCCC[N:2])[C:1](O)=O'}

    AA_SMILES = {'A': '[N:2][C@@H](C)[C:1](O)=O', 'R': '[N:2][C@@H](CCCNC(N)=N)[C:1](O)=O',
                 'N': '[N:2][C@@H](CC(N)=O)[C:1](O)=O', 'D': '[N:2][C@@H](CC(O)=O)[C:1](O)=O',
                 'C': '[N:2][C@@H](CS)[C:1](O)=O', 'Q': '[N:2][C@@H](CCC(N)=O)[C:1](O)=O',
                 'E': '[N:2][C@@H](CCC(O)=O)[C:1](O)=O', 'G': '[N:2]C[C:1](O)=O',
                 'H': '[N:2][C@@H](CC1=CNC=N1)[C:1](O)=O', 'I': '[N:2][C@@H]([C@@H](C)CC)[C:1](O)=O',
                 'K': '[N:2][C@@H](CCCCN)[C:1](O)=O', 'L': '[N:2][C@@H](CC(C)C)[C:1](O)=O',
                 'M': '[N:2][C@@H](CCSC)[C:1](O)=O', 'F': '[N:2][C@@H](CC1=CC=CC=C1)[C:1](O)=O',
                 'P': 'C1CC[N:2][C@@H]1[C:1](O)=O', 'S': '[N:2][C@@H](CO)[C:1](O)=O',
                 'T': '[N:2][C@@H]([C@H](O)C)[C:1](O)=O', 'W': '[N:2][C@@H](CC1=CNC2=CC=CC=C12)[C:1](O)=O',
                 'Y': '[N:2][C@@H](CC1=CC=C(C=C1)O)[C:1](O)=O', 'V': '[N:2][C@@H](C(C)C)[C:1](O)=O'}

    T_SMILES = {'+': '[N:2]'}

    C_SMILES = {'&': 'C[C:1](=O)'}

    # GA class var
    mut_n = 1
    b_insert_rate = 0.1
    selec_strategy = 'Elitist'
    rndm_newgen_fract = 10
    fitness = 'MXFP'

    # initiatization of class variables updated by the GA
    dist_dict_old = {}
    gen_n = 0
    found_identity = 0
    steady_min = 0
    timelimit_seconds = None
    cbd_av = None
    cbd_min = None
    dist_dict = None
    surv_dict = None
    time = 0
    min_dict = {}
    # used internally to recognize a methylated aa:
    metbond = False
    # can be set with exclude or allow methylation,
    # it refers to the possibility of having methylation in the entire GA:
    methyl = True

    # debug
    verbose = False

    def __init__(self):
        pass

    def split_seq_components(self, seq):
        """split seq in generations and branching units

        Arguments:
            seq {string} -- dendrimer sequence

        Returns:
            lists -- generations(gs, from 0 to..), branching units, terminal and capping
        """

        g = []
        gs = []
        bs = []
        t = []
        c = []

        for ix, i in enumerate(seq):
            if i not in ['1', '2', '3', '4', '5', '6', '7', '8']:
                if i in self.CT:
                    t.append(i)
                elif i in self.NT:
                    c.append(i)
                elif i == 'X':
                    continue
                elif i == '-':
                    if seq[ix - 1] in ['1', '2', '3', '4', '5', '6', '7', '8']:
                        bs.append(i)
                    else:
                        g.append(i)
                else:
                    g.append(i)
            else:
                gs.append(g[::-1])
                bs.append(i)
                g = []
        gs.append(g[::-1])
        gs = gs[::-1]
        bs = bs[::-1]

        return gs, bs, t, c

    def connect_mol(self, mol1, mol2):
        """it is connecting all Nterminals of mol1 with the Cterminal
        of the maximum possible number of mol2s

        Arguments:
            mol1 {rdKit mol object} -- first molecule to be connected
            mol2 {rdKit mol object} -- second molecule to be connected

        Returns:
            rdKit mol object -- mol1 updated (connected with mol2, one or more)
        """
        count = 0

        # detects all the N terminals in mol1
        for atom in mol1.GetAtoms():
            atom.SetProp('Cterm', 'False')
            atom.SetProp('methyl', 'False')
            if atom.GetSmarts() == '[N:2]' or atom.GetSmarts() == '[NH2:2]' or atom.GetSmarts() == '[NH:2]':
                count += 1
                atom.SetProp('Nterm', 'True')
            else:
                atom.SetProp('Nterm', 'False')

        # detects all the C terminals in mol2 (it should be one)
        for atom in mol2.GetAtoms():
            atom.SetProp('Nterm', 'False')
            atom.SetProp('methyl', 'False')
            if atom.GetSmarts() == '[C:1]' or atom.GetSmarts() == '[CH:1]':
                atom.SetProp('Cterm', 'True')
            else:
                atom.SetProp('Cterm', 'False')

        # mol2 is addes to all the N terminal of mol1
        for i in range(count):
            combo = rdmolops.CombineMols(mol1, mol2)
            Nterm = []
            Cterm = []

            # saves in two different lists the index of the atoms which has to be connected
            for atom in combo.GetAtoms():
                if atom.GetProp('Nterm') == 'True':
                    Nterm.append(atom.GetIdx())
                if atom.GetProp('Cterm') == 'True':
                    Cterm.append(atom.GetIdx())

            # creates the amide bond
            edcombo = rdchem.EditableMol(combo)
            edcombo.AddBond(Nterm[0], Cterm[0],
                            order=Chem.rdchem.BondType.SINGLE)
            edcombo.RemoveAtom(Cterm[0] + 1)
            clippedMol = edcombo.GetMol()

            # removes tags and lables form c term atoms which reacted
            clippedMol.GetAtomWithIdx(Cterm[0]).SetProp('Cterm', 'False')
            clippedMol.GetAtomWithIdx(Cterm[0]).SetAtomMapNum(0)

            # methylates amide bond
            if self.metbond == True and self.methyl == True:
                Nterm = []
                Met = []
                methyl = rdmolfiles.MolFromSmiles('[C:4]')
                for atom in methyl.GetAtoms():
                    atom.SetProp('methyl', 'True')
                    atom.SetProp('Nterm', 'False')
                    atom.SetProp('Cterm', 'False')
                metcombo = rdmolops.CombineMols(clippedMol, methyl)
                for atom in metcombo.GetAtoms():
                    if atom.GetProp('Nterm') == 'True':
                        Nterm.append(atom.GetIdx())
                    if atom.GetProp('methyl') == 'True':
                        Met.append(atom.GetIdx())
                metedcombo = rdchem.EditableMol(metcombo)
                metedcombo.AddBond(
                    Nterm[0], Met[0], order=Chem.rdchem.BondType.SINGLE)
                clippedMol = metedcombo.GetMol()
                clippedMol.GetAtomWithIdx(Met[0]).SetProp('methyl', 'False')
                clippedMol.GetAtomWithIdx(Met[0]).SetAtomMapNum(0)

            # removes tags and lables form the atoms which reacted
            clippedMol.GetAtomWithIdx(Nterm[0]).SetProp('Nterm', 'False')
            clippedMol.GetAtomWithIdx(Nterm[0]).SetAtomMapNum(0)

            # uptades the 'core' molecule
            mol1 = clippedMol
        self.metbond = False
        return mol1

    def smiles_from_seq(self, seq):
        """Calculates the smiles of a given peptide dendrimer sequence

        Arguments:
            seq {string} -- peptide dendrimer sequence
        Returns:
            string -- molecule_smile - SMILES of the peptide
        """

        gs, bs, terminal, capping = self.split_seq_components(seq)

        # modifies the Cterminal
        if terminal:
            molecule = rdmolfiles.MolFromSmiles(self.T_SMILES[terminal[0]])
        else:
            molecule = ''

        # creates the dendrimer structure
        for gen in gs:
            for aa in gen:
                if aa == '-':
                    self.metbond = True
                    continue
                if molecule == '':
                    molecule = rdmolfiles.MolFromSmiles(self.AA_SMILES[aa])
                else:
                    molecule = self.connect_mol(
                        molecule, rdmolfiles.MolFromSmiles(self.AA_SMILES[aa]))

            if bs:
                if bs[0] == '-':
                    self.metbond = True
                    bs.pop(0)
                if molecule == '':
                    molecule = rdmolfiles.MolFromSmiles(self.B_SMILES[bs[0]])
                else:
                    molecule = self.connect_mol(
                        molecule, rdmolfiles.MolFromSmiles(self.B_SMILES[bs[0]]))
                bs.pop(0)

        # adds capping to the N-terminal (the called clip function is different, cause the listed smiles
        # for the capping are already without OH, it is not necessary removing any atom after foming the new bond)
        if capping:
            molecule = attach_capping(
                molecule, rdmolfiles.MolFromSmiles(self.C_SMILES[capping[0]]))

        # clean the smile from all the tags
        for atom in molecule.GetAtoms():
            atom.SetAtomMapNum(0)

        molecule_smile = rdmolfiles.MolToSmiles(
            molecule, isomericSmiles=True).replace('[N]', 'N').replace('[C]', 'C')
        return molecule_smile


import pathlib
from setuptools import setup, find_packages

# The directory containing this file
HERE = pathlib.Path(__file__).parent

# The text of the README file
README = (HERE / "README.md").read_text()

setup(name='monrp',
      version='1.0',
      author="Víctor Pérez-Piqueras",
      author_email="victor.perezpiqueras@uclm.es",
      description="Multi-Ojective Next Release Problem set of algorithms",
      long_description=README,
      long_description_content_type="text/markdown",
      url="https://github.com/UCLM-SIMD/MONRP",
      license="MIT",
      classifiers=[
          "Programming Language :: Python :: 3",
          "Programming Language :: Python :: 3.8"
      ],
      packages=find_packages(exclude=("test_*",)),
      install_requires=[
          'imageio',
          "pandas",
          "numpy",
          "scipy",
          "scikit-learn",
          "matplotlib"
      ],
      python_requires=">=3.8",
      include_package_data=True,
      )


# -*- coding: utf-8 -*-
"""
TRAINEE BURGING GOOSE AERO DESEING UFPR.

Gráfico tração em função da velocidade hélice 20x8E
"""

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

file = 'dados.txt'  # importando arquivo txt com os dados convertidos
columnsnames = ['x1', 'y1']  # Nomeando as colunas

df = pd.read_csv(file, skiprows=1, names=columnsnames, sep='\s+')

lista_x = df["x1"].tolist()  # Transformando coluna x em lista
lista_y = df["y1"].tolist()  # Transformando coluna y em lista

x = np.array(lista_x)  # Inserindo pontos da velocidade em x

y = np.array(lista_y)  # Inserindo pontos da tração em y

poly = np.polyfit(x, y, 2)  # Ajustando polinomio

a, b, c = poly

print("Coeficientes:")
print(a, b, c)  # Print dos coeficientes

p = 1.206  # Densidade local calculada
p0 = 1.225  # Densidade nível do mar

#plt.scatter(x, y, label='tração')
plt.plot(x, a*x**2 + b*x + c, color='k', label='h nível do mar')
plt.plot(x, (a*x**2 + b*x + c)*(p/p0), color='darkcyan', label='h calculado')
plt.title('Scorpion 4020KV, APC20X8E a 5000RPM')
plt.legend()
plt.xlabel('V (m/s)')
plt.ylabel('T (N)')
plt.ylim(0, 50)
plt.xlim(0, 26)
plt.yticks(np.arange(0, 55, 5))
plt.xticks(np.arange(0, 28, 2))


from .environment import (
    LOG_FORMAT,
    LOG_DATETIME_FORMAT,
)
from logging import (
    getLogger,
    Formatter,
    StreamHandler,
    getLevelName,
    DEBUG,
    INFO,
    WARNING,
)
from logging.handlers import RotatingFileHandler
from sys import stdout, stderr
from typing import Optional
from pathlib import Path

class Log:
    """
    Logger class, optionally binds to a log file and writes logs with timestamp
    and PID.
    """

    __slots__ = ["name", "level", "tag", "file", "logger"]

    def __init__(
        self,
        name,
        level: str = "INFO",
        tag: Optional[str] = None,
        file: Optional[Path] = None,
        clear: bool = False,
    ) -> None:
        """
        Initializes an object of type 'Log'.

        Parameters
        ----------
        name : str
            The name of the logger, usually '__name__' is best praxis.
        level : str
            The level to log on (DEBUG, INFO, WARNING, ERROR or CRITICAL).
        tag : str, optional
            Tag of the logging object (e.g. local or remote), by default None.
        file : Path, optional
            Path to the log file, by default None.
        clear : bool, optional
            Clear the provided log file, by default False.
        """

        self.name = name
        self.level = level
        self.tag = tag
        self.file = file

        # Clear log file if exists
        if clear:
            self.clear_log_file()

        # Logger config
        numeric_level = self._level_from_str(level)
        level = level.upper()
        formatter = Formatter(LOG_FORMAT, LOG_DATETIME_FORMAT)
        logger = getLogger(name)
        if logger.hasHandlers():
            logger.debug(self._format("Clear handlers"))
            logger.handlers.clear()
        logger.setLevel(numeric_level)

        # Stream handler
        stream_handler_stdout = StreamHandler(stdout)
        stream_handler_stdout.setFormatter(formatter)
        stream_handler_stdout.setLevel(DEBUG)
        stream_handler_stdout.addFilter(lambda record: record.levelno <= INFO)
        stream_handler_stderr = StreamHandler(stderr)
        stream_handler_stderr.setFormatter(formatter)
        stream_handler_stderr.setLevel(WARNING)
        logger.addHandler(stream_handler_stdout)
        logger.addHandler(stream_handler_stderr)

        # File handler
        if file is not None:
            file_handler = RotatingFileHandler(
                Path(file), maxBytes=512, backupCount=0
            )
            file_handler.setFormatter(formatter)
            file_handler.setLevel(DEBUG)
            logger.addHandler(file_handler)
            logger.debug(
                self._format(
                    f"Set file handler to '{file}'"
                )
            )

        # Complete initialization
        logger.debug(self._format("Set stream handler to 'STDOUT/STDERR'"))
        logger.debug(self._format(f"Set log level to '{level}'"))
        self.logger = logger
        self.logger.debug(self._format("Logger initialized"))

    def __repr__(self) -> str:
        return (
            f"Log({self.name!r}, {self.level!r}, "
            + f"{self.tag!r}, {self.file!r})"
        )

    def __str__(self) -> str:
        return f"Logger {self.name!r} with level {self.level!r}"

    def clear_log_file(self) -> None:
        """
        Clears the log file.
        """
        if self.file is not None:
            open(self.file, "w").close()

    def debug(self, msg: str) -> None:
        """
        Log a message with level "DEBUG"

        Parameters
        ----------
        msg : str
            Message to log.
        """
        self.logger.debug(self._format(msg))

    def info(self, msg: str) -> None:
        """
        Log a message with level "INFO"

        Parameters
        ----------
        msg : str
            Message to log.
        """
        self.logger.info(self._format(msg))

    def warning(self, msg: str) -> None:
        """
        Log a message with level "WARNING"

        Parameters
        ----------
        msg : str
            Message to log.
        """
        self.logger.warning(self._format(msg))

    def error(self, msg: str) -> None:
        """
        Log a message with level "ERROR"

        Parameters
        ----------
        msg : str
            Message to log.
        """
        self.logger.error(self._format(msg))

    def critical(self, msg: str) -> None:
        """
        Log a message with level "CRITICAL"

        Parameters
        ----------
        msg : str
            Message to log.
        """
        self.logger.critical(self._format(msg))

    def exception(self, msg: str) -> None:
        """
        Log an exception with message.

        Parameters
        ----------
        msg : str
            Message to log.
        """
        self.logger.exception(self._format(msg))

    @staticmethod
    def _level_from_str(level_name: str) -> int:
        numeric_level = getLevelName(level_name.upper())
        if not isinstance(numeric_level, int):
            raise ValueError(f"Invalid log level: {level_name}")
        return numeric_level

    def _format(self, msg: str) -> str:
        if self.tag is not None:
            return f"{self.tag} - {msg}"
        else:
            return msg


import argparse

import src

def parse_args():
    parser = argparse.ArgumentParser(description="Kaggle time-series competition")
    parser.add_argument("--task", "-t", type=str)
    parser.add_argument("--config", "-c", type=str, default="conf.yaml")
    parser.add_argument("--model", "-m", type=str, default="baseline")
    parser.add_argument("--data", "-d", type=str, default="")

    args = parser.parse_args()

    return args

if __name__ == "__main__":
    args = parse_args()

    if args.task == "train":
        if args.model == "baseline":
            src.models.run_baseline()
        else:
            trainer = src.models.Trainer()

            trainer.train()
    elif args.task == "test":
        trainer = src.models.Trainer()

        trainer.test()

    elif args.task == "build_feats":
        src.features.build_feats()
    else:
        raise NotImplementedError()


import requests
url="https://images.pexels.com/photos/674010/pexels-photo-674010.jpeg?auto=compress&cs=tinysrgb&dpr=1&w=500"
resp=requests.get(url)
resp.status_code

with open('image.jpg', 'wb') as fp:
    fp.write(resp.content)

def download(url, filename):
    try:
        resp = requests.get(url)
        if resp.status_code==200:
            with open(filename, 'wb') as fp:
                fp.write(resp.content)
                print("Image downloaded Successfully")
        else:
            return "Invalid Url"
    except Exception as error:
        return error
# url1=input("Enter the url= ")
# format=".jpg"
# filename1= input("Enter the file name you want= ")
# download(url1,filename1)

download("https://cdn.pixabay.com/photo/2015/04/23/22/00/tree-736885__480.jpg","tree.jpg")


from collections import defaultdict
class Solution:
    def numFactoredBinaryTrees(self, arr: List[int]) -> int:
        # sumi = len(arr)
        mod = 10**9 + 7
        n = len(arr)
        arr.sort()
        d = defaultdict()
        for num in arr:
            d[num] = d.get(num, 0) + 1

        for i in range(1,n):
            for j in range(i+1):
                if arr[i] % arr[j] == 0 and (arr[i] / arr[j]) in d:
                    d[arr[i]] += d[arr[j]] * d[arr[i]/arr[j]]

        return sum(d.values()) % mod


import pandas as pd
import ast
import click
from transformers import pipeline
from tqdm import tqdm

@click.command()
@click.option("--datafile", type=str, required=True, help="Path to the datafile")
@click.option(
    "--model_name_or_path",
    type=str,
    default=None,
    help="Model to be used for the generation.",
)
@click.option(
    "--output_path", type=str, required=True, help="Path to save the generations."
)
def execute(datafile, output_path, model_name_or_path=None):

    if model_name_or_path is not None:
        classifier = pipeline("zero-shot-classification", model=model_name_or_path)
    else:
        classifier = pipeline("zero-shot-classification")

    data = pd.read_csv(datafile)
    data = data.to_dict("records")

    utterances = []
    classes = []
    generations = []

    for case in tqdm(data):

        labels = ast.literal_eval(case["classes"])

        utterances.append(case["utterance"])
        classes.append(labels)

        res = classifier(case["utterance"], labels)

        label = "Unknown"
        if res["scores"][0] > 0.5:
            label = res["labels"][0]

        generations.append(label)

    df = pd.DataFrame.from_dict(
        {"utterance": utterances, "classes": classes, "snli": generations}
    )
    df.to_csv(output_path)

if __name__ == "__main__":
    execute()


import logging
import os
import sys
from datetime import datetime
from typing import Callable, Optional

import pytest
from pytest_embedded.plugin import multi_dut_argument, multi_dut_fixture

DEFAULT_SDKCONFIG = 'default'

@pytest.fixture(scope='session', autouse=True)
def session_tempdir() -> str:
    _tmpdir = os.path.join(
        os.path.dirname(__file__),
        'pytest_embedded_log',
        datetime.now().strftime('%Y-%m-%d_%H-%M-%S'),
    )
    os.makedirs(_tmpdir, exist_ok=True)
    return _tmpdir

@pytest.fixture
@multi_dut_argument
def config(request: pytest.FixtureRequest) -> str:
    return getattr(request, 'param', None) or DEFAULT_SDKCONFIG

@pytest.fixture
@multi_dut_fixture
def build_dir(request: pytest.FixtureRequest, app_path: str, target: Optional[str], config: Optional[str]) -> str:
    """
    Check local build dir with the following priority:

    1. build_<target>_<config>
    2. build_<target>
    3. build_<config>
    4. build

    Returns:
        valid build directory
    """
    check_dirs = []
    if target is not None and config is not None:
        check_dirs.append(f'build_{target}_{config}')
    if target is not None:
        check_dirs.append(f'build_{target}')
    if config is not None:
        check_dirs.append(f'build_{config}')
    check_dirs.append('build')

    for check_dir in check_dirs:
        binary_path = os.path.join(app_path, check_dir)
        if os.path.isdir(binary_path):
            logging.info(f'find valid binary path: {binary_path}')
            return check_dir

        logging.warning(
            'checking binary path: %s... missing... try another place', binary_path
        )

    recommend_place = check_dirs[0]
    logging.error(
        f'no build dir valid. Please build the binary via "idf.py -B {recommend_place} build" and run pytest again'
    )
    sys.exit(1)

@pytest.fixture(autouse=True)
@multi_dut_fixture
def junit_properties(test_case_name: str, record_xml_attribute: Callable[[str, object], None]) -> None:
    """
    This fixture is autoused and will modify the junit report test case name to <target>.<config>.<case_name>
    """
    record_xml_attribute('name', test_case_name)


from django.db import models

class GlobalDB(models.Model):
    """
    Global Database includes all registered and personal contacts
    """
    phone_number = models.CharField(max_length=13, blank=False, null=True)
    country_code = models.CharField(max_length=4, blank=False, null=True)
    name = models.CharField(max_length=30, blank=False, null=True)
    email = models.EmailField(max_length=255, blank=True)
    is_registered = models.BooleanField(default=False)
    is_personal_contact = models.BooleanField(default=False)
    is_marked_as_spam = models.BooleanField(default=False)
    created_at = models.DateTimeField(auto_now_add=True)

    def __str__(self):
        return self.country_code + '-' +self.phone_number+ ': ' +self.name

    class Meta:
        ordering = [
            'id', 'name', 'is_registered', 'phone_number',
            'country_code', 'email', 'is_personal_contact', 'is_marked_as_spam'
        ]

class PersonalContact(models.Model):
    """
    Includes relation of a registered number and its personal contact
    """
    phone_number = models.CharField(max_length=13, blank=False, null=True)
    country_code = models.CharField(max_length=4, blank=False, null=True)
    contact_of = models.ForeignKey(GlobalDB,related_name='Personal2Global', blank=False, on_delete=models.CASCADE)
    created_at = models.DateTimeField(auto_now_add=True)

    def __str__(self):
        return str(self.personal_contact) + ' : ' + str(self.contact_of)

class SpamDB(models.Model):
    """
    Includes number of times a number is marked as spam
    """
    phone_number = models.CharField(max_length=13, blank=False, null=True)
    country_code = models.CharField(max_length=4, blank=False, null=True)
    marked_by = models.ForeignKey(GlobalDB,related_name='Spam2Global', blank=False, on_delete=models.CASCADE)
    marked_at = models.DateTimeField(auto_now_add=True)

    def __str__(self):
        return str(self.spam_phone_number)+ ': ' +str(self.marked_by)