# -*- coding: utf-8 -*-
"""
Created on Wed Jun 30 11:11:25 2021

@author: pearsonra
"""

import unittest
import json
import pathlib
import geopandas
import shapely
import shutil
import sys
import pytest
import logging
import numpy
import rioxarray
import gc

from src.geofabrics import processor

class Test(unittest.TestCase):
    """A class to test the basic measured river interpolation functionality
    contained in processor.MeasuredRiverGenerator.

    Tests run include:
        1. test_river_polygon(linux/windows) - Test that the expected river polygon is
        created
        2. test_river_elevations(linux/windows) - Test that the expected river
        bathymetry is created
    """

    @classmethod
    def setUpClass(cls):
        """Create a CatchmentGeometry object and then run the DemGenerator processing
        chain to download remote files and produce a DEM prior to testing."""

        test_path = pathlib.Path().cwd() / pathlib.Path(
            "tests/test_many_stages_westport"
        )

        # Setup logging
        logging.basicConfig(
            filename=test_path / "test.log",
            encoding="utf-8",
            level=logging.INFO,
            force=True,
        )
        logging.info("In test_many_stages_westport")

        # load in the test instructions
        instruction_file_path = test_path / "instruction.json"
        with open(instruction_file_path, "r") as file_pointer:
            cls.instructions = json.load(file_pointer)

        # Remove any files from last test, then create a results directory
        cls.cache_dir = test_path / "data"
        cls.results_dir = cls.cache_dir / "results"
        cls.tearDownClass()
        cls.results_dir.mkdir()

        # create fake catchment boundary
        x0 = 1482951
        y0 = 5375247
        x1 = 1484180
        y1 = 5373303
        catchment = shapely.geometry.Polygon([(x0, y0), (x1, y0), (x1, y1), (x0, y1)])
        catchment = geopandas.GeoSeries([catchment])
        catchment = catchment.set_crs(
            cls.instructions["dem"]["output"]["crs"]["horizontal"]
        )

        # save faked catchment boundary - used as land boundary as well
        catchment_file = cls.results_dir / "catchment.geojson"
        catchment.to_file(catchment_file)

        # Run pipeline - download files and generated DEM
        runner = processor.MeasuredRiverGenerator(
            cls.instructions["measured"], debug=False
        )
        runner.run()
        runner = processor.RawLidarDemGenerator(cls.instructions["dem"], debug=False)
        runner.run()
        runner = processor.HydrologicDemGenerator(cls.instructions["dem"], debug=False)
        runner.run()

    @classmethod
    def tearDownClass(cls):
        """Remove created cache directory and included created and downloaded files at
        the end of the test."""

        cls.clean_data_folder()

    @classmethod
    def clean_data_folder(cls):
        """Remove all generated or downloaded files from the data directory"""

        assert cls.cache_dir.exists(), (
            "The data directory that should include the comparison benchmark dem file "
            "doesn't exist"
        )

        # Cycle through all folders within the cache dir deleting their contents
        for path in cls.cache_dir.iterdir():
            if path.is_dir():
                for file in path.glob("*"):  # only files
                    if file.is_file():
                        file.unlink()
                    elif file.is_dir():
                        shutil.rmtree(file)
                shutil.rmtree(path)

    @pytest.mark.skipif(sys.platform != "win32", reason="Windows test - this is strict")
    def test_result_dem_windows(self):
        """A basic comparison between the generated and benchmark DEM"""

        file_path = (
            self.cache_dir / self.instructions["dem"]["data_paths"]["benchmark_dem"]
        )
        with rioxarray.rioxarray.open_rasterio(file_path, masked=True) as benchmark_dem:
            benchmark_dem.load()
        # Load in test DEM
        file_path = (
            self.results_dir / self.instructions["dem"]["data_paths"]["result_dem"]
        )
        with rioxarray.rioxarray.open_rasterio(file_path, masked=True) as test_dem:
            test_dem.load()
        # compare the generated and benchmark DEMs
        diff_array = (
            test_dem.z.data[~numpy.isnan(test_dem.z.data)]
            - benchmark_dem.z.data[~numpy.isnan(benchmark_dem.z.data)]
        )
        logging.info(f"DEM array diff is: {diff_array[diff_array != 0]}")
        numpy.testing.assert_array_almost_equal(
            test_dem.z.data[~numpy.isnan(test_dem.z.data)],
            benchmark_dem.z.data[~numpy.isnan(benchmark_dem.z.data)],
            err_msg="The generated result_dem has different data from the "
            "benchmark_dem",
        )

        # explicitly free memory as xarray seems to be hanging onto memory
        del test_dem
        del benchmark_dem
        gc.collect()

    @pytest.mark.skipif(
        sys.platform != "linux", reason="Linux test - this is less strict"
    )
    def test_result_dem_linux(self):
        """A basic comparison between the generated and benchmark DEM"""

        file_path = (
            self.cache_dir / self.instructions["dem"]["data_paths"]["benchmark_dem"]
        )
        with rioxarray.rioxarray.open_rasterio(file_path, masked=True) as benchmark_dem:
            benchmark_dem.load()
        # Load in test DEM
        file_path = (
            self.results_dir / self.instructions["dem"]["data_paths"]["result_dem"]
        )
        with rioxarray.rioxarray.open_rasterio(file_path, masked=True) as test_dem:
            test_dem.load()
        # compare the generated and benchmark DEMs
        diff_array = (
            test_dem.z.data[~numpy.isnan(test_dem.z.data)]
            - benchmark_dem.z.data[~numpy.isnan(benchmark_dem.z.data)]
        )
        logging.info(f"DEM array diff is: {diff_array[diff_array != 0]}")

        threshold = 10e-2
        allowable_number_above = 3
        self.assertTrue(
            len(diff_array[numpy.abs(diff_array) > threshold])
            <= allowable_number_above,
            "more than "
            f"{allowable_number_above} DEM values differ by more than {threshold} on"
            f" Linux test run: {diff_array[numpy.abs(diff_array) > threshold]}",
        )
        threshold = 10e-6
        self.assertTrue(
            len(diff_array[numpy.abs(diff_array) > threshold]) < len(diff_array) / 100,
            f"{len(diff_array[numpy.abs(diff_array) > threshold])} or more than 1% of "
            f"DEM values differ by more than {threshold} on Linux test run: "
            f"{diff_array[numpy.abs(diff_array) > threshold]}",
        )

        # explicitly free memory as xarray seems to be hanging onto memory
        del test_dem
        del benchmark_dem
        gc.collect()

if __name__ == "__main__":
    unittest.main()


import urllib.request
from bs4 import BeautifulSoup

'''
urllib.error.URLError:
SSL認証エラー発生
macOSが原因
https://qiita.com/orangain/items/0a641d980019fd7e0c52

'''

class Scraper:
    def __init__(self, site):
        self.site = site

    def scrape(self):
        response = urllib.request.urlopen(self.site)
        html = response.read()
        parser = "html.parser"
        soup = BeautifulSoup(html, parser)

        with open("news_headline.txt", "w") as file:
            for tag in soup.find_all("a"):
                url = tag.get("href")
                # 日本語のニュースページにはhtmlを含むhref属性がない
                if url and  "articles" in url:
                    file.write("/n" + url)
        print("file save end")

news = "https://news.google.com/"
Scraper(news).scrape()


# The setup script installs all the required dependencies
# and create a virtual environment to run the code

import argparse
import os
import subprocess

parser = argparse.ArgumentParser(description='This script is for first time setup on windows machine.\
                                              It creates a python virtual environment and installs \
                                              all the required packages. For help type: python setup.py --help.\
                                              To run this setup script type: python setup.py')
args = parser.parse_args()

def install_cmd(cmd):
    command_run = subprocess.call(cmd, shell=True)
    if command_run == 0:
        # Success
        return 0
    else:
        # Cmd failed
        return 1

if __name__ == "__main__":
    print("Installing python-pip\n")
    pip_cmd = install_cmd("python3 -m ensurepip --upgrade")
    if pip_cmd:
        print("Unable to install python-pip. Terminating ...")
        quit()

    print("Installing python-virtualenv\n")
    pip_cmd = install_cmd("pip install --trusted-host pypi.org --trusted-host files.pythonhosted.org virtualenv")
    if pip_cmd:
        print("Unable to install python-virtualenv. Terminating ...")
        quit()

    print("Creating virtual enviroment virtual-env\n")
    pip_cmd = install_cmd("python3 -m venv virtual-env")
    if pip_cmd:
        print("Unable to craete python virtual environment. Terminating ...")
        quit()

    pip_path = "virtual-env{sep}Scripts{sep}pip3.exe".format(sep=os.sep)
    print("Installing all the required dependencies ...\n")
    pip_cmd = install_cmd("{pip_path} install --trusted-host pypi.org --trusted-host files.pythonhosted.org -r requirements.txt"
                        .format(pip_path=pip_path))
    if pip_cmd:
        print("Unable to install dependencies. Terminating ...")
        quit()

    print("Virtual environment created.\n")
    print("To activate the virtual environment type the following command:\n")
    print("$ virtual-env\\Scripts\\activate")


from pyspark.sql import SparkSession
from pyspark.sql.functions import *

spark = SparkSession.builder.appName("wiki").config("spark.sql.streaming.checkpointLocation", "/tmp/checkpoint").getOrCreate()

df = spark \
  .readStream \
  .format("kafka") \
  .option("kafka.bootstrap.servers", "localhost:9092") \
  .option("subscribe", "click") \
  .option("failOnDataLoss", "false") \
  .option("startingOffsets", "earliest") \
  .load() \
  .selectExpr("CAST(value AS STRING)")

schema = "prev_title STRING, timestamp TIMESTAMP"
df = df.select(from_json(col("value"), schema).alias("data")).select("data.*")

watermarkedDF = df \
    .withWatermark("timestamp", "10 minutes") \
    .groupBy(window("timestamp", "5 minutes"), "prev_title") \
    .agg(count("*").alias("count"), max("timestamp").alias("timestamp"))

watermarkedDF.createOrReplaceTempView("wiki")

sqlDF = spark.sql("SELECT prev_title as source, count, timestamp FROM wiki")

query = sqlDF.writeStream \
    .outputMode("append") \
    .format("org.apache.spark.sql.cassandra") \
    .option("keyspace", "wiki") \
    .option("table", "realtime") \
    .start()

query.awaitTermination()


import newspaper
import pymongo
from deep_translator import (GoogleTranslator,
                             ChatGptTranslator,
                             MicrosoftTranslator,
                             PonsTranslator,
                             LingueeTranslator,
                             MyMemoryTranslator,
                             YandexTranslator,
                             PapagoTranslator,
                             DeeplTranslator,
                             QcriTranslator,
                             single_detection,
                             batch_detection)
import time
import os, sys
import requests

client = pymongo.MongoClient('localhost', 27017)
db = client["gnews"]
headlines_collection = db["headlines"]
articles_collection = db["Post"]

my_translator = GoogleTranslator(source='auto', target='zh-CN')

def GT(text, batch=False):
    time.sleep(3)
    if batch:
        #print("list len:", len(text))
        words = 0;
        for x in text:
            words += len(x)
        if words < 5000:
            return my_translator.translate_batch(text)
        words = 0
        tmp = []
        index = 0
        for x in text:
            words += len(x)
            tmp.append(x)
            index += 1
            if words > 4500:
                break
        if words>=5000:
            tmp.pop()
            index -= 1
        return GT(tmp, True) + GT(text[index:], True)
    else:
        return my_translator.translate(text=text)

def save_article(x, a):
    path = "/root/GNewsFront/public/news_resource/pics/"+x['link_hash']
    if not os.path.exists(path):
        os.mkdir(path, 0o666)
    if not os.path.exists(path+"/figure.webp"):
        if x['figure'] is not None:
            data = requests.get(x['figure'], timeout=(5,5)).content
            if data is not None:
                with open(path+"/figure.webp", 'wb') as f:
                    f.write(data)
                    f.close()
            else:
                x['figure'] = None

    file = "/root/GNewsFront/public/news_resource/pics/"+x['link_hash']+"/article.html"
    with open(file, "w") as f:
        f.write(a.article_html)
        f.close()

def fetch_img(x, article):
    url = article.top_img
    if url is None or not article.has_top_image():
        return
    file = "/root/GNewsFront/public/news_resource/pics/"+x['link_hash']+"/top_img.jpg"
    try:
        res = requests.get(url, timeout=(5,5))
    except requests.exceptions.RequestException as e:
        print("fetch top_image timeout", url)
        return False
    if res.status_code == 200:
        with open(file, 'wb') as f:
            f.write(res.content)
            f.close()
        return True
    else:
        return False

def get_icon(x):
    icon = x['icon']
    alt_icon = x['alt_icon']
    icon_title = x['icon_title']

    path = "/root/GNewsFront/public/news_resource/icons/" + icon_title
    if not os.path.exists(path):
        print(icon_title)
        os.makedirs(path, 0o666)

    url = ""
    file = ""
    if icon is not None:
        file = path + "/icon.webp"
        url = icon
    else:
        file = path + "/alt_icon.webp"
        url = alt_icon

    if os.path.exists(file):
        return

    data = requests.get(url, timeout=(5,5)).content
    if data is not None:
        with open(file, 'wb') as f:
            f.write(data)
            f.close()
    else:
        print("get icon failed", url, icon_title)

def split_string(string, length):
    return [string[i:i + length] for i in range(0, len(string), length)]

def get_headline():
    myquery={"downloaded":False,"skip":False}
    c = newspaper.Config()
    c.browser_user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36'
    for x in headlines_collection.find(myquery):
        article = newspaper.Article(x['final_link'], keep_article_html=True, config=c)
        try:
            article.download()
            article.parse()
            article.nlp()
        except Exception as e:
            headlines_collection.update_one({"_id":x["_id"]},{"$set":{"skip":True}})
            print("download or update article failed,set skip:", x["final_link"])
            continue

        text_lines_tmp = article.text.splitlines()
        text_lines = []
        #text_lines = [x for x in text_lines if len(x) != 0]
        for i,t in enumerate(text_lines_tmp):
            if len(t) == 0:
                continue
            if len(t)>5000:
                text_lines+=split_string(t, 4900)
            else:
                text_lines.append(t)

        #for i,t in enumerate(text_lines):
        #    print(i,len(t),t)
        if not text_lines:
            headlines_collection.update_one({"_id":x["_id"]},{"$set":{"skip":True}})
            print("text_lines empty,set skip:", x["final_link"], text_lines)
            continue

        #print(x['title'])
        text_lines_cn = GT([x['title']]+text_lines, True)
        #print(text_lines_cn)
        title_cn = text_lines_cn[0]
        del text_lines_cn[0]

        save_article(x, article)

        text = ""
        for l in text_lines:
            text += l
            text += "\n\n"
        text_cn = ""
        for l in text_lines_cn:
            if l is None:
                continue
            text_cn += l
            text_cn += "\n\n"
        #print(text,text_cn)
        has_top_image = fetch_img(x, article)
        get_icon(x)
        a = {
            "figure": "/news_resource/pics/"+x['link_hash']+"/figure.webp" if x['figure'] is not None else None,
            "published": True,
            "title": x['title'],
            "title_cn": title_cn,
            "icon": "/news_resource/icons/" + x['icon_title'] + ("/alt_icon.webp" if x['icon'] is None else "/icon.webp"),
            "time": x['time'],
            "top_image": "/news_resource/pics/"+x['link_hash']+"/top_img.jpg" if article.has_top_image() and has_top_image else None,
            "summary": article.summary,
            "text": text,
            "text_cn": text_cn,
            "url": x['final_link'],
            }
        try:
            articles_collection.insert_one(a)
            headlines_collection.update_one({"_id":x["_id"]},{"$set":{"downloaded":True}})
        except Exception as e:
            print("insert article or update headline failed:", a["url"], str(e))

        print("Inserted one article")
        #break
    #for p in articles_collection.find():
    #    print(p)

get_headline()
client.close()

#docker network create mongoCluster
#docker run --name mongo --network mongoCluster -d --restart unless-stopped -p 127.0.0.1:27017:27017 -v /root/mongo-data/:/data/db mongodb/mongodb-community-server --replSet rs0 --bind_ip localhost,mongo
#docker exec -it mongo mongosh
#db.headlines.createIndex({"final_link":1},{unique:true})
#db.Post.createIndex({"url":1},{unique:true})
#mkdir /root/GNewsFront/public/news_resource/pics/
#mkdir /root/GNewsFront/public/news_resource/icons/


import os
import random
import string
import time

from selenium.common.exceptions import NoSuchElementException, ElementNotVisibleException, InvalidElementStateException, \
    StaleElementReferenceException
from selenium.webdriver import ActionChains
from selenium.webdriver.common import keys
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.wait import WebDriverWait
from wheel.signatures import assertTrue
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import json
from pprint import pprint
from features.pages.page_selector import LoginPageLocator, GeneralLocator, \
    AddManager, AddCustomer, AddWorker
from selenium.webdriver.support import expected_conditions as EC

def login(context, email, password):
    wait = WebDriverWait(context.browser, 10)
    wait.until(EC.element_to_be_clickable((LoginPageLocator.EMAIL_FIELD)))\
        .send_keys(email)
    wait.until(EC.element_to_be_clickable((LoginPageLocator.PASSWORD_FIELD)))\
        .send_keys(password)
    wait.until(EC.element_to_be_clickable((LoginPageLocator.SIGNIN_BUTTON)))\
        .click()

ROOT_DIR = os.path.dirname(os.path.abspath(__file__))

def create_managers(context, count):
    wait = WebDriverWait(context.browser, 10)

    # wait.until(EC.element_located_to_be_selected((GeneralLocator.MENU_MANAGER))).click()
    # context.browser.find_element(*GeneralLocator.MENU_MANAGER).click()
    with open(ROOT_DIR + '/managers.json') as data_file:
        data = json.load(data_file)
    i = 0

    while i < int(count):
        time.sleep(1)
        context.browser.find_element(*GeneralLocator.ADD_MANAGER_BTN).click()
        wait.until(EC.presence_of_element_located((AddManager.F_NAME))).\
            send_keys(data["data"][i][0])
        context.browser.find_element(*AddManager.L_NAME).send_keys(data["data"][i][1])
        context.browser.find_element(*AddManager.PHONE).send_keys(data["data"][i][2])
        context.browser.find_element(*AddManager.EMAIL).send_keys(data["data"][i][3])
        context.browser.find_element(*AddManager.PASSWORD).send_keys("Go1234")
        context.browser.find_element(*AddManager.CONFIRM_PASSWORD).send_keys("Go1234")
        context.browser.find_element(*AddManager.SAVE_NEW_MANAGER_BTN).click()
        time.sleep(1)
        i = i + 1

def create_customers(context, count):
    context.browser.find_element(*GeneralLocator.MENU_CUSTOMER).click()
    with open(ROOT_DIR + '/customers.json') as data_file:
        data = json.load(data_file)
    i = 0

    while i < int(count):
        context.browser.find_element(*GeneralLocator.ADD_MANAGER_BTN).click()
        time.sleep(1)
        context.browser.find_element(*AddCustomer.COMPANY).send_keys(data[i]['company'])
        context.browser.find_element(*AddCustomer.VATIN).send_keys(data[i]['vatin'])
        context.browser.find_element(*AddCustomer.COUNTRY).send_keys(data[i]['country'])
        context.browser.find_element(*AddCustomer.CITY).send_keys(data[i]['city'])
        context.browser.find_element(*AddCustomer.STREET).send_keys(data[i]['street'])
        context.browser.find_element(*AddCustomer.HOUSE).send_keys(data[i]['house'])
        context.browser.find_element(*AddCustomer.PHONE).send_keys(data[i]['phone'])
        context.browser.find_element(*AddCustomer.SAVE_NEW_CUSTOMER_BTN).click()
        time.sleep(0.5)
        i = i + 1

def create_workers(context, count):
    # context.browser.find_element(*GeneralLocator.MENU_WORKER).click()
    with open(ROOT_DIR + '/workers.json') as data_file:
        data = json.load(data_file)
    i = 0
    time.sleep(5)

    while i < int(count):
        time.sleep(1)
        context.browser.find_element(*GeneralLocator.ADD_MANAGER_BTN).click()
        time.sleep(1)
        context.browser.find_element(*AddWorker.FIRST_NAME).send_keys(data[i]['first'])
        context.browser.find_element(*AddWorker.LAST_NAME).send_keys(data[i]['last'])
        context.browser.find_element(*AddWorker.PHONE).send_keys(data[i]['phone'])
        context.browser.find_element(*AddWorker.DROPDOWN_LIST).click()
        context.browser.find_element(By.XPATH, ".//ul[@id='managerForNewWorker-menu']/li[2]").click()
        context.browser.find_element(By.ID, "breakTimeForNewWorker-button").click()
        context.browser.find_element(By.XPATH, ".//ul[@id='breakTimeForNewWorker-menu']/li[2]").click()
        context.browser.find_element(*AddWorker.SAVE_NEW_WORKER_BTN).click()
        time.sleep(0.5)
        i = i + 1

def generate_any_word(lenght_word_digit):
    from random import choice
    from string import lowercase
    n = lenght_word_digit

    string_val = "".join(choice(lowercase) for i in range(n))

    return string_val

def generate_digits(lenght_digit):
    from random import choice
    from string import lowercase
    n = lenght_digit

    string_val = "".join(choice(string.digits) for i in range(n))

    return string_val

def check_exists_by_xpath(context, xpath):
    try:
        context.browser.find_element(By.XPATH, xpath)
    except NoSuchElementException:
        return False
    return True

def sort_by_column_name(context, name):

    context.browser.implicitly_wait(10)

    array = []
    get_index = context.browser.find_elements(By.XPATH, './/div[@class="table-head-row"]/div/span')
    for i in get_index:
        array.append(i.text)

    column = array.index(name)

    column_text = context.browser.find_element(By.XPATH,
                                 './/span[text()[contains(.,"{}")]]'.format(name))
    text_color = column_text.value_of_css_property('color')

    if (text_color == 'rgba(83, 83, 83, 1)'):
        context.browser.find_element(By.XPATH, './/span[text()[contains(.,"{}")]]'.format(name)).click()

    else:
        context.browser.find_element(By.XPATH, './/span[text()[contains(.,"{}")]]'.format(name)).click()
        context.browser.find_element(By.XPATH, './/span[text()[contains(.,"{}")]]'.format(name)).click()

    time.sleep(1)

    managers_list = context.browser.find_elements(By.XPATH, ".//div[@class='table-row']")

    all_managers_after_filter_by_name = []
    for i in managers_list:
        a = i.text
        b = a.split("\n")
        all_managers_after_filter_by_name.append(b[column])

    print(all_managers_after_filter_by_name)
    actual_result = sorted(all_managers_after_filter_by_name, reverse=True)
    assert all_managers_after_filter_by_name == actual_result

def getToastMessage(context):
    try:
        search_result = context.browser.find_element(*GeneralLocator.TOAST)
        return search_result
    except NoSuchElementException:
        time.sleep(0.5)

def setEvents(context, start, end):
    def _move_and_click(context, elem):
        Hover = ActionChains(context.browser).move_to_element(elem)
        Hover.click().perform()

    max_page = int(context.browser.find_element(By.XPATH, ".//ul[@id='pagination-indicators']/li[last()]").text)
    counter = 2
    while counter <= max_page:
        s = context.browser.find_elements(By.XPATH,
                                          ".//button[1][contains(@class, 'start') and not(contains(@class ,'hide'))]")
        e = context.browser.find_elements(By.XPATH,
                                          ".//button[2][contains(@class, 'end') and not(contains(@class ,'hide'))]")

        if len(s) == 0 and len(e) == 0:
            page = context.browser.find_element(By.XPATH, ".//ul[@id='pagination-indicators']/li[.='{}']".
                                         format(counter))
            page.click()
            print ("click on other page " + str(counter))
            time.sleep(2)
        else:
            for test in s:
                start_list = context.browser.find_elements(By.XPATH, ".//button[1][contains(@class, 'start') and not(contains(@class ,'hide'))]")
                start_input = context.browser.find_elements(By.XPATH, ".//button[1][contains(@class, 'start') and not(contains(@class ,'hide'))]/parent::div//input[contains(@class, 'input-1')]")
                for a in start_list:
                    _move_and_click(context, a)
                    time.sleep(0.3)
                    for i in start_input:
                        i.send_keys(start)
                        time.sleep(0.1)
                        i.send_keys(Keys.ENTER)
                        time.sleep(1)
                        break
                    break

            for test1 in e:
                end_list = context.browser.find_elements(By.XPATH, ".//button[2][contains(@class, 'end') and not(contains(@class ,'hide'))]")
                end_input = context.browser.find_elements(By.XPATH, ".//button[2][contains(@class, 'end') and not(contains(@class ,'hide'))]/parent::div//input[contains(@class, 'input-2')]")
                for a in end_list:
                    _move_and_click(context, a)
                    time.sleep(0.3)
                    for i in end_input:
                        i.send_keys(end)
                        time.sleep(0.1)
                        i.send_keys(Keys.ENTER)
                        time.sleep(1.5)
                        break
                    break
        page = context.browser.find_element(By.XPATH, ".//ul[@id='pagination-indicators']/li[.='{}']".
                                            format(counter))
        page.click()
        print ("click on other page")
        time.sleep(2)
        counter = counter + 1


import numpy as np
import pandas as pd
import inspect
import GPy
import scipy.optimize as opt
import bokeh.plotting as bp
from bokeh.embed import components
from bokeh.models import Legend
from bokeh.layouts import widgetbox
from bokeh.layouts import gridplot

import seaborn as sns

from acquisitions import LocalMove, SGD, Explore, Exploit, Phase, UCB, EI
from decisions import Propto, Softmax

"""
Get mean/variances of GP posterior predictive given a kernel and observations
Parameters:
    kernel: GPy kernel
    observed_x: list of observed actions
    observed_y: list of observed function values
    all_x: range over possible actions
"""

def gp(kernel, observed_x, observed_y, all_x):

    if len(observed_x) == 0:
        return np.zeros(len(all_x))[:,None], np.ones(len(all_x))[:,None]
    else:
        X = observed_x[:,None]
        Y = observed_y[:,None]
        m = GPy.models.GPRegression(X, Y, kernel)
        m.Gaussian_noise.variance = .00001
        mean, var = m.predict(all_x[:,None])
        return mean, var

"""
Get means and variances for each trial given a kernel and a sequence of actions
Parameters:
    kernel: GPy kernel to be used
    function: Function on which the given responses were made
    actions: sequence of actions given by participant
"""

def get_mean_var(kernel, function, actions):

    all_mean = []
    all_var = []
    all_x = np.arange(len(function))
    for i in range(len(actions)):
        observed_x = np.array(actions[:i])
        if len(observed_x) == 0:
            observed_y = []
        else:
            observed_y = function[observed_x]
        mean, var = gp(kernel, observed_x, observed_y, all_x)
        all_mean.append(mean.ravel())
        all_var.append(var.ravel())
    return np.array(all_mean), np.array(all_var)

"""
Get utilities for all actions on all trials given a sequence actions
Parameters:
    list actions: sequence of actions given by participant
    acquisition_type: acquisition function class
    list acq_params: list of paramaters to pass to acquisition function
    array all_means: trials x actions array of GP means. Pass None if acquisition
                     function does not use a GP
    array all_vars: trials x actions array of GP variances. Pass None if
                    acquisition function does not use a GP
"""

def get_utility(all_x, actions, rewards, acquisition_type, kernel, acq_params, all_means = [], all_vars = [], replace = True):

    utility_data = pd.DataFrame()
    for i in range(len(actions)):
        trial = 'trial_' + str(i + 1)
        if len(all_means) == 0:
            mean = None
        else:
            mean = all_means[i]
        if len(all_vars) == 0:
            var = None
        else:
            var = all_vars[i]

        if i > 0:
            last_x = actions[i - 1]
            last_y = rewards[i - 1]
        else:
            last_x = None
            last_y = None
        if i > 1:
            second_last_x = actions[i - 2]
            second_last_y = rewards[i - 2]
            idx = [j for j in range(len(actions[:i])) if actions[j] != last_x]
            if len(idx) > 0:
                unique_second_last_x = actions[idx[-1]]
                unique_second_last_y = rewards[idx[-1]]
            else:
                unique_second_last_x = None
                unique_second_last_y = None
                second_last_x = None
                second_last_y = None
        else:
            unique_second_last_x = None
            unique_second_last_y = None
            second_last_x = None
            second_last_y = None

        args = {'all_x': all_x, 'mean': mean, 'var': var, 'trial': i,
                'last_x': last_x, 'last_y': last_y,
                'second_last_x': second_last_x, 'second_last_y': second_last_y,
                'unique_second_last_x': unique_second_last_x, 'unique_second_last_y': unique_second_last_y,
                'ntrials': len(actions), 'trial': i, 'actions': actions[:i], 'rewards': rewards[:i],
                'kernel': kernel}
        acq_arg_names = list(inspect.signature(acquisition_type.__init__).parameters.keys())
        acq_args = {arg_name: args[arg_name] for arg_name in args.keys() if arg_name in acq_arg_names}
        acquisition = acquisition_type(**acq_args)
        utility = acquisition(*acq_params)
        if not replace:
            utility = [utility[j] if j not in actions[:i] else np.NaN for j in range(len(utility))]
        utility_data[trial] = utility
    return utility_data

"""
Get log likelihoods for all actions on all trials given a sequence actions
Parameters:
    DataFrame utility_data: action x trial data frame of utilities
    list actions: sequence of actions given by participant
    decision_type: decision function class
    list dec_params: list of paramaters to pass to acquisition function
"""

def get_likelihood(utility_data, actions, decision_type, dec_params):

    likelihood_data = pd.DataFrame()
    for i in range(len(actions)):
        trial = utility_data.columns[i]
        args = {'trial': i, 'x2': actions[i - 1] if i > 0 else None}
        dec_arg_names = list(inspect.signature(decision_type.__init__).parameters.keys())
        dec_args = {arg_name: args[arg_name] for arg_name in args.keys() if arg_name in dec_arg_names}
        decision = decision_type(**dec_args)
        utility = utility_data[trial]
        likelihood = np.log(decision(utility, *dec_params))

        likelihood_data[trial] = likelihood
    likelihood_data.index = utility_data.index
    return likelihood_data

"""
Get log likelihood given a set of actions and the log likelihoods associated with
a particular model
Parameters:
    list actions: sequence of actions given by participant
    DataFrame likelihood_data: action x trial data frame of log likelihoods
"""

def joint_log_likelihood(actions, likelihood_data):

    return np.nansum([likelihood_data[likelihood_data.columns[i]][actions[i]] for i in range(len(actions))])

"""
Find MLE acquisition and decision function parameters given a sequence of actions. Returns
a dictionary including all utilities/likelihoods across trials and values for model comparison.
Parameters:
    list actions: sequence of actions given by participant
    acquisition_type: acquisition function class
    decision_type: decision function class
    array all_means: trials x actions array of GP means. Pass None if acquisition
                     function does not use a GP
    array all_vars: trials x actions array of GP variances. Pass None if
                    acquisition function does not use a GP
"""

def fit_strategy(all_x, actions, rewards, acquisition_type, decision_type, replace, kernel, all_means = [], all_vars = [], method = 'DE', restarts = 5):

    init_acq_params = acquisition_type.init_params
    init_dec_params = decision_type.init_params
    nacq_params = len(init_acq_params)

    if nacq_params == 0:
        utility_data = get_utility(all_x, actions, rewards, acquisition_type, [], kernel, all_means = all_means, all_vars = all_vars, replace = replace)
        def obj(params):
            likelihood_data = get_likelihood(utility_data, actions, decision_type, params)
            jll = joint_log_likelihood(actions, likelihood_data)
            return -jll

    else:
        def obj(params):
            acq_params = params[:nacq_params]
            dec_params = params[nacq_params:]
            utility_data = get_utility(all_x, actions, rewards, acquisition_type, acq_params, kernel, all_means, all_vars, replace)
            likelihood_data = get_likelihood(utility_data, actions, decision_type, dec_params)
            jll = joint_log_likelihood(actions, likelihood_data)
            return -jll

    init_params = init_acq_params + init_dec_params
    bounds = acquisition_type.bounds + decision_type.bounds
    fun = np.inf
    final_x = None
    for i in range(restarts):
        if method == 'DE':
            x = opt.differential_evolution(obj, bounds)
        else:
            x = opt.minimize(obj, init_params, method = method, bounds = bounds)
        if x.fun < fun:
            fun = x.fun
            final_x = x
    print (final_x)
    params = final_x.x
    acq_params = params[:nacq_params]
    dec_params = params[nacq_params:]
    if nacq_params > 0:
        utility_data = get_utility(all_x, actions, rewards, acquisition_type, acq_params, kernel, all_means, all_vars, replace)
    likelihood_data = get_likelihood(utility_data, actions, decision_type, dec_params)
    n = len(actions)
    k = len(params)
    data = {'acquisition': acquisition_type.__name__, 'acquisition_params': acq_params.tolist(),
            'decision': decision_type.__name__, 'decision_params': dec_params.tolist(),
            'all_utilities': utility_data.to_dict(), 'all_likelihoods': likelihood_data.to_dict(),
            'MLE_likelihood': -fun, 'AIC': -2 * -fun + 2 * k, 'BIC': -2 * -fun + np.log(n) * k}
    random_ic = -2 * (np.log(1. / len(all_x)) * n)
    data['pseudo_r2'] = 1 - (data['AIC'] / random_ic)
    data['approx_bf'] = np.exp(random_ic - data['BIC'])
    return data

"""
fits combinations of kernels and acquisition/decision function strategies to the actions
of one participant
Parameters:
    list actions: sequence of actions given by participant
    Series function: Function on which the given responses were made
    list training_functions: List of functions on which to fit kernel parameters
    list kernels: List of GPy kernels to fit to actions
    list strategies: list of (acquisition_type, decision_type) tuples to fit to actions
"""

def fit_participant(participant_id, goal, actions, function, function_samples, kernels, strategies, method = 'DE', restarts = 5):

    all_x = np.arange(len(function))[:,None]
    stacked_function_samples_x = np.hstack([np.arange(len(f)) for f in function_samples])[:,None]
    stacked_function_samples_y = np.hstack(function_samples)[:,None]
    rewards = [function[a] for a in actions]
    if goal == 'find_max_last':
        replace = False
    else:
        replace = True
    all_data = []

    gp_strategies = [strategy for strategy in strategies if strategy[0].isGP]
    non_gp_strategies = [strategy for strategy in strategies if not strategy[0].isGP]

    for i in range(len(kernels)):
        kernel = kernels[i]
        m = GPy.models.GPRegression(X = stacked_function_samples_x, Y = stacked_function_samples_y, kernel = kernel)
        m.optimize()
        all_means, all_vars = get_mean_var(kernel, function, actions)
        for acquisition_type, decision_type in gp_strategies:
            data = fit_strategy(all_x, actions, rewards, acquisition_type, decision_type, replace, all_means = all_means, all_vars = all_vars, method = method, restarts = restarts)
            data['kernel'] = type(kernel).__name__
            data['all_means'] = {'trial_' + str(i + 1): all_means[i].ravel().tolist() for i in range(len(all_means))}
            data['all_vars'] = {'trial_' + str(i + 1): all_vars[i].ravel().tolist() for i in range(len(all_vars))}
            all_data.append(data)

    for acquisition_type, decision_type in non_gp_strategies:
        data = fit_strategy(all_x, actions, rewards, acquisition_type, decision_type, replace, all_means = [], all_vars = [], method = method, restarts = restarts)
        all_data.append(data)

    participant_data = {'id': participant_id, 'function': function.ravel().tolist(), 'actions': actions, 'rewards': rewards}
    participant_data['models'] = all_data
    return participant_data

def add_viz_data(data):

    for i in range(len(data)):
        participant = data[i]
        colormap = np.array(sns.color_palette("hls", len(participant['models']) + 1).as_hex())
        plots = []
        for trial in range(1, len(participant['models'][0]['all_utilities'].keys()) + 1):
            u_plot = bp.figure(title = "Utility of Next Action", plot_width = 700, plot_height = 400, tools = "")
            u_plot.toolbar.logo = None
            l_plot = bp.figure(title = "Log Likelihood of Next Action", plot_width = 700, plot_height = 400, tools = "")
            l_plot.toolbar.logo = None
            legend_items = []

            actions = participant['actions'][:trial - 1]
            rewards = participant['rewards'][:trial - 1]
            if len(rewards) == 0:
                min_reward = 0
                max_reward = 0
            else:
                min_reward = np.min(rewards)
                max_reward = np.max(rewards)
            next_action = participant['actions'][trial - 1]
            if len(actions) > 0:
                u_plot.circle(actions, rewards, color = colormap[-1])
                a = l_plot.circle(actions, rewards, color = colormap[-1])
                legend_items.append(('Actions', [a]))
            utilities = [[model['all_utilities']['trial_' + str(trial)][k] for k in range(len(model['all_utilities']['trial_' + str(trial)]))] for model in participant['models']]
            likelihoods = [[model['all_likelihoods']['trial_' + str(trial)][k] for k in range(len(model['all_likelihoods']['trial_' + str(trial)]))] for model in participant['models']]
            u_valmin = min(np.nanmin(np.array(utilities).ravel()), min_reward)
            u_valmax = max(np.nanmax(np.array(utilities).ravel()), max_reward)
            l_valmin = min(np.nanmin([l for l in np.array(likelihoods).ravel() if not np.isinf(l)]), min_reward) - .01
            l_valmax = max(np.nanmax(np.array(likelihoods).ravel()), max_reward) + .01

            u_plot.vbar(x = next_action, width = 0.1, bottom = u_valmin, top = u_valmax, color = "black")
            b = l_plot.vbar(x = next_action, width = 0.1, bottom = l_valmin, top = l_valmax, color = "black")
            legend_items.append(('Next Action             | p(Next)   | p(1,...,Next - 1)', [b]))
            for j in range(len(utilities)):
                utility = utilities[j]
                likelihood = likelihoods[j]
                index = [utility[k] for k in range(len(utility)) if not np.isnan(utility[k])]
                utility = [u if not np.isnan(u) and not np.isinf(u) else np.nanmin([ui for ui in utility if not np.isinf(ui)]) for u in utility]
                likelihood = [l if not np.isnan(l) and not np.isinf(l) else np.nanmin([li for li in likelihood if not np.isinf(li)]) for l in likelihood]
                acq = participant['models'][j]['acquisition']
                if 'kernel' in participant['models'][j]:
                    kern = '/' + participant['models'][j]['kernel']
                else:
                    kern = ''
                params = '(' + ','.join([str(np.round(p, 2)) for p in participant['models'][j]['acquisition_params'] + participant['models'][j]['decision_params']]) + ')'

                u_plot.line(list(range(len(utility))), utility, color = colormap[j])
                c = l_plot.line(list(range(len(likelihood))), likelihood, color = colormap[j])

                legend_string = acq + kern + params
                trial_l = np.round(likelihood[next_action], 2)
                likelihood_data = pd.DataFrame(participant['models'][j]['all_likelihoods'])
                total_l = np.round(joint_log_likelihood(actions[:trial], likelihood_data), 2)
                legend_string = legend_string + (' ' * (14 - len(legend_string))) + ' | ' + str(trial_l) + '      | ' + str(total_l)

                legend_items.append((legend_string, [c]))
            u_script, u_div = components(u_plot)
            l_script, l_div = components(l_plot)
            legend = Legend(items = legend_items)
            l_plot.add_layout(legend, 'right')

            gp_plot = bp.figure(title = "Expected Reward", plot_width = 400, plot_height = 400, tools = "")
            gp_plot.toolbar.logo = None
            gp_plot.circle(actions, rewards, color = colormap[-1], legend = "Actions")
            for j in range(len(data[i]['models'])):
                if 'kernel' in data[i]['models'][j].keys():
                    kernel = data[i]['models'][j]['kernel']
                    mean = data[i]['models'][j]['all_means']['trial_' + str(trial)]
                    var = data[i]['models'][j]['all_vars']['trial_' + str(trial)]
                    std = np.sqrt(np.array(var))
                    upper = np.array(mean) + 2 * std
                    lower = np.array(mean) - 2 * std
                    index = np.arange(len(mean))
                    gp_plot.line(index, mean, color = colormap[j], legend = kernel)

                    band_x = np.append(index, index[::-1])
                    band_y = np.append(lower, upper[::-1])
                    gp_plot.patch(band_x, band_y, color = colormap[j], fill_alpha = 0.2)
            gp_plot.legend.location = "top_right"
            f_script, f_div = components(gp_plot)

            p = gridplot([[gp_plot, u_plot], [l_plot]], toolbar_location=None)
            p_script, p_div = components(p)

            plots.append({'u_script': u_script, 'u_div': u_div, 'l_script': l_script, 'l_div': l_div,
                          'f_script': f_script, 'f_div': f_div, 'p_script': p_script, 'p_div': p_div})
        data[i]['plots'] = plots
    return data

def fit_all_participants(results, method = 'DE', restarts = 5):

    data = []
    for index, participant in results.iterrows():
        actions = participant['response']
        function = participant['function']
        function_n = ((function - np.mean(function)) / np.std(function))
        function_samples = list(participant['function_samples'].values())
        function_samples_n = np.array([((f - np.mean(f)) / np.std(f)) for f in function_samples])
        participant_id = participant['somataSessionId']
        function_name = participant['function_name']
        goal = participant['goal']
        score = participant['total_score']

        if function_name == 'pos_linear':
            kernels = [GPy.kern.RBF(1), GPy.kern.Linear(1) + GPy.kern.Bias(1)]
        elif function_name == 'neg_quad':
            kernels = [GPy.kern.RBF(1), GPy.kern.Poly(1, order = 2)]
        elif function_name == 'sinc_compressed':
            kernels = [GPy.kern.RBF(1), GPy.kern.StdPeriodic(1) + GPy.kern.RBF(1)]
        strategies = [(LocalMove, Softmax), (SGD, Softmax), (Phase, Softmax), (UCB, Softmax)]
        strategies = [(LocalMove, Softmax), (SGD, Softmax), (Explore, Softmax), (Exploit, Softmax), (Phase, Softmax), (UCB, Softmax)]
        kernels = [GPy.kern.RBF(1)]

        participant_data = fit_participant(participant_id, goal, actions, function_n, function_samples_n, kernels, strategies, method = method, restarts = restarts)
        participant_data['function_name'] = function_name
        participant_data['goal'] = goal
        participant_data['score'] = score
        data.append(participant_data)
    return data


#!/usr/bin/python
# -*- coding: utf-8 -*-

from typing import TYPE_CHECKING, List

from terminaltables import DoubleTable, AsciiTable, build
from terminaltables.width_and_alignment import max_dimensions

from semantic_modeling.algorithm.string import auto_wrap
from transformation.models.table_schema import Schema

if TYPE_CHECKING:
    from transformation.models.data_table import DataTable

def visualize(tbl: 'DataTable', format: str="ascii", inner_row_border: bool=True, max_text_width: int=40):
    if format == 'double':
        clazz = DoubleTable
    elif format == 'ascii':
        clazz = AsciiTable
    else:
        assert False

    def render_tbl(title, arrays, meta_dim):
        inst = clazz(arrays)
        inst.title = title
        inst.inner_heading_row_border = False
        inst.inner_row_border = inner_row_border
        dimensions = max_dimensions(inst.table_data, inst.padding_left, inst.padding_right)[:3]
        return build.flatten(inst.gen_table(meta_dim['current_dim'][0], dimensions[1], meta_dim['current_dim'][1]))

    def wrap_text(schema: Schema, row):
        object = {}
        for attr, val in schema.attributes.items():
            if isinstance(val, Schema):
                if val.is_list_of_objects:
                    object[attr] = [wrap_text(val, r) for r in row[attr]]
                else:
                    object[attr] = wrap_text(val, row[attr])
            elif val == Schema.LIST_VALUE:
                if row[attr] is None:
                    object[attr] = None
                else:
                    object[attr] = [auto_wrap(s, max_text_width) if isinstance(s, str) else s for s in row[attr]]
            else:
                if isinstance(row[attr], str):
                    object[attr] = auto_wrap(row[attr], max_text_width)
                else:
                    object[attr] = row[attr]
        return object

    def cal_dimensions(schema: Schema, rows: List[dict]):
        """Calculate dimensions of a table so that we can render it properly"""
        meta = {
            "current_dim": None,
            "nested_dims": {}
        }
        placeholder = {}
        for attr, val in schema.attributes.items():
            if isinstance(val, Schema):
                if val.is_list_of_objects:
                    meta['nested_dims'][attr], sample_str = cal_dimensions(val, [r for row in rows for r in row[attr]])
                else:
                    meta['nested_dims'][attr], sample_str = cal_dimensions(val, [row[attr] for row in rows])
                placeholder[attr] = sample_str
            elif val == Schema.LIST_VALUE:
                nested_arrays = [[r] for row in rows if row[attr] is not None for r in row[attr]]
                instance = clazz(nested_arrays[:1])
                dimensions = max_dimensions(nested_arrays, instance.padding_left, instance.padding_right)[:3]
                meta['nested_dims'][attr] = {
                    "current_dim": (dimensions[0], dimensions[2]),
                    "nested_dims": {}
                }
                placeholder[attr] = build.flatten(instance.gen_table(dimensions[0], dimensions[1][:1], dimensions[2]))

        arrays = [[]]
        for attr, val in schema.attributes.items():
            if isinstance(val, Schema):
                arrays[-1].append(placeholder[attr])
            else:
                arrays[-1].append(attr)

        for row in rows:
            array = []
            for attr, val in schema.attributes.items():
                if isinstance(val, Schema) or val == Schema.LIST_VALUE:
                    array.append(placeholder[attr])
                else:
                    array.append(row[attr])
            arrays.append(array)

        instance = clazz([arrays[0]])
        dimensions = max_dimensions(arrays, instance.padding_left, instance.padding_right)[:3]
        meta['current_dim'] = dimensions[0], dimensions[2]

        sample_str = build.flatten(instance.gen_table(dimensions[0], dimensions[1][:1], dimensions[2]))
        return meta, sample_str

    def schema2array(schema: Schema, meta_dims: dict):
        array = []
        for attr, val in schema.attributes.items():
            if isinstance(val, Schema):
                tbl_str = render_tbl(attr, [schema2array(val, meta_dims['nested_dims'][attr])], meta_dims['nested_dims'][attr])
                array.append(tbl_str)
            else:
                array.append(attr)
        return array

    def row2array(schema: Schema, row: dict, meta_dims: dict):
        array = []
        for attr, val in schema.attributes.items():
            if isinstance(val, Schema):
                dim = meta_dims['nested_dims'][attr]
                if val.is_list_of_objects:
                    tbl_str = render_tbl(None, [row2array(val, r, dim) for r in row[attr]], dim)
                    array.append(tbl_str)
                else:
                    tbl_str = render_tbl(None, [row2array(val, row[attr], dim)], dim)
                    array.append(tbl_str)
            elif val == Schema.LIST_VALUE:
                dim = meta_dims['nested_dims'][attr]
                if row[attr] is not None:
                    tbl_str = render_tbl(None, [[str(x)] for x in row[attr]], dim)
                    array.append(tbl_str)
                else:
                    array.append(row[attr])
            else:
                array.append(row[attr])

        return array

    rows = [wrap_text(tbl.schema, r) for r in tbl.rows]
    meta_dims = cal_dimensions(tbl.schema, rows)[0]

    arrays = []
    arrays.append(schema2array(tbl.schema, meta_dims))
    for row in rows:
        arrays.append(row2array(tbl.schema, row, meta_dims))

    return render_tbl(tbl.id, arrays, meta_dims)
    # x = clazz(arrays)
    # x.title = tbl.title
    # x.inner_row_border = inner_row_border
    # x.inner_heading_row_border = False
    # return str(x.table)


from django.conf.urls import url
from .views import (
    login_view,
    logout_view,
    polls_view,
    poll_vote_view,
    poll_results_view,
)

urlpatterns = [
    url(r'^$', login_view, name='login'),
    url(r'^logout$', logout_view, name='logout'),
    url(r'^polls/$', polls_view, name='public_polls'),
    url(r'^polls/(?P<poll_id>[0-9]+)$', polls_view, name='poll'),
    url(r'^polls/(?P<poll_id>[0-9]+)/vote$', poll_vote_view, name='poll_vote'),
    url(r'^polls/(?P<poll_id>[0-9]+)/results$', poll_results_view, name='poll_results'),
]


import uuid

from aws_lambda_powertools import Logger
from aws_lambda_powertools.utilities.data_classes import APIGatewayProxyEvent
from pydantic import ValidationError

from api.models import Action, EnumerationQueryArgs, LambdaResponse
from api.repository import DynamoActionRepository
from api.responses import BadRequest, NotFound, Ok

logger = Logger(service="gw-api", utc=True)

def introspect(context) -> LambdaResponse:
    return Ok.as_json({"version": context.function_version, "schema": ""})

def enumerate(event: APIGatewayProxyEvent) -> LambdaResponse:
    raw_qargs = (
        {} if event["queryStringParameters"] is None else event["queryStringParameters"]
    )
    try:
        qargs = EnumerationQueryArgs(**raw_qargs)
    except ValidationError as ve:
        logger.info("Unable to parse query args", extra={"errors": ve.errors()})
        return BadRequest.as_json(ve.errors())

    uid = str(uuid.UUID(int=0))
    repo = DynamoActionRepository()
    if qargs.status:
        result = repo.get_actions_by_status(uid, qargs.status)
    elif qargs.created_at:
        result = repo.get_actions_by_created_at(
            uid, since=qargs.created_at.since, until=qargs.created_at.until
        )
    elif qargs.completed_at:
        result = repo.get_actions_by_completed_at(
            uid, since=qargs.completed_at.since, until=qargs.completed_at.until
        )
    else:
        result = repo.enumerate_actions_for_user(uid)
    return Ok.as_json(result)

def run(event: APIGatewayProxyEvent) -> LambdaResponse:
    # Do something interesting
    repo = DynamoActionRepository()
    action = Action(details={"endpoint": "run"}, created_by=uuid.UUID(int=0))
    repo.store_actions(action)
    return Ok.as_json(action)

def status(event: APIGatewayProxyEvent) -> LambdaResponse:
    uid = str(uuid.UUID(int=0))
    repo = DynamoActionRepository()

    assert event.path_parameters
    action_id = event.path_parameters["action_id"]
    action = repo.get_action_by_id(uid, action_id)

    if action is None:
        logger.info(
            "Unable to find Action", extra={"user_id": uid, "action_id": action_id}
        )
        return NotFound.as_json(f"Action with ID {action_id} was not found.")
    return Ok.as_json(action)

def cancel(event: APIGatewayProxyEvent) -> LambdaResponse:
    return Ok.as_json({"Endpoint": "cancel"})

def release(event: APIGatewayProxyEvent) -> LambdaResponse:
    return Ok.as_json({"Endpoint": "release"})


# Crie um programa que leia o ano de nascimento de sete pessoas. No final, mostre quantas pessoas ainda não atingiram a maioridade e quantas já são maiores.

from datetime import date

maioridade = 0
menoridade = 0
ano_atual = date.today().year

for i in range(1, 8):
    ano_nascimento = int(input(f'Em que ano a {i}ª pessoa nasceu? '))
    if ano_atual - ano_nascimento >= 21:
        maioridade += 1
    else:
        menoridade += 1

print(f'Ao todo, tivemos {maioridade} pessoas maiores de idade')
print(f'E também tivemos {menoridade} pessoas menores de idade')


import copy
import datetime
import os
import re
import uuid
from typing import List

from fastapi import APIRouter, Response, UploadFile

from registry.models.module import Module
from registry.models.modules import Meta, ModuleResponse

router = APIRouter()

module_db: List[Module] = []

def _filter_by_namespace(modules: List[Module], namespace: str) -> List[Module]:
    for module in modules:
        if module.namespace != namespace:
            modules.remove(module)
    return modules

def _filter_by_provider(modules: List[Module], provider: str) -> List[Module]:
    for module in modules:
        if module.provider != provider:
            modules.remove(module)
    return modules

def _filter_by_verified(modules: List[Module]) -> List[Module]:
    for module in modules:
        if not module.verified:
            modules.remove(module)
    return modules

@router.get("/")
async def list_modules(offset: int = 0, provider: str = "", verified: bool = False):
    modules = copy.deepcopy(module_db)

    if provider:
        modules = _filter_by_provider(modules, provider)

    if verified:
        modules = _filter_by_verified(modules, verified)

    return ModuleResponse(
        meta=Meta(
            limit=offset,
            current_offset=0,
            next_offset=offset,
            next_url=f"/v1/modules?limit={offset}&offset={offset}&verified={str(verified).lower()}",
        ),
        modules=modules,
    )

@router.get("/{namespace}")
async def list_modules_by_namespace(
    namespace, offset: int = 0, provider: str = "", verified: bool = False
):
    modules: List[Module] = []

    for module in module_db:
        if module.namespace == namespace:
            modules.append(module)

    if provider:
        modules = _filter_by_provider(modules, provider)

    if verified:
        modules = _filter_by_verified(modules, verified)

    return ModuleResponse(
        meta=Meta(
            limit=offset,
            current_offset=0,
            next_offset=offset,
            next_url=f"/v1/modules?limit={offset}&offset={offset}&verified={str(verified)}",
        ),
        modules=modules,
    )

@router.get("/search")
async def search_modules(
    q: str,
    offset: int = 0,
    provider: str = "",
    namespace: str = "",
    verified: bool = False,
):
    modules: List[Module] = []

    for module in module_db:
        if re.search(q, module.name):
            modules.append(module)

    if namespace:
        modules = _filter_by_namespace(modules, namespace)

    if provider:
        modules = _filter_by_provider(modules, provider)

    if verified:
        modules = _filter_by_verified(modules, verified)

    return ModuleResponse(
        meta=Meta(
            limit=offset,
            current_offset=0,
            next_offset=offset,
            next_url=f"/v1/modules?limit={offset}&offset={offset}&verified={str(verified)}",
        ),
        modules=modules,
    )

@router.get("/{namespace}/{name}/{provider}/versions")
async def list_module_versions(namespace: str, name: str, provider: str):
    modules = [
        module
        for module in module_db
        if module.namespace == namespace
        and module.name == name
        and module.provider == provider
    ]
    return {"modules": modules}

@router.get("/{namespace}/{name}")
async def list_latest_module_version_foreach_provider(namespace: str, name: str, offset: int = 0):
    modules = [
        module
        for module in module_db
        if module.namespace == namespace
        and module.name == name
    ]

    return ModuleResponse(Meta(limit=0, current_offset=0), modules)

@router.get("/{namespace}/{name}/{provider}/{version}/download")
async def download_specific_module_version(
    namespace: str, name: str, provider: str, version: str
):
    for module in module_db:
        if (
            module.namespace == namespace
            and module.name == name
            and module.provider == provider
            and module.version == version
        ):
            url = module.url
            break

    return Response(status_code=204, headers={"X-Terraform-Get": url})

@router.get("/{namespace}/{name}/{provider}/download")
async def download_latest_module_version(namespace: str, name: str, provider: str):
    for module in module_db:
        if (
            module.namespace == namespace
            and module.name == name
            and module.provider == provider
            and module.latest
        ):
            url = module.url
            break

    return Response(status_code=204, headers={"X-Terraform-Get": url})

@router.get("/{namespace}/{name}/{provider}")
async def get_latest_module_version_by_provider(
    namespace: str, name: str, provider: str
):
    for module in module_db:
        if (
            module.namespace == namespace
            and module.name == name
            and module.provider == provider
            and module.latest
        ):
            return module
    return {}

@router.get("/{namespace}/{name}/{provider}/{version}")
async def get_module(namespace: str, name: str, provider: str, version: str):
    for module in module_db:
        if (
            module.namespace == namespace
            and module.name == name
            and module.provider == provider
            and module.version == version
        ):
            return module
    return {}

@router.post("/{namespace}/{name}/{provider}/vcs")
def create_module_vcs(namespace: str, name: str, provider: str):
    return {}

@router.put("/{namespace}/{name}/{provider}/novcs")
def create_module_latest_no_vcs(namespace: str, name: str, provider: str, file: UploadFile):
    try:
        path = f"data/{namespace}/{name}/{provider}/latest/"
        os.makedirs(path, exist_ok=True)

        filepath = f"{path}/archive.tar.gz"
        with open(filepath, "wb") as f:
            f.write(file.file.read())

    except Exception as err:
        return {"error": err}

    return {"filename": file.filename}

@router.put("/{namespace}/{name}/{provider}/{version}/novcs")
def create_module_version_no_vcs(namespace: str, name: str, provider: str, version: str, file: UploadFile):
    try:
        # module = Module(
        #     id=uuid.uuid4(),
        #     owner="",
        #     namespace=namespace,
        #     name=name,
        #     provider=provider,
        #     version=version,
        #     is_latest=False,
        #     description="",
        #     source="upload",
        #     created_at=datetime.fromtimestamp(datetime.timestamp(datetime.now()), tz=datetime.tzinfo).strftime("%Y-%m-%dT%H:%M:%S.%fZ%z"),
        #     download_url=f"http://127.0.0.1:8000/v1/modules/{namespace}/{name}/{provider}/{version}/download")

        path = f"data/{namespace}/{name}/{provider}/{version}"
        os.makedirs(path, exist_ok=True)

        filepath = f"{path}/archive.tar.gz"
        with open(filepath, "wb") as f:
            f.write(file.file.read())

    except Exception as err:
        return {"error": err}

    return {"filename": file.filename}


#! /usr/local/bin/python3.7
"""
Consider the following "magic" 3-gon ring, filled with the numbers 1 to 6, and
each line adding to nine.

       4
        \
         \
          3
         / \
        /   \
       1-----2-----6
      /
     /
    5

Working clockwise, and starting from the group of three with the numerically
lowest external node (4,3,2 in this example), each solution can be described
uniquely. For example, the above solution can be described by the
set {4,3,2; 6,2,1; 5,1,3}.

It is possible to complete the ring with four different totals: 9, 10, 11, and
12. There are eight solutions in total:

    Total    Solution set
     9       4,2,3; 5,3,1; 6,1,2
     9       4,3,2; 6,2,1; 5,1,3
    10       2,3,5; 4,5,1; 6,1,3
    10       2,5,3; 6,3,1; 4,1,5
    11       1,4,6; 3,6,2; 5,2,4
    11       1,6,4; 5,4,2; 3,2,6
    12       1,5,6; 2,6,4; 3,4,5
    12       1,6,5; 3,5,4; 2,4,6

By concatenating each group it is possible to form 9-digit strings; the maximum
string for a 3-gon ring is 432621513.

Using the numbers 1 to 10, and depending on arrangements, it is possible to form
16- and 17-digit strings. What is the maximum 16-digit string for a "magic"
5-gon ring?

Solution: 6531031914842725
"""
from itertools import permutations

def PE_68():
    """
    There are 10 x 5 x 4 = 120 options for the first 3-node set
    There are 3 x 2 x 1 = 10 options for the second 3-node set
    There are 1 x 1 x 1 = 1 options for the third 3-node set
    So there are 120 x 10 x 1 = 720 total options
    Check each to see if test condition holds, and if so add result to output

    >>> PE_68()
    6531031914842725
    """
    known_strings = []
    base = set(range(1, 11))
    for a, b, c in permutations(base, 3):
        subtotal = a + b + c
        for e, d in generate_next(base, (a, b, c), 2, subtotal - c):
            for f, g in generate_next(base, (a, b, c, d, e), 2, subtotal - e):
                for h, i in generate_next(base, (a, b, c, d, e, f, g), 2, subtotal - g):
                    for (j,) in generate_next(base, (a, b, c, d, e, f, g, h, i), 1, subtotal - b - i):
                        string = ''.join(map(str, (a, b, c, d, c, e, f, e, g, h, g, i, j, i, b)))
                        if not_a_rotation(string, known_strings):
                            known_strings.append(string)
    return int(known_strings[-1])

def generate_next(base, known, size, control_subtotal):
    return (digits for digits in permutations(base - set(known), size)
            if sum(digits) == control_subtotal)

def not_a_rotation(string, known_strings):
    return not any(map(lambda known_string: are_rotations(string, known_string),
                       known_strings))

def are_rotations(string1, string2):
    for _ in range(len(string1)):
        string2 = string2[1:] + string2[0]
        if string1 == string2:
            return True
    return False

if __name__ == '__main__':
    import doctest; doctest.testmod()


"""Onionr - Private P2P Communication.

DBCreator, creates sqlite3 databases used by Onionr
"""
"""
    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <https://www.gnu.org/licenses/>.
"""
import sqlite3, os
from coredb import dbfiles
import filepaths

def createAddressDB():
    '''
        Generate the address database

        types:
            1: I2P b32 address
            2: Tor v2 (like facebookcorewwwi.onion)
            3: Tor v3
    '''
    if os.path.exists(dbfiles.address_info_db):
        raise FileExistsError("Address database already exists")
    conn = sqlite3.connect(dbfiles.address_info_db)
    c = conn.cursor()
    c.execute('''CREATE TABLE adders(
        address text,
        type int,
        knownPeer text,
        speed int,
        success int,
        powValue text,
        failure int,
        lastConnect int,
        lastConnectAttempt int,
        trust int,
        introduced int
        );
    ''')
    conn.commit()
    conn.close()

def createPeerDB():
    '''
        Generate the peer sqlite3 database and populate it with the peers table.
    '''
    if os.path.exists(dbfiles.user_id_info_db):
        raise FileExistsError("User database already exists")
    # generate the peer database
    conn = sqlite3.connect(dbfiles.user_id_info_db)
    c = conn.cursor()
    c.execute('''CREATE TABLE peers(
        ID text not null,
        name text,
        adders text,
        dateSeen not null,
        trust int,
        hashID text);
    ''')
    c.execute('''CREATE TABLE forwardKeys(
    peerKey text not null,
    forwardKey text not null,
    date int not null,
    expire int not null
    );''')
    conn.commit()
    conn.close()
    return

def createBlockDB():
    '''
        Create a database for blocks

        hash         - the hash of a block
        dateReceived - the date the block was recieved, not necessarily when it was created
        decrypted    - if we can successfully decrypt the block (does not describe its current state)
        dataType     - data type of the block
        dataFound    - if the data has been found for the block
        dataSaved    - if the data has been saved for the block
        sig    - optional signature by the author (not optional if author is specified)
        author       - multi-round partial sha3-256 hash of authors public key
        dateClaimed  - timestamp claimed inside the block, only as trustworthy as the block author is
        expire int   - block expire date in epoch
    '''
    if os.path.exists(dbfiles.block_meta_db):
        raise FileExistsError("Block database already exists")
    conn = sqlite3.connect(dbfiles.block_meta_db)
    c = conn.cursor()
    c.execute('''CREATE TABLE hashes(
        hash text not null,
        dateReceived int,
        decrypted int,
        dataType text,
        dataFound int,
        dataSaved int,
        sig text,
        author text,
        dateClaimed int,
        expire int
        );
    ''')
    conn.commit()
    conn.close()
    return

def createBlockDataDB():
    if os.path.exists(dbfiles.block_data_db):
        raise FileExistsError("Block data database already exists")
    else:
        if not os.path.exists(filepaths.block_data_location):
            os.mkdir(filepaths.block_data_location)
    conn = sqlite3.connect(dbfiles.block_data_db)
    c = conn.cursor()
    c.execute('''CREATE TABLE blockData(
        hash text not null,
        data blob not null
        );
    ''')
    conn.commit()
    conn.close()

def createForwardKeyDB():
    '''
        Create the forward secrecy key db (*for *OUR* keys*)
    '''
    if os.path.exists(dbfiles.forward_keys_db):
        raise FileExistsError("Block database already exists")
    conn = sqlite3.connect(dbfiles.forward_keys_db)
    c = conn.cursor()
    c.execute('''CREATE TABLE myForwardKeys(
        peer text not null,
        publickey text not null,
        privatekey text not null,
        date int not null,
        expire int not null
        );
    ''')
    conn.commit()
    conn.close()
    return

def create_blacklist_db():
    if os.path.exists(dbfiles.blacklist_db):
        raise FileExistsError("Blacklist db already exists")
    conn = sqlite3.connect(dbfiles.blacklist_db, timeout=10)
    c = conn.cursor()
    # Create table
    c.execute('''CREATE TABLE blacklist(
            hash text primary key not null,
            dataType int,
            blacklistDate int,
            expire int
            );
        ''')
    conn.commit()
    conn.close()

create_funcs = [createAddressDB, createPeerDB,
                createBlockDB, createBlockDataDB,
                createForwardKeyDB, create_blacklist_db]


import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import yfinance as yf
from dataset import SlidingWindowTransformer
from models.lstm import LSTMForecaster
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from training_arguments import TrainingArguments

AMZN = yf.download('AMZN', start='2013-01-01', end='2019-12-31', progress=False)
all_data = AMZN[['Adj Close', 'Open', 'High', 'Low', "Close", "Volume"]].round(2)

train_df, test_df = train_test_split(all_data, test_size=0.2, shuffle=False)

input_cols = ['Adj Close', 'Open', 'High', 'Low', "Close", "Volume"]
output_cols = ['Adj Close']

X_train, y_train = train_df[input_cols], train_df[output_cols]
X_test, y_test = test_df[input_cols], test_df[output_cols]

window_size = 24
forecast_size = 1
step_size = 1

lstm = LSTMForecaster(
    window_size=window_size,
    forecast_size=forecast_size,
    hidden_size=128,
    num_layers=1,
    in_features=len(input_cols),
    out_features=len(output_cols),
    training_args=TrainingArguments(
        criterion=nn.MSELoss,
        optimizer=torch.optim.Adam,
        lr=0.003,
        max_epochs=150,
        batch_size=32,
        device='cuda',
    ),
)

model = Pipeline(
    steps=[
        ('scaler', StandardScaler()),
        ('slding', SlidingWindowTransformer(window_size=window_size, forecast_size=forecast_size, step_size=step_size)),
        ('lstm', lstm),
    ]
)
model.fit(X=X_train, y=y_train)
y_pred = model.predict(X=X_test)
_, y_true = model['slding'].transform(X=None, y=y_test)

mse_list = [mean_squared_error(true, pred) for true, pred in zip(y_true, y_pred)]
average_mse = np.mean(mse_list)

print(average_mse)
# loss = F.mse_loss(torch.from_numpy(y_pred), torch.from_numpy(y_true)).item()
# print(loss)


from feature_process import *
from pose_cluster import *
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay, accuracy_score, roc_curve, RocCurveDisplay
from sklearn.inspection import permutation_importance
import tensorflow as tf
import csv

def train_balance(x,y):
    health = np.where(y==0)[0]
    pain = np.where(y==1)[0]
    sng = np.where(y==2)[0]
    mins = min([len(health),len(pain),len(sng)])
    health = np.random.choice(health, mins, replace=False)
    pain = np.random.choice(pain, mins, replace=False)
    sng = np.random.choice(sng, mins, replace=False)
    newidx = np.concatenate([health,pain,sng])
    return x[newidx],y[newidx]

class DataSet:
    '''
    Storing dataset to train/to test, root of related files, info of each single mice
    '''
    def __init__(self, dlc, bsoid=None, vidc=None, vids=None, dep=None, specific=[]):
        self.specific = specific
        self.all_treatment = ['Capbasal','Cap','pH5.2basal','pH5.2','pH7.4basal','pH7.4',
                                'pH5.2ASIC3KObasal','pH5.2ASIC3KO','CapTV1KObasal','CapTV1KO']
        self.files = {}
        self.files['dlc'] = self.load_paths(dlc, True)
        self.files['bsoid'] = self.load_paths(bsoid)
        self.files['vids'] = self.load_paths(vids)
        self.files['vidc'] = self.load_paths(vidc)
        self.files['dep'] = self.load_paths(dep)
        self.data_config()
        self.mclf=None

    def load_paths(self, root, sav_treat=False):
        if not root:
            return []
        files = os.listdir(root)
        sav_files = []
        treatments = []
        names = []
        for file in files:
            sav = True
            for sp in self.specific:
                if file.find(sp)==-1:
                    sav = False
                    break
            if not sav:
                continue
            if sav_treat:
                treatment = file.split('-')[0]
                name = file.split('-')[1]
                if file.find('basal')!=-1:
                    treatments.append(treatment+'basal')
                else:
                    treatments.append(treatment)
                names.append(name)
            sav_files.append(root+'/'+file)
        if sav_treat:
            self.names = names
            self.treatments = treatments
        return sav_files

    def data_config(self):
        self.ind={}
        self.ind['basal'] = np.array([i for i, j in enumerate(self.treatments) if j.find('basal')!=-1])
        for t in self.all_treatment:
            self.ind[t] = np.array([i for i, j in enumerate(self.treatments) if j == t])
        print('basal:',len(self.ind['basal']),' ,pain:',len(self.ind['Cap']),
                ' sng:',len(self.ind['pH5.2']),' pH7.4:',len(self.ind['pH7.4']),
                ' sngKO:',len(self.ind['pH5.2ASIC3KO']),' CapKO:',len(self.ind['CapTV1KO']))

    def sel_file(self, filetype='dlc', treatment='Cap'):
        if treatment == 'basal':
            return [self.files[filetype][i] for i, j in enumerate(self.treatments) if j.find('basal')!=-1]
        return [self.files[filetype][i] for i, j in enumerate(self.treatments) if j==treatment]

    def sel_feat(self, treatment='all'):
        if treatment == 'basal':
            return [self.mice_feat[i] for i, j in enumerate(self.treatments) if j.find('basal')!=-1]
        return [self.mice_feat[i] for i, j in enumerate(self.treatments) if j==treatment]

    def sel_data(self, treatment='all', sel_type='x_train'):
        if treatment == 'basal':
            return [self.data[sel_type][i] for i, j in enumerate(self.treatments) if j.find('basal')!=-1]
        return [self.data[sel_type][i] for i, j in enumerate(self.treatments) if j==treatment]

    def generate_feature(self, feat_type='frame'):
        self.mice_feat = []
        for i in range(len(self.files['dlc'])):
            if feat_type[:-1] == 'bs':
                tmp = miceFeature(self.treatments[i], bsoid=self.files['bsoid'][i], feat_type=feat_type)
            else:
                tmp = miceFeature(self.treatments[i], self.files['dlc'][i], feat_type=feat_type)#,self.files['vidc'][i],self.files['vids'][i],self.files['dep'][i])
            self.mice_feat.append(tmp)

    def generate_train_test(self, split=0.5, motion_del=False, k=1):
        '''
        validation setting as last k-th of each three treatment mice
        '''
        # config for mice_feat
        for miceF in self.mice_feat:
            if self.mclf:
                miceF.labeling(self.mclf,self.motion_score)
            else:
                miceF.labeling()
            miceF.train_config(split=split, motion_del=motion_del)

        # start
        all_sets = ['x_train','y_train','x_test','y_test','x_val','y_val']
        self.data = {}
        for s in all_sets:
            self.data[s] = []

        for t in self.all_treatment:
            inds = self.ind[t]
            k = k%len(inds)
            for i in range(len(inds)):
                if i==len(inds)-k-1:
                    continue
                ind = inds[i]
                self.data['x_train'].append(self.mice_feat[ind].x_train)
                self.data['y_train'].append(self.mice_feat[ind].y_train)
                self.data['x_test'].append(self.mice_feat[ind].x_test)
                self.data['y_test'].append(self.mice_feat[ind].y_test)
            ind = inds[len(inds)-k-1]
            self.data['x_val'].append(self.mice_feat[ind].feature)
            self.data['y_val'].append(self.mice_feat[ind].label)

    def pose_cls(self, sel=['random'], sel_num=20, embed=False, k=10, cls_type='km', clf_type='svm'):
        # miceF : miceFeature class object
        # get feature
        feat = []
        if sel[0]=='random':
            miceFs = self.mice_feat
            ind = np.random.choice(np.arange(len(miceFs)), sel_num, replace=False)
            for i in ind:
                feat.append(miceFs[i].feature)
        else:
            miceFs = []
            for s in sel:
                miceFs.extend(self.sel_feat(s))
            for miceF in miceFs:
                feat.append(miceF.feature)
        feat = np.concatenate(feat)
        # if is lstm => flatten to 2d feature
        if len(feat.shape)>2:
            feat = feat.reshape(len(feat), feat.shape[1]*feat.shape[2])
        # cluster
        if embed:
            embeder, embeddings = embedfeat(feat)
            motions, mclf = motion_cluster(embeddings, k, cls_type)
            self.embeder = embeder
        else:
            motions, mclf = motion_cluster(feat, k, cls_type)
        motion_num = len(np.unique(motions))
        if not mclf:
            mclf = motion_clf(feat, motions, clf_type=clf_type)
        # cluster predict and save result
        motionsB = [0]*motion_num
        motionsT = [0]*motion_num
        miceFsB, miceFsT = self.sel_feat('Capbasal'), self.sel_feat('Cap')
        for i in range(len(miceFsB)):
            miceFB = miceFsB[i]
            miceFT = miceFsT[i]
            if embed:
                motionB = motion_predict(miceFB.feature, mclf, embeder)
                motionT = motion_predict(miceFT.feature, mclf, embeder)
            else:
                motionB = motion_predict(miceFB.feature, mclf)
                motionT = motion_predict(miceFT.feature, mclf)
            for i in np.unique(motions):
                motionsB[i]+= len(np.where(motionB==i)[0])
                motionsT[i]+= len(np.where(motionT==i)[0])
        # motion score
        motion_num = len(motionsB)
        ratio = np.zeros((motion_num), dtype=float)
        for i in range(motion_num):
            if (motionsB[i]+motionsT[i])>0:
                ratio[i] = motionsT[i]/(motionsB[i]+motionsT[i])
        motion_score = np.zeros((motion_num), dtype=float)
        th = 0.4
        motion_score[(ratio<=th) | (ratio>=1-th)] = 1
        motion_score[(ratio>th) & (ratio<1-th)] = -1
        print("bad motions:", len(np.where(motion_score==-1)[0]))
        # plot
        x = np.arange(motion_num)
        width = 0.3
        plt.bar(x, motionsB, width, color='green', label='basal')
        plt.bar(x + width, motionsT, width, color='red', label='treat')
        plt.xticks(x + width / 2, x)
        plt.legend(bbox_to_anchor=(1,1), loc='upper left')
        #plt.show()
        #plt.savefig()
        self.mclf = mclf
        self.motionsB = motionsB
        self.motionsT = motionsT
        self.motion_score = motion_score

class miceFeature:
    '''
    Storing All data(file paths, landmarks, features ...) of single mice(file)
    '''
    def __init__(self, treatment, dlc=None, bsoid=None, vidc=None, vids=None, dep=None, feat_type='frame'):
        self.treatment = treatment
        if(dlc):
            self.dlcfile = dlc
            self.read_dlc()
        if(bsoid):
            self.bsoidfile = bsoid
        if(vidc):
            self.vidcfile = vidc
        if(vids):
            self.vidsfile = vids
        if(dep):
            self.depfile = dep

        self.count_feature(feat_type=feat_type)

    ### DLC functions #############################################################################
    def read_dlc(self):
        if not os.path.isfile(self.dlcfile):
            print("no file")
            return
        raw = np.genfromtxt(self.dlcfile, delimiter=",",dtype=int)[3:]
        getcol = tuple(np.arange(len(raw[0]))[np.arange(len(raw[0]))%3!=0])
        self.dlc_index = np.expand_dims(raw[:,0], axis=1)
        self.dlc_raw = raw[:,getcol]
        #remove nan
        notnan = ~np.isnan(self.dlc_raw).any(axis=1)
        self.dlc_raw = self.dlc_raw[notnan]
        self.dlc_index = self.dlc_index[notnan]
    def dlc_wrap(self):
        return np.resize(self.dlc_raw,(len(self.dlc_raw),int(self.dlc_raw.shape[1]/2),2))
    ###############################################################################################

    ### generate feature ##########################################################################
    def count_feature(self, feat_type='frame'):
        # config
        sel_dist=[[0,1],[0,2],[1,3],[2,3],[3,4],[3,5],[4,6],[5,6]]
        sel_ang=[[1,3,2],[0,3,6],[4,3,5]]
        sel_coord=[]
        normalize_range=(0,1)
        include_index = False
        window = 10
        if feat_type[-1]=='F':
            step = 10
        else:
            step = 5

########### frame ################################################
        if feat_type == 'frame':
            # config frame
            sel_dist=[[0,3],[3,4],[1,2]]
            sel_ang=[[0,1,3],[1,3,4]]
            # frame feature pre
            dist = count_dist(self.dlc_raw, sel_dist)
            ang = count_angle(self.dlc_raw, sel_ang)
            # disp = count_disp(self.dlc_raw, step=1, threshold=None)
            # frame feature
            feat = dist
            feat = np.hstack([feat, ang])
            # feat = np.hstack([feat, disp[:,0:1]])
            # normalize
            feat = feature_normalize(feat, normalize_range=normalize_range)
#########bsoid + cwt (full/half)##################################
        if feat_type[:-1] == 'bscwt':
            # frame feature pre
            dist = count_dist(self.dlc_raw, sel_dist)[1:]
            ang = count_angle(self.dlc_raw, sel_ang)[1:]
            disp = count_disp(self.dlc_raw, step=1, threshold=None)
            # frame feature
            feat = dist[:,5:6]
            feat = np.hstack([feat, dist[:,7:8]])
            feat = np.hstack([feat, ang[:,2:3]])
            feat = np.hstack([feat, disp[:,2:3]])
            # segment feature
            # seg = abs(fft_signal(feat, window=seg_window, flat=True))
            seg = cwt_signal(feat, window=window, step=step)
            # combine
            tmp = np.hstack([disp, ang])
            feat = np.hstack([seg, seg_statistic(tmp, count_types=['avg'], window=window, step=step)])
            feat = np.hstack([feat, seg_statistic(dist, count_types=['sum'], window=window, step=step)])
            # normalize
            feat = feature_normalize(feat, normalize_range=normalize_range)
########## bsoid ##########################################################
        if feat_type[:-1] == 'bs':
            savfile = joblib.load(self.bsoidfile)
            if len(savfile) > 10:
                feat = savfile
            else:
                feat = savfile[0]
########### bsoid LSTM ############################################
        if feat_type[:-1] == 'bsLSTM':
            # frame feature pre
            dist = count_dist(self.dlc_raw, sel_dist)[1:]
            ang = count_angle(self.dlc_raw, sel_ang)[1:]
            disp = count_disp(self.dlc_raw, step=1, threshold=None)
            # segment feature combine
            tmp = np.hstack([disp, ang, disp])
            tmp = feature_normalize(tmp, normalize_range=normalize_range)
            feat = generate_tmpfeat(tmp, window=window, step=step)
########### bsoid + cwt LSTM ############################################
        if feat_type[:-1] == 'bscwtLSTM':
            # frame feature pre
            dist = count_dist(self.dlc_raw, sel_dist)[1:]
            ang = count_angle(self.dlc_raw, sel_ang)[1:]
            disp = count_disp(self.dlc_raw, step=1, threshold=None)
            # frame feature
            feat = dist[:,5:6]
            feat = np.hstack([feat, dist[:,7:8]])
            feat = np.hstack([feat, ang[:,2:3]])
            feat = np.hstack([feat, disp[:,2:3]])
            # segment feature combine
            seg = cwt_signal(feat, window=window, step=step, flat=False)
            tmp = np.hstack([dist, ang, disp])
            tmp = feature_normalize(tmp, normalize_range=normalize_range)
            feat = generate_tmpfeat(tmp, window=window, step=step)
            feat = np.concatenate([feat, seg], axis=2)
#########################################################################
        self.feature = feat

    ### train test config ##########################################################################
    def labeling(self, mclf=None, motion_score=None):
        # pain:1 sng:2 health:0
        labels = np.zeros((self.feature.shape[0]), dtype=int)
        if self.treatment == 'pH5.2':
            labels[:] = 2
        elif self.treatment == 'pH7.4' or self.treatment.find('basal')!=-1 or \
                self.treatment.find('pH5.2ASIC3KO')!=-1 or self.treatment.find('CapTV1KO')!=-1:
            labels[:] = 0
        elif self.treatment == 'Cap':
            labels[:] = 1
        if mclf:
            motions = motion_predict(self.feature, mclf)
            for i in range(len(motion_score)):
                if motion_score[i] == -1:
                    labels[np.where(motions==i)] = 0 ## bad motion label
        self.label=labels

    def train_config(self, split=0.5, shuffle=True, motion_del=False):
        # select sample
        if motion_del:
            feat = self.feature[np.where(self.label!=-1)]
            label = self.label[np.where(self.label!=-1)]
        else:
            feat = self.feature
            label = self.label
        # shuffle
        ind = np.arange(len(feat))
        np.random.shuffle(ind)
        self.shuffle = ind
        # split
        if shuffle:
            feat = feat[ind]
            label = label[ind]
        if split==0:
            self.x_train = feat
            self.y_train = label
            self.x_test = []
            self.y_test = []
        else:
            # split : training portion
            sp = int(len(label)*split)
            self.x_train = feat[:sp,:]
            self.y_train = label[:sp]
            self.x_test = feat[sp:,:]
            self.y_test = label[sp:]

class Analysis:
    def __init__(self, model_type='svm', classes=3):
        if model_type == 'svm':
            self.model = SVC(kernel='rbf', C=1000)
        elif model_type == 'rf':
            self.model = RandomForestClassifier(random_state=42)
        elif model_type == 'dnn':
            self.model = DNN_model(classes)
        elif model_type == 'lstm':
            self.model = LSTM_model(classes)

    def train(self, x, y):
        self.model = self.model.fit(x,y)

    def test(self, x, y, show=False):
        sc = self.model.score(x, y)
        if show:
            print('accuracy = ',sc)
        return sc

    def analysis(self, x, y, seperate=False):
        pred = self.model.predict(x)

        # non-seperate
        tp = np.count_nonzero(((y==1) & (pred==1)) | ((y==2) & (pred==2)))
        tn = np.count_nonzero(((y==0) & (pred==0)) | ((y==-1) & (pred==-1)))
        fp = np.count_nonzero(((y==0) & ((pred==1)|(pred==2))) | ((y==-1) & ((pred==1)|(pred==2)))  |((y==1) & (pred==2)) | ((y==2) & (pred==1)) )  # mis postive
        fn = np.count_nonzero(((y==1) & ((pred==0)|(pred==-1)|(pred==2))) | ((y==2) & ((pred==0)|(pred==-1)|(pred==1))))  #|((y==0) & (pred==-1)) | ((y==-1) & (pred==0)) ) # mis negative
        toler = np.count_nonzero(((y==0) & (pred==-1)) | ((y==-1) & (pred==0))) # miss negative is useless
        if (fp+tn)==0:
            fa = 0
        else:
            fa = fp/(fp+tn)
        if (tp+fn)==0:
            dr = 0
        else:
            dr = tp/(tp+fn)
        acc = (tp+tn)/(tp+tn+fp+fn)
        print('accuracy = ', acc)
        print("false alarm: ", fa)
        print("detection rate: ", dr)
        return [acc,fa,dr]

    def analysis2(self, x, y):
        '''
        seperate detection rate of pain/sng
        '''
        pred = self.model.predict(x)
        tn = np.count_nonzero(((y==0) & (pred==0)) | ((y==-1) & (pred==-1)))
        fn_p = np.count_nonzero((y==1) & ((pred==0)|(pred==-1)|(pred==2)))
        tp_p = np.count_nonzero((y==1) & (pred==1))
        # fp_p = np.count_nonzero(((y==0) & (pred==1)) | ((y==-1) & (pred==1)) | ((y==2) & (pred==1)) )
        fn_s = np.count_nonzero((y==2) & ((pred==0)|(pred==-1)|(pred==1)))
        tp_s = np.count_nonzero((y==2) & (pred==2))
        # fp_s = np.count_nonzero(((y==0) & (pred==2)) | ((y==-1) & (pred==2)) | ((y==1) & (pred==2)) )
        if (tp_p+fn_p)==0:
            dr_p = 0
        else:
            dr_p = tp_p/(tp_p+fn_p)
        if (tp_s+fn_s)==0:
            dr_s = 0
        else:
            dr_s = tp_s/(tp_s+fn_s)
        return [dr_p, dr_s]

    def feat_importance(self, x, y, save_path=None):
        r = permutation_importance(self.model, x, y, n_repeats=10, random_state=0)
        feature_names = np.arange(len(x[0]))
        features = np.array(feature_names)
        # sorted_idx = r.importances_mean.argsort()
        plt.barh(features, r.importances_mean)
        plt.xlabel("Permutation Importance")
        if save_path:
            plt.savefig(save_path)
        return r.importances_mean

class LSTM_model:
    def __init__(self, classes):
        self.classes = classes
        self.build_model()

    def build_model(self):
        self.model = tf.keras.models.Sequential()
        self.model.add(tf.keras.layers.LSTM(32, return_sequences=False))
        self.model.add(tf.keras.layers.Dropout(0.2))
        self.model.add(tf.keras.layers.BatchNormalization())
        self.model.add(tf.keras.layers.Dense(self.classes, activation='softmax'))
        opt = tf.keras.optimizers.Adam(learning_rate=0.01)
        self.model.compile(optimizer=opt,
                  loss='categorical_crossentropy',
                  metrics=['accuracy'])

    def fit(self, x, y):
        self.classes_ = np.arange(self.classes)
        if self.classes == 4:
            self.classes_ = self.classes_-1
        callbacks = [tf.keras.callbacks.EarlyStopping(monitor='accuracy', patience=20, mode='max')]
        self.model.fit(x, tf.one_hot(y,self.classes), epochs=200, batch_size=16) #,callbacks=callbacks)
        return self

    def predict(self, x):
        if self.classes == 4:
            return np.argmax(self.model.predict(x), axis=1)-1
        return np.argmax(self.model.predict(x), axis=1)

class DNN_model:
    def __init__(self, classes):
        self.classes = classes
        self.build_model()
    def build_model(self):
        self.model = tf.keras.models.Sequential()
        self.model.add(tf.keras.layers.Dense(32),activation='relu')
        self.model.add(tf.keras.layers.Dropout(0.2))
        self.model.add(tf.keras.layers.Dense(32),activation='relu')
        self.model.add(tf.keras.layers.Dropout(0.2))
        self.model.add(tf.keras.layers.BatchNormalization())
        self.model.add(tf.keras.layers.Dense(self.classes, activation='softmax'))
        opt = tf.keras.optimizers.Adam(learning_rate=0.01)
        self.model.compile(optimizer=opt,
                  loss='categorical_crossentropy',
                  metrics=['accuracy'])

    def fit(self, x, y):
        self.classes_ = np.arange(self.classes)
        if self.classes == 4:
            self.classes_ = self.classes_-1
        callbacks = [tf.keras.callbacks.EarlyStopping(monitor='accuracy', patience=20, mode='max')]
        self.model.fit(x, tf.one_hot(y,self.classes), epochs=200, batch_size=16) #,callbacks=callbacks)
        return self

    def predict(self, x):
        if self.classes == 4:
            return np.argmax(self.model.predict(x), axis=1)-1
        return np.argmax(self.model.predict(x), axis=1)


import pygame

class Enemy:

    color = (255,255,255)

    # Constructor
    def __init__(self, x_, y_, w_, h_, aSpeed, aWaypointList) -> None:
        """Constructor

        Args:
            x_ (int): initializes x pos
            y_ (int): initializes y pos
            w_ (int): initializes width
            h_ (int): initializes height
            aSpeed (float): initializes speed
            aWaypointList (TBD): [description]
        """
        # initialize vars
        self.hitbox = pygame.rect.Rect(x_, y_, w_, h_)
        self.speed = aSpeed
        # make it a copy so the original list is not later changed
        self.waypointList = aWaypointList.copy()

        # get the starting position
        startingX = self.waypointList[0].x
        startingY = self.waypointList[0].y
        self.currentPos = pygame.Vector2(startingX, startingY)
        # get the index of the target position
        self.waypointIndex = 1

        # boolean representing if the bullet should exist
        self.exists = True

    def updatePos(self):

        # if you have not reached the last waypoint
        if self.waypointIndex < len(self.waypointList):

            # get the target pos
            targetPos = self.waypointList[self.waypointIndex]

            # if you have not reached the current position yet
            # then move towards the target
            if self.currentPos.distance_to(targetPos) >= self.speed:
                # get the velocity
                velocity = targetPos - self.currentPos
                # normalize the velocity to get a unit vector
                velocity = velocity.normalize()
                # update the current position so that it moves towards the target
                self.currentPos += (velocity*self.speed)

            # if it is close enough to the target, then move on to the next target
            else:
                self.waypointIndex += 1

            # update the hitbox position to match currentPos
            self.hitbox.x = self.currentPos.x
            self.hitbox.y = self.currentPos.y

        # if you have reached the last waypoint
        else:
            self.exists = False

    def renderHitbox(self, aSurface):
        """Displays the hitbox of the game object

        Args:
            aSurface (pygame surface object): surface to draw on
        """
        pygame.draw.rect(aSurface, self.color, self.hitbox, 1)

    def display(self, aSurface, imageToDisplay):
        """Displays the hitbox of the game object

        Args:
            aSurface (pygame surface object): surface to draw on
            imageToDisplay (pygame image): image to show at the bullet's location
        """
        aSurface.blit(imageToDisplay, (self.hitbox.x, self.hitbox.y))


import sys
import os
sys.path.append(os.getcwd())
import datetime
import json
import torch

from gensim.corpora import Dictionary
from torchtext.vocab import Vectors
from util.CoMa_Model import CoMaModel
from website.model_predict import predict
from flask import Flask, escape, request, send_from_directory, jsonify

from gevent.pywsgi import WSGIServer

app = Flask(__name__)

print("Loading config. ")
if len(sys.argv) > 1:
    config_file = sys.argv[1]
else:
    config_file = "./data/config.json"
with open(config_file, "r", encoding="utf-8") as f:
    args = json.loads(f.readlines()[0])
for k, v in args.items():
    print(k, ":", v)
print("Loading model and vectors. ")
device = torch.device(args['device'])
model = CoMaModel(args).to(device)
print(os.path.join(args['corpus_path'], "..", "models", args['dataset'], "CoMa.model"))
model.load_state_dict(torch.load(os.path.join(args['corpus_path'], "..", "models", args['dataset'], 'CoMa.model'),
                                 map_location=device))
print("Model: ", device, datetime.datetime.now())
model.eval()
vectors = Vectors(args['corpus_path'] + args['dataset'] + "/word_embeddings.bin").stoi
print("Embeddings: ", datetime.datetime.now())
output_vectors = Dictionary.load(args['corpus_path'] + args['dataset'] + "/venue_dict")
print("Output vectors: ", datetime.datetime.now())
print("Website running now: ", datetime.datetime.now())

@app.route('/')
def index():
    return app.send_static_file('index.html')

@app.route('/static/<path:path>')
def send_static(path):
    return send_from_directory('static', path)

@app.route('/match', methods=['POST'])
def match():
    title = request.json.get("title", "")
    abstract = request.json.get("abstract", "")
    keywords = request.json.get("keywords", "")

    # Process title, abstract, and keywords.
    # keywords = " ".join(re.split('[^\w-]+', keywords.lower()))
    # abstract = " ".join(re.split('[^\w-]+', abstract.lower()))
    # title = " ".join(re.split('[^\w-]+', title.lower()))

    print(f"Request started: {datetime.datetime.now()}")
    # Return JSON with the same structure as in example.json
    return predict(input_abstract=abstract, input_title=title, input_keywords=keywords, model=model, vectors=vectors,
                   output_vectors=output_vectors, device=device)

if __name__ == '__main__':
    http_server = WSGIServer(('', 5000), app)
    http_server.serve_forever()
    print("Page running. ")