import argparse
import glob
import magic # package is called 'python-libmagic' in pip, and it also requires libmagic-dev to be installed in the system
import numpy as np
import os
import pickle
import re
import subprocess
import sys
import tempfile

def check_model(model_path):
    mimetype = magic.Magic().from_file(model_path)

    basename = os.path.basename(model_path)
    (filename, file_extension) = os.path.splitext(basename)

    if not (mimetype == "application/octet-stream" and file_extension == ".tflite"):
        print("Model file: " + model_path + " is of unsupported type " + mimetype)
        exit(1)

def check_beeswax(beeswax_path):
    if not os.path.isfile(beeswax_path):
        print("An application named 'beeswax' was not found in " + beeswax_directory)
        exit(1)

def read_classes(image_directory, classes_to_test):
    classes = [os.path.basename(d[0]) for d in os.walk(image_directory)]
    if len(classes) <= 1:
        print(image_directory + " is empty")
        exit(1)
    classes = classes[1:]
    classes = classes[0:min(len(classes), classes_to_test)]
    return classes

def read_images(classes, image_directory, images_per_class):
    images = []
    for c in classes:
        class_dir = image_directory + "/" + c + "/"
        class_images = glob.glob(class_dir + "*.bmp")
        class_images = class_images[0:min(len(class_images), images_per_class)]
        images += class_images
    return images

def get_output(command):
    return subprocess.run(command, shell=True, stderr=subprocess.STDOUT, stdout=subprocess.PIPE).stdout.decode("utf-8")

def create_temp_list(images):
    temp_list = tempfile.NamedTemporaryFile(mode="w+", prefix="tmp_", suffix=".txt", delete=True)
    # temp_list = open("tmp_foo.txt", "w+")
    temp_list.write("\n".join(images) + "\n")
    temp_list.seek(0)
    return temp_list

def run_beeswax(beeswax_path, model_path, images):
    temp_list = create_temp_list(images)
    split_output = get_output(beeswax_path
        + " -m " + model_path
        + " -l " + "labels.txt"
        + " -f " + temp_list.name).split("\n")

    temp_list.close()

    return split_output

def filter_output(output):
    return [line for line in output if line.startswith("top-5:") or line.startswith("image-path:")]

def count_hits(output):
    top1_hits = 0
    top5_hits = 0

    image_path_pattern = re.compile(r"image-path: (?P<image_path>/.*/(?P<class_name>[\w'\-]+)/\w+\.bmp)$")
    # top_pattern = re.compile(r"top-5: (?P<foo>( \| )?(?P<class_name>[\w']+) \(\d+\.\d+%\))*")
    top_pattern = re.compile(r"top-5: ?(( \| )?[\w'\-]+ \(\d+\.\d+%\))*")
    top_classes_pattern = re.compile(r"([\w'\-]+ \(\d+\.\d+%\))")

    next_class = None
    image_path = None

    for line in output:
        if next_class is None:
            match = image_path_pattern.match(line)
            if match is not None:
                next_class = match["class_name"]
                image_path = match["image_path"]
            else:
                print("Class not found in line: " + line)
                exit(1)
        elif next_class is not None:
            match = top_pattern.match(line)
            if match is not None:
                top_classes = [match.split(" ")[0] for match in top_classes_pattern.findall(line)]
                # print("Guessed: " + str(top_classes) + " for class " + next_class.upper())
                if len(top_classes) > 0:
                    if next_class == top_classes[0]:
                        top1_hits += 1
                        top5_hits += 1
                        # print(image_path)
                    elif next_class in top_classes:
                        top5_hits += 1
                else:
                    print("The following line does not contain a top list:\n" + line)
                next_class = None
            else:
                print("Top predictions not found in line: " + line)
                # exit(1)

    return [top1_hits, top5_hits]

def batchify(original_list, batch_size):
    for i in range(0, (len(original_list) // batch_size) + (len(original_list) % batch_size > 0)):
        yield original_list[i * batch_size : min((i + 1) * batch_size, len(original_list))]
    return

def test_model(model_path, image_directory, beeswax_directory, classes_to_test=100, images_per_class=2, batch_size=800, threads=1, seed=None):
    if None in [image_directory, beeswax_directory]:
        paths_filename = "xorapu_paths.pkl"

        if os.path.isfile(paths_filename):
            paths = pickle.load(open(paths_filename, "rb"))
            image_directory = paths["image_directory"]
            beeswax_directory = paths["beeswax_directory"]
        else:
            print("Error: One or more paths were not set previously, %s was not found." % (paths_filename))
            print("Run ./xorapu.py -h for info on how to set them.")
            exit(1)

    check_model(model_path)

    beeswax_path = beeswax_directory + "/beeswax"
    check_beeswax(beeswax_path)

    image_directory = os.path.abspath(image_directory)
    classes = read_classes(image_directory, classes_to_test)

    images = read_images(classes, image_directory, images_per_class)

    # print(len(images))

    split_output = []
    for subset in batchify(images, batch_size):
        split_output += run_beeswax(beeswax_path, model_path, subset)

    # [print(line) for line in split_output]

    filtered_output = filter_output(split_output)

    (top1_hits, top5_hits) = count_hits(filtered_output)

    top1_accuracy = float(top1_hits) / len(images) * 100.0
    top5_accuracy = float(top5_hits) / len(images) * 100.0

    return [top1_accuracy, top5_accuracy]

if __name__ == "__main__":

    # Preparing environment

    parser = argparse.ArgumentParser(description="Execute the beeswax benchmark and report accuracy.")

    parser.add_argument("--model", "-m", help="Path to the model that will be tested. Always required.")
    parser.add_argument("--images", "-i", help="Path to the directory of the input images. Required only once.")
    parser.add_argument("--beeswax", "-b", help="Path to the directory that contains the beeswax executable. Required only once.")

    args = parser.parse_args()

    model_path = None
    image_directory = None
    beeswax_directory = None

    paths_filename = "xorapu_paths.pkl"

    if os.path.isfile(paths_filename):
        paths = pickle.load(open(paths_filename, "rb"))
        if "image_directory" in paths.keys():
            image_directory = paths["image_directory"]
        if "beeswax_directory" in paths.keys():
            beeswax_directory = paths["beeswax_directory"]

    model_path = args.model
    if args.images is not None:
        image_directory = args.images
    if args.beeswax is not None:
        beeswax_directory = args.beeswax

    paths = {"image_directory":image_directory, "beeswax_directory":beeswax_directory}
    with open(paths_filename, "wb") as f:
        pickle.dump(paths, f)

    if model_path is None:
        print("Path to model not set. Saved dataset and beeswax directories. Now exiting.")
        exit(0)

    # Done preparing environment

    print("Running xorapu with images from %s and engine from %s" % (image_directory, beeswax_directory))

    (top1_accuracy, top5_accuracy) = test_model(model_path, image_directory, beeswax_directory)

    # Printing result

    print("Top 1 accuracy: %.02f%%" % (top1_accuracy))
    print("Top 5 accuracy: %.02f%%" % (top5_accuracy))

    # Done printing result


# -*- coding: utf-8 -*-
# Author: Litre WU
# E-mail: litre-wu@tutanota.com
# Software: PyCharm
# File: 看准-企业工商信息查询.py
# Time: 1月 06, 2022
import asyncio
from aiohttp import ClientSession, ClientTimeout, TCPConnector
from user_agent import generate_user_agent
from random import randint
from time import sleep
from sys import platform
from json import loads
from bs4 import BeautifulSoup
import pandas as pd
from fastapi import FastAPI
from pydantic import BaseModel, Field
import socket
from boltons.cacheutils import LRU
from hashlib import md5

if platform == "win32":
    asyncio.set_event_loop(asyncio.ProactorEventLoop())
    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())

host = socket.gethostbyname(socket.gethostname())

lru_cache = LRU(max_size=100)

tags_metadata = [
    {
        "name": "看准-企业工商信息查询接口",
        "description": "看准-企业工商信息查询",
        "externalDocs": {
            "description": "More",
            "url": f"http://{host}/docs",
        },
    },
]

contact = {
    "name": "Litre",
    "url": "http://121.37.209.113",
    "email": "litre-wu@tutanota.com",
}

app = FastAPI(openapi_url="/api/v1/api.json", title="看准-企业工商信息查询接口", contact=contact, openapi_tags=tags_metadata)

class SearchItem(BaseModel):
    query: str = Field(..., example='哔哩哔哩')
    cityCode: str = Field(..., example=0)
    industryCodes: str = Field(..., example='')
    pageNum: str = Field(..., example=1)
    limit: str = Field(..., example=15)

# 查询接口
@app.post("/search", tags=["看准-工商信息查询"])
async def search(data: SearchItem):
    kwargs = data.dict()
    key = md5(str(kwargs).encode()).hexdigest()
    if lru_cache.get(key): return lru_cache[key]
    result = await query(**kwargs)
    if result: lru_cache[key] = result
    return result

class InfoItem(BaseModel):
    encCompanyId: str = Field(..., example='0XN_2dW7Fw~~')

# 工商信息接口
@app.post("/compInfo", tags=["看准-工商信息查询"])
async def info(data: InfoItem):
    kwargs = data.dict()
    key = md5(str(kwargs).encode()).hexdigest()
    if lru_cache.get(key): return lru_cache[key]
    result = await compInfo(**kwargs)
    if result: lru_cache[key] = result
    return result

# 公共请求函数
async def pub_req(**kwargs):
    method = kwargs.get("method", "GET")
    url = kwargs.get("url", "")
    params = kwargs.get("params", {})
    data = kwargs.get("data", {})
    headers = {**{"User-Agent": generate_user_agent()}, **kwargs.get("headers", {})}
    proxy = kwargs.get("proxy", "")
    timeout = kwargs.get("timeout", 10)
    try:
        async with asyncio.Semaphore(20):
            async with ClientSession(timeout=ClientTimeout(total=3),
                                     connector=TCPConnector(ssl=False),
                                     trust_env=True) as client:
                async with client.request(method=method, url=url, params=params, data=data, headers=headers,
                                          proxy=proxy,
                                          timeout=timeout) as rs:
                    if rs.status == 200 or 201:
                        content = await rs.read()
                        return content
                    else:
                        sleep(randint(1, 2))
                        retry = kwargs.get("retry", 0)
                        retry += 1
                        if retry >= 2:
                            return None
                        kwargs["retry"] = retry
                        return await pub_req(**kwargs)
    except Exception as e:
        print(e)
        sleep(randint(1, 2))
        retry = kwargs.get("retry", 0)
        retry += 1
        if retry >= 2:
            return None
        kwargs["retry"] = retry
        return await pub_req(**kwargs)

# 查询
async def query(**kwargs):
    meta = {
        "url": "https://www.kanzhun.com/search/company_v2.json",
        "params": {
            "query": kwargs.get("query", "哔哩哔哩"),
            "cityCode": kwargs.get("cityCode", 0),
            "industryCodes": kwargs.get("industryCodes", ""),
            "pageNum": kwargs.get("pageNum", 1),
            "limit": kwargs.get("limit", 15),
        },
        "headers": {
            "Accept-Encoding": "gzip, deflate, br"
        }
    }
    res = await pub_req(**meta)
    if not res: return None
    # print(res.decode())
    return loads(res)

# 工商信息
async def compInfo(**kwargs):
    meta = {
        "url": f'https://www.kanzhun.com/firm/info/{kwargs.get("encCompanyId", "")}.html',
        "headers": {
            "Accept-Encoding": "gzip, deflate, br"
        }
    }
    res = await pub_req(**meta)
    if not res: return None
    soup = BeautifulSoup(res.decode(), 'html.parser')
    div = soup.find_all("div", class_="kz-company-desc")
    if div:
        table = div[0].table
        table = pd.read_html(str(table))
        info = {}
        for x in table[0].values.tolist():
            for i in range(0, len(x), 2):
                if x[i].strip("-"):
                    info[x[i]] = x[i + 1]
        return info

async def main():
    rs = await query()
    # rs = await compInfo(**{"encCompanyId": "0XN_2dW7Fw~~"})
    print(rs)

if __name__ == '__main__':
    asyncio.get_event_loop().run_until_complete(main())

看准-企业工商信息查询.py

看准-企业工商信息查询.py


from selenium.webdriver.common.by import By

class ProductDetailPage:

    def __init__(self, driver):
        self.driver = driver

        # Define locators for elements related to the Product Detail page
        self.return_to_product_link = (By.LINK_TEXT, "Return to FI-SW-01")
        self.add_to_cart_button = (By.XPATH, "//a[contains(@class, 'Button')]")

    def click_return_to_product(self):
        self.driver.find_element(*self.return_to_product_link).click()

    def click_add_to_cart_by_item_id(self, item_id):
        """
        Click the "Add to Cart" button for the item specified by item_id.

        Args:
            item_id (str): The ID of the item to add to the cart.

        Raises:
            NoSuchElementException: If the elements are not found.
        """
        try:
            item_elements = self.driver.find_elements(By.XPATH, "//table//tr//td[1]")
            add_to_cart_buttons = self.driver.find_elements(By.XPATH, "//table//tr//td[5]/a")

            for index, item_element in enumerate(item_elements):
                if item_element.text == item_id:
                    add_to_cart_buttons[index].click()
                    return
        except Exception as e:
            print(f"Exception occurred while adding item {item_id} to cart: {e}")
            raise


# COVID-19 Statistics Twitter Bot
# Contributors: ammar-s847 | shenbenson | JunZheng-Dev

import tweepy
import time
import requests
import json
import threading

CONSUMER_KEY = ''
CONSUMER_SECRET = ''
ACCESS_KEY = ''
ACCESS_SECRET = ''

FILE_NAME = 'last_seen_id.txt'
COUNTRY_URL = "https://covid-19-data.p.rapidapi.com/country"
TOTALS_URL = "https://covid-19-data.p.rapidapi.com/totals"

headers = {
    'x-rapidapi-host': "covid-19-data.p.rapidapi.com",
    'x-rapidapi-key': ""
}

auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_KEY, ACCESS_SECRET)
api = tweepy.API(auth)

def retrieve_last_seen_id(file_name):
    f_read = open(file_name, 'r')
    last_seen_id = int(f_read.read().strip())
    f_read.close()
    return last_seen_id

def store_last_seen_id(last_seen_id, file_name):
    f_write = open(file_name, 'w')
    f_write.write(str(last_seen_id))
    f_write.close()
    return

def world_stats() :
    querystring = {"format":"json"}
    response = requests.request("GET", TOTALS_URL, headers=headers, params=querystring)
    data = json.loads(response.text[1:-1])
    time = str(data['lastUpdate']).replace('T', ' ')[0:-6] + ' CEST'
    status = (' 𝗪𝗼𝗿𝗹𝗱 𝗗𝗮𝘁𝗮\n\nTotal Cases: ' + f"{data['confirmed']:,}" +
            '\nRecovered Cases: ' + f"{data['recovered']:,}" + '\nCurrent Cases: ' +
            f"{(data['confirmed'] - data['recovered'] - data['deaths']):,}" +
            '\nTotal Deaths: ' + f"{data['deaths']:,}" + '\nMortality Rate: ' +
            str("{:.2f}".format(data['deaths'] / data['confirmed'] * 100)) +
            '%\n\nUpdated ' + time)
    return status

def country_stats(country) :
    querystring = {"format":"json","name":country}
    response = requests.request("GET", COUNTRY_URL, headers=headers, params=querystring)
    data = json.loads(response.text[1:-1])
    time = str(data['lastUpdate']).replace('T', ' ')[0:-6] + ' CEST'
    reply = (' 𝗗𝗮𝘁𝗮 𝗳𝗼𝗿 ' + country.upper() + '\n\nTotal Cases: ' +
            f"{data['confirmed']:,}" + '\nRecovered Cases: ' +
            f"{data['recovered']:,}" + '\nCurrent Cases: ' +
            f"{(data['confirmed'] - data['recovered'] - data['deaths']):,}" +
            '\nTotal Deaths: ' + f"{data['deaths']:,}" + '\nMortality Rate: ' +
            str("{:.2f}".format(data['deaths'] / data['confirmed'] * 100)) +
            '%\n\nUpdated ' + time)
    return reply

def reply_to_tweets():
    print('expecting new tweets...', flush = True)
    last_seen_id = retrieve_last_seen_id(FILE_NAME)
    mentions = api.mentions_timeline(last_seen_id, tweet_mode='extended')
    for mention in reversed(mentions):
        last_seen_id = mention.id
        store_last_seen_id(last_seen_id, FILE_NAME)
        textholder = mention.full_text.lower()
        print(mention.user.screen_name + ' - ' + textholder, flush = True)
        if 'covid:' in textholder:
            country = textholder[textholder.index(':') + 1:]
            if 'world' in country.lower():
                try:
                    api.update_status('@' + mention.user.screen_name +
                        world_stats(), mention.id)
                except:
                    reply = ' Uh oh! Something went wrong.'
                    api.update_status('@' + mention.user.screen_name +
                            reply, mention.id)
            else:
                try:
                    api.update_status('@' + mention.user.screen_name +
                            country_stats(country), mention.id)
                except:
                    reply = ' Uh oh! Something went wrong.'
                    api.update_status('@' + mention.user.screen_name +
                            reply, mention.id)

def dailyTweet():
    threading.Timer(86400.0, dailyTweet).start()
    try:
        api.update_status(world_stats())
    except Exception as e:
        print(e)

dailyTweet()

while True:
    reply_to_tweets()
    time.sleep(15)


import joblib
from sklearn.model_selection import GridSearchCV
from sklearn.tree import DecisionTreeClassifier

import ipal_iids.settings as settings
from ids.featureids import FeatureIDS

class DecisionTree(FeatureIDS):
    _name = "DecisionTree"
    _description = "Decision tree classifier."
    _decisiontree_default_settings = {
        # DecisionTree GridSearch Parameters
        # TODO Better sample random values?
        "criterion": ["gini", "entropy"],
        "splitter": ["best", "random"],
        "max_depth": [None],
        "min_samples_split": [2],
        "min_samples_leaf": [1],
        "min_weight_fraction_leaf": [0],
        "max_features": ["sqrt", "log2", None],
        "max_leaf_nodes": [None],
        "min_impurity_decrease": [0.0],
        "random_state": [None],
        "class_weight": ["balanced", "balanced_subsample", None],
        "ccp_alpha": [0.0],
        # GridSearch Options
        "scoring": None,  # accuracy ..
        "jobs": 4,
        "verbose": 10,
        "no-probability": False,
    }

    def __init__(self, name=None):
        super().__init__(name=name)
        self._add_default_settings(self._decisiontree_default_settings)

        self.dtc = None
        self.classes = None

    # the IDS is given the path to file(s) containing its requested training data
    def train(self, ipal=None, state=None):
        if ipal and state:
            settings.logger.error("Only state or message supported")
            exit(1)

        if state is None:
            state = ipal

        events, annotation, _ = super().train(state=state)
        annotation = [a is not False for a in annotation]

        if len(set(annotation)) <= 1:
            settings.logger.warning(
                "Training with a single class ({}) only!".format(set(annotation))
            )

        # Learn DecisionTree
        settings.logger.info("Learning DecisionTree")
        tuned_parameters = {
            "criterion": self.settings["criterion"],
            "splitter": self.settings["splitter"],
            "max_depth": self.settings["max_depth"],
            "min_samples_split": self.settings["min_samples_split"],
            "min_samples_leaf": self.settings["min_samples_leaf"],
            "min_weight_fraction_leaf": self.settings["min_weight_fraction_leaf"],
            "max_features": self.settings["max_features"],
            "max_leaf_nodes": self.settings["max_leaf_nodes"],
            "min_impurity_decrease": self.settings["min_impurity_decrease"],
            "random_state": self.settings["random_state"],
            "class_weight": self.settings["class_weight"],
            "ccp_alpha": self.settings["ccp_alpha"],
        }
        settings.logger.info(tuned_parameters)

        # Test if gridsearch is neccesary
        if max([len(v) for v in tuned_parameters.values()]) > 1:
            settings.logger.info("Finding best parameteres with GirdSearchCV")
            dtc = GridSearchCV(
                DecisionTreeClassifier(),
                [tuned_parameters],
                scoring=self.settings["scoring"],
                n_jobs=self.settings["jobs"],
                verbose=self.settings["verbose"],
            )

            dtc.fit(events, annotation)

            settings.logger.info("Best parameters set found on development set:")
            settings.logger.info(dtc.best_params_)
            settings.logger.info("Grid scores on development set:")
            means = dtc.cv_results_["mean_test_score"]
            stds = dtc.cv_results_["std_test_score"]
            for mean, std, params in zip(means, stds, dtc.cv_results_["params"]):
                settings.logger.info(
                    "%0.3f (+/-%0.03f) for %r" % (mean, std * 2, params)
                )

            # Save best estimator
            self.dtc = dtc.best_estimator_

        else:
            tuned_parameters = {k: v[0] for k, v in tuned_parameters.items()}
            self.dtc = DecisionTreeClassifier(**tuned_parameters)
            self.dtc.fit(events, annotation)

        self.classes = list(self.dtc.classes_)

    def new_state_msg(self, msg):
        state = super().new_state_msg(msg)
        if state is None:
            return False, None

        alert = bool(self.dtc.predict([state])[0])

        if self.settings["no-probability"]:  # takes less time
            return alert, 1 if alert else 0

        else:
            probability = self.dtc.predict_proba([state])[0][self.classes.index(True)]
            return alert, probability

    def new_ipal_msg(self, msg):
        # There is no difference for this IDS in state or message format! It only depends on the configuration which features are used.
        return self.new_state_msg(msg)

    def save_trained_model(self):
        if self.settings["model-file"] is None:
            return False

        model = {
            "_name": self._name,
            "preprocessors": super().save_trained_model(),
            "settings": self.settings,
            "classifier": self.dtc,
            "classes": self.classes,
        }

        joblib.dump(model, self._resolve_model_file_path(), compress=3)

        return True

    def load_trained_model(self):
        if self.settings["model-file"] is None:
            return False

        try:  # Open model file
            model = joblib.load(self._resolve_model_file_path())
        except FileNotFoundError:
            settings.logger.info(
                "Model file {} not found.".format(str(self._resolve_model_file_path()))
            )
            return False

        # Load model
        assert self._name == model["_name"]
        super().load_trained_model(model["preprocessors"])
        self.settings = model["settings"]
        self.dtc = model["classifier"]
        self.classes = model["classes"]

        return True

    def visualize_model(self, max_depth=3):
        import matplotlib.pyplot as plt
        from sklearn.tree import plot_tree

        fig, ax = plt.subplots(nrows=1, ncols=1)

        ax.axis("off")

        settings.logger.info("Plotting tree.")

        plot_tree(
            self.dtc,
            max_depth=max_depth,
            filled=True,
            impurity=True,
            rounded=True,
            ax=ax,
        )

        return plt, fig


# -*- coding: utf-8 -*-
import pytest

@pytest.mark.parametrize("cmd,expected_output", [
    ('install', "I'm install"),
    ('i', "I'm install"),
    ('uninstall', "I'm uninstall"),
    ('un', "I'm uninstall"),
    ('u', "Too many matches: uninstall, update"),
    ('XXX', 'No such command'),
])
def test_aliased_group(cmd, expected_output):
    import click
    from click.testing import CliRunner
    from snakepit.groups import AliasedGroup

    @click.group(cls=AliasedGroup)
    def pit():
        click.echo("I'm snakepit")

    @pit.command()
    def install():
        click.echo("I'm install")

    @pit.command()
    def uninstall():
        click.echo("I'm uninstall")

    @pit.command()
    def update():
        click.echo("I'm update")

    runner = CliRunner()
    result = runner.invoke(pit, [cmd])
    assert expected_output in result.output


import scrapy
from demo.items import BookItem

class MySpider(scrapy.Spider):
    name = "mySpider"
    start_urls = ['http://127.0.0.1:5000']

    def parse(self, response):
        try:
            data = response.body.decode()
            selector = scrapy.Selector(text=data)
            books = selector.xpath("//book")
            for book in books:
                item = BookItem()
                item["title"] = book.xpath("./title/text()").extract_first()
                item["author"] = book.xpath("./author/text()").extract_first()
                item["publisher"] = book.xpath("./publisher/text()").extract_first()
                yield item
        except Exception as err:
            print(err)


import pygame
import pygame.gfxdraw

from mygame.client.config.colors import MENU_SELECTION_COLOR

class Menu:
    def __init__(self, hexgrid):
        self.hexgrid = hexgrid
        self.actions = None
        self.position = None
        self.visible = False

    def display_actions(self, actions, position):
        if len(actions) > 0 and position:
            self.actions = {}
            self.position = position
            for action in actions:
                if action.name in self.actions:
                    # Add the action to the list
                    self.actions[action.name] += action
                else:
                    self.actions[action.name] = [action]
            self.visible = True

    def clear_actions(self):
        self.actions = None
        self.position = None
        self.visible = False

    def draw(self):
        if self.visible:
            pygame.gfxdraw.aacircle(self.hexgrid.surface, self.position[0], self.position[1],
                                    int(self.hexgrid.hex_width_half), MENU_SELECTION_COLOR)


import numpy as np
import matplotlib.pyplot as plt
from skimage import io
from skimage import transform
board = io.imread('imp-brd-643-1_1.jpg')

src = np.array([[300,0], [0, 0], [0, 300], [300, 300]])
dst = np.array([[810, 230], [180, 460], [775, 850], [1410, 490]])

tform3 = transform.ProjectiveTransform()
tform3.estimate(src, dst)
warped = transform.warp(board, tform3, output_shape=(300, 300))

fig, ax = plt.subplots(nrows=2, figsize=(8, 3))

ax[0].imshow(board, cmap=plt.cm.gray)
ax[0].plot(dst[:, 0], dst[:, 1], '.r')
ax[1].imshow(warped, cmap=plt.cm.gray)

for a in ax:
    a.axis('off')

plt.tight_layout()
plt.show()


from fastapi import FastAPI, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel

from Twitter_Scrapper import Get_Scrapped_Data
from Text_Normalization import Text_Normalization
from Database_Firebase import uploadDataToFirebase, getFirebaseAuth, uploadTopicDataToFirebase
from Topic_Modelling import trainTopicModel

# ------------------------------------------------------------------------------------

class ScrapperKeyword(BaseModel):
  keyword : str

# ------------------------------------------------------------------------------------

originsList = ["http://localhost:3000"]

apiApp = FastAPI()

apiApp.add_middleware(
  CORSMiddleware,
  allow_origins = originsList,
  allow_credentials = True,
  allow_methods = ["*"],
  allow_headers = ["*"]
)

fbAuth = getFirebaseAuth()
dbName = "Twitter Data Before Modelling"
topicDb = "Twitter Topic Modelling"
maxCollectionLength = 2500
# ------------------------------------------------------------------------------------

# async def getDataForDay(_dbName, _keyword, _date):
#   data = await fbAuth.collection(_dbName).Document(_keyword)
#   return data

@apiApp.get("/")
def readRoot():
  return {"Connection" : "Is Working"}

# @apiApp.get("/api/tweets")
# async def getTweetDataForDay():
#   response = 2

#   if response:
#     return "Operation Successful"
#   raise HTTPException(404, "Operation Failed")

@apiApp.post("/api/keyword")
async def post_keyword(_keyword : ScrapperKeyword):

  scrappedData = Get_Scrapped_Data(_keyword.keyword)
  normalizedText = Text_Normalization(scrappedData, 'Tweet')
  topicDataDict = trainTopicModel(normalizedText)

  response = await uploadDataToFirebase(dbName, _keyword.keyword, normalizedText, maxCollectionLength, fbAuth)

  resp2 = await uploadTopicDataToFirebase(topicDb, _keyword.keyword, topicDataDict, maxCollectionLength, fbAuth)

  if response:
    return "Operation Successful"
  return HTTPException(404, "Operation Failed")


import math
import random
import numpy
import pandas
import os
from datetime import datetime
from scipy.integrate import odeint

class Gillespie:

    # const is an array of numbers that represent the constants of the reactions
    def __init__(self, tmax=10, m0=0, p0=0, const=None):
        if const is None:
            const = [1, 1, .1, .1]
        self.tmax = tmax
        self.k0 = const[0]
        self.k1 = const[1]
        self.dm = const[2]
        self.dp = const[3]
        self.Nm = m0
        self.Np = p0

    def initial_state(self):
        return [self.Nm, self.Np]

    def update_propensities(self, Nm, Np):
        a1 = self.k0
        a2 = self.k1 * Nm
        a3 = self.dm * Nm
        a4 = self.dp * Np

        total = a1 + a2 + a3 + a4

        r1 = a1/total
        r2 = a2/total
        r3 = a3/total
        r4 = a4/total

        return [r1, r2, r3, r4, total]

    def next_reaction(self, r):
        n = random.uniform(0, 1)
        # Create an mRNA
        if 0 <= n <= r[0]:
            return [1, 0]
        # Create a protein
        elif r[0] < n <= r[1] + r[0]:
            return [0, 1]
        # Delete an mRNA
        elif r[1] + r[0] < n <= r[2] + r[1] + r[0]:
            return [-1, 0]
        # Delete a protein
        elif r[2] + r[1] + r[0] < n <= r[3] + r[2] + r[1] + r[0]:
            return [0, -1]
        # no reaction occurs
        elif r[3] + r[2] + r[1] + r[0] < n < 1:
            return [0, 0]

    def time_to_next_rxn(self, a, t):
        dt = -math.log(1 - random.uniform(0, 1))/a[4]
        t = t + dt
        return t, dt

    def update_reaction_vector(self, current_state, update_vector):
        current_state = numpy.array(current_state)
        update_vector = numpy.array(update_vector)
        updated = current_state + update_vector
        return updated

class CellDivision(Gillespie):
    def __init__(self, tmax=10, m0=0, p0=0, const=None, number_of_sims=1, cell_cycle=6300):
        super().__init__(tmax, m0, p0, const)
        self.number_of_sims = number_of_sims
        self.two_genes = cell_cycle/2
        self.divide = cell_cycle

    def two_genes_propensities(self, Nm, Np):
        a1 = 2 * self.k0
        a2 = self.k1 * Nm
        a3 = self.dm * Nm
        a4 = self.dp * Np

        total = a1 + a2 + a3 + a4

        r1 = a1 / total
        r2 = a2 / total
        r3 = a3 / total
        r4 = a4 / total

        return [r1, r2, r3, r4, total]

    def division_of_molecules(self, current_state):
        parent_cell = []
        current_mrna, current_protein = current_state[0], current_state[1]
        mrna = 0
        prot = 0
        for i in range(current_mrna):
            stays = bool(random.randint(0, 1))
            if stays == 0:
                mrna = mrna + 1
            else:
                pass
        parent_cell.append(mrna)
        for i in range(current_protein):
            stays = bool(random.randint(0, 1))
            if stays == 0:
                prot = prot + 1
            else:
                pass
        parent_cell.append(prot)
        return parent_cell

    def sim(self):
        time = []
        protein = []
        mrna = []
        genes = []
        divide = ["No"]
        counter0 = [0]
        counter = 0
        clock = 0
        cell_cycle = [0]
        t = 0  # start time
        r0 = self.initial_state()
        g0 = 1
        genes.append(g0)
        mrna.append(r0[0])
        protein.append(r0[1])
        time.append(t)
        while t <= self.tmax:
            # Refactor this. Not doing what I want it to.
            if self.two_genes <= counter <= self.divide:
                genes.append(2)
                a = self.two_genes_propensities(r0[0], r0[1])
                next_rxn = self.next_reaction(a)
                sim_time = self.time_to_next_rxn(a, t)
                t = sim_time[0]
                if counter == self.divide:
                    r = self.division_of_molecules(r0)
                    mrna.append(r[0])
                    protein.append(r[1])
                    time.append(t)
                    r0 = r
                    # Mark relevant data
                    counter = 0
                    clock = 0
                    counter0.append(counter)
                    cell_cycle.append(clock)
                    divide.append("Yes")
                else:
                    r = self.update_reaction_vector(r0, next_rxn)
                    mrna.append(r[0])
                    protein.append(r[1])
                    time.append(t)
                    r0 = r
                    # Mark relevant data
                    counter = counter + 1
                    clock = sim_time[1] + clock
                    counter0.append(counter)
                    cell_cycle.append(clock)
                    divide.append("No")
            else:
                a = self.update_propensities(r0[0], r0[1])
                next_rxn = self.next_reaction(a)
                sim_time = self.time_to_next_rxn(a, t)
                t = sim_time[0]
                r = self.update_reaction_vector(r0, next_rxn)
                mrna.append(r[0])
                protein.append(r[1])
                time.append(t)
                r0 = r
                # Mark relevant data
                counter = counter + 1
                clock = sim_time[1] + clock
                counter0.append(counter)
                cell_cycle.append(clock)
                genes.append(1)
                divide.append("No")

        sim_df = pandas.DataFrame()
        sim_df["Seconds"] = time
        time = [x / 3600 for x in time]
        sim_df["Hours"] = time
        sim_df["Time"] = time
        sim_df["Proteins"] = protein
        sim_df["mRNA"] = mrna
        sim_df["Gene Number"] = genes
        sim_df["Divide"] = divide
        sim_df["Counter"] = counter0
        sim_df["Clock"] = cell_cycle
        return sim_df

    def multiple_cells(self):
        timestamp = datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
        directory = "Dir_with_runs-{time}".format(time=timestamp)
        os.mkdir(directory)
        for i in range(0, self.number_of_sims):
            path = os.path.join(directory, "Run{n}.csv".format(n=i))
            sim = self.sim()
            sim.to_csv(path, index=False, encoding='utf-8', sep='\t',)
        return directory

class DeterministicCellDivision:
    # t is a number
    # const is an array of numbers that represent the constants of the reactions
    def __init__(self, tmax=10, num_of_datapoints=1000, m0=0, p0=0, const=(1, 1, .1, .1)):
        self.n = num_of_datapoints
        self.tmax = tmax
        self.m0 = m0
        self.p0 = p0
        self.k0 = const[0]
        self.k1 = const[1]
        self.dm = const[2]
        self.dp = const[3]
        self.t = numpy.linspace(0, self.tmax, self.n)  # time points

    def numerical_solution_1_gene(self, z, t):
        m0 = z[0]
        p0 = z[1]
        dmdt = self.k0 - self.dm * m0
        dpdt = self.k1 * m0 - self.dp * p0
        dzdt = dmdt, dpdt
        return dzdt

    def numerical_solution_2_gene(self, z, t):
        m0 = z[0]
        p0 = z[1]
        dmdt = 2 * self.k0 - self.dm * m0
        dpdt = self.k1 * m0 - self.dp * p0
        dzdt = dmdt, dpdt
        return dzdt

    def numerical_sim(self):
        # store solutions
        m = numpy.empty_like(self.t)
        p = numpy.empty_like(self.t)
        # record initial conditions
        z0 = [self.m0, self.p0]
        m[0] = self.m0
        p[0] = self.p0
        counter = 0
        counters = []
        counters.append(counter)
        # solve ODE
        for i in range(1, self.n):
            if counter < 1800:
                # span for next time step
                tspan = [self.t[i - 1], self.t[i]]

                # solve for next step
                z = odeint(self.numerical_solution_1_gene, z0, tspan)

                # store solution for plotting
                m[i] = z[1][0]
                p[i] = z[1][1]
                # next initial condition
                z0 = z[1]
                counter = counter + 1
                counters.append(counter)
            elif 1800 <= counter < 3600:
                tspan = [self.t[i - 1], self.t[i]]
                # solve for next step
                z = odeint(self.numerical_solution_2_gene, z0, tspan)
                # store solution for plotting
                m[i] = z[1][0]
                p[i] = z[1][1]
                # next initial condition
                z0 = z[1]
                counter = counter + 1
                counters.append(counter)
            elif counter >= 3600:
                m[i] = m[i-1]/2
                p[i] = p[i-1]/2
                z0 = m[i], p[i]
                counter = 0
                counters.append(counter)

        time = [x / 3600 for x in self.t]
        sim_df = pandas.DataFrame()
        sim_df["Counters"] = counters
        sim_df["Time"] = time
        sim_df["Proteins"] = p
        sim_df["mRNA"] = m
        return sim_df


import logging
from flask import session, abort
from kgeditor import db
from kgeditor.models import Domain

class DomainDAO:
    def __init__(self):
        pass

    def get(self, id):
        user_id = session.get('user_id')
        try:
            domain = Domain.query.filter_by(id=id, creator_id=user_id).first()
        except Exception as e:
            logging.error(e)
            return abort(500, 'Database error.')
        data = domain.to_dict()

        return {'data': data, 'message':'Fetch domain succeed.'}, 200

    def create(self, data):
        user_id = session.get('user_id')
        logging.info(data)
        try:
            domain = Domain.query.filter_by(name=data['name']).first()
        except Exception as e:
            logging.error(e)
            return abort(500, 'Database error.')
        else:
            if domain is not None:
                return abort(500, 'Domain already exist.')

        domain = Domain(name=data['name'], creator_id=user_id)
        try:
            db.session.add(domain)
            db.session.commit()
            # arango_conn.createDatabase(name="domain_{}".format(domain.id))
        except Exception as e:
            db.session.rollback()
            logging.error(e)
            return abort(500, 'Database error.')

        return {'message': 'Create domain succeed.'}, 201

    def update(self, id, data):
        user_id = session.get('user_id')
        try:
            domain = Domain.query.filter_by(id=id, creator_id=user_id).first()
            domain.name = data['name']
            db.session.commit()
        except Exception as e:
            logging.error(e)
            db.session.rollback()
            return abort(500, 'Database error.')

        return {'message': 'Updata domain succeed.'}, 200

    def delete(self, id):
        user_id = session.get('user_id')
        try:
            domain = Domain.query.filter_by(id=id, creator_id=user_id).first()
        except Exception as e:
            logging.error(e)
            db.session.rollback()
            return abort(500, 'Database error.')
        else:
            if domain is None:
                return abort(500, 'Domain not exist.')
        try:
            db.session.delete(domain)
            db.session.commit()
        except Exception as e:
            db.session.rollback()
            logging.error(e)
            return abort(500, 'Database error.')
        return {'message':'Delete domain succeed.'}, 200

    @property
    def all(self):
        try:
            domain_list = Domain.query.all()
        except Exception as e:
            logging.error(e)
            return abort(500, 'Database error.')
        domain_dict_list = []
        for domain in domain_list:
            domain_dict_list.append(domain.to_dict())

        return {'data': domain_dict_list, 'message': 'Query succeed.'}, 200


import json as js
import numpy as np
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.naive_bayes import MultinomialNB
from sklearn.neural_network import MLPClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.tree import DecisionTreeClassifier

#Part 2.5
f = open('goemotions.json')
file = js.load(f)
split = np.array(file)
x = split[:, 0]
y1 = split[:, 1]
y2 = split[:, 2]

x1_train, x1_test, y1_train, y1_test = train_test_split(x, y1, train_size=0.6, test_size=0.4)
x2_train, x2_test, y2_train, y2_test = train_test_split(x, y2, train_size=0.6, test_size=0.4)

#Part 2.3.1
print("Using Naive Bayes Model with default parameters")
vec = CountVectorizer()
x1_train = vec.fit_transform(x1_train)
x1_test = vec.transform(x1_test)
x2_train = vec.fit_transform(x2_train)
x2_test = vec.transform(x2_test)
classifier = MultinomialNB()

model = classifier.fit(x1_train, y1_train)
y_pred = model.predict(x1_test)
print("For emotions : \n")
print(classification_report(y1_test, y_pred))
print("Confusion Matrix: \n", confusion_matrix(y1_test, y_pred))

model = classifier.fit(x2_train, y2_train)
y_pred = model.predict(x2_test)
print("For sentiments : \n")
print(classification_report(y2_test, y_pred))
print("Confusion Matrix: \n", confusion_matrix(y2_test, y_pred))

# Part 2.3.2
print("Using DTC model with default parameters")
dtc = DecisionTreeClassifier()
dtc.fit(x1_train, y1_train)
y1_pred = dtc.predict(x1_test)
print("For emotions : \n")
print(classification_report(y1_test, y1_pred))
print("Confusion Matrix: \n", confusion_matrix(y1_test, y1_pred))
dtc1 = DecisionTreeClassifier()
dtc1.fit(x2_train, y2_train)
y2_pred = dtc1.predict(x2_test)
print("For sentiments : \n")
print(classification_report(y2_test, y2_pred))
print("Confusion Matrix: \n", confusion_matrix(y2_test, y2_pred))

#Part 2.3.3
print("Using MLP model with default parameters")
mlp_clf = MLPClassifier(max_iter=1)
mlp_clf.fit(x1_train, y1_train)
y1_pred = mlp_clf.predict(x1_test)
print("For emotions : \n")
print(classification_report(y1_test, y1_pred))
print("Confusion Matrix: \n", confusion_matrix(y1_test, y1_pred))
mlp2 = MLPClassifier(max_iter=1)
mlp2.fit(x2_train, y2_train)
y2_pred = mlp2.predict(x2_test)
print("For sentiments : \n")
print(classification_report(y2_test, y2_pred))
print("Confusion Matrix: \n", confusion_matrix(y2_test, y2_pred))

#Part 2.3.4
print("Using a better performing Naive Bayes CLassifier using GridSearchCV. ")
nb_classifier = MultinomialNB()
param_dict={"alpha":[0.5, 0, 1, 10.0]}
grid1 = GridSearchCV(estimator=nb_classifier,param_grid=param_dict,n_jobs=-1,error_score='raise')
grid1.fit(x1_train, y1_train)
y1_pred = grid1.predict(x1_test)
print("For emotions : \n")
print(classification_report(y1_test, y1_pred))
print("Confusion Matrix: \n", confusion_matrix(y1_test, y1_pred))

grid2=GridSearchCV(estimator=nb_classifier,param_grid=param_dict,n_jobs=-1,error_score='raise')
grid2.fit(x2_train, y2_train)
y2_pred = grid2.predict(x2_test)
print("For sentiments : \n")
print(classification_report(y2_test, y2_pred))
print("Confusion Matrix: \n", confusion_matrix(y2_test, y2_pred))

#Part 2.3.5
print("Using a better performing Decision Tree found using GridSearchCV. ")
dtc = DecisionTreeClassifier()
param_dict={"criterion":["entropy"], "max_depth": [2,11],"min_samples_split": [12,40,6]}
grid=GridSearchCV(estimator=dtc,param_grid=param_dict,n_jobs=-1,error_score='raise')
grid.fit(x1_train, y1_train)
y1_pred = grid.predict(x1_test)
print("For emotions : \n")
print(classification_report(y1_test, y1_pred))
print("Confusion Matrix: \n", confusion_matrix(y1_test, y1_pred))
print("\n The best estimator across ALL searched params:\n", grid.best_estimator_)
print("\n The best score across ALL searched params:\n", grid.best_score_)
print("\n The best parameters across ALL searched params:\n", grid.best_params_)

dtc1 = DecisionTreeClassifier()
grid1=GridSearchCV(estimator=dtc1,param_grid=param_dict,n_jobs=-1,error_score='raise')
grid1.fit(x2_train, y2_train)
y2_pred = grid1.predict(x2_test)
print("For sentiments : \n")
print(classification_report(y2_test, y2_pred))
print("Confusion Matrix: \n", confusion_matrix(y2_test, y2_pred))
print("\n The best estimator across ALL searched params:\n", grid1.best_estimator_)
print("\n The best score across ALL searched params:\n", grid1.best_score_)
print("\n The best parameters across ALL searched params:\n", grid1.best_params_)

#Part 2.3.6
print("Using a better performing MLP found using GridSearchCV. ")
mlp = MLPClassifier(early_stopping=True, verbose=True, max_iter=1)
sc = StandardScaler(with_mean=False)
scaler_x = sc.fit(x1_train)
x1_train_scaled = scaler_x.transform(x1_train)
scaler_y = sc.fit(y1_train)
y1_train_scaled = scaler_y.transform(y1_train)

parameters= {
    'hidden_layer_sizes': ((5,5), (5,10)),
    'activation': ['logistic', 'tanh', 'relu', 'identity'],
    'solver': ['adam', 'sgd'],
    }
clf = GridSearchCV(estimator = mlp, param_grid=parameters, n_jobs=-1,error_score='raise')

clf.fit(x1_train_scaled, y1_train)
y1_pred= clf.predict(x1_test)
print('accuracy: {:.2f}'.format(accuracy_score(y1_test, y1_pred)))
print("For emotions : \n")
print(classification_report(y1_test, y1_pred,zero_division=1))
print("Confusion Matrix: \n", confusion_matrix(y1_test, y1_pred))
x2_train = vec.fit_transform(x2_train)
x2_test = vec.transform(x2_test)
y2_train = vec.transform(y2_train)
y2_test = vec.transform(y2_test)
scaler_x2 = sc.fit(x2_train)
x2_train_scaled = scaler_x2.transform(x2_train)
scaler_y2 = sc.fit(y2_train)
y2_train_scaled = scaler_y2.transform(y2_train)

mlp1 = MLPClassifier(max_iter=1)
grid1=GridSearchCV(estimator=mlp1,param_grid=parameters,n_jobs=-1,error_score='raise')
grid1.fit(x2_train_scaled, y2_train)
y2_pred = grid1.predict(x2_test)
print("For sentiments : \n")
print(classification_report(y2_test, y2_pred, zero_division=1))
print("Confusion Matrix: \n", confusion_matrix(y2_test, y2_pred))

f.close()


#!/usr/bin/env python
# coding: utf-8

# In[3]:

import matplotlib.pyplot as plt
import numpy as np
x = np.array([12,14,16,18,20,22,24])
y = np.array([100,200,250,400,300,450,500])
plt.plot(x,y)
plt.xlabel("Temperature in degree Celsius")
plt.ylabel("Sales")
plt.show()

# In[ ]:


from collections import deque

s=input()
S=[]
for i in range(len(s)):
    S.append(s[i])
S=deque(S)
q=int(input())
flag=False
for i in range(q):
    t=input()
    if t=='1':
        if flag:
            flag=False
        else:
            flag=True
    else:
        if t[2]=='1':
            if flag:
                S.append(t[4])
            else:
                S.appendleft(t[4])
        elif t[2]=='2':
            if flag:
                S.appendleft(t[4])
            else:
                S.append(t[4])

if flag:
    x=''.join(S)
    print(x[::-1])
else:
    print(''.join(S))


## This file is module for face_det_video.py
import numpy as np
import math
import cv2
import dlib
import os, shutil, subprocess
import time
import base64
from PIL import Image
from io import BytesIO
import pickle
import socketio

from keras.models import load_model
from fer import FER

src = os.getcwd()
sio = socketio.Client()

blank_head_x =[]#[0]*5
blank_head_y =[]#[0]*5
blank_head_z =[]#[0]*5
param_check = False
# seq_emo = [[0] * 7] * 5
time_step_data = 15

class Face_dec():
    detector_emo = FER(mtcnn=True)
    def __init__(self,weight_model="src/weight/model_15frame_adddata.h5",pickle_file="src/rpy_tee.pickle",mtcnn=True,rotate_angle=450,dlib_file="src/weight/shape_predictor_68_face_landmarks.dat"):
        self.test_load_model = load_model(weight_model) #"src/weight/model_15frame_adddata.h5"
        self.shape_predictor = dlib.shape_predictor(dlib_file)
        self.cnn_face_detector = dlib.get_frontal_face_detector()
        self.rotage_head_frame = rotate_angle #450
        self.face_img = np.zeros((500, 500, 3))
        self.face4unity = False
        self.facereg_new = False
        self.facelock = [True]
        self.facecd = [0.00]
        self.seq_emo = [[0] * 7] * 5
        with open(pickle_file, 'rb') as handle: # "src/rpy_tee.pickle"
            self.rotage_head = pickle.load(handle)

        # encode image to string base64
    def string_encode(self,fimg):
        fimg_rgb = cv2.cvtColor(fimg, cv2.COLOR_BGR2RGB)
        img = Image.fromarray(fimg_rgb.astype(np.uint8))
        im_file = BytesIO()
        img.save(im_file, format="JPEG")
        # im_bytes: image in binary format.
        im_bytes = im_file.getvalue()
        face_encode = base64.b64encode(im_bytes)
        # print(face_encode)
        print(type(face_encode))
        return face_encode

    # convert rect object(dlib) to tuple
    def rect_to_bb(self,rect):
        # take a bounding predicted by dlib and convert it
        # to the format (x, y, w, h) as we would normally do
        # with OpenCV
        x = rect.left()
        y = rect.top()
        w = rect.right() - x
        h = rect.bottom() - y
        # return a tuple of (x, y, w, h)
        return (x, y, w, h)

    # convert shape object(dlib) to facial landmark numpy
    def shape_to_np(self,shape, dtype="int"):
        # initialize the list of (x, y)-coordinates
        coords = np.zeros((68, 2), dtype=dtype)
        # loop over the 68 facial landmarks and convert them
        # to a 2-tuple of (x, y)-coordinates
        for i in range(0, 68):
            coords[i] = (shape.part(i).x, shape.part(i).y)
        # return the list of (x, y)-coordinates
        return coords

    # compute angle degree
    def getAngleDegree(self,p1x, p1y, p2x, p2y):
        angleDeg = math.fabs(math.atan2(p2y - p2y, p2x - p1x) * 180 / math.pi);
        return angleDeg

    # check straight face
    def isStraightFace(self,landmarkP):
        straightFace_bool = False
        # find angle degree from landmark point
        # l = left, m = mid, r = right
        dis_lr = self.getAngleDegree(landmarkP[0][0], landmarkP[0][1], landmarkP[16][0], landmarkP[16][1])
        dis_lm = self.getAngleDegree(landmarkP[0][0], landmarkP[0][1], landmarkP[27][0], landmarkP[27][1])
        dis_mr = self.getAngleDegree(landmarkP[27][0], landmarkP[27][1], landmarkP[16][0], landmarkP[16][1])
        dis_mid = self.getAngleDegree(landmarkP[27][0], landmarkP[27][1], landmarkP[8][0], landmarkP[8][1])

        # find distance
        distL = math.fabs(landmarkP[27][0] - landmarkP[0][0])
        distR = math.fabs(landmarkP[27][0] - landmarkP[16][0])
        maxX = max(distL, distR)
        distX = math.fabs(distL - distR)
        print(distL)
        print(distR)
        print(maxX)
        print(distX)
        print(dis_mid, 'dis_mid')
        if (dis_lr <= 7 and dis_lm <= 7 and dis_mr <= 7 and distX <= maxX * 0.3):
            straightFace_bool = True

        return straightFace_bool

    # set path
    # path_root = os.getcwd()
    # path_src = os.path.join(path_root, "src")
    # path_weight = os.path.join(path_src, "weight")

    # load face and facial landmark detection model

    # change state for send face image to unity
    def chg_face_lock_stat(self,face_lock_stat):
        self.facelock[0] = face_lock_stat
        self.facecd[0] = time.time()

    # set value x to min <= x <= max
    def clamp(self,n, minn, maxn):
        return max(min(maxn, n), minn)

    def detectemotion(self,img, codinates):
        res = self.detector_emo.detect_emotions(img, codinates)
        return res

    def change_param_check(self,param_check_):
        param_check = param_check_
        return param_check

    def send_rpy(self,blank_head_x=blank_head_x,blank_head_y=blank_head_y,blank_head_z=blank_head_z): # ลบ rotage_head_frame
        blank_head_x.append(max(min(self.rotage_head[self.rotage_head_frame]['angle'][0],2),-2))
        blank_head_y.append(self.rotage_head[self.rotage_head_frame]['angle'][1]-4)
        blank_head_z.append(max(min(self.rotage_head[self.rotage_head_frame]['angle'][2],3),-3))
        self.rotage_head_frame = self.rotage_head_frame + 5
        print(blank_head_x,self.rotage_head_frame)
        if len(blank_head_x) > 5 :
            param_x = sum(blank_head_x)/len(blank_head_x) #* 3
            param_y = sum(blank_head_y)/len(blank_head_y) * 3
            param_z = sum(blank_head_z)/len(blank_head_z) #* 3
            # sio.emit("send_rpy_estimation","x:{}|y:{}|z:{}".format(param_x,param_y,param_z))
            # print("x:{}|y:{}|z:{}".format(sum(blank_head_x)/len(blank_head_x),sum(blank_head_y)/len(blank_head_y),sum(blank_head_z)/len(blank_head_z)))
            blank_head_x = blank_head_x[:3]
            blank_head_y = blank_head_y[:3]
            blank_head_z = blank_head_z[:3]
        return self.rotage_head_frame

    def playVideo(self,video_file=0): #
        cap = cv2.VideoCapture(video_file)
        while (cap.isOpened()):
            start_time = time.time()

            # Read Image
            ret, img = cap.read()
            if ret != True:
                print('read frame failed')
                # continue
                break
            size = img.shape

            # if image size is large -> resize image for less compute cost
            if size[0] > 900:
                h = size[0] / 3
                w = size[1] / 3
                img = cv2.resize(img, (int(w), int(h)), interpolation=cv2.INTER_CUBIC)
                size = img.shape

            # send image if facelock[0] is false
            if (not self.facelock[0]):
                # send to "recv_face_img" in Unity
                # sio.emit("face_img", self.string_encode(img))
                pass
            cv2.imshow('preview',img)

            # face detection
            dets = self.cnn_face_detector(img, 1)
            # find face image with maximum bounding box size and assign value w h x y
            if len(dets) > 0:
                bb_size = []
                max_bb_size = 0
                max_bb_size_idx = 0
                face_w = size[1]
                face_h = size[0]
                face_x = size[1] / 2
                face_y = size[0] / 2
                for idx_bb in range(len(dets)):
                    face = dets[idx_bb]
                    (x, y, w, h) = self.rect_to_bb(face)
                    if w * h > max_bb_size:
                        max_bb_size = w * h
                        max_bb_size_idx = idx_bb
                        face_w = w
                        face_h = h
                        face_x = x
                        face_y = y

                # get face max size and find facial landmark
                face = dets[max_bb_size_idx]
                shape = self.shape_predictor(img, face)
                landmark = self.shape_to_np(shape)

                # set value min <= x <= max
                face_w = self.clamp(face_w, 0, img.shape[1])
                face_x = self.clamp(face_x, 0, img.shape[1])
                face_h = self.clamp(face_h, 0, img.shape[0])
                face_y = self.clamp(face_y, 0, img.shape[0])

                # send center of face detection position to unity for control 3D model look at sphere
                face_pos_x = face_x + face_w / 2
                face_pos_y = face_y + face_h / 2
                face_pos_x = (face_pos_x - 320) / 640
                face_pos_y = (face_pos_y - 320) / 640

                # sio.emit("send_face_pos", "start:" + str(face_pos_x) + ":" + str(face_pos_y) + ":end")

                self.face_img = img[face_y: face_y + face_h, face_x: face_x + face_w]
                face_codinate = [[face_x, face_y, face_w, face_h]]
                if param_check:
                    print(param_check,'param_check')
                    rotage_head_frame = self.send_rpy(rotage_head_frame)

                emotion_res = self.detectemotion(img, face_codinate)
                data_emotion = list(emotion_res[0]['emotions'].values())
                list_emotion = list(emotion_res[0]['emotions'].keys())
                self.seq_emo.append(data_emotion)

                if len(self.seq_emo) > time_step_data - 1:
                    seq_emo_ = np.asarray(self.seq_emo)
                    seq_emo_ = np.expand_dims(seq_emo_, axis=0)
                    res_test = self.test_load_model.predict(seq_emo_)
                    res_test = list(np.clip(res_test, 0, 1))
                    self.seq_emo = self.seq_emo[2:]  # edit
                    test_result_pred = dict(zip(list_emotion, res_test[0][-1]))
                    # sio.emit("send_pose_estimation",
                    #         "More_Angry:{}|Fear:{}|Happy:{}|More_Sad:{}|Surprise:{}".format(res_test[0][-1][0],
                    #                                                                         res_test[0][-1][2],
                    #                                                                         res_test[0][-1][3] * 0.6,
                    #                                                                         res_test[0][-1][4],
                    #                                                                         res_test[0][-1][5]))
                if cv2.waitKey(1) & 0xFF == ord('q'):
                    cv2.destroyAllWindows()
                    break

            # if face not lock it will send straight face image to unity
                # print(facelock[0])
                if (not self.facelock[0]):
                    self.face4unity = True
                    sio.emit("face_img", self.string_encode(img))

                    # countdown 3s for send straight face image
                    if ((time.time() - self.facecd[0]) < 3.0):
                        time.sleep(0.1)
                        continue

                    # check if is straight face image
                    faceSF = self.isStraightFace(landmark)
                    print(faceSF)

                    # if it send to unity already it will be stop send next image
                    if faceSF and self.face4unity:

                        self.facereg_new = True;
                        if (self.facereg_new):
                            self.facelock[0] = True
                            self.face4unity = False
                            self.facereg_new = False
                            self.face_img = img[face_y: face_y + face_h, face_x: face_x + face_w]
                            # send to "recv_face_img" in Unity
                            sio.emit("face_img", self.string_encode(self.face_img))
                            # send to "recv_face_conf" in Unity
                            sio.emit("open_face_conf", "open");
                            # clear face image to null
                            self.face_img = np.zeros((500, 500, 3))
                            print("SF")

                # if send image time sleep for delay
                time.sleep(0.1)
            else:
                # if not send image time sleep for delay but longer than send image time
                time.sleep(0.2)
                continue


import pandas as pd
import numpy as np

df_hair = pd.read_csv("hair_dryer.tsv",sep='\t')
df_microwave=pd.read_csv("microwave.tsv",sep='\t')
df_pacifier=pd.read_csv("pacifier.tsv",sep='\t')

# print(df_hair["verified_purchase"])

YorN=['Y','N']

"""
    check verified purchase : Y/N
"""
df_hair=df_hair[df_hair["verified_purchase"].isin(YorN)]
df_microwave=df_microwave[df_microwave["verified_purchase"].isin(YorN)]
df_pacifier=df_pacifier[df_pacifier["verified_purchase"].isin(YorN)]
"""
    check vine : Y/N
"""
df_hair=df_hair[df_hair["vine"].isin(YorN)]
df_microwave=df_microwave[df_microwave["vine"].isin(YorN)]
df_pacifier=df_pacifier[df_pacifier["vine"].isin(YorN)]

# print(df_microwave)
"""
    check total_votes : if it is numeric
"""
df_hair=df_hair[df_hair["total_votes"].map(str).str.isnumeric()]
df_microwave=df_microwave[df_microwave["total_votes"].map(str).str.isnumeric()]
df_pacifier=df_pacifier[df_pacifier["total_votes"].map(str).str.isnumeric()]

"""
    check helpful_votes : if it is numeric
"""
df_hair=df_hair[df_hair["helpful_votes"].map(str).str.isnumeric()]
df_microwave=df_microwave[df_microwave["helpful_votes"].map(str).str.isnumeric()]
df_pacifier=df_pacifier[df_pacifier["helpful_votes"].map(str).str.isnumeric()]

"""
    check review : if it contains empty strings
"""

def check_empty(s):
    if not s:
        print("empty string!")
        return False
    return True

df_hair=df_hair[df_hair["review_body"].astype(str).apply(check_empty)]
df_microwave=df_microwave[df_microwave["review_body"].astype(str).apply(check_empty)]
df_pacifier=df_pacifier[df_pacifier["review_body"].astype(str).apply(check_empty)]

def Del_list(list_of_list_of_tokens,htmlname):
    from gensim import corpora, models

    import nltk
    # nltk.download('stopwords')
    from nltk.corpus import stopwords
    stop_words = stopwords.words('english')
    stop_words.extend(['from', 'subject', 're', 'edu', 'use','br','and','to','a','it','We','It','would','the','one','This','A','2','The','I','When'])
    stop_words.extend(['1','3','4','5','6','7','8','9'])

    for list_member in list_of_list_of_tokens:
        for stop_item in stop_words:
            while stop_item in list_member:
                list_member.remove(stop_item)

    return list_of_list_of_tokens

def LDA(list_of_list_of_tokens,htmlname):
    from gensim import corpora, models

    import nltk
    # nltk.download('stopwords')
    from nltk.corpus import stopwords
    stop_words = stopwords.words('english')
    stop_words.extend(['from', 'subject', 're', 'edu', 'use','br','and','to','a','it','We','It','would','the','one','This','A','2','The','I'])

    for list_member in list_of_list_of_tokens:
        for stop_item in stop_words:
            while stop_item in list_member:
                list_member.remove(stop_item)

    # list_of_list_of_tokens = [["a","b","c"], ["d","e","f"]]
    # ["a","b","c"] are the tokens of document 1, ["d","e","f"] are the tokens of document 2...
    dictionary_LDA = corpora.Dictionary(list_of_list_of_tokens)
    dictionary_LDA.filter_extremes(no_below=3)
    corpus = [dictionary_LDA.doc2bow(list_of_tokens) for list_of_tokens in list_of_list_of_tokens]

    num_topics = 15
    lda_model = models.LdaModel(corpus, num_topics=num_topics, \
                                    id2word=dictionary_LDA, \
                                    passes=4, alpha=[0.01]*num_topics, \
                                    eta=[0.01]*len(dictionary_LDA.keys()))

    import pyLDAvis
    import pyLDAvis.gensim_models
    vis = pyLDAvis.gensim_models.prepare(topic_model=lda_model, corpus=corpus, dictionary=dictionary_LDA)
    pyLDAvis.save_html(vis, htmlname)

    return

import re

# list_of_list_of_tokens=[]
# for s in df_hair["review_body"].astype(str):
#     list_of_tokens = re.findall(r'\w+', s)
#     list_of_list_of_tokens.append(list_of_tokens)

# LDA(list_of_list_of_tokens,'hair_drier_html_file.html')

df_microwave["review_date"]=pd.to_datetime(df_microwave["review_date"])

df_microwave= df_microwave.sort_values(by=["product_id","review_date"],ascending=[True,True])

df_microwave["diff_date"]=pd.to_datetime(df_microwave["review_date"]).diff().apply(lambda x: x/np.timedelta64(1, 'D')).fillna(0).astype('int64')

df_microwave.groupby("product_id").agg({'star_rating':'count'}).to_csv("number_over_each_product.csv")

products_microwave=df_microwave.groupby("product_id").agg({'star_rating':'count'}).index.tolist()

"""
    tokenize comments
"""
list_of_list_of_tokens=[]
for s in df_microwave["review_body"].astype(str):
    list_of_tokens = re.findall(r'\w+', s)
    list_of_list_of_tokens.append(list_of_tokens)

list_of_list_of_tokens = Del_list(list_of_list_of_tokens,'microwave_html_file.html')

"""
    create array
"""

array_for_LSTM=[]
element_ind=0 # since it's sorted, you can extract element from token_list in this way

sample_DEBUG=0 # turn on <- =1

for product in products_microwave:
    array_item=[]
    df_part=df_microwave[df_microwave["product_id"] == product].copy()

    df_part.loc[df_part.index[0],"diff_date"]=0

    df_len=[*range(len(df_part))]
    df_part.index=df_len # "reindexing"

    if sample_DEBUG != 0: # debug
        print(df_part["diff_date"])
        # print(diff_days)
        # print((pd.to_datetime(row_en["review_date"])-last_date).dt.days)
        # print(array_item)
        # df_part.to_csv("check_part.csv")
        sample_DEBUG=sample_DEBUG-1
    for row_n in df_len:
        row_en=df_part[df_part.index==row_n]
        a = (map(lambda x: x.lower(), list_of_list_of_tokens[element_ind]))
        row_list = [list(a)]
        row_list.extend(row_en["star_rating"])
        row_list.extend(row_en["helpful_votes"])
        row_list.extend(row_en["total_votes"])
        row_list.extend(row_en["verified_purchase"])
        row_list.extend(row_en["review_date"])
        row_list.extend(row_en["diff_date"])

        last_date=pd.to_datetime(row_en["review_date"])

        array_item.append(row_list)
        element_ind=element_ind+1

    array_for_LSTM.append(array_item)

def Cook_data(array_for_LSTM,del_word_freq=70,Format_Words="Word_List",if_train_rating=True,
if_train_helpful_vote=True,if_train_total_vote=True,if_train_vine=True,if_train_diff_date=True,remove_date=False,
del_product_review_num_threshold=-1
):
    """
    api:
        array_for_LSTM - a list of array to train
        array_for_LSTM[i] - the array to train
        array_for_LSTM[i][j] - the j-th item of an array
        for every item it looks like
        [[string containing all the words except for stop words],star rating,total votes,verified purchase,review_date,diff_date(in days)]

        del_word_freq=70, Choose a word frequency to select only high freq words
        Format_Words="Word_List", If "Encoded_List" then it's encoded, then you can directly train them
        if_train_rating=True,     You can select whether you want to train/normalize this
        if_train_helpful_vote=True,
        if_train_total_vote=True,
        if_train_vine=True,
        if_train_diff_date=True,
        remove_date=False,        Choose to remove the dates or not
        del_product_review_num_threshold=-1 Choose a threshold to sift the products,
            eg. if this = 40 then you will only get products with >40 reviews

        Example:
            print(array_for_LSTM[0][0])
            output:
               See it your self
               After cooking and before cooking
        Note:
            the date difference is in (days)
            You MUST!!!!!!!!!!! insure the date difference is at the end.
    """

    # Filter Products

    for product_index,product in enumerate(array_for_LSTM):
        if len(product)<=del_product_review_num_threshold:
            del array_for_LSTM[product_index]

    # Filter Words using del_word_freq
    dict_for_words={}
    dict_for_word_index={}
    for product_index,product in enumerate(array_for_LSTM):
        for review_index,review in enumerate(product):
            for word_in_review in review[0]:
                if word_in_review in dict_for_words:
                    dict_for_words[word_in_review]+=1
                else:
                    dict_for_words[word_in_review]=0

    dict_for_words=dict(filter(lambda elem: elem[1]>=del_word_freq,dict_for_words.items()))

    for word_ind,dic_key in enumerate(dict_for_words.keys()):
        dict_for_word_index[dic_key]=word_ind

    for product_index,product in enumerate(array_for_LSTM):
        for review_index,review in enumerate(product):
            word_list=[]
            for word_in_review in review[0]:
                if word_in_review in dict_for_words:
                    word_list.append(word_in_review)
            review[0]=word_list

    # print(len(dict_for_word_index),len(dict_for_words))
    Review_mean=[]
    Review_std=[]

    if Format_Words == "Encoded_List":
        for product_index,product in enumerate(array_for_LSTM):
            for review_index,review in enumerate(product):
                word_list=[0]*len(dict_for_words)
                for word_in_review in review[0]:
                    if word_in_review in dict_for_word_index:
                       word_list[dict_for_word_index[word_in_review]]+=1
                review[0]=word_list
    # Normalize every number

        for word_index in range(len(dict_for_words)):
            list_of_n_th_word=[review[0][word_index] for product in array_for_LSTM for review in product]
            list_of_n_th_word=np.array(list_of_n_th_word)
            mean_list=list_of_n_th_word.mean()
            std_dev_list=list_of_n_th_word.std()

            Review_mean.append(mean_list)
            Review_std.append(std_dev_list)

            if std_dev_list!=0:
                list_of_n_th_word=(list_of_n_th_word-mean_list)/std_dev_list

            rev_index=0
            for product in array_for_LSTM:
                for review in product:
                    review[0][word_index]=list_of_n_th_word[rev_index]
                    rev_index+=1

        # Package the first dimension
        Review_mean=[Review_mean]
        Review_std=[Review_std]

    if if_train_rating:
        list_of_train_data=[review[1] for product in array_for_LSTM for review in product]
        list_of_train_data=np.array(list_of_train_data)
        mean_list=list_of_train_data.mean()
        std_dev_list=list_of_train_data.std()

        Review_mean.append(mean_list)
        Review_std.append(std_dev_list)

        # print(mean_list,std_dev_list)

        if std_dev_list!=0:
            list_of_train_data=(list_of_train_data-mean_list)/std_dev_list
        rev_index=0
        for product in array_for_LSTM:
            for review in product:
                review[1]=list_of_train_data[rev_index]
                rev_index+=1

    if if_train_helpful_vote:
        list_of_train_data=[review[2] for product in array_for_LSTM for review in product]
        list_of_train_data=np.array(list_of_train_data)
        mean_list=list_of_train_data.mean()
        std_dev_list=list_of_train_data.std()

        Review_mean.append(mean_list)
        Review_std.append(std_dev_list)

        if std_dev_list!=0:
            list_of_train_data=(list_of_train_data-mean_list)/std_dev_list
        rev_index=0
        for product in array_for_LSTM:
            for review in product:
                review[2]=list_of_train_data[rev_index]
                rev_index+=1

    if if_train_total_vote:
        list_of_train_data=[review[3] for product in array_for_LSTM for review in product]
        list_of_train_data=np.array(list_of_train_data)
        mean_list=list_of_train_data.mean()
        std_dev_list=list_of_train_data.std()

        Review_mean.append(mean_list)
        Review_std.append(std_dev_list)

        if std_dev_list!=0:
            list_of_train_data=(list_of_train_data-mean_list)/std_dev_list
        rev_index=0
        for product in array_for_LSTM:
            for review in product:
                review[3]=list_of_train_data[rev_index]
                rev_index+=1

    map_Y_N={}
    map_Y_N['Y']=1
    map_Y_N['N']=0
    for product in array_for_LSTM:
        for review in product:
            review[4]=map_Y_N[review[4]]

    if if_train_vine:
        list_of_train_data=[review[4] for product in array_for_LSTM for review in product]
        list_of_train_data=np.array(list_of_train_data)
        mean_list=list_of_train_data.mean()
        std_dev_list=list_of_train_data.std()

        Review_mean.append(mean_list)
        Review_std.append(std_dev_list)

        if std_dev_list!=0:
            list_of_train_data=(list_of_train_data-mean_list)/std_dev_list
        rev_index=0
        for product in array_for_LSTM:
            for review in product:
                review[4]=list_of_train_data[rev_index]
                rev_index+=1

    removed_date=[]
    if remove_date:
        for product in array_for_LSTM:
            for review in product:
                removed_date.append(review[5])
                del review[5]

    if if_train_diff_date:
        list_of_train_data=[review[-1] for product in array_for_LSTM for review in product]
        list_of_train_data=np.array(list_of_train_data)
        mean_list=list_of_train_data.mean()
        std_dev_list=list_of_train_data.std()

        Review_mean.append(mean_list)
        Review_std.append(std_dev_list)

        if std_dev_list!=0:
            list_of_train_data=(list_of_train_data-mean_list)/std_dev_list
        rev_index=0
        for product in array_for_LSTM:
            for review in product:
                review[-1]=list_of_train_data[rev_index]
                rev_index+=1

    return array_for_LSTM,removed_date,dict_for_word_index,Review_mean,Review_std

# print(array_for_LSTM[0][0])
array_for_LSTM,removed_date,dict_for_words,Review_mean,Review_std= \
Cook_data(array_for_LSTM,70,"Encoded_List",True,True,True,True,True,True,-1)

def Reverse_Cook(array_for_LSTM,removed_date,dict_for_word_index,Review_mean,Review_std,Format_Words="Word_List",if_train_rating=True,
if_train_helpful_vote=True,if_train_total_vote=True,if_train_vine=True,if_train_diff_date=True,remove_date=False):
    if remove_date:
        rem_id=0
        for product in array_for_LSTM:
            for review in product:
                review.insert(5,removed_date[rem_id])
                rem_id+=1

    mean_std_ind=0

    if Format_Words == "Encoded_List":

        for word_index in range(len(dict_for_word_index)):
            mean_list=Review_mean[0][word_index]
            std_dev_list=Review_std[0][word_index]

            list_of_n_th_word=[review[0][word_index] for product in array_for_LSTM for review in product]
            list_of_n_th_word=np.array(list_of_n_th_word)

            if std_dev_list!=0:
                list_of_n_th_word=list_of_n_th_word*std_dev_list+mean_list

            rev_index=0
            for product in array_for_LSTM:
                for review in product:
                    review[0][word_index]=list_of_n_th_word[rev_index]
                    rev_index+=1

        dict_for_word_index = dict((y,x) for x,y in dict_for_word_index.items())
        # print(dict_for_word_index)
        #RecoverWord
        for product_index,product in enumerate(array_for_LSTM):
            for review_index,review in enumerate(product):
                word_list=[]
                for wd_ind,word_freq in enumerate(review[0]):
                    while word_freq>=0:
                        if word_freq>=0.3:
                            word_list.append(dict_for_word_index[wd_ind])
                        word_freq-=1
                review[0]=word_list

    if if_train_rating:
        list_of_train_data=[review[1] for product in array_for_LSTM for review in product]
        list_of_train_data=np.array(list_of_train_data)

        mean_list=Review_mean[1]
        std_dev_list=Review_std[1]

        if std_dev_list!=0:
            list_of_train_data=list_of_train_data*std_dev_list+mean_list

        rev_index=0
        for product in array_for_LSTM:
            for review in product:
                review[1]=list_of_train_data[rev_index]
                rev_index+=1

    if if_train_helpful_vote:
        list_of_train_data=[review[2] for product in array_for_LSTM for review in product]
        list_of_train_data=np.array(list_of_train_data)

        mean_list=Review_mean[2]
        std_dev_list=Review_std[2]

        if std_dev_list!=0:
            list_of_train_data=list_of_train_data*std_dev_list+mean_list

        rev_index=0
        for product in array_for_LSTM:
            for review in product:
                review[2]=list_of_train_data[rev_index]
                rev_index+=1

    if if_train_total_vote:
        list_of_train_data=[review[3] for product in array_for_LSTM for review in product]
        list_of_train_data=np.array(list_of_train_data)

        mean_list=Review_mean[3]
        std_dev_list=Review_std[3]

        if std_dev_list!=0:
            list_of_train_data=list_of_train_data*std_dev_list+mean_list

        rev_index=0
        for product in array_for_LSTM:
            for review in product:
                review[3]=list_of_train_data[rev_index]
                rev_index+=1

    if if_train_vine:
        list_of_train_data=[review[4] for product in array_for_LSTM for review in product]
        list_of_train_data=np.array(list_of_train_data)

        mean_list=Review_mean[4]
        std_dev_list=Review_std[4]

        if std_dev_list!=0:
            list_of_train_data=list_of_train_data*std_dev_list+mean_list

        rev_index=0
        for product in array_for_LSTM:
            for review in product:
                if list_of_train_data[rev_index]>=0.5:
                    review[4]='Y'
                else:
                    review[4]='N'
                rev_index+=1

    if if_train_diff_date:
        list_of_train_data=[review[-1] for product in array_for_LSTM for review in product]
        list_of_train_data=np.array(list_of_train_data)

        mean_list=Review_mean[-1]
        std_dev_list=Review_std[-1]

        if std_dev_list!=0:
            list_of_train_data=list_of_train_data*std_dev_list+mean_list

        rev_index=0
        for product in array_for_LSTM:
            for review in product:
                review[-1]=list_of_train_data[rev_index]
                rev_index+=1

    return array_for_LSTM
# print(Review_mean)
array_for_LSTM=Reverse_Cook(array_for_LSTM,removed_date,dict_for_words,Review_mean,Review_std,"Encoded_List",True,True,True,True,True,True)
# print(array_for_LSTM[0][0])

from tqdm import tqdm
from numpy import array
from sklearn.preprocessing import LabelEncoder

input_size=209
output_size=input_size
train_window=10
train_windows=train_window*input_size
hidden_size=500
words_min_repeatence=70
preseved_space=5
max_value_of_each_preseved_space=[-100]*6
max_value_of_each_preseved_space[3]=1

dic_for_words=[]
real_dic={}
k=-1
for i in range(len(array_for_LSTM)):
    i=k+1
    if i == len(array_for_LSTM):
        break
    if len(array_for_LSTM[i]) > train_window:
        for j in array_for_LSTM[i]:
            dic_for_words.extend(j[0])
            for l in range(len(j[1:])):
                try:
                    if j[l+1]>max_value_of_each_preseved_space[l]:
                        max_value_of_each_preseved_space[l]=j[l+1]
                except:
                    continue
    else:
        del array_for_LSTM[i]
        i-=1
    k=i

print(max_value_of_each_preseved_space)
values = array(dic_for_words)
label_encoder = LabelEncoder()
integer_encoded = label_encoder.fit_transform(values)
real_dic=real_dic.fromkeys(dic_for_words,0)
for i in dic_for_words:
    real_dic[i]+=1
real_dic=dict((k,integer_encoded[dic_for_words.index(k)]) for k,v in real_dic.items() if v>=words_min_repeatence)
j=0
for i in real_dic.keys():
    real_dic[i]=j
    j+=1

reversed_dic={v:k for k,v in real_dic.items()}

dic_for_words=[]
i=0
j=0
k=0
values=[]
label_encoder=[]
integer_encoded=[]

'''
list_for_words=[[],[]]
list_for_words[0]=list(real_dic.keys())
list_for_words[1]=list(real_dic.values())
list_for_words = list(map(list, zip(*list_for_words)))
list_for_words.sort(key=lambda x:x[1],reverse=True)
print((list_for_words))
'''

import torch
import torch.nn as nn
import torch.optim as optim

input_dataset=[]
data_for_single_comment=[]
data_for_each_brand=[]

def create_inout_sequences(input_data, tw,input_size):
    inout_seq = []
    L = len(input_data)
    for i in range(0,L-tw-input_size+1,input_size):
        train_seq = input_data[i:i+tw]
        train_label = input_data[i+tw:i+tw+input_size]
        inout_seq.append((train_seq ,train_label))
    return inout_seq

for i in tqdm(array_for_LSTM):
    data_for_each_brand=[]
    for j in i:
        data_for_single_comment=[0.0]*(input_size-preseved_space)
        for k in j[0]:
            try:
                data_for_single_comment[real_dic[k]]=1.0
            except:
                continue
        for l in range(6):
            if l == 3:
                data_for_single_comment.append(1-float(YorN.index(j[l+1])))
            elif l != 4:
                data_for_single_comment.append(j[l+1]/max_value_of_each_preseved_space[l])
        data_for_each_brand.extend(data_for_single_comment)
    data_for_each_brand=torch.FloatTensor(data_for_each_brand).cuda()
    input_dataset.extend(create_inout_sequences(data_for_each_brand,train_windows,input_size))

print(input_dataset[1][1])

class LSTM(nn.Module):
    def __init__(self, input_size=1, hidden_layer_size=200, output_size=1):
        super().__init__()
        self.hidden_layer_size = hidden_layer_size

        self.lstm = nn.LSTM(input_size, hidden_layer_size).cuda()

        self.linear = nn.Linear(hidden_layer_size, output_size).cuda()

        self.ReLU = nn.ReLU()

        self.hidden_cell = (torch.zeros(1,1,self.hidden_layer_size).cuda(),
                            torch.zeros(1,1,self.hidden_layer_size).cuda())

    def forward(self, input_seq):
        lstm_out, self.hidden_cell = self.lstm(input_seq.view(train_window ,1, -1), self.hidden_cell)
        predictions = self.linear(lstm_out.view(train_window, -1))
        predictions = self.ReLU(predictions)
        return predictions[-1]

model = LSTM(input_size,hidden_size,output_size)
loss_function = nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

epochs = 50

for i in tqdm(range(epochs)):
    for seq, labels in input_dataset:
        optimizer.zero_grad()
        model.hidden_cell = (torch.zeros(1, 1, model.hidden_layer_size).cuda(),
                            torch.zeros(1, 1, model.hidden_layer_size).cuda())

        y_pred = model(seq)

        single_loss = loss_function(y_pred, labels)
        single_loss.backward()
        optimizer.step()

    if i%5 == 4:
        print(f'epoch: {i+1:3} loss: {single_loss.item():10.8f}')

print(f'epoch: {i+1:3} loss: {single_loss.item():10.10f}')
torch.save(model, f'LSTM_model_{input_size-preseved_space}_{train_window}_{hidden_size}_{epochs}.pth')
# print(array_for_LSTM[1][1])

# list_of_list_of_tokens=[]
# for s in df_pacifier["review_body"].astype(str):
#     list_of_tokens = re.findall(r'\w+', s)
#     list_of_list_of_tokens.append(list_of_tokens)
# LDA(list_of_list_of_tokens,'pacifier_html_file.html')

# df_hair.to_csv("hair_dryer_clean.csv")
# df_microwave.to_csv("microwave_clean.csv")
# df_pacifier.to_csv("pacifier_clean.csv")


import os
import json
import soundfile as sf

script_dir = os.path.dirname(os.path.abspath(__file__))
python_root_dir = os.path.dirname(script_dir)
app_root_dir = os.path.dirname(python_root_dir)

def read_json(dir_name: str, json_name: str):
    file_path = os.path.join(python_root_dir, dir_name, json_name + ".json")
    with open(file_path, "r") as f:
        data = json.load(f)
    return data

def write_json(dir_name: str, json_name: str, data: dict):
    file_path = os.path.join(python_root_dir, dir_name, json_name + ".json")
    with open(file_path, "w") as f:
        json.dump(data, f)

def write_audio(dir_name: str, file_name: str, data):
    file_path = os.path.join(app_root_dir, dir_name, file_name + ".wav")

    # If a file with the same name already exists, remove it to forcefully write
    if os.path.exists(file_path):
        os.remove(file_path)

    sf.write(file_path, data, 16000)


import lib.logging as logging
LOG = logging.getLogger(__name__)

def _get_heartbeats(heartbeat_file):
    heartbeats = dict()
    import tarfile
    with tarfile.open(heartbeat_file, 'r') as tar:
        for tarinfo in tar:
            #heartbeats have the format: -1639960620.2999249
            #where the `-` indicates downtime, and a `+` indicates uptime
            #the number is a timestamp taken by time.time()
            _heartbeat = tarinfo.name
            _header = _heartbeat[0]
            _timestamp = _heartbeat[1:]

            import datetime
            _key = datetime.datetime.fromtimestamp(float(_timestamp))
            if _header == "+":
                _value = True
            else:
                _value = False

            heartbeats[_key] = _value

    return heartbeats

def _get_input():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("start")
    parser.add_argument("stop")
    args = parser.parse_args()

    return {"start": args.start, "stop": args.stop}

def _get_timeline(_heartbeats, start, stop):
    '''
    input a dict of heartbeats and output a list of True and False indicating
    uptimes and downtimes

    ----

    _heartbeats; a dict mapping datetime objects to boolean
        indicating uptime (True) and downtime (False)
        at the times indicated by the datetime objects
    start; a timestamp indicating the start time of a span of time we're interested
        in
    stop; a timestamp indicating the end of a span of time we're interested in
    '''
    heartbeats = _heartbeats.copy()

    #then insert the time values of start and stop into this sequence of heartbeats
    #so we can make an ordering of events
    #   marking start and stop with None values
    import datetime
    _start = datetime.datetime.fromtimestamp(float(start))
    _stop = datetime.datetime.fromtimestamp(float(stop))
    heartbeats[_start] = None
    heartbeats[_stop] = None

    #then sort by times to form a timeline of heartbeats showing uptime and
    #downtime; as well as showing where the start and stop times fell in the
    #timeline
    #   see:
    #   https://towardsdatascience.com/sorting-a-dictionary-in-python-4280451e1637
    heartbeats_sorted_by_time = sorted(heartbeats.items(), key = lambda kv: kv[0])

    #then discarding the times to only show a sequence of uptimes and downtimes
    return [t[1] for t in heartbeats_sorted_by_time]

def _partition_timeline(_timeline):
    #check for the empty case, where we have a timeline ONLY of the start and
    #stop events
    if all([(t is None)for t in _timeline]):
        res = {
            "pre_start": [],
            "post_stop": [],
            "between_start_and_stop": [],
        }

    timeline = _timeline.copy()

    #then removing and saving the uptimes and downtimes before start and after
    #stop
    #since we only want to check for downtime between start and stop
    #(except in the special case below)
    i_start = timeline.index(None)
    pre_start = timeline[0:i_start]
    timeline2 = timeline[i_start:]

    timeline2.reverse()

    j_stop = timeline2.index(None)
    post_stop = timeline2[0:j_stop]
    post_stop.reverse()
    timeline3 = timeline2[j_stop:]

    #then discarding the None's that mark start and stop,
    #to allow checking for any downtime (False values) in the span between
    #start and stop
    timeline3.pop(0)
    timeline3.pop(-1)

    res = {
        "pre_start": pre_start,
        "post_stop": post_stop,
        "between_start_and_stop": timeline3
    }

    return res

def _return(val):
    import json
    print( json.dumps( val ) )
    import sys
    sys.exit()

def main():
    log_path = "./log"
    archive_name = "heartbeats.tar"

    from pathlib import Path
    log_dir = Path(log_path)
    heartbeat_file = log_dir / archive_name

    heartbeats = _get_heartbeats(heartbeat_file)

    #BUG, in the rare case where we have 0 or 1 entries in the heartbeat file
    #just return as if we were connected
    if len(heartbeats) <= 1:
        _return(True)

    res = _get_input()
    start = res["start"]
    stop = res["stop"]

    timeline = _get_timeline(heartbeats, start, stop)
    res = _partition_timeline(timeline)

    #check for the special case if there is no gap between start and stop
    #this should produce an empty list
    if len(res["between_start_and_stop"]) == 0:
        '''
        In this special case, find the heartbeats immediately before the start
        time
        and
        after the stop time
        Then if they both show downtime, assume the span is downtime
        In the extra-special case that there are no heartbeats before
        or after the start and stop times, then assume downtime for each
        and return accordingly

        L, R neighbors

        T, T, assume up
        T, F, assume up
        F, T, assume up
        F, F, assume down
        and assume if there are no neighbors, that it's downtime; False
        '''
        pre_start = res["pre_start"]
        post_stop = res["post_stop"]

        LOG.debug("special case handling")

        _left_neighbor = False
        if len(pre_start) != 0:
            _left_neighbor = pre_start[-1]

        _right_neighbor = False
        if len(post_stop) != 0:
            _right_neighbor = post_stop[0]

        _return( _left_neighbor | _right_neighbor )

    #if there is any downtime in the span, then the network was not up during
    #the span marked by start and stop
    #and if there was no downtime, then the network was functioning during the
    #span
    _return( all(res["between_start_and_stop"]) )

if __name__ == '__main__': main()


################################################################################################################
## Streamlit App 007 - 01 - 01/01
################################################################################################################
## App 007 - Iris Classification
################################################################################################################
import numpy as np
import pandas as pd
import streamlit as st
from sklearn import datasets
from sklearn.ensemble import RandomForestClassifier

st.write("""
         # Iris Flower Classification App
         ## This App predicts the different types of varieties from the classic Iris Dataset.
         """)

st.sidebar.header('User Input Parameters')

def user_input_features():
    sepal_length = st.sidebar.slider('Sepal length', 4.3, 7.9, 5.4)
    sepal_width = st.sidebar.slider('Sepal width', 2.0, 4.4, 3.4)
    petal_length = st.sidebar.slider('Petal length', 1.0, 6.9, 1.3)
    petal_width = st.sidebar.slider('Petal width', 0.1, 2.5, 0.2)
    data = {'sepal_length':sepal_length,
            'sepal_width':sepal_width,
            'petal_length':petal_length,
            'petal_width':petal_width}
    features = pd.DataFrame(data, index=[0])
    return features

df = user_input_features()

st.subheader('User Input Parameters')
## Print out the DataFrame
st.write(df)

iris  = datasets.load_iris()
X = iris.data
y = iris.target

clf = RandomForestClassifier()
### Progress Bar - Not used
#import time
#progress_bar = st.progress(0)
#for perc_completed in range(100):
#    time.sleep(0.005)
#    progress_bar.progress(perc_completed+1)
clf.fit(X, y)

prediction = clf.predict(df)
prediction_proba = clf.predict_proba(df)

#st.subheader('Class labels and their corresponding index number')
#st.write(iris.target_names)

#st.subheader('Prediction')
#st.write(iris.target_names[prediction])

#st.subheader('Prediction Probability')
#st.write(prediction_proba)

st.write('')

st.write('##### Prediction Probabilities Index Number and Class Label')
pred_probs_names = {"0":['setosa'],
                    "1":['versicolor'],
                    "2":['virginica']}

df_pred_probs_names = pd.DataFrame(pred_probs_names)
st.write(df_pred_probs_names)

st.write('')

#st.subheader("Prediction Results")
st.markdown("<h3 style='text-align:center;'>Prediction Results </h3>", unsafe_allow_html=True)
st.write('')

df_02_data = {"Prediction Probabilities":str(prediction_proba),
              "Index of Prediction":[prediction],
              "Prediction Name": iris.target_names[prediction],
              "Prediction Probability":str(prediction_proba.max())+' %'
              }

df_02 = pd.DataFrame(df_02_data)
st.write(df_02)

st.write('')

name_prediction = iris.target_names[prediction][0]

#st.markdown(f"""
#         ##### Given the input parameters, the model predicts that the species is: *{name_prediction}* with {str(prediction_proba.max()*100)} % confidence
#         """)

## Rounding
st.markdown(f"""
         ##### Given the input parameters, the model predicts that the species is: *{name_prediction}* with {np.round(prediction_proba.max()*100,2)} % confidence
         """)

st.write('---')
st.write('')
st.write('')
st.write('')
st.write('')
st.write('')
st.write('')
st.write('')
st.write('')
st.write('')
st.write('')
st.write('')
st.write('')
st.write('')
st.write('')
st.write('')
st.write('')
st.write('')
st.write('')
st.write('')
st.write('')
st.write('')
st.write('')
st.write('')
st.write('')

st.write('### Resources:')
st.write("""
         Data Source: [Scikit-Learn Datasets](https://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_iris.html#)\n
         [Fisher,R. A.. (1988). Iris. UCI Machine Learning Repository. https://doi.org/10.24432/C56C76.](https://archive.ics.uci.edu/dataset/53/iris)\n
         [Streamlit](https://streamlit.io/)
         """)
st.write('###### [Special Thanks to Free Code Camp and Chanin Nantasenamat](https://www.freecodecamp.org/).')
st.write('###### Thank you kindly to all who make information and knowledge available for free.')


# coding: utf-8

import requests
import json
import random

def command_man(body):

    request_temp_str = 'http://api.oboobs.ru/boobs/%d/%d/'
    count_boobs_req = 'http://api.oboobs.ru/boobs/count/'

    try:
        c = requests.get(count_boobs_req, verify=False)
        boobs_count = json.loads(c.text)[0]['count']
        c.close()
    except:
        boobs_count = 0

    if body.startswith('всего'):
        answer = 'Фоток с сиськами всего %d.' % boobs_count
        return answer
    i = int(body) if body.isdigit() else 1

    id_bs = random.choice(xrange(boobs_count))

    if id_bs < i:
        id_bs = i

    print(boobs_count)

    rs = request_temp_str % (id_bs,i)

    r = requests.get(rs, verify=False)
    print(r.text)
    rq_ans = json.loads(r.text)

    ans = []
    for x in rq_ans:
        id_str = '0'*(1 + len(str(boobs_count)) - len(str(x['id']))) + str(x['id'])
        ans.append('http://media.oboobs.ru/boobs/' + id_str + '.jpg')

    answer = '\n'.join(ans)

    return answer

print(command_man(''))


from math import sqrt, ceil
from utils import square_root_mod as sqrtm
from utils import modular_exponentiation as me
from utils import modinv

def bsgs(h, g, order, p):
    """
    Baby-Step-Giant-Step algorithm
    :param h: g^a
    :param g: g
    :param order: order of the group
    :return: a
    """

    max_list_size = 50000000
    m = min(sqrtm(order, p), max_list_size)  # list size
    m_giant = order // m + 1
    l = {}

    # baby steps
    power = 1
    for i in range(m + 1):
        if i > 0:
            # faster than calling me every iteration
            power *= g
            power %= p
        l[power] = i

    # giant steps
    # g^(-j*m) = (g^-m)^j
    gminv = modinv(me(g, m, p) % p, p)
    gminv_j = gminv
    power = 1
    for j in range(m_giant + 1):
        if j > 0:
            # faster than calling me every iteration
            power *= gminv_j
            power %= p

        z = (power * h) % p
        if z in l:
            # found!
            i = l[z]
            a = (i + j * m) % order
            return a

    # not found
    return None