#UIAHandler.py
#A part of NonVisual Desktop Access (NVDA)
#Copyright (C) 2008-2018 NV Access Limited, Joseph Lee
#This file is covered by the GNU General Public License.
#See the file COPYING for more details.

from comtypes import COMError
import config
from logHandler import log

handler=None
isUIAAvailable=False

if config.conf and config.conf["UIA"]["enabled"]:
	# Because Windows 7 SP1 (NT 6.1) or later is supported, just assume UIA can be used unless told otherwise.
	try:
		from _UIAHandler import *
		isUIAAvailable=True
	except ImportError:
		log.debugWarning("Unable to import _UIAHandler",exc_info=True)
		pass

def initialize():
	global handler
	if not isUIAAvailable:
		raise NotImplementedError
	try:
		handler=UIAHandler()
	except COMError:
		handler=None
		raise RuntimeError("UIA not available")

def terminate():
	global handler
	if handler:
		handler.terminate()
		handler=None


# -*- coding: utf8 -*-
import copy
import hashlib
import coincurve

from py_ecc.secp256k1 import N as secp256k1n

from mythril.laser.ethereum.helper import ALL_BYTES, bytearray_to_int, concrete_int_to_bytes, sha3, zpad

def int_to_32bytes(i):   #used because int can't fit as bytes function's input
    o = [0] * 32
    for x in range(32):
        o[31 - x] = i & 0xff
        i >>= 8
    return bytes(o)

def ecrecover_to_pub(rawhash, v, r, s):
    try:
        pk = coincurve.PublicKey.from_signature_and_message(
            zpad(concrete_int_to_bytes(r), 32) + zpad(concrete_int_to_bytes(s), 32) +
            ALL_BYTES[v - 27],
            rawhash,
            hasher=None,
        )
        pub = pk.format(compressed=False)[1:]
    except BaseException:
        pub = b"\x00" * 64

    return pub

def extract32(data, i):
    if i >= len(data):
        return 0
    o = data[i: min(i + 32, len(data))]
    o.extend(bytearray(32 - len(o)))
    return bytearray_to_int(o)

def ecrecover(data):
    data = bytearray(data)
    message = b''.join(map(lambda x: ALL_BYTES[x], data[0:32]))
    v = extract32(data, 32)
    r = extract32(data, 64)
    s = extract32(data, 96)
    if r >= secp256k1n or s >= secp256k1n or v < 27 or v > 28:
        return []
    try:
        pub = ecrecover_to_pub(message, v, r, s)
    except Exception as e:
        return []
    o = [0] * 12 + [x for x in sha3(pub)[-20:]]
    return o

def sha256(data):
    data = bytes(data)
    return hashlib.sha256(data).digest()

def ripemd160(data):
    data = bytes(data)
    return 12*[0]+[i for i in hashlib.new('ripemd160', data).digest()]

def identity(data):
    return copy.copy(data)

def native_contracts(address, data):
    '''
    takes integer address 1, 2, 3, 4
    '''

    functions = (ecrecover, sha256, ripemd160, identity)
    return functions[address-1](data)


from collections import defaultdict

def solution(s):
    temp = [a.split(',') for a in s[2:-2].split("},{")]
    elements = defaultdict(int)
    for t in temp:
        for x in t:
            elements[x] += 1
    return list(map(int, sorted(elements, key=lambda k: elements[k], reverse=True)))


from sqlite3 import connect

class Yui_Database():
    """
    Database of Yui Chat bot
        This database won't be used if you're hosting this on heroku
    """

    def __init__(self) -> None:
        self.yui_db = connect("yui-x.db")
        self.curs = self.yui_db.cursor()
        # Creates a table for storing ai engine details
        self.curs.execute(
            """
            CREATE TABLE
            IF NOT EXISTS
            engine (engine_name text)
            """
        )

    async def set_engine(self, engine_name):
        is_exists = await self.get_engine()
        if not is_exists:
            self.curs.execute(
                """
                INSERT INTO engine(engine_name)
                VALUES (?)
                """, (engine_name,)
            )
        else:
            self.curs.execute(
                """
                UPDATE engine
                SET engine_name=?
                """, (engine_name,)
            )
        return self.yui_db.commit()

    async def get_engine(self):
        selct = self.curs.execute(
            """
            SELECT engine_name
            FROM engine
            """
        ).fetchone()
        if selct:
            return selct[0]
        else:
            return None


from bs4 import BeautifulSoup
import requests

URL = "https://www.empireonline.com/movies/features/best-movies-2/"
response = requests.get(URL)
empire_web = response.text
soup = BeautifulSoup(empire_web, "html.parser")
all_tags = soup.find_all(name="img", class_="jsx-952983560")
titles = []
for tag in all_tags:
    title = tag.get("alt")
    titles.append(title)
cut_list_titles = titles[12:]
movies = cut_list_titles[::-1]

with open("movies.txt", mode="w") as file:
    for movie in movies:
        number = movies.index(movie) + 1
        file.write(f"{number}) {movie}\n")


import logging
from pathlib import Path
from typing import List

import joblib
import pandas as pd
from sklearn.pipeline import Pipeline

from model import __version__ as _version
from model.config.core import DATASET_DIR, TRAINED_MODEL_DIR, config

logger = logging.getLogger(__name__)

def load_dataset(*, client_file_name: str, price_file_name: str) -> pd.DataFrame:
    # Loading the datasets.
    dataframe_client = pd.read_csv(Path(f"{DATASET_DIR}/{client_file_name}"))
    dataframe_price = pd.read_csv(Path(f"{DATASET_DIR}/{price_file_name}"))

    # Converting to datetime.
    dataframe_client = datetime_conversion_client(df=dataframe_client)
    dataframe_price = datetime_conversion_price(df=dataframe_price)

    # Transforming the price data.
    dataframe_price = price_data_trans(df=dataframe_price)

    # Merging the client and price datasets.
    dataframe = merging_datasets(df=dataframe_client, df_1=dataframe_price)

    # Getting time and consumption features.
    dataframe = time_features(df=dataframe)
    dataframe = consum_features(df=dataframe)

    dataframe = dataframe.drop("Unnamed: 0", axis=1)
    dataframe = dataframe.dropna()

    # Creating the new categorical features.
    dataframe["price_change_energy"] = dataframe[
        "offpeak_diff_dec_january_energy"
    ].apply(price_change)
    dataframe["price_change_power"] = dataframe["offpeak_diff_dec_january_power"].apply(
        price_change
    )

    # Dropping the first feature.
    dataframe = dataframe.drop(
        ["offpeak_diff_dec_january_energy", "offpeak_diff_dec_january_power"], axis=1
    )

    dataframe = dataframe.dropna()

    return dataframe

def save_pipeline(*, pipeline_to_persist: Pipeline) -> None:
    """Persist the pipeline.
    Saves the versioned model, and overwrites any previous
    saved models. This ensures that when the package is
    published, there is only one trained model that can be
    called, and we know exactly how it was built.
    """

    # Prepare versioned save file name
    save_file_name = f"{config.app_config.pipeline_save_file}{_version}.pkl"
    save_path = TRAINED_MODEL_DIR / save_file_name

    remove_old_pipelines(files_to_keep=[save_file_name])
    joblib.dump(pipeline_to_persist, save_path)

def load_pipeline(*, file_name: str) -> Pipeline:
    """Load a persisted pipeline."""

    file_path = TRAINED_MODEL_DIR / file_name
    return joblib.load(filename=file_path)

def remove_old_pipelines(*, files_to_keep: List[str]) -> None:
    """
    Remove old model pipelines.
    This is to ensure there is a simple one-to-one
    mapping between the package version and the model
    version to be imported and used by other applications.
    """
    do_not_delete = files_to_keep + ["__init__.py"]
    for model_file in TRAINED_MODEL_DIR.iterdir():
        if model_file.name not in do_not_delete:
            model_file.unlink()

def datetime_conversion_client(df: pd.DataFrame) -> pd.DataFrame:

    df["date_activ"] = pd.to_datetime(df["date_activ"], format="%Y-%m-%d")
    df["date_end"] = pd.to_datetime(df["date_end"], format="%Y-%m-%d")
    df["date_modif_prod"] = pd.to_datetime(df["date_modif_prod"], format="%Y-%m-%d")
    df["date_renewal"] = pd.to_datetime(df["date_renewal"], format="%Y-%m-%d")

    return df

def datetime_conversion_price(df: pd.DataFrame) -> pd.DataFrame:

    df["price_date"] = pd.to_datetime(df["price_date"], format="%Y-%m-%d")

    return df

def price_data_trans(df: pd.DataFrame) -> pd.DataFrame:

    # Group off-peak prices by companies and month
    monthly_price_by_id = (
        df.groupby(["id", "price_date"])
        .agg({"price_off_peak_var": "mean", "price_off_peak_fix": "mean"})
        .reset_index()
    )

    # Get january and december prices
    jan_prices = monthly_price_by_id.groupby("id").first().reset_index()
    dec_prices = monthly_price_by_id.groupby("id").last().reset_index()

    # Calculate the difference
    diff = pd.merge(
        dec_prices.rename(
            columns={"price_off_peak_var": "dec_1", "price_off_peak_fix": "dec_2"}
        ),
        jan_prices.drop(columns="price_date"),
        on="id",
    )
    diff["offpeak_diff_dec_january_energy"] = diff["dec_1"] - diff["price_off_peak_var"]
    diff["offpeak_diff_dec_january_power"] = diff["dec_2"] - diff["price_off_peak_fix"]
    diff = diff[
        ["id", "offpeak_diff_dec_january_energy", "offpeak_diff_dec_january_power"]
    ]

    return diff

def merging_datasets(df: pd.DataFrame, df_1: pd.DataFrame) -> pd.DataFrame:

    new_df = df.merge(df_1, how="left", left_on="id", right_on="id")

    return new_df

# A function to create the new feature "price_change"
def price_change(x: float) -> str:
    if x > 0:
        return "increase"
    elif x < 0:
        return "decrease"
    else:
        return "stable"

def time_features(df: pd.DataFrame) -> pd.DataFrame:
    # Getting the activation year
    df["activ_year"] = df["date_activ"].dt.year

    # Getting the activation month
    df["activ_month"] = df["date_activ"].dt.month

    # Getting the ending year
    df["end_year"] = df["date_end"].dt.year

    # Getting the ending month
    df["end_month"] = df["date_end"].dt.month

    # Getting the year of the last modification of the product
    df["modif_prod_year"] = df["date_modif_prod"].dt.year

    # Getting the month of the last modification of the product
    df["modif_prod_month"] = df["date_modif_prod"].dt.month

    # Getting the renewal year
    df["renewal_year"] = df["date_renewal"].dt.year

    # Getting the renewal month
    df["renewal_month"] = df["date_renewal"].dt.month

    # Difference between activation and renewal in days
    df["diff_act_renew"] = (df["date_renewal"] - df["date_activ"]).dt.days

    # Difference between activation and end in days
    df["diff_act_end"] = (df["date_end"] - df["date_activ"]).dt.days

    # Difference between activation and production modification.
    df["diff_act_modif"] = (df["date_modif_prod"] - df["date_activ"]).dt.days

    # Difference between end and production modification.
    df["diff_end_modif"] = (df["date_end"] - df["date_modif_prod"]).dt.days

    return df

def consum_features(df: pd.DataFrame) -> pd.DataFrame:

    # Getting the average consumption per month for the past 12 months.
    df["avrg_month_cons"] = df["cons_12m"] / 12

    # Getting the average consumption per month of gaz for the past 12 months.
    df["avrg_month_cons_gaz"] = df["cons_gas_12m"] / 12

    # Getting the average forcasted consumption per month for the next 12 months.
    df["forcast_avrg_month_cons"] = df["forecast_cons_12m"] / 12

    # Getting the ratio of the last month consumption to the last 12m consumption
    df["ratio_last_month_last12m_cons"] = df["cons_last_month"] / df["cons_12m"]

    # Getting the ratio of the last month consumption to the the average consumption per month for the past 12 months.
    df["ratio_last_month_avg_cons"] = df["cons_last_month"] / df["avrg_month_cons"]

    return df


from flask import Flask, request, jsonify
from helpers import *
import pigpio
app = Flask(__name__)

#rgb 22, 27, 17
#base teal 40 97 15

pi = pigpio.pi()

@app.route("/")
def home():
    return "Hello World!"

@app.route("/color")
def set_color():
    args = request.args.to_dict()
    r = to_PWM_dutycycle(args['r'])
    g = to_PWM_dutycycle(args['g'])
    b = to_PWM_dutycycle(args['b'])
    pi.set_PWM_dutycycle(GPIO_RED, r)
    pi.set_PWM_dutycycle(GPIO_GREEN, g)
    pi.set_PWM_dutycycle(GPIO_BLUE, b)
    return str(r) + ' ' + str(g) + ' ' + str(b)

@app.route("/get_color")
def get_color():
    color = get_current_color(pi)
    return jsonify(color)

if __name__ == "__main__":
    #app.run(debug=True)
    app.run(host='0.0.0.0', port=80, debug=True)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import asyncio
from pathlib import Path
import sys
import unittest

from syncer import sync

from pyppeteer.errors import NetworkError, PageError

from .base import BaseTestCase

class TestNetworkEvent(BaseTestCase):
    @sync
    async def test_request(self):
        requests = []
        self.page.on('request', lambda req: requests.append(req))
        await self.page.goto(self.url + 'empty')
        self.assertEqual(len(requests), 1)
        req = requests[0]
        self.assertEqual(req.url, self.url + 'empty')
        self.assertEqual(req.resourceType, 'document')
        self.assertEqual(req.method, 'GET')
        self.assertTrue(req.response)
        self.assertEqual(req.frame, self.page.mainFrame)
        self.assertEqual(req.frame.url, self.url + 'empty')

    @sync
    async def test_request_post(self):
        await self.page.goto(self.url + 'empty')

        from tornado.web import RequestHandler

        class PostHandler(RequestHandler):
            def post(self):
                self.write('')

        self.app.add_handlers('localhost', [('/post', PostHandler)])
        requests = []
        self.page.on('request', lambda req: requests.append(req))
        await self.page.evaluate('fetch("/post", {method: "POST", body: JSON.stringify({foo: "bar"})})')  # noqa: E501
        self.assertEqual(len(requests), 1)
        req = requests[0]
        self.assertTrue(req)
        self.assertEqual(req.postData, '{"foo":"bar"}')

    @sync
    async def test_response(self):
        responses = []
        self.page.on('response', lambda res: responses.append(res))
        await self.page.goto(self.url + 'empty')
        self.assertEqual(len(responses), 1)
        response = responses[0]
        self.assertEqual(response.url, self.url + 'empty')
        self.assertEqual(response.status, 200)
        # self.assertTrue(response.ok)
        self.assertFalse(response.fromCache)
        self.assertFalse(response.fromServiceWorker)
        self.assertTrue(response.request)
        self.assertEqual(response.securityDetails, {})

    @sync
    async def test_response_https(self):
        responses = []
        self.page.on('response', lambda res: responses.append(res))
        await self.page.goto('https://example.com/')
        self.assertEqual(len(responses), 1)
        response = responses[0]
        self.assertEqual(response.url, 'https://example.com/')
        self.assertEqual(response.status, 200)
        self.assertTrue(response.ok)
        self.assertFalse(response.fromCache)
        self.assertFalse(response.fromServiceWorker)
        self.assertTrue(response.request)
        self.assertTrue(response.securityDetails)
        self.assertEqual(response.securityDetails.protocol, 'TLS 1.2')

    @sync
    async def test_from_cache(self):
        responses = {}

        def set_response(resp):
            basename = resp.url.split('/').pop()
            responses[basename] = resp

        self.page.on('response', set_response)

        await self.page.goto(self.url + 'static/cached/one-style.html')
        await self.page.reload()

        self.assertEqual(len(responses), 2)
        self.assertEqual(responses['one-style.html'].status, 304)
        self.assertFalse(responses['one-style.html'].fromCache)
        self.assertEqual(responses['one-style.css'].status, 200)
        self.assertTrue(responses['one-style.css'].fromCache)

    @sync
    async def test_response_from_service_worker(self):
        responses = {}

        def set_response(resp):
            basename = resp.url.split('/').pop()
            responses[basename] = resp

        self.page.on('response', set_response)

        await self.page.goto(
            self.url + 'static/serviceworkers/fetch/sw.html',
            waitUntil='networkidle2',
        )
        await self.page.evaluate('async() => await window.activationPromise')
        await self.page.reload()

        self.assertEqual(len(responses), 2)
        self.assertEqual(responses['sw.html'].status, 200)
        self.assertTrue(responses['sw.html'].fromServiceWorker)
        self.assertEqual(responses['style.css'].status, 200)
        self.assertTrue(responses['style.css'].fromServiceWorker)

    @unittest.skipIf(sys.platform.startswith('msys'), 'Fails on MSYS')
    @sync
    async def test_response_body(self):
        responses = []
        self.page.on('response', lambda res: responses.append(res))
        await self.page.goto(self.url + 'static/simple.json')
        self.assertEqual(len(responses), 1)
        res = responses[0]
        self.assertTrue(res)
        self.assertEqual(await res.text(), '{"foo": "bar"}\n')
        self.assertEqual(await res.json(), {'foo': 'bar'})

    @sync
    async def test_fail_get_redirected_body(self):
        response = await self.page.goto(self.url + 'redirect1')
        redirectChain = response.request.redirectChain
        self.assertEqual(len(redirectChain), 1)
        redirected = redirectChain[0].response
        self.assertEqual(redirected.status, 302)
        with self.assertRaises(NetworkError) as cm:
            await redirected.text()
        self.assertIn(
            'Response body is unavailable for redirect response',
            cm.exception.args[0],
        )

    @unittest.skip('This test hangs')
    @sync
    async def test_not_report_body_unless_finished(self):
        await self.page.goto(self.url + 'empty')
        serverResponses = []

        from tornado.web import RequestHandler

        class GetHandler(RequestHandler):
            def get(self):
                serverResponses.append(self)
                self.write('hello ')

        self.app.add_handlers('localhost', [('/get', GetHandler)])
        pageResponse = asyncio.get_event_loop().create_future()
        finishedRequests = []
        self.page.on('response', lambda res: pageResponse.set_result(res))
        self.page.on('requestfinished', lambda: finishedRequests.append(True))

        asyncio.ensure_future(
            self.page.evaluate('fetch("./get", {method: "GET"})'))
        response = await pageResponse
        self.assertTrue(serverResponses)
        self.assertTrue(response)
        self.assertEqual(response.status, 200)
        self.assertFalse(finishedRequests)

        responseText = response.text()
        serverResponses[0].write('wor')
        serverResponses[0].finish('ld!')
        self.assertEqual(await responseText, 'hello world!')

    @sync
    async def test_request_failed(self):
        await self.page.setRequestInterception(True)

        async def interception(req):
            if req.url.endswith('css'):
                await req.abort()
            else:
                await req.continue_()

        self.page.on(
            'request', lambda req: asyncio.ensure_future(interception(req)))

        failedRequests = []
        self.page.on('requestfailed', lambda req: failedRequests.append(req))
        await self.page.goto(self.url + 'static/one-style.html')
        self.assertEqual(len(failedRequests), 1)
        self.assertIn('one-style.css', failedRequests[0].url)
        self.assertIsNone(failedRequests[0].response)
        self.assertEqual(failedRequests[0].resourceType, 'stylesheet')
        self.assertEqual(
            failedRequests[0].failure()['errorText'], 'net::ERR_FAILED')
        self.assertTrue(failedRequests[0].frame)

    @sync
    async def test_request_finished(self):
        requests = []
        self.page.on('requestfinished', lambda req: requests.append(req))
        await self.page.goto(self.url + 'empty')

        self.assertEqual(len(requests), 1)
        req = requests[0]
        self.assertEqual(req.url, self.url + 'empty')
        self.assertTrue(req.response)
        self.assertEqual(req.frame, self.page.mainFrame)
        self.assertEqual(req.frame.url, self.url + 'empty')

    @sync
    async def test_events_order(self):
        events = []
        self.page.on('request', lambda req: events.append('request'))
        self.page.on('response', lambda res: events.append('response'))
        self.page.on(
            'requestfinished', lambda req: events.append('requestfinished'))
        await self.page.goto(self.url + 'empty')
        self.assertEqual(events, ['request', 'response', 'requestfinished'])

    @sync
    async def test_redirects(self):
        events = []
        self.page.on('request', lambda req: events.append(
            '{} {}'.format(req.method, req.url)))
        self.page.on('response', lambda res: events.append(
            '{} {}'.format(res.status, res.url)))
        self.page.on('requestfinished', lambda req: events.append(
            'DONE {}'.format(req.url)))
        self.page.on('requestfailed', lambda req: events.append(
            'FAIL {}'.format(req.url)))
        response = await self.page.goto(self.url + 'redirect1')
        self.assertEqual(events, [
            'GET {}'.format(self.url + 'redirect1'),
            '302 {}'.format(self.url + 'redirect1'),
            'DONE {}'.format(self.url + 'redirect1'),
            'GET {}'.format(self.url + 'redirect2'),
            '200 {}'.format(self.url + 'redirect2'),
            'DONE {}'.format(self.url + 'redirect2'),
        ])

        # check redirect chain
        redirectChain = response.request.redirectChain
        self.assertEqual(len(redirectChain), 1)
        self.assertIn('redirect1', redirectChain[0].url)

class TestRequestInterception(BaseTestCase):
    @sync
    async def test_request_interception(self):
        await self.page.setRequestInterception(True)

        async def request_check(req):
            self.assertIn('empty', req.url)
            self.assertTrue(req.headers.get('user-agent'))
            self.assertEqual(req.method, 'GET')
            self.assertIsNone(req.postData)
            self.assertTrue(req.isNavigationRequest())
            self.assertEqual(req.resourceType, 'document')
            self.assertEqual(req.frame, self.page.mainFrame)
            self.assertEqual(req.frame.url, 'about:blank')
            await req.continue_()

        self.page.on('request',
                     lambda req: asyncio.ensure_future(request_check(req)))
        res = await self.page.goto(self.url + 'empty')
        self.assertEqual(res.status, 200)

    @sync
    async def test_referer_header(self):
        await self.page.setRequestInterception(True)
        requests = list()

        async def set_request(req):
            requests.append(req)
            await req.continue_()

        self.page.on('request',
                     lambda req: asyncio.ensure_future(set_request(req)))
        await self.page.goto(self.url + 'static/one-style.html')
        self.assertIn('/one-style.css', requests[1].url)
        self.assertIn('/one-style.html', requests[1].headers['referer'])

    @sync
    async def test_response_with_cookie(self):
        await self.page.goto(self.url + 'empty')
        await self.page.setCookie({'name': 'foo', 'value': 'bar'})

        await self.page.setRequestInterception(True)

        async def continue_(req):
            await req.continue_()

        self.page.on('request', lambda r: asyncio.ensure_future(continue_(r)))
        response = await self.page.reload()
        self.assertEqual(response.status, 200)

    @sync
    async def test_request_interception_stop(self):
        await self.page.setRequestInterception(True)
        self.page.once('request',
                       lambda req: asyncio.ensure_future(req.continue_()))
        await self.page.goto(self.url + 'empty')
        await self.page.setRequestInterception(False)
        await self.page.goto(self.url + 'empty')

    @sync
    async def test_request_interception_custom_header(self):
        await self.page.setExtraHTTPHeaders({'foo': 'bar'})
        await self.page.setRequestInterception(True)

        async def request_check(req):
            self.assertEqual(req.headers['foo'], 'bar')
            await req.continue_()

        self.page.on('request',
                     lambda req: asyncio.ensure_future(request_check(req)))
        res = await self.page.goto(self.url + 'empty')
        self.assertEqual(res.status, 200)

    @sync
    async def test_request_interception_custom_referer_header(self):
        await self.page.goto(self.url + 'empty')
        await self.page.setExtraHTTPHeaders({'referer': self.url + 'empty'})
        await self.page.setRequestInterception(True)

        async def request_check(req):
            self.assertEqual(req.headers['referer'], self.url + 'empty')
            await req.continue_()

        self.page.on('request',
                     lambda req: asyncio.ensure_future(request_check(req)))
        res = await self.page.goto(self.url + 'empty')
        self.assertEqual(res.status, 200)

    @sync
    async def test_request_interception_abort(self):
        await self.page.setRequestInterception(True)

        async def request_check(req):
            if req.url.endswith('.css'):
                await req.abort()
            else:
                await req.continue_()

        failedRequests = []
        self.page.on('request',
                     lambda req: asyncio.ensure_future(request_check(req)))
        self.page.on('requestfailed', lambda e: failedRequests.append(e))
        res = await self.page.goto(self.url + 'static/one-style.html')
        self.assertTrue(res.ok)
        self.assertIsNone(res.request.failure())
        self.assertEqual(len(failedRequests), 1)

    @sync
    async def test_request_interception_custom_error_code(self):
        await self.page.setRequestInterception(True)

        async def request_check(req):
            await req.abort('internetdisconnected')

        self.page.on('request',
                     lambda req: asyncio.ensure_future(request_check(req)))
        failedRequests = []
        self.page.on('requestfailed', lambda req: failedRequests.append(req))
        with self.assertRaises(PageError):
            await self.page.goto(self.url + 'empty')
        self.assertEqual(len(failedRequests), 1)
        failedRequest = failedRequests[0]
        self.assertEqual(
            failedRequest.failure()['errorText'],
            'net::ERR_INTERNET_DISCONNECTED',
        )

    @unittest.skip('Need server-side implementation')
    @sync
    async def test_request_interception_amend_http_header(self):
        pass

    @sync
    async def test_request_interception_abort_main(self):
        await self.page.setRequestInterception(True)

        async def request_check(req):
            await req.abort()

        self.page.on('request',
                     lambda req: asyncio.ensure_future(request_check(req)))
        with self.assertRaises(PageError) as cm:
            await self.page.goto(self.url + 'empty')
        self.assertIn('net::ERR_FAILED', cm.exception.args[0])

    @sync
    async def test_request_interception_redirects(self):
        await self.page.setRequestInterception(True)
        requests = []

        async def check(req):
            await req.continue_()
            requests.append(req)

        self.page.on('request', lambda req: asyncio.ensure_future(check(req)))
        response = await self.page.goto(self.url + 'redirect1')
        self.assertEqual(response.status, 200)

    @sync
    async def test_redirect_for_subresource(self):
        await self.page.setRequestInterception(True)
        requests = list()

        async def check(req):
            await req.continue_()
            requests.append(req)

        self.page.on('request', lambda req: asyncio.ensure_future(check(req)))
        response = await self.page.goto(self.url + 'one-style.html')
        self.assertEqual(response.status, 200)
        self.assertIn('one-style.html', response.url)
        self.assertEqual(len(requests), 5)
        self.assertEqual(requests[0].resourceType, 'document')
        self.assertEqual(requests[1].resourceType, 'stylesheet')

        # check redirect chain
        redirectChain = requests[1].redirectChain
        self.assertEqual(len(redirectChain), 3)
        self.assertIn('/one-style.css', redirectChain[0].url)
        self.assertIn('/three-style.css', redirectChain[2].url)

    @unittest.skip('This test is not implemented')
    @sync
    async def test_request_interception_abort_redirects(self):
        pass

    @unittest.skip('This test is not implemented')
    @sync
    async def test_request_interception_equal_requests(self):
        pass

    @sync
    async def test_request_interception_data_url(self):
        await self.page.setRequestInterception(True)
        requests = []

        async def check(req):
            requests.append(req)
            await req.continue_()

        self.page.on('request', lambda req: asyncio.ensure_future(check(req)))
        dataURL = 'data:text/html,
yo
'
        response = await self.page.goto(dataURL)
        self.assertEqual(response.status, 200)
        self.assertEqual(len(requests), 1)
        self.assertEqual(requests[0].url, dataURL)

    @sync
    async def test_request_interception_abort_data_url(self):
        await self.page.setRequestInterception(True)

        async def request_check(req):
            await req.abort()

        self.page.on('request',
                     lambda req: asyncio.ensure_future(request_check(req)))
        with self.assertRaises(PageError) as cm:
            await self.page.goto('data:text/html,No way!')
        self.assertIn('net::ERR_FAILED', cm.exception.args[0])

    @sync
    async def test_request_interception_with_hash(self):
        await self.page.setRequestInterception(True)
        requests = []

        async def check(req):
            requests.append(req)
            await req.continue_()

        self.page.on('request', lambda req: asyncio.ensure_future(check(req)))
        response = await self.page.goto(self.url + 'empty#hash')
        self.assertEqual(response.status, 200)
        self.assertEqual(response.url, self.url + 'empty')
        self.assertEqual(len(requests), 1)
        self.assertEqual(requests[0].url, self.url + 'empty')

    @sync
    async def test_request_interception_encoded_server(self):
        await self.page.setRequestInterception(True)

        async def check(req):
            await req.continue_()

        self.page.on('request', lambda req: asyncio.ensure_future(check(req)))
        response = await self.page.goto(self.url + 'non existing page')
        self.assertEqual(response.status, 404)

    @unittest.skip('Need server-side implementation')
    @sync
    async def test_request_interception_badly_encoded_server(self):
        pass

    @unittest.skip('Need server-side implementation')
    @sync
    async def test_request_interception_encoded_server_2(self):
        pass

    @unittest.skip('This test is not implemented')
    @sync
    async def test_request_interception_invalid_interception_id(self):
        pass

    @sync
    async def test_request_interception_disabled(self):
        error = None

        async def check(req):
            try:
                await req.continue_()
            except Exception as e:
                nonlocal error
                error = e

        self.page.on('request', lambda req: asyncio.ensure_future(check(req)))
        await self.page.goto(self.url + 'empty')
        self.assertIsNotNone(error)
        self.assertIn('Request interception is not enabled', error.args[0])

    @sync
    async def test_request_interception_with_file_url(self):
        await self.page.setRequestInterception(True)
        urls = []

        async def set_urls(req):
            urls.append(req.url.split('/').pop())
            await req.continue_()

        self.page.on(
            'request', lambda req: asyncio.ensure_future(set_urls(req)))

        def pathToFileURL(path: Path):
            pathName = str(path).replace('\\', '/')
            if not pathName.startswith('/'):
                pathName = '/{}'.format(pathName)
            return 'file://{}'.format(pathName)

        target = Path(__file__).parent / 'static' / 'one-style.html'
        await self.page.goto(pathToFileURL(target))
        self.assertEqual(len(urls), 2)
        self.assertIn('one-style.html', urls)
        self.assertIn('one-style.css', urls)

    @sync
    async def test_request_respond(self):
        await self.page.setRequestInterception(True)

        async def interception(req):
            await req.respond({
                'status': 201,
                'headers': {'foo': 'bar'},
                'body': 'intercepted',
            })

        self.page.on(
            'request', lambda req: asyncio.ensure_future(interception(req)))
        response = await self.page.goto(self.url + 'empty')
        self.assertEqual(response.status, 201)
        self.assertEqual(response.headers['foo'], 'bar')
        body = await self.page.evaluate('() => document.body.textContent')
        self.assertEqual(body, 'intercepted')

    @unittest.skip('Sending binary object is not implemented')
    @sync
    async def test_request_respond_bytes(self):
        pass

class TestNavigationRequest(BaseTestCase):
    @sync
    async def test_navigation_request(self):
        requests = dict()

        def set_request(req):
            requests[req.url.split('/').pop()] = req

        self.page.on('request', set_request)
        await self.page.goto(self.url + 'redirect3')
        self.assertTrue(requests['redirect3'].isNavigationRequest())
        self.assertTrue(requests['one-frame.html'].isNavigationRequest())
        self.assertTrue(requests['frame.html'].isNavigationRequest())
        self.assertFalse(requests['script.js'].isNavigationRequest())
        self.assertFalse(requests['style.css'].isNavigationRequest())

    @sync
    async def test_interception(self):
        requests = dict()

        async def on_request(req):
            requests[req.url.split('/').pop()] = req
            await req.continue_()

        self.page.on('request',
                     lambda req: asyncio.ensure_future(on_request(req)))

        await self.page.setRequestInterception(True)
        await self.page.goto(self.url + 'redirect3')
        self.assertTrue(requests['redirect3'].isNavigationRequest())
        self.assertTrue(requests['one-frame.html'].isNavigationRequest())
        self.assertTrue(requests['frame.html'].isNavigationRequest())
        self.assertFalse(requests['script.js'].isNavigationRequest())
        self.assertFalse(requests['style.css'].isNavigationRequest())

    @sync
    async def test_image(self):
        requests = []
        self.page.on('request', lambda req: requests.append(req))
        await self.page.goto(self.url + 'static/huge-image.png')
        self.assertEqual(len(requests), 1)
        self.assertTrue(requests[0].isNavigationRequest())


import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np

def calculate_birds_eye_view_parameters(x_bounds, y_bounds, z_bounds):
    """
    Parameters
    ----------
        x_bounds: Forward direction in the ego-car.
        y_bounds: Sides
        z_bounds: Height

    Returns
    -------
        bev_resolution: Bird's-eye view bev_resolution
        bev_start_position Bird's-eye view first element
        bev_dimension Bird's-eye view tensor spatial dimension
    """
    bev_resolution = torch.tensor(
        [row[2] for row in [x_bounds, y_bounds, z_bounds]])
    bev_start_position = torch.tensor(
        [row[0] + row[2] / 2.0 for row in [x_bounds, y_bounds, z_bounds]])
    bev_dimension = torch.tensor([(row[1] - row[0]) / row[2]
                                 for row in [x_bounds, y_bounds, z_bounds]], dtype=torch.long)

    return bev_resolution, bev_start_position, bev_dimension

def gen_dx_bx(xbound, ybound, zbound):
    dx = torch.Tensor([row[2] for row in [xbound, ybound, zbound]])
    bx = torch.Tensor([row[0] + row[2]/2.0 for row in [xbound, ybound, zbound]])
    nx = torch.LongTensor([(row[1] - row[0]) / row[2] for row in [xbound, ybound, zbound]])

    return dx, bx, nx

# Instance utils
def update_instance_ids(instance_seg, old_ids, new_ids):
    """
    Parameters
    ----------
        instance_seg: torch.Tensor arbitrary shape
        old_ids: 1D tensor containing the list of old ids, must be all present in instance_seg.
        new_ids: 1D tensor with the new ids, aligned with old_ids

    Returns
        new_instance_seg: torch.Tensor same shape as instance_seg with new ids
    """
    indices = torch.arange(old_ids.max() + 1, device=instance_seg.device)
    for old_id, new_id in zip(old_ids, new_ids):
        indices[old_id] = new_id

    return indices[instance_seg].long()

def make_instance_seg_consecutive(instance_seg):
    # Make the indices of instance_seg consecutive
    unique_ids = torch.unique(instance_seg)  # include background
    new_ids = torch.arange(len(unique_ids), device=instance_seg.device)
    instance_seg = update_instance_ids(instance_seg, unique_ids, new_ids)
    return instance_seg

def predict_instance_segmentation_and_trajectories(
                                    foreground_masks,
                                    ins_sigmoid,
                                    vehicles_id=1,
                                    ):
    if foreground_masks.dim() == 5 and foreground_masks.shape[2] == 1:
        foreground_masks = foreground_masks.squeeze(2)  # [b, t, h, w]
    foreground_masks = foreground_masks == vehicles_id  # [b, t, h, w]  Only these places have foreground id

    argmax_ins = ins_sigmoid.argmax(dim=1)  # long, [b, t, h, w], ins_id starts from 0
    argmax_ins = argmax_ins + 1 # [b, t, h, w], ins_id starts from 1
    instance_seg = (argmax_ins * foreground_masks.float()).long()  # bg is 0, fg starts with 1

    # Make the indices of instance_seg consecutive
    instance_seg = make_instance_seg_consecutive(instance_seg).long()

    return instance_seg


"""
Description: Train emotion classification model
"""

from keras.callbacks import CSVLogger, ModelCheckpoint, EarlyStopping
from keras.callbacks import ReduceLROnPlateau
from keras.preprocessing.image import ImageDataGenerator
from load_and_process import load_fer2013
from load_and_process import preprocess_input
from models.cnn import mini_XCEPTION
from sklearn.model_selection import train_test_split

# parameters
batch_size = 32
num_epochs = 10000
input_shape = (48, 48, 1)
validation_split = .2
verbose = 1
num_classes = 7
patience = 50
base_path = 'models/'

# data generator
data_generator = ImageDataGenerator(
                        featurewise_center=False,
                        featurewise_std_normalization=False,
                        rotation_range=10,
                        width_shift_range=0.1,
                        height_shift_range=0.1,
                        zoom_range=.1,
                        horizontal_flip=True)

# model parameters/compilation
model = mini_XCEPTION(input_shape, num_classes)
model.compile(optimizer='adam', loss='categorical_crossentropy',
              metrics=['accuracy'])
model.summary()

    # callbacks
log_file_path = base_path + '_emotion_training.log'
csv_logger = CSVLogger(log_file_path, append=False)
early_stop = EarlyStopping('val_loss', patience=patience)
reduce_lr = ReduceLROnPlateau('val_loss', factor=0.1,
                                  patience=int(patience/4), verbose=1)
trained_models_path = base_path + '_mini_XCEPTION'
model_names = trained_models_path + '.{epoch:02d}-{val_acc:.2f}.hdf5'
model_checkpoint = ModelCheckpoint(model_names, 'val_loss', verbose=1,
                                                    save_best_only=True)
callbacks = [model_checkpoint, csv_logger, early_stop, reduce_lr]

# loading dataset
faces, emotions = load_fer2013()
faces = preprocess_input(faces)
num_samples, num_classes = emotions.shape
xtrain, xtest,ytrain,ytest = train_test_split(faces, emotions,test_size=0.2,shuffle=True)
model.fit_generator(data_generator.flow(xtrain, ytrain,
                                            batch_size),
                        steps_per_epoch=len(xtrain) / batch_size,
                        epochs=num_epochs, verbose=1, callbacks=callbacks,
                        validation_data=(xtest,ytest))


import numpy as np
import scipy as sc
import scipy.sparse as scs

class DiscreteCircularRadonTransform:
    """
    %SPHERICALTOMO  Creates a 2D spherical Radon tomography test problem
    %
    % This function genetates a tomography test problem based on the spherical
    % Radon tranform where data consists of integrals along circles.  This type
    % of problem arises, e.g., in photoacoustic imaging.
    %
    % The image domain is a square centered at the origin.  The centers for the
    % integration circles are placed on a circle just outside the image domain.
    % For each circle center we integrate along a number of concentric circles
    % with equidistant radii, using the periodic trapezoidal rule.
    %
    % Input:
    %   N           Scalar denoting the number of pixels in each dimesion, such
    %               that the image domain consists of N^2 cells.
    %   angles      Vector containing the angles to the circle centers in
    %               degrees. Default: angles = 0:2:358.
    %   numCircles  Number of concentric integration circles for each center.
    %               Default: numCircles = round(sqrt(2)*N).
    %   asMatrix    If True, a sparse matrix is returned in A (default).
    %               If False, instead a function handle is returned.
    %
    % Output:
    %   A           If input isMatrix is True (default): coefficient matrix with
    %               N^2 columns and length(angles)*p rows.
    %               If input isMatrix is False: A function handle representing a
    %               matrix-free version of A in which the forward and backward
    %               operations are computed as A.fwd(x) and A.bwd(y),
    %               respectively, for column vectors x and y of appropriate size.
    %               The matrix is never formed explicitly, thus saving memory.
    %
    %
    % Based on Matlab code written: Per Christian Hansen, Jakob Sauer Jorgensen, and
    % Maria Saxild-Hansen, 2010-2017 & Juergen Frikel, OTH Regensburg.
    %
    % This file is part of the AIR Tools II package and is distributed under
    % the 3-Clause BSD License. A separate license file should be provided as
    % part of the package.
    %
    % Copyright 2017 Per Christian Hansen, Technical University of Denmark and
    % Jakob Sauer Jorgensen, University of Manchester.
    """
    def __init__(self, N,angles=None,numCircles=None,asMatrix=True):
        self.N = N
        # Default value of the angles to the circle centers.
        if angles is None:
            self.angles = np.arange(0, 359, 2)*np.pi/180.
        else:
            self.angles = angles

        if numCircles is None:
            self.numCircles = int( np.round(np.sqrt(2.)*N) )
        else:
            self.numCircles = int( numCircles )

        if asMatrix:
            self.A = self._get_or_apply_system_matrix(self.N, self.angles, self.numCircles)
        else:
            self.A = None

        self.shape = [self.numCircles*self.angles.shape[0], self.N*self.N]

    def fwd(self, x):
        if self.A is not None:
            return self.A*(x)
        else:
            return self._get_or_apply_system_matrix(self.N,self.angles,self.numCircles, x, False)

    def bwd(self, y):
        if self.A is not None:
            return self.A.T*y
        else:
            return self._get_or_apply_system_matrix(self.N,self.angles,self.numCircles, y, True)

    def _get_or_apply_system_matrix(self, N, angles, numCircles, u=None, adjoint=None):
        # Define the number of angles.
        nA = angles.shape[0]

        # Radii for the circles.
        radii  = np.linspace(0,2,numCircles+1)
        radii  = radii[1:]

        # Image coordinates.
        centerImg = np.ceil(N/2)

        # Determine the quarature parameters.
        dx = np.sqrt(2.)/N
        nPhi = np.ceil((4.*np.pi/dx)*radii)
        dPhi = 2*np.pi/nPhi

        II = np.arange(nA)
        JJ = np.arange(numCircles)

        if u is None:
            isMatrix = True
        else:
            isMatrix = False

        if isMatrix:
            # Initialize vectors that contains the row numbers, the column numbers
            # and the values for creating the matrix A effiecently.
            nnz = int(2*N*nA*numCircles)
            rows = np.zeros( nnz, dtype=np.int32)
            cols = np.zeros( nnz, dtype=np.int32)
            vals = np.zeros( nnz, dtype=np.float64)

            idxend = 0

        else:
            if adjoint == False:
                assert u.shape[0] == self.N*self.N
                A = np.zeros(numCircles*nA, dtype=np.float64)
            else:
                assert u.shape[0] == numCircles*nA
                A = np.zeros(self.N*self.N, dtype=np.float64)

        # Loop over angles.
        for m in II:
            # Angular position of source.
            xix = np.cos(angles[m])
            xiy = np.sin(angles[m])

            # Loop over the circles.
            for n in JJ:
                # (x,y) coordinates of circle.
                k = np.arange(nPhi[n])*dPhi[n]
                xx = (xix + radii[n]*np.cos(k))/dx + centerImg
                yy = (xiy + radii[n]*np.sin(k))/dx + centerImg

                # Round to get pixel index.
                col = np.round( xx )-1
                row = np.round( yy )-1

                # Discard if outside box domain.
                IInew = np.logical_and(col>=0, col<N) & np.logical_and(row>=0, row<N)
                row = row[IInew]
                col = col[IInew]
                J = (N-row-1) + col*N

                # Convert to linear index and bin
                Ju, w = np.unique(J, return_counts=True)

                # Determine rows, columns and weights.
                i = m*numCircles + n
                ii = np.array([i]*Ju.shape[0])
                jj = Ju
                aa = (2.*np.pi*radii[n]/nPhi[n])*w

                # Store the values, if any.
                if jj.shape[0] > 0:
                    if isMatrix:
                        # Create the indices to store the values to vector for
                        # later creation of A matrix.
                        idxstart = idxend
                        idxend = idxstart + jj.shape[0]
                        idx = np.arange(idxstart, idxend)

                        # Store row numbers, column numbers and values.
                        rows[idx] = ii
                        cols[idx] = jj
                        vals[idx] = aa
                    else:
                        # If any nonzero elements, apply forward or back operator
                        if adjoint==False:
                            A[i] = np.dot(aa, u[jj])
                        else:
                            A[jj] += u[i]*aa

        if isMatrix:
            # Truncate excess zeros.
            rows = rows[:idxend]
            cols = cols[:idxend]
            vals = vals[:idxend]

            # Create sparse matrix A from the stored values.
            A = scs.csr_matrix((vals, (rows,cols)), (self.numCircles*nA,self.N*self.N) )/np.sqrt(2)
        return A


# Выполнить эти комманды в консоли питона перед запуском, загрузка модулей nltk
# import nltk
# nltk.download('punkt')
# nltk.download('stopwords')

import csv
import string
import pymorphy2
from nltk.corpus import stopwords
from nltk.tokenize import sent_tokenize, word_tokenize
from nltk.stem import SnowballStemmer
import pandas as pd

# Предварительные настройки
stop_words = stopwords.words("russian")  # nltk загрузка стоп-слов
snowball = SnowballStemmer(language="russian")  # nltk стемматизатор
morph = pymorphy2.MorphAnalyzer(lang='ru')  # Морфический анализатор

#  text = 'В МЭР рассказали о проблемах в реализации совместных проектов с Кубой'

def tokenizer(text):
    result = []
    for element in sent_tokenize(text, language="russian"):  # Токенизация предложения
        tokens = word_tokenize(element, language="russian")  # Токенизация слов
        result_tokens = []
        tokens = [''.join(c for c in s if c not in string.punctuation) for s in tokens]  # удаляем знаки препинания
        tokens = [s for s in tokens if s]  # удаляем пустые элементы
        for token in tokens:
            if token not in stop_words and token != "—" and token != "–":  # удаление стоп-слов и дефисов
                token = morph.parse(token)[0].normal_form  # Нормализация
                token = snowball.stem(token)  # Стемматизация
                result_tokens.append(token)
        result.append(result_tokens)
    #  if len(result) > 1:  # проверка на мультитокеновость
    print(result)
    return result

"""
# Тестовые данные
with open('input.csv', newline='', encoding="UTF-8") as inputfile:
    spamreader = csv.reader(inputfile, delimiter='|')
    for row in spamreader:
        if row:
            linkrow = row[0]
            textrow = row[1]
            daterow = row[2]
            tokenizer(textrow)
            print(linkrow, textrow, daterow)
"""

df = pd.read_csv("input2.csv", error_bad_lines=False, names=["link", "text", "date"])
products_list = df.values.tolist()
for row in products_list:
    if row:
        linkrow = row[0]
        textrow = row[1]
        daterow = row[2]
        tokenized = tokenizer(textrow)
        print(linkrow, textrow, daterow)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from io import StringIO, BytesIO

__author__ = 'Mr.Huo'

def main():
    # 内存中读写字符串
    strio = StringIO()
    strio.write('hello world\nhello world\nhello world\n')
    print(strio.getvalue())

    strio1 = StringIO('hello world\nhello world\nhello world\n')
    while True:
        s = strio1.readline()
        if s == '':
            break
        print(s, end='')
    #内存中读写二进制
    byteio = BytesIO()
    byteio.write('中华'.encode("utf-8"))
    print(byteio.getvalue())

    byteio = BytesIO('中华'.encode("utf-8"))
    print(byteio.read())

if __name__ == '__main__':
    main()


from flask import Flask, request
import ktrain
from ktrain import text
import numpy as np

app = Flask(__name__)

@app.route('/json-example', methods=['GET','POST']) #GET requests will be blocked

def json_example():
    req_data = request.get_json()

#define fields
    topics = req_data["topics"] #topics to be discussed, the last one should be other
    texts = req_data["text"] #all the text in the order of speech
    people = req_data["people"] #people who talked the texts in the order of speech

	##Find off-topic text
	#create offtopic binary array for each text
    off_topic= []
        #create zeroshotclassifier
    zsl = text.ZeroShotClassifier()
        #go through texts and find the topics
    for i in texts:
        doc = i
        scores= zsl.predict(doc, labels=topics, include_labels=False)
        if scores[-1] > 0.5 or np.array(scores)[0:-1].max() < 0.5:
            off_topic.append(1)
        else:
            off_topic.append(0)

        #percentage of text that is off topic
    off= np.sum(np.array(off_topic))/ len(off_topic)

##Create summaries for the whole text
    #create summarizer
    ts = text.TransformerSummarizer()
    #create concatenated text to be summarized
    text_for_summarize= " ".join(texts)
    #get summary
    summary= ts.summarize(text_for_summarize)

    return off_topic, summary


from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.core.paginator import Paginator, PageNotAnInteger, EmptyPage
from django.http import Http404
from django.http import HttpResponse
# from django.http import JsonResponse
from django.shortcuts import render, redirect, get_object_or_404
from .forms import ItemForm
from django.utils import timezone
from .models import Item

# from units.forms import UnitForm

# Create your views here.

@login_required
def item_create(request):
    initial_data = {
        'created_date': timezone.now().date(),
    }
    # unit_form = UnitForm(request.POST or None, initial=initial_data)
    item_form = ItemForm(request.POST or None, initial=initial_data)
    if request.method == 'POST':
        if item_form.is_valid():
            item = item_form.save(commit=False)
            item.user = request.user
            # if unit_form.is_valid():
            #     unit = unit_form.save(commit=False)
            #     unit.user = request.user
            #     unit.save()
            #     item.unit_id = unit.pk
            item.save()
            messages.success(request, "Expense added.")
            return redirect("item_create")

    context = {
        'form': item_form,
        # 'unit_form': unit_form
    }
    return render(request, 'items/item_edit.html', context)

@login_required
def item_edit(request, pk):
    item_data = get_object_or_404(Item, id=pk)
    form = ItemForm(data=request.POST or None, instance=item_data)
    if request.method == 'POST':
        if form.is_valid():
            item = form.save(commit=False)
            item.user = request.user
            item.save()
            messages.success(request, "Item %s edited" % (item.name))
            return redirect('item_list')
    context = {
        'form': form
    }
    return render(request, 'items/item_edit.html', context)

@login_required
def item_list(request):
    item_list_data = Item.objects.all().order_by("name")
    per_page = 10
    paginator = Paginator(item_list_data, per_page)
    page = request.GET.get('page')

    try:
        items = paginator.page(page)
    except PageNotAnInteger:
        items = paginator.page(1)
    except EmptyPage:
        items = paginator.page(paginator.num_pages)

    return render(request, 'items/item_list.html', {'items': items})

@login_required
def item_delete(request, pk):
    try:
        item = Item.objects.get(user=request.user, id=pk)  # item is a database
    except Item.DoesNotExist:
        raise Http404()

    if request.method == 'POST':
        item.delete()
        # messages.success(request, "Transaction %s deleted." % (item.title))
        return redirect('item_list')

    context = {
        'item': item
    }
    return render(request, 'items/delete.html', context)

@login_required
def item_unit_price(request, pk):
    if request.is_ajax():
        data = Item.objects.get(id=pk)
        price = data.unit_price
        return HttpResponse(price)


from django.db import models
from core.models import Gender, Religion, MartialStatus, MilitaryStatus, Nationalitie
from django.db.models import Max
from uuid import uuid4

class PersonalInfo(models.Model):
    uuid = models.UUIDField(
        primary_key=True, default=uuid4, unique=True, editable=False
    )
    name_ar = models.CharField(
        max_length=100, verbose_name="Name (Arabic)", unique=True
    )
    name_en = models.CharField(
        max_length=100, verbose_name="Name (English)", unique=True
    )
    gender = models.ForeignKey(
        Gender, on_delete=models.CASCADE, related_name="Personal_Info"
    )
    insurance_id = models.CharField(
        max_length=50, unique=True, verbose_name="Insurance ID"
    )
    religion = models.ForeignKey(
        Religion, on_delete=models.CASCADE, related_name="Personal_Info"
    )
    martial_status = models.ForeignKey(
        MartialStatus,
        on_delete=models.CASCADE,
        related_name="PersonalInfo",
        verbose_name="Martial Status",
    )
    military_status = models.ForeignKey(
        MilitaryStatus,
        on_delete=models.CASCADE,
        related_name="PersonalInfo",
        verbose_name="Military Service Status",
    )
    birth_date = models.DateField(verbose_name="Birth Date")
    address = models.CharField(max_length=200)
    created_at = models.DateTimeField(auto_now_add=True)

    def __str__(self):
        return f"{self.name_ar} - {self.name_en}"

class NationalID(models.Model):
    uuid = models.UUIDField(
        primary_key=True, default=uuid4, unique=True, editable=False
    )
    nationality = models.ForeignKey(
        Nationalitie, on_delete=models.CASCADE, related_name="National_IDs"
    )
    personal_info = models.ForeignKey(
        PersonalInfo, on_delete=models.CASCADE, related_name="National_IDs", null=True
    )
    nationalId = models.CharField(max_length=50, unique=True)
    expire_date = models.DateField()

    def __str__(self) -> str:
        return f"{self.nationalId}"

class MobileNumber(models.Model):
    uuid = models.UUIDField(
        primary_key=True, default=uuid4, unique=True, editable=False
    )
    personal_info = models.ForeignKey(
        PersonalInfo, on_delete=models.CASCADE, related_name="MobileNumbers"
    )
    mobile_number = models.CharField(max_length=15)

    def __str__(self) -> str:
        return f"{self.mobile_number}"

# class Educational_Info(models.Model):
#     uuid = models.UUIDField(
#         primary_key=True, default=uuid4, unique=True, editable=False
#     )
#     personal_info = models.ForeignKey(
#         Personal_Info, on_delete=models.CASCADE, related_name="Educational_Info"
#     )

#     def __str__(self) -> str:
#         return f"{self.personal}"

class Employment_Info(models.Model):
    uuid = models.UUIDField(
        primary_key=True, default=uuid4, unique=True, editable=False
    )
    hr_code = models.BigIntegerField(verbose_name="HR Code", blank=True, null=False)
    personal_info = models.OneToOneField(
        PersonalInfo, on_delete=models.CASCADE, related_name="Employment_Info"
    )

    def save(self, *args, **kwargs):
        if self.hr_code is None:
            max_hrcode = Employment_Info.objects.aggregate(Max("hr_code"))[
                "hr_code__max"
            ]
            if max_hrcode is not None:
                self.hr_code = max_hrcode + 1
            else:
                self.hr_code = 1
        super(Employment_Info, self).save(*args, **kwargs)

    def __str__(self) -> str:
        return f"{self.personal_info}"

# class Department(models.Model):
#     uuid = models.UUIDField(
#         primary_key=True, default=uuid4, unique=True, editable=False
#     )
#     department = models.CharField(max_length=100)
#     department_manager = models.ForeignKey(
#         Employment_Info,
#         on_delete=models.SET_NULL,
#         null=True,
#         related_name="Departments",
#     )
#     is_active = models.BooleanField(default=True)

# class Employee_Status(models.Model):
#     uuid = models.UUIDField(
#         primary_key=True, default=uuid4, unique=True, editable=False
#     )
#     employee = models.ForeignKey(
#         Employment_Info, on_delete=models.CASCADE, related_name="Employee_Status"
#     )
#     notes = models.TextField(max_length=400)
#     is_active = models.BooleanField(default=True)
#     status_change_date = models.DateTimeField(auto_now_add=True)

# class Documentation(
#     models.Model,
# ):
#     def get_employee_images_filepath(self, filname):
#         return f"employees/{self.employee.hr_code}/{filname}"

#     uuid = models.UUIDField(
#         primary_key=True, default=uuid4, unique=True, editable=False
#     )
#     employee = models.ForeignKey(
#         Employment_Info, on_delete=models.CASCADE, related_name="Documentation"
#     )
#     profile_picture = models.ImageField(
#         max_length=255, upload_to=get_employee_images_filepath, null=True, blank=True
#     )
#     job_offer = models.ImageField(
#         max_length=255, upload_to=get_employee_images_filepath, null=True, blank=True
#     )
#     labor_picture = models.ImageField(
#         max_length=255, upload_to=get_employee_images_filepath, null=True, blank=True
#     )
#     birth_cert = models.ImageField(
#         max_length=255, upload_to=get_employee_images_filepath, null=True, blank=True
#     )
#     military_status = models.ImageField(
#         max_length=255, upload_to=get_employee_images_filepath, null=True, blank=True
#     )
#     insurance_picture = models.ImageField(
#         max_length=255, upload_to=get_employee_images_filepath, null=True, blank=True
#     )
#     id_picture = models.ImageField(
#         max_length=255, upload_to=get_employee_images_filepath, null=True, blank=True
#     )
#     education_picture = models.ImageField(
#         max_length=255, upload_to=get_employee_images_filepath, null=True, blank=True
#     )


import gym
import numpy as np

# Learning rate
alpha = 1e-4
# Discounting factor
gamma = 0.9999
# Exploration rate
epsilon = 0.01

state_size = (9, 9, 9, 9, 9, 9, 2, 2)
min_state = [-1.2, -0.3, -2.4, -2.4, -3.5, -6.0, 0., 0.]
max_state = [ 1.2,  1.2,  2.4,  0.7,  3.8,  7.9, 1., 1.]
bins = [np.arange(min, max, (max - min) / sz)[1:]
        for sz, min, max in zip(state_size, min_state, max_state)]

def discretise(state):
    res = []
    for s, b in zip(state, bins):
        res.append(np.digitize(s, b).item())
    return tuple(res)

def rollout(env, Q, train=False):
    episode_reward = 0

    state = env.reset()
    state = discretise(state)
    for t in range(env.unwrapped.spec.max_episode_steps):
        # Choose action
        if train and np.random.uniform(0., 1.) < epsilon:
            # Select a random action
            action = np.random.choice(env.action_space.n)
        else:
            # Select the greedy action
            action = np.argmax(Q[state])

        next_state, reward, done, _ = env.step(action)
        next_state = discretise(next_state)

        episode_reward += reward

        if train:
            # Read the current state-action value
            Q_sa = Q[state + (action,)]

            # Find the value of the next state
            next_Q_s = np.max(Q[next_state])

            # Calculate the updated state action value
            new_Q_sa = Q_sa + alpha * (reward + gamma * next_Q_s - Q_sa)

            # Write the updated value
            Q[state + (action,)] = new_Q_sa

        state = next_state

        if done:
            break

    return episode_reward

def main():

    env = gym.make('LunarLander-v2')

    Q = np.ones(state_size + (env.action_space.n,))

    episodes = int(1e4)
    average_reward = 0.
    for e in range(episodes):
        # Train
        rollout(env, Q, train=True)

        # Test
        episode_reward = rollout(env, Q, train=False)
        average_reward = 0.99 * average_reward + 0.01 * episode_reward

        if average_reward >= env.spec.reward_threshold:
            print('Solved!')
            break

        print("{}: {}".format(e, average_reward))

if __name__ == "__main__":
    main()


from __future__ import annotations

from typing import TYPE_CHECKING

from actions.attack.split_damage import SplitDamageAction
from constants.enums import Location
from custom_exceptions import Impossible

if TYPE_CHECKING:
    from entity import Entity

class ArrowAction(SplitDamageAction):
    def __init__(self, entity: Entity, direction: Location):
        """
        Arrow action hits all adjacent targets as well as entities at the attacker's location
        This is a split damage attack action that divides total damage by number of targets hit
        :param entity: acting Entity
        :param direction: key pressed to make attack
        """
        self.entity = entity
        ammo = {'arrows': len(self.entity.crew.roster) // 4}
        enough_ammo = True
        for ammo_type in ammo.keys():
            if ammo_type not in self.entity.cargo.manifest.keys():
                enough_ammo = False
            elif self.entity.cargo.manifest[ammo_type] - ammo[ammo_type] < 0:
                enough_ammo = False
        if not enough_ammo:
            raise Impossible(f"Not enough arrows!")

        targets = []
        neighbor_tiles = self.engine.game_map.get_neighbors_at_elevations(self.entity.x,
                                                                          self.entity.y,
                                                                          elevations='all')
        neighbor_tiles.append((entity.x, entity.y))
        for tile_x, tile_y in neighbor_tiles:
            targets.extend(self.engine.game_map.get_targets_at_location(tile_x, tile_y))
        if self.entity in targets:
            targets.remove(self.entity)
        if len(targets) < 1:
            raise Impossible(f"No adjacent targets")
        damage = (len(self.entity.crew.roster) // 4) // len(targets)
        if self.entity.crew.has_occupation("archer"):
            damage += 1
        super().__init__(entity, targets, damage, direction, ammo)

    def perform(self) -> bool:
        """
        perform a split damage attack
        :return: bool
        """
        return super().perform()


from multiprocessing import Process, Queue
import random, time, os

def proc_write(q, urls):
    print("process %s is writing..." % os.getpid())
    for url in urls:
        q.put(url)
        print("put %s to queue." % url)
        time.sleep(random.random())

def proc_read(q):
    print("process %s is reading..." % os.getpid())
    while True:
        url = q.get(True)
        print("get %s from queue." % url)

def main():
    q = Queue()
    w1 = Process(target=proc_write, args=(q, ['url_1', 'url_2', 'url_3']))
    w2 = Process(target=proc_write, args=(q, ['url_4', 'url_5', 'url_6']))
    p1 = Process(target=proc_read, args=(q, ))
    w1.start()
    w2.start()
    p1.start()
    w1.join()
    w2.join()
    p1.terminate()

if __name__ == "__main__":
    main()


import os
import time
import random
import numpy as np
import torch
import metaworld

from collections import OrderedDict

import maml_trpo.utils.pytorch_utils as ptu

from maml_trpo.envs import AsyncVectorEnv
from maml_trpo.policies import GaussianPolicy, CategoricalPolicy
from maml_trpo.critics import ValueFun
from maml_trpo.samplers import Sampler, MetaWorldTaskSampler
from maml_trpo.utils import Logger
from maml_trpo.meta_learner import MetaLearner

def init_metaworld_env(benchmark, seed, num_workers):
    """
    initializes metaworld env across num_workers

    Args:
    benchmark : Metworld Benchmark(one of ML1, ML10, ML45 from metaworld)
    seed : random seed
    num_workers : number of parallel workers

    Returns
    MetaWorld Env distributed across workers
    """
    _, env_cls = list(benchmark.train_classes.items())[0]
    seeds = [ seed * (i+1) for i in range(num_workers)]
    env = AsyncVectorEnv([env_cls for _ in range(num_workers)], seeds)
    return env

def set_random_seed(seed):
    """
    Sets random seed for reproducibility of experiments
    """
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)

def set_benchmark(benchmark_type):
    """
    initializes metaworld benchmark
    Args:
    benchmark_type : str input one of ('ml1', 'ml10', 'ml45')

    Returns
    MetaWorld Benchmark
    """
    if benchmark_type == 'ml1':
        # pick-place-v2 is chosen arbitarily, can choose any other env
        return metaworld.ML1('pick-place-v2')
    elif benchmark_type == 'ml10':
        return metaworld.ML10()
    elif benchmark_type == 'ml45':
        return metaworld.ML45()

class Trainer:
    """
    Trainer class to run train and evaluation experiments
    """

    def __init__(self, params):

        self.params = params
        self.benchmark = set_benchmark(self.params['benchmark_type'])

        set_random_seed(self.params['seed'])

        self.env = init_metaworld_env(self.benchmark, self.params['seed'], self.params['num_workers'])
        self.eval_env = init_metaworld_env(self.benchmark, self.params['seed'], 1)

        self.task_sampler = MetaWorldTaskSampler(self.benchmark, mode='train')
        self.test_task_sampler = MetaWorldTaskSampler(self.benchmark, mode='test')

        self.sampler = Sampler(self.env, num_workers=self.params['num_workers'])
        self.render_sampler = Sampler(self.eval_env, num_workers=1)

        obs_space, ac_space = self.env.get_spaces()

        self.device = ptu.init_gpu(self.params['use_gpu'], self.params['which_gpu'])

        self.policy = GaussianPolicy(
            in_size = int(np.prod(obs_space.shape)),
            out_size = int(np.prod(ac_space.shape)),
            hidden_units=[100, 100],
            activation='tanh',
            device=self.device)

        self.value_fn = ValueFun(
            in_size = int(np.prod(obs_space.shape)),
            out_size=1,
            hidden_units=[32, 32],
            activation='tanh',
            device=self.device)

        self.logger = Logger(self.params['logdir'])

        self.meta_train_itrs = self.params['n_iters']
        self.meta_train_batch_size = self.params['meta_batch_size']

        # meta-train and meta-test envs have same number of task variations in Metaworld benchmark
        # Eval-itrs is to evaluate for each task variation across meta-train and meta-test envs
        self.eval_itrs =  int(len(self.benchmark.test_tasks)/len(self.benchmark.test_classes))

        self.metalearner = MetaLearner(
            policy = self.policy,
            value_fn = self.value_fn,
            inner_lr = self.params['inner_lr'],
            num_adapt_steps = self.params['num_adapt_steps'],
            episodes_per_task = self.params['episodes_per_task'],
            gamma = self.params['gamma'],
            gae_lam = self.params['gae_lam'],
            max_kl = self.params['max_kl_increment'],
            max_backtracks = self.params['max_backtracks'],
            cg_steps = self.params['cg_steps'],
            cg_damping =self.params['cg_damping'],
            device = self.device,
            restore = self.params['restore'],
            ckpt_path = self.params['ckpt_path'])

        self.start_itr = self.metalearner.itr

        self.meta_eval_freq =  params['eval_freq']

    def meta_train(self):

        if self.start_itr!=0:
            self.start_itr+= 1

        for itr in range(self.start_itr, self.meta_train_itrs):

            print(f"At Iteration {itr}")

            logs = OrderedDict()

            tasks = self.task_sampler.sample_tasks(self.meta_train_batch_size)
            train_logs = self.metalearner.train_step(tasks, self.sampler)
            logs.update(train_logs)

            if itr==0 or ((itr+1)% self.meta_eval_freq)==0:
                meta_batch_size = len(self.benchmark.test_classes)
                test_tasks = self.test_task_sampler.sample_tasks(meta_batch_size)
                test_logs = self.metalearner.evaluate_step(test_tasks, self.sampler)
                logs.update(test_logs)

                self.metalearner.save('{}/maml_ckpt_itr_{}.pt'.format(self.params['logdir'], itr))

            for key, value in logs.items():
                print('{} : {}'.format(key, value))
                self.logger.log_scalar(value, key, itr)

            self.logger.flush()

    def meta_evaluate(self, eval_on='train_tasks', render=False):

        if eval_on == 'train_tasks':
            task_sampler = self.task_sampler
            meta_batch_size = len(self.benchmark.train_classes)
            prefix ='mt_tr'

        else:
            task_sampler = self.test_task_sampler
            meta_batch_size = len(self.benchmark.test_classes)
            prefix ='mt_ts'

        if render==True:
            sampler = self.render_sampler
        else:
            sampler = self.sampler

        for itr in range(self.eval_itrs):
            tasks = task_sampler.sample_tasks(meta_batch_size, shuffle=False)
            logs = self.metalearner.evaluate_step(tasks, sampler, log_videofn=self.logger.log_videos,
                                                                render=render, prefix=prefix,video_itr=itr)
            for key, value in logs.items():
                print('{} : {}'.format(key, value))
                self.logger.log_scalar(value, key, itr)

            self.logger.flush()

def main():

    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--train', action='store_true')
    parser.add_argument('--evaluate', action='store_true')
    parser.add_argument('--eval_on', type=str, choices=['train_tasks', 'test_tasks'], default='meta_train',
                                    help='test meta-trained policy on either meta_train/meta_test tasks')
    parser.add_argument('--render', action='store_true')

    parser.add_argument('--benchmark_type', type=str, default='ml10', choices=['ml1', 'ml10', 'ml45'])
    parser.add_argument('--exp_name', type=str, default='todo')
    parser.add_argument('--n_iters', '-n', type=int, default=300)
    parser.add_argument('--num_workers', '-n_w', type=int, default=1)

    parser.add_argument('--use_gae', action='store_true')
    parser.add_argument('--gae_lam', type=float, default=0.95)
    parser.add_argument('--gamma', type=float, default=0.99)

    parser.add_argument('--meta_batch_size', '-mb', type=int, default=20) # tasks collected per meta train iteration
    parser.add_argument('--episodes_per_task', type=int, default=10) # rollouts collected per a single task

    parser.add_argument('--num_adapt_steps', type =int, default=1)

    parser.add_argument('--inner_lr', type=float, default=1e-4, help='learning_rate for inner-loop/adaptation steps')

    # TRPO update params
    parser.add_argument('--cg_steps', type=int, default=10, help='num cg steps')
    parser.add_argument('--cg_damping', type=float, default=1e-1, help='damping to use in cg')
    parser.add_argument('--max_backtracks', type=int, default=10, help='no of backtracks for line search')
    parser.add_argument('--max_kl_increment', type=float, default=1e-2,
                                           help='max_kl divergence between old and new policy')

    parser.add_argument('--seed', type=int, default=1)
    parser.add_argument('--use_gpu', '-ngpu', action='store_true')
    parser.add_argument('--which_gpu', '-gpu_id', default=0)
    parser.add_argument('--scalar_log_freq', type=int, default=1)
    parser.add_argument('--eval_freq', type=int, default=10)

    parser.add_argument('--restore', action='store_true')
    parser.add_argument('--ckpt_path', type=str, help='checkpoint path to restore from')

    args = parser.parse_args()

    # convert to dictionary
    params = vars(args)

    ##################################
    ### CREATE DIRECTORY FOR LOGGING
    ##################################

    data_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../data')

    if not (os.path.exists(data_path)):
        os.makedirs(data_path)

    logdir = args.benchmark_type + '_' + args.exp_name + '_' + time.strftime("%d-%m-%Y-%H-%M-%S")
    logdir = os.path.join(data_path, logdir)
    params['logdir'] = logdir
    if not(os.path.exists(logdir)):
        os.makedirs(logdir)

    ###################
    ### RUN TRAINING
    ###################

    trainer = Trainer(params)

    if args.train:
        trainer.meta_train()

    elif args.evaluate:
        trainer.meta_evaluate(params['eval_on'], params['render'])

if __name__ == "__main__":
    main()


#!/usr/bin/env python

import numpy as np
import matplotlib.pyplot as plt
from scipy import random

#compute m draws from the dirichlet distribition with data given in vector a
def rdirich(m, a):
    th = np.zeros((m, len(a)+1))

    #generates m random gamma draws based on each of the data points
    for j in range(len(a)):
        th[:,j] = random.gamma(a[j], 1, m)
    th = th/np.sum(th, axis=1)[:,np.newaxis] #normalizes
    th[:,-1] = th[:,0] - th[:,1] #calculates gamma
    return th

if __name__ == '__main__':
    #define parameters
    p = 3
    m = int(1e6)

    #define prior and data, calculate posterior
    prior = np.ones(p) * .01
    data = np.asarray((727, 583, 137))
    post = rdirich(m, prior+data)

    #calculate confidence interval size and print with mean
    ci =  np.percentile(post[:,-1], (2.5, 97.5))
    print(np.average(post[:,-1]), ci, np.sum(post[:,-1]>0)/m)

    #plot posterior histograms
    plt.figure(figsize=(10,4))
    plt.subplot(121)
    plt.hist(post[:,0], label = r'$\theta_1$', bins = 50)
    plt.hist(post[:,1], label = r'$\theta_2$', bins = 50)
    plt.xlabel('Vote Fraction')
    plt.ylabel('Counts')
    plt.legend()
    plt.subplot(122)
    plt.hist(post[:,-1], label = r'$\gamma$', bins = 50, color = 'g')
    plt.axvline(x=ci[0], color = 'k', ls = '--')
    plt.axvline(x=ci[1], color = 'k', ls = '--')
    plt.xlabel('Vote Fraction')
    plt.ylabel('Counts')
    plt.legend()
    plt.tight_layout()
    plt.show()


from django.urls import path
from . import views

urlpatterns = [
    path('', views.index, name='horoscope-index'),
    path('type/', views.index2),
    path('<int:sign_zodiac>', views.get_horoscope_by_number),
    path('<str:sign_zodiac>', views.get_horoscope_by_sign, name='horoscope-name'),
    path("type/<str:element>", views.type_element, name='element-name'),

]


# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load in

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory
import os
print(os.listdir("../input"))

# Any results you write to the current directory are saved as output.

df = pd.read_csv('../input/PUBG_Player_Statistics.csv')
df.head()

df.info()

df.shape

df.describe()

df['solo_KillDeathRatio'].value_counts()
# From this we can see solo kd ratio on avg is high for 1.0

df['solo_RoundsPlayed'].value_counts()

solo=df.filter(regex='solo')
duo=df.filter(regex='duo')
squad=df.filter(regex='squad')

print(len(df)
      ,len(solo)
      ,len(duo)
      ,len(squad))
print(len(df.columns)
      ,len(solo.columns)
      ,len(duo.columns)
      ,len(squad.columns))

corr = df.corr()
corr.info(verbose=True)

import matplotlib.pyplot as plt
import seaborn as sns
cols_to_drop = ['Id', 'groupId', 'matchId', 'matchType','matchTypeReduced']
cols_to_fit = [col for col in df.columns if col not in cols_to_drop]
corr = df[cols_to_fit].corr()

plt.figure(figsize=(9,7))
sns.heatmap(
    corr,
    xticklabels=corr.columns.values,
    yticklabels=corr.columns.values,
    linecolor='white',
    linewidths=0.1,
    cmap="RdBu"
)
plt.show()


import ast
import json
import os

import pandas as pd
from sklearn.model_selection import train_test_split

from classifier.TfClassifier import TfClassifier
from ranker.tfidfRanker import TfidfRanker
from reader.PtReader import PtReader
from reader.TfReader import TfReader

from silence_tensorflow import silence_tensorflow
silence_tensorflow()

LABEL_PATH = 'chatBot/data/news2/classes.json'
DB_PATH = 'chatBot/data/news2/split_contexts/'

CLASSIFIER_PATH = 'chatBot/models/MCT/tf_bert_base_1.h5'
READER_PATH = 'chatBot/models/QA/electra_squad2'

def read_data(path, test_size_split=0.15):
    df = pd.read_csv(path)

    train_df, eval_df = train_test_split(df, test_size=test_size_split)

    return train_df, eval_df

def get_labels():
    with open(LABEL_PATH) as json_file:
        mappings = json.load(json_file)
        inv_map = {v: k for k, v in mappings.items()}

    return inv_map

input_lock = False

def process_input(shown):
    global input_lock
    if input_lock or shown == 0 or shown % 5 != 0:
        return 0

    print('s: show more, n: new question, q: quit')

    inpt = input()

    if inpt.lower() == 's':
        input_lock = True
        return 0
    if inpt.lower() == 'n':
        return 1
    elif inpt.lower() == 'q':
        exit(code=0)
    else:
        process_input(shown)
        return 0

def main():

    labels = get_labels()

    classifier = TfClassifier()
    classifier.load(CLASSIFIER_PATH)
    ranker = TfidfRanker()
    reader = TfReader()
    reader.load(
        model_path=READER_PATH)

    # Infer on any question
    while True:
        print("Ready to roll")
        question = input()
        label_id = classifier.predict(question)
        group = labels[label_id]
        print(group)

        contexts = []

        shown = 0
        predicted = 0
        global input_lock
        with open(DB_PATH + group + '.txt', 'r', encoding='utf-8') as f:
            lines = f.readlines()

            for line in lines:
                context = ast.literal_eval(line)
                contexts.append(context)

            print(len(contexts))

            ranked_contexts = ranker.rank(question, contexts, 0)

            for _, row in ranked_contexts.iterrows():
                date = row['date']
                text = row['text']

                answer, probability = reader.predict(
                    question, text)
                predicted += 1

                if answer != '':
                    pretty_probability = str(probability*100)[0:3] + '%'

                    print(f'{date}: {answer} - {pretty_probability}')
                    print(f'context: "{text}"\n')

                    input_lock = False
                    shown += 1

                elif predicted % 50 == 0:
                    print(predicted)

                new_question = process_input(shown)
                if new_question:
                    break

if __name__ == '__main__':
    main()


import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import axes3d, Axes3D
from random import randrange
import math
import open3d as o3d
import trimesh
from scipy.spatial import Delaunay
from colormap import rgb2hex
import random
def createPoints():
    numberOfPoints=100
    odleglosc=1
    depth=createDepth(numberOfPoints)
    full_x=[]
    full_y=[]
    full_z=[]
    ranges=np.linspace(0,2*math.pi,np.shape(depth)[0])
    for idx,angle in enumerate(ranges):
        d_0=depth[idx]
        xs = np.array([np.cos(angle)*(odleglosc-d_0[numberOfPoints-1-i]) for i in range(0,numberOfPoints)])#wspolrzedne do rysowania
        ys = np.array([np.sin(angle)*(odleglosc-d_0[numberOfPoints-1-i]) for i in range(0,numberOfPoints)])#wspolrzedne do rysowania
        zs=np.linspace(0, 1  ,numberOfPoints)
        full_x.append(xs)
        full_y.append(ys)
        full_z.append(zs)
    return full_x,full_y,full_z

def createDepth(numberOfPointsInLine):
    depthArray=[]
    for i in range(0,100):
        row=np.linspace(0,1,numberOfPointsInLine)
        depthArray.append(row)
    return depthArray
# def createDepth(numberOfPointsInLine):
#     depthArray=[]
#     angles=np.linspace(0,6*math.pi,1000)
#     for angle in angles:
#         distance=math.sin(angle)
#         row=[distance]*numberOfPointsInLine
#         depthArray.append(row)
#     return depthArray
def showDiagram(x,y,z,color):
    fig = plt.figure()
    ax = Axes3D(fig)
    ax.scatter(x,y,z,c=color)
    ax.set_xlabel('X Label')
    ax.set_ylabel('Y Label')
    ax.set_zlabel('Z Label')
    plt.show()
def flatCoords(x,y,z):
    fullArr=[]
    colors=[]
    red=np.linspace(0,1,np.shape(x)[0])
    green=np.linspace(0,1,np.shape(x)[1])

    for (rowx, rowy, rowz,color_red) in zip(x, y, z,red):
        for (it_x, it_y, it_z,color_green) in zip(rowx, rowy, rowz,green):
            fullArr.append([it_x, it_y, it_z] )
            colors.append([color_red,color_green,0])
    return fullArr,colors
def createDelaunayMesh(flat,color):
    tri = Delaunay(flat)
    faces=[]
    for i in tri.simplices:
        faces.append([i[0],i[1],i[2]])
    probnymesh = trimesh.Trimesh(vertices=flat,faces=faces,vertex_colors=color)
    probnymesh.export('testDelaunayMesh.ply')
    pcd_load = o3d.io.read_triangle_mesh("testDelaunayMesh.ply")
    o3d.visualization.draw_geometries([pcd_load],window_name='delaunay')

    pcd = o3d.geometry.PointCloud()
    pcd.points = o3d.utility.Vector3dVector(flat)
    pcd.colors =  o3d.utility.Vector3dVector(color)
    o3d.visualization.draw_geometries([pcd],window_name='point cloud')

    # pcd.estimate_normals()
    # distances = pcd.compute_nearest_neighbor_distance()
    # avg_dist = np.mean(distances)
    # radius =3*avg_dist
    # bpa_mesh = o3d.geometry.TriangleMesh.create_from_point_cloud_ball_pivoting(pcd,o3d.utility.DoubleVector([radius, radius * 2] ))
    # tri_mesh = trimesh.Trimesh(np.asarray(bpa_mesh.vertices), np.asarray(bpa_mesh.triangles),vertex_normals=np.asarray(bpa_mesh.vertex_normals),vertex_colors=pcd.colors)
    # tri_mesh.export('testbpamesh.ply')
    # pcd_load = o3d.io.read_triangle_mesh("testbpamesh.ply")
    # o3d.visualization.draw_geometries([pcd_load],window_name='bpa cloud')

x,y,z=createPoints()
flat,color=flatCoords(x,y,z)
# showDiagram(x,y,z,color)
createDelaunayMesh(flat,color)


from __future__ import annotations
from typing import List, Generic, TypeVar, Callable, Union, Any
import functools
import itertools
from ldm.modules.attention import SpatialTransformer

from ldm.modules.diffusionmodules.openaimodel import UNetModel
from ldm.modules.attention import CrossAttention
import torch.nn as nn

__all__ = ['ObjectHooker', 'ModuleLocator', 'AggregateHooker', 'UNetCrossAttentionLocator']

ModuleType = TypeVar('ModuleType')
ModuleListType = TypeVar('ModuleListType', bound=List)

class ModuleLocator(Generic[ModuleType]):
    def locate(self, model: nn.Module) -> List[ModuleType]:
        raise NotImplementedError

class ObjectHooker(Generic[ModuleType]):
    def __init__(self, module: ModuleType):
        self.module: ModuleType = module
        self.hooked = False
        self.old_state = dict()

    def __enter__(self):
        self.hook()
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.unhook()

    def hook(self):
        if self.hooked:
            raise RuntimeError('Already hooked module')

        self.old_state = dict()
        self.hooked = True
        self._hook_impl()

        return self

    def unhook(self):
        if not self.hooked:
            raise RuntimeError('Module is not hooked')

        for k, v in self.old_state.items():
            if k.startswith('old_fn_'):
                setattr(self.module, k[7:], v)

        self.hooked = False
        self._unhook_impl()

        return self

    def monkey_patch(self, fn_name, fn):
        self.old_state[f'old_fn_{fn_name}'] = getattr(self.module, fn_name)
        setattr(self.module, fn_name, functools.partial(fn, self.module))

    def monkey_super(self, fn_name, *args, **kwargs):
        return self.old_state[f'old_fn_{fn_name}'](*args, **kwargs)

    def _hook_impl(self):
        raise NotImplementedError

    def _unhook_impl(self):
        pass

class AggregateHooker(ObjectHooker[ModuleListType]):
    def _hook_impl(self):
        for h in self.module:
            h.hook()

    def _unhook_impl(self):
        for h in self.module:
            h.unhook()

    def register_hook(self, hook: ObjectHooker):
        self.module.append(hook)

class UNetCrossAttentionLocator(ModuleLocator[CrossAttention]):
    def locate(self, model: UNetModel, layer_idx: int) -> List[CrossAttention]:
        """
        Locate all cross-attention modules in a UNetModel.

        Args:
            model (`UNetModel`): The model to locate the cross-attention modules in.

        Returns:
            `List[CrossAttention]`: The list of cross-attention modules.
        """
        blocks = []
        i = 0

        for unet_block in itertools.chain(model.input_blocks, model.output_blocks, [model.middle_block]):
            # if 'CrossAttn' in unet_block.__class__.__name__:
            if not layer_idx or i == layer_idx:
                for module in unet_block.modules():
                    if type(module) is SpatialTransformer:
                        spatial_transformer = module
                        for basic_transformer_block in spatial_transformer.transformer_blocks:
                            blocks.append(basic_transformer_block.attn2)
            i += 1

        if layer_idx:
            blocks = [blocks[0]]

        return blocks


# coding: utf-8

## Random Forest

from sklearn.tree import DecisionTreeClassifier
from sklearn.tree import export_graphviz
from sklearn.model_selection import cross_val_score
from pydotplus import graph_from_dot_data
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, precision_score, recall_score,f1_score,confusion_matrix

## Change path

import sys
sys.path.append("/Users/deepthisen/Desktop/Courses/stat656/MidTerm Exam/")

from sklearn.model_selection import cross_validate
from sklearn.metrics import roc_auc_score

from Class_tree import DecisionTree
from Class_replace_impute_encode import ReplaceImputeEncode

## Change Path
file_path = '/Users/deepthisen/Desktop/Courses/stat656/MidTerm Exam/'
df = pd.read_excel(file_path+"CreditCard_Defaults.xlsx")

attribute_map = {
    'Default':[1,(0,1),[0,0]],
    'Gender':[1,(1, 2),[0,0]],
    'Education':[2,(0,1,2,3,4,5,6),[0,0]],
    'Marital_Status':[2,(0,1,2,3), [0,0]],
    'card_class':[2,(1,2,3),[0,0]],
    'Age':[0,(20,80),[0,0]],
    'Credit_Limit':[0,(100, 80000),[0,0]],
    'Jun_Status':[0,(-2,8),[0,0]],
    'May_Status':[0,(-2,8),[0,0]],
    'Apr_Status':[0,(-2,8),[0,0]],
    'Mar_Status':[0,(-2,8),[0,0]],
    'Feb_Status':[0,(-2,8),[0,0]],
    'Jan_Status':[0,(-2,8),[0,0]],
    'Jun_Bill':[0,(-12000, 32000),[0,0]],
    'May_Bill':[0,(-12000, 32000),[0,0]],
    'Apr_Bill':[0,(-12000, 32000),[0,0]],
    'Mar_Bill':[0,(-12000, 32000),[0,0]],
    'Feb_Bill':[0,(-12000, 32000),[0,0]],
    'Jan_Bill':[0,(-12000, 32000),[0,0]],
    'Jun_Payment':[0,(0, 60000),[0,0]],
    'May_Payment':[0,(0, 60000),[0,0]],
    'Apr_Payment':[0,(0, 60000),[0,0]],
    'Mar_Payment':[0,(0, 60000),[0,0]],
    'Feb_Payment':[0,(0, 60000),[0,0]],
    'Jan_Payment':[0,(0, 60000),[0,0]],
    'Jun_PayPercent':[0,(0, 1),[0,0]],
    'May_PayPercent':[0,(0, 1),[0,0]],
    'Apr_PayPercent':[0,(0, 1),[0,0]],
    'Mar_PayPercent':[0,(0, 1),[0,0]],
    'Feb_PayPercent':[0,(0, 1),[0,0]],
    'Jan_PayPercent':[0,(0, 1),[0,0]]
}

df.drop(['Customer'], axis=1)

rie = ReplaceImputeEncode(data_map=attribute_map, drop=False, nominal_encoding='one-hot', display=True)
encoded_df = rie.fit_transform(df)

## Splitting into Target and Predictors

varlist = df['Default']
X = encoded_df.drop('Default', axis=1)
y = encoded_df['Default']
np_y = np.ravel(y)

## Cross Validation RANDOM FOREST

from sklearn.ensemble import RandomForestClassifier
estimators_list   = [20,30,50,70,90]
max_features_list = ['auto','sqrt',0.3, 0.5, 0.7]
score_list = ['accuracy', 'recall', 'precision', 'f1']
max_f1 = 0
max_AUC = 0
mean_list = []
std_list = []

for e in estimators_list:
    for f in max_features_list:
        print("\nNumber of Trees: ", e, " Max_features: ", f)
        rfc = RandomForestClassifier(n_estimators=e, criterion="gini", max_depth=None, min_samples_split=20,min_samples_leaf=1, max_features=f,\
		                             n_jobs=-1, bootstrap=True, random_state=12345)
        rfc= rfc.fit(X, np_y)
        scores = cross_validate(rfc, X, np_y, scoring=score_list,return_train_score=False, cv=10)

        print("{:.<13s}{:>6s}{:>13s}".format("Metric", "Mean", "Std. Dev."))
        for s in score_list:
            var = "test_"+s
            mean = scores[var].mean()
            std  = scores[var].std()
            mean_list.append(mean)
            std_list.append(std)
            print("{:.<13s}{:>7.4f}{:>10.4f}".format(s, mean, std))
        if mean > max_AUC:
            max_AUC = mean
            best_estimator    = e
            best_max_features = f

print("\nBest based on F1-Score")
print("Best Number of Estimators (trees) = ", best_estimator)
print("Best Maximum Features = ", best_max_features)

## Splitting and fitting the model
X_train, X_validate, y_train, y_validate = train_test_split(X, np_y,test_size = 0.3, random_state=12345)

rfc = RandomForestClassifier(n_estimators=50, criterion="gini", max_depth=None, min_samples_split=20,min_samples_leaf=1, max_features=0.7, \
                               n_jobs=1, bootstrap=True, random_state=12345)
rfc= rfc.fit(X_train, y_train)

# Printing Test Results
DecisionTree.display_binary_split_metrics(rfc, X_train, y_train,X_validate, y_validate)
features = X_train.columns.values.tolist()
DecisionTree.display_importance(rfc, features)

# Code to find the best threshold under a given set of conditions
# Enhancements for threshold adjustment for better sensitivity
threshold_array = np.linspace(0.15,0.85,num =140)
best_metric = 0
min_sensitivity = 0.7
min_accuracy = 0.8
i = 0

best_threshold_acc = "infeasible under given conditions"
best_threshold_sen = "infeasible under given conditions"
best_threshold = "infeasible under given conditions"
predict_probabilities = rfc.predict_proba(X_validate)

for i in threshold_array:

    predict_mine = np.where(predict_probabilities < i,0,1)
    k = confusion_matrix(y_validate, predict_mine[:,1])

    True_Predictions = k[0,0]+k[1,1]
    False_pos = k[1,0]
    True_pos = k[1,1]
    True_neg = k[0,0]
    False_neg = k[0,1]
    Total = True_Predictions + False_pos + False_neg
    Accuracy =(True_Predictions/Total)
    Sensitivity =(True_pos)/(k[1,0]+k[1,1])

    if (Accuracy + Sensitivity) > best_metric and Accuracy > min_accuracy and Sensitivity > min_sensitivity:
        best_metric = (Accuracy + Sensitivity)
        best_threshold_acc = Accuracy
        best_threshold_sen = Sensitivity
        best_threshold = i

print("\nBest Model")
print("Accuracy....",best_threshold_acc)
print("Sensitivity..",best_threshold_sen)
print("Threshold....",best_threshold)


import matplotlib
import numpy as np
import matplotlib.cm as cm
import matplotlib.pyplot as plt
import sys

with open("structure3D.dat") as file:
    data = [[float(digit) for digit in line.split()] for line in file]

nx=len(data[0][:])
ny=len(data[:])

print(nx)
print(ny)

x=np.arange(-180, 180.0, (360.0+360.0/float(nx))/float(nx))
y=np.arange(-90, 90.0, (180.0+180.0/float(ny))/float(ny))

fig, ax = plt.subplots()
ax.set_aspect(1.0)
#CS = ax.contourf(x, y, data, 100,cmap='coolwarm',levels=[0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1.0])
CS = ax.contourf(x, y, data, 100,cmap='coolwarm')
CS = ax.contour(x, y, data, colors='k')
ax.clabel(CS, inline=1, fontsize=10)
ax.set_xlabel('longitude')
ax.set_ylabel('lattitude')

ax.axvline(90., color='blue')
ax.axvline(-90., color='blue')
ax.axvline(0., color='red')

fig.savefig("Structure3D.pdf")


from __future__ import annotations

import os
from tqdm import tqdm
from shutil import copy
import argparse

def rename(src_dir, trg_dir, str_name):
    print(src_dir, trg_dir, str_name)
    os.makedirs(trg_dir, exist_ok=True)
    img_names = [x for x in os.listdir(src_dir) if not x.endswith(".json")]

    for img_n in tqdm(img_names):
        name = os.path.splitext(img_n)[0]
        src_img_path = os.path.join(src_dir, img_n)
        tar_img_path = os.path.join(trg_dir, str_name + "_" + img_n)
        copy(src_img_path, tar_img_path)
        if os.path.exists(os.path.join(src_dir, name + ".json")):
            src_path = os.path.join(src_dir, name + ".json")
            tar_path = os.path.join(trg_dir, str_name + "_" + name + ".json")
            copy(src_path, tar_path)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('-s', '--src_dir', type=str, help="")
    parser.add_argument('-t', '--trg_dir', type=str, help="")
    parser.add_argument('-n', '--name', type=str, help="")

    args = parser.parse_args()

    rename(args.src_dir, args.trg_dir, args.name)


from XGCN.data import io, csr
from XGCN.utils.parse_arguments import parse_arguments
from XGCN.utils.utils import ensure_dir

import numpy as np
import os.path as osp
from tqdm import tqdm

def main():

    config = parse_arguments()

    ppr_data_root = config['ppr_data_root']
    results_root = config['results_root']
    ensure_dir(results_root)
    graph_type = config['graph_type']

    nei = io.load_pickle(osp.join(ppr_data_root, "nei.pkl"))
    wei = io.load_pickle(osp.join(ppr_data_root, "wei.pkl"))

    topk = config['topk']
    nei = nei[:,:topk]
    wei = wei[:,:topk]

    # from ppr neighbor to edges
    num_nodes = len(nei)
    max_num_edges = topk * num_nodes
    E_src = np.empty(max_num_edges, dtype=np.int32)
    E_dst = np.empty(max_num_edges, dtype=np.int32)

    start = 0
    for u in tqdm(range(num_nodes)):
        u_nei = nei[u][wei[u] != 0]
        d = len(u_nei)
        u_self = np.full(d, fill_value=u, dtype=np.int32)

        E_src[start : start + d] = u_self
        E_dst[start : start + d] = u_nei
        start += d
    E_src = E_src[:start]
    E_dst = E_dst[:start]

    # from edges to csr graph
    indptr, indices = csr.from_edges_to_csr(E_src, E_dst, num_nodes)
    num_nodes = len(indptr) - 1
    num_edges = len(indices)
    info = {
        'graph_type': graph_type,
        'num_nodes': num_nodes,
        'num_edges': num_edges
    }

    io.save_yaml(osp.join(results_root, 'info.yaml'), info)
    io.save_pickle(osp.join(results_root, 'indptr.pkl'), indptr)
    io.save_pickle(osp.join(results_root, 'indices.pkl'), indices)

if __name__ == '__main__':

    main()