from __future__ import division

import six

import datetime
import os
import random
import shutil
import string
import tempfile
import subprocess
import sys

import requests

import verta
from verta import Client
from verta._internal_utils import _utils

import hypothesis
import pytest
from . import utils

RANDOM_SEED = 0
INPUT_LENGTH = 12  # length of iterable input fixture

DEFAULT_HOST = None
DEFAULT_PORT = None
DEFAULT_EMAIL = None
DEFAULT_DEV_KEY = None

# hypothesis on Jenkins is apparently too slow
hypothesis.settings.register_profile("default", suppress_health_check=[hypothesis.HealthCheck.too_slow])
hypothesis.settings.load_profile("default")

def pytest_addoption(parser):
    parser.addoption("--oss", action="store_true", help="run OSS-compatible tests")

def pytest_collection_modifyitems(config, items):
    if config.getoption("--oss"):
        skip_not_oss = pytest.mark.skip(reason="not available in OSS")
        for item in items:
            if 'not_oss' in item.keywords:
                item.add_marker(skip_not_oss)
    else:
        skip_oss = pytest.mark.skip(reason="only applicable to OSS")
        for item in items:
            if 'oss' in item.keywords:
                item.add_marker(skip_oss)

@pytest.fixture(scope='session')
def host():
    return os.environ.get("VERTA_HOST", DEFAULT_HOST)

@pytest.fixture(scope='session')
def port():
    return os.environ.get("VERTA_PORT", DEFAULT_PORT)

@pytest.fixture(scope='session')
def email():
    return os.environ.get("VERTA_EMAIL", DEFAULT_EMAIL)

@pytest.fixture(scope='session')
def dev_key():
    return os.environ.get("VERTA_DEV_KEY", DEFAULT_DEV_KEY)

@pytest.fixture(scope='session')
def email_2():
    # For collaboration tests
    return os.environ.get("VERTA_EMAIL_2")

@pytest.fixture(scope='session')
def dev_key_2():
    # For collaboration tests
    return os.environ.get("VERTA_DEV_KEY_2")

@pytest.fixture
def seed():
    return RANDOM_SEED

@pytest.fixture
def nones():
    return [None]*INPUT_LENGTH

@pytest.fixture
def bools(seed):
    random.seed(seed)
    return [bool(random.randint(0, 1)) for _ in range(INPUT_LENGTH)]

@pytest.fixture
def floats(seed):
    random.seed(seed)
    return [random.uniform(-3**2, 3**3) for _ in range(INPUT_LENGTH)]

@pytest.fixture
def ints(seed):
    random.seed(seed)
    return [random.randint(-3**4, 3**5) for _ in range(INPUT_LENGTH)]

@pytest.fixture
def strs(seed):
    """no duplicates"""
    random.seed(seed)
    gen_str = lambda: ''.join(random.choice(string.ascii_letters) for _ in range(INPUT_LENGTH))
    result = set()
    while len(result) < INPUT_LENGTH:
        single_str = gen_str()
        while single_str in result:
            single_str = gen_str()
        else:
            result.add(single_str)
    return list(result)

@pytest.fixture
def flat_lists(seed, nones, bools, floats, ints, strs):
    random.seed(seed)
    values = (nones, bools, floats, ints, strs)
    return [
        [
            values[random.choice(range(len(values)))][i]
            for i in range(INPUT_LENGTH)
        ]
        for _ in range(INPUT_LENGTH)
    ]

@pytest.fixture
def flat_dicts(seed, nones, bools, floats, ints, strs):
    random.seed(seed)
    values = (nones, bools, floats, ints, strs)
    return [
        {
            strs[i]: values[random.choice(range(len(values)))][i]
            for i in range(INPUT_LENGTH)
        }
        for _ in range(INPUT_LENGTH)
    ]

@pytest.fixture
def nested_lists(seed, nones, bools, floats, ints, strs):
    random.seed(seed)
    values = (nones, bools, floats, ints, strs)
    flat_values = [value for type_values in values for value in type_values]
    def gen_value(p=1):
        if random.random() < p:
            return [
                gen_value(p/2)
                for _ in range(random.choice(range(4)))
            ]
        else:
            return random.choice(flat_values)
    return [
        [
            gen_value()
            for _ in range(random.choice(range(3))+1)
        ]
        for _ in range(INPUT_LENGTH)
    ]

@pytest.fixture
def nested_dicts(seed, nones, bools, floats, ints, strs):
    random.seed(seed)
    values = (nones, bools, floats, ints, strs)
    flat_values = [value for type_values in values for value in type_values]
    def gen_value(p=1):
        if random.random() < p:
            return {
                key: gen_value(p/2)
                for key, _ in zip(strs, range(random.choice(range(4))))
            }
        else:
            return random.choice(flat_values)
    return [
        {
            key: gen_value()
            for key, _ in zip(strs, range(random.choice(range(3))+1))
        }
        for _ in range(INPUT_LENGTH)
    ]

@pytest.fixture
def scalar_values(nones, bools, floats, ints, strs):
    return [type_values[0]
            for type_values in (nones, bools, floats, ints, strs)]

@pytest.fixture
def collection_values(flat_lists, flat_dicts, nested_lists, nested_dicts):
    return [type_values[0]
            for type_values in (flat_lists, flat_dicts, nested_lists, nested_dicts)]

@pytest.fixture
def all_values(scalar_values, collection_values):
    return scalar_values + collection_values

@pytest.fixture(scope='session')
def output_path():
    dirpath = ".outputs"
    while len(dirpath) < 1024:
        try:  # avoid name collisions
            os.mkdir(dirpath)
        except OSError:
            dirpath += '_'
        else:
            yield os.path.join(dirpath, "{}")
            break
    else:
        raise RuntimeError("dirpath length exceeded 1024")
    shutil.rmtree(dirpath)

@pytest.fixture
def dir_and_files(strs, tmp_path):
    """
    Creates nested directory of empty files.

    Returns
    -------
    dirpath : str
    filepaths : set of str

    """
    filepaths = {
        os.path.join(strs[0], strs[1], strs[2]),
        os.path.join(strs[0], strs[1], strs[3]),
        os.path.join(strs[0], strs[2]),
        os.path.join(strs[0], strs[4]),
        os.path.join(strs[2]),
        os.path.join(strs[5]),
    }

    for filepath in filepaths:
        p = tmp_path / filepath
        p.parent.mkdir(parents=True, exist_ok=True)
        p.touch()

    return str(tmp_path), filepaths

@pytest.fixture
def tempdir_root():
    return os.environ.get("TEMPDIR_ROOT")

@pytest.fixture
def in_tempdir(tempdir_root):
    """Moves test to execute inside a temporary directory."""
    dirpath = tempfile.mkdtemp(dir=tempdir_root)
    try:
        with utils.chdir(dirpath):
            yield dirpath
    finally:
        shutil.rmtree(dirpath)

@pytest.fixture
def client(host, port, email, dev_key):
    print("[TEST LOG] test setup begun {} UTC".format(datetime.datetime.utcnow()))
    client = Client(host, port, email, dev_key, debug=True)

    yield client

    proj = client._ctx.proj
    if proj is not None:
        proj.delete()

    print("[TEST LOG] test teardown completed {} UTC".format(datetime.datetime.utcnow()))

@pytest.fixture
def client_2(host, port, email_2, dev_key_2):
    """For collaboration tests."""
    if not (email_2 and dev_key_2):
        pytest.skip("second account credentials not present")
    print("[TEST LOG] test setup begun {} UTC".format(datetime.datetime.utcnow()))

    client = Client(host, port, email_2, dev_key_2, debug=True)

    yield client

    print("[TEST LOG] test teardown completed {} UTC".format(datetime.datetime.utcnow()))

@pytest.fixture
def experiment_run(client):
    proj = client.set_project()
    print("[TEST LOG] Project ID is {}".format(proj.id))
    client.set_experiment()
    run = client.set_experiment_run()
    print("[TEST LOG] Run ID is {}".format(run.id))

    return run

@pytest.fixture
def model_for_deployment(strs):
    np = pytest.importorskip("numpy")
    pd = pytest.importorskip("pandas")
    sklearn = pytest.importorskip("sklearn")
    from sklearn import linear_model

    num_rows, num_cols = 36, 6

    data = pd.DataFrame(np.tile(np.arange(num_rows).reshape(-1, 1),
                                num_cols),
                        columns=strs[:num_cols])
    X_train = data.iloc[:,:-1]  # pylint: disable=bad-whitespace
    y_train = data.iloc[:, -1]

    return {
        'model': sklearn.linear_model.LogisticRegression(),
        'model_api': verta.utils.ModelAPI(X_train, y_train),
        'requirements': six.StringIO("scikit-learn=={}".format(sklearn.__version__)),
        'train_features': X_train,
        'train_targets': y_train,
    }

@pytest.fixture
def repository(client):
    name = _utils.generate_default_name()
    repo = client.get_or_create_repository(name)

    yield repo

    repo.delete()

@pytest.fixture
def commit(repository):
    commit = repository.get_commit()

    yield commit

@pytest.fixture
def created_datasets(client):
    """Container to track and clean up Datasets created during tests."""
    created_datasets = []

    yield created_datasets

    datasets_conn = {}  # prevent duplication
    for dataset in created_datasets:
        datasets_conn[dataset.id] = dataset._conn

    for dataset_id in datasets_conn:
        utils.delete_datasets([dataset_id], datasets_conn[dataset_id])

@pytest.fixture
def registered_model(client):
    model = client.get_or_create_registered_model()
    yield model
    model.delete()

@pytest.fixture
def created_registered_models():
    """Container to track and clean up `RegisteredModel`s created during tests."""
    to_delete = []

    yield to_delete

    for registered_model in to_delete:
        registered_model.delete()

@pytest.fixture
def model_version(registered_model):
    yield registered_model.get_or_create_version()

@pytest.fixture
def created_endpoints():
    to_delete = []

    yield to_delete

    for endpoint in to_delete:
        endpoint.delete()

@pytest.fixture
def endpoint(client, created_endpoints):
    path = _utils.generate_default_name()
    endpoint = client.create_endpoint(path)
    created_endpoints.append(endpoint)

    yield endpoint

@pytest.fixture
def created_endpoints():
    to_delete = []

    yield to_delete

    for endpoint in to_delete:
        endpoint.delete()

@pytest.fixture
def organization(client):
    workspace_name = _utils.generate_default_name()
    org = client._create_organization(workspace_name)

    yield org

    org.delete()

@pytest.fixture
def requirements_file():
    with tempfile.NamedTemporaryFile('w+') as tempf:
        # create requirements file from pip freeze
        pip_freeze = subprocess.check_output([sys.executable, '-m', 'pip', 'freeze'])
        pip_freeze = six.ensure_str(pip_freeze)
        tempf.write(pip_freeze)
        tempf.flush()  # flush object buffer
        os.fsync(tempf.fileno())  # flush OS buffer
        tempf.seek(0)

        yield tempf


# -*- coding: UTF-8 -*-
import re,json,os,collections
from pathlib import Path
from datetime import datetime
import pandas as pd

def mkdir(path):
    # 去除首位空格
    path=path.strip()
    # 去除尾部 \ 符号
    path=path.rstrip("\\")
    path=path.encode('utf-8')
    # 判断路径是否存在
    # 存在     True
    # 不存在   False
    isExists=os.path.exists(path)
    # 判断结果
    if not isExists:
        os.makedirs(path)
        return True
    else:
        # 如果目录存在则不创建，并提示目录已存在
        return False

def find_time(file_path):
    with open (file_path, 'r',encoding='UTF-8') as f:
        lines = f.readlines()
        a = ''
        for line in lines:
            a += line.strip()
    b = a.split("*****")
    res = []
    for post in b:
        post_time = ''.join(re.findall(r"^.*?发表于\s(\d{4}-\d{1,2}-\d{1,2})", post))
        res.append(post_time)
    return res

def filter_time(troll,input_list):
    return_dict = dict(collections.Counter(input_list))
    for i in list(return_dict.keys()):
        if return_dict[i] < 45 or i not in date_list:
            return_dict.pop(i)
        #外野数据量太大，提高筛选标准
        elif troll =='troll' and return_dict[i] < 90:
            return_dict.pop(i)
    if(len(return_dict) <2):
        return None
    else:
        return return_dict

year = '2023'
current_date = datetime.now().strftime("%Y%m%d")
date_list = [pd.Timestamp(x).strftime("%Y-%-m-%-d") for x in pd.date_range(year+'0101',current_date)]
# date_list = [pd.Timestamp(x).strftime("%Y-%-m-%-d") for x in pd.date_range(year+'0101',year+'1231')]
forum_dict ={'troll':['外野'],'game':['手游专楼','游戏区'],'anime':['漫区'],'vtb':['虚拟主播区专楼']}

result_dict = {'troll':{},'game':{},'anime':{},'vtb':{}}

for forum in forum_dict.keys():
    for sub_forum in forum_dict[forum]:
        for pa in Path('./'+sub_forum+'/').iterdir():
            if Path(pa).is_dir():
                total_time = []
                thread_title = re.findall(r'\d{5,7}-?(.*)$',str(pa))[0]
                for sub_pa in Path(pa).iterdir():
                    total_time = total_time + find_time(sub_pa)
                result = filter_time(forum,total_time)
            else:
                thread_title = re.findall(r'\d{5,7}-\d{1,3}-?(.*).md$',str(pa))[0]
                result = filter_time(forum,find_time(pa))
            if(result):
                result_dict[forum][thread_title] = result

mkdir('./S1B/')
with open('./S1B/S1B-'+year+'.json', "w", encoding="utf-8") as f:
    f.write(json.dumps(result_dict,indent=2,ensure_ascii=False))


from typing import Optional, List, Union

from fastapi import Depends, HTTPException
from fastapi.openapi.models import OAuthFlows as OAuthFlowsModel
from fastapi.security import OAuth2
from fastapi.security import SecurityScopes
from fastapi.security.utils import get_authorization_scheme_param
from jwt.exceptions import InvalidTokenError
from starlette.requests import Request
from starlette.status import HTTP_401_UNAUTHORIZED

from api.adapter.s3_adapter import S3Adapter
from api.application.services.authorisation.token_utils import parse_token
from api.application.services.authorisation.dataset_access_evaluator import (
    DatasetAccessEvaluator,
)
from api.application.services.permissions_service import PermissionsService
from api.common.config.auth import (
    IDENTITY_PROVIDER_TOKEN_URL,
    RAPID_ACCESS_TOKEN,
    Action,
)
from api.common.config.layers import Layer
from api.common.custom_exceptions import (
    AuthorisationError,
    UserCredentialsUnavailableError,
    NotAuthorisedToViewPageError,
    AuthenticationError,
)
from api.common.logger import AppLogger
from api.domain.token import Token
from api.domain.dataset_metadata import DatasetMetadata

class OAuth2ClientCredentials(OAuth2):
    def __init__(
        self,
        token_url: str,
        scheme_name: str = None,
        allowed_scopes: dict = None,
        auto_error: bool = True,
    ):
        if not allowed_scopes:
            allowed_scopes = {}
        flows = OAuthFlowsModel(
            clientCredentials={"tokenUrl": token_url, "scopes": allowed_scopes}
        )
        super().__init__(flows=flows, scheme_name=scheme_name, auto_error=auto_error)

    async def __call__(self, request: Request) -> Optional[str]:
        return get_client_token(request)

class OAuth2UserCredentials:
    async def __call__(self, request: Request) -> Optional[str]:
        return get_user_token(request)

oauth2_scheme = OAuth2ClientCredentials(token_url=IDENTITY_PROVIDER_TOKEN_URL)
oauth2_user_scheme = OAuth2UserCredentials()
s3_adapter = S3Adapter()
permission_service = PermissionsService()
dataset_access_evaluator = DatasetAccessEvaluator()

def is_browser_request(request: Request) -> bool:
    accept_type = request.headers.get("Accept")
    return accept_type.startswith("text/html")

def user_logged_in(request: Request) -> bool:
    token = request.cookies.get(RAPID_ACCESS_TOKEN, None)
    return token is not None

def validate_token(
    browser_request: bool = Depends(is_browser_request),
    client_token: Optional[str] = Depends(oauth2_scheme),
    user_token: Optional[str] = Depends(oauth2_user_scheme),
) -> Token:
    check_credentials_availability(browser_request, client_token, user_token)
    token = client_token if client_token else user_token
    return parse_token(token)

def handle_authorisation_error(
    user_token: Optional[str], error: Union[AuthorisationError, InvalidTokenError]
):
    if isinstance(error, InvalidTokenError):
        raise HTTPException(status_code=HTTP_401_UNAUTHORIZED, detail=str(error))
    else:
        if user_token:
            raise NotAuthorisedToViewPageError()
        else:
            raise error

def secure_endpoint(
    security_scopes: SecurityScopes,
    browser_request: bool = Depends(is_browser_request),
    client_token: Optional[str] = Depends(oauth2_scheme),
    user_token: Optional[str] = Depends(oauth2_user_scheme),
):
    try:
        token = validate_token(browser_request, client_token, user_token)
        check_permissions(token.subject, security_scopes.scopes)
    except (InvalidTokenError, AuthorisationError) as error:
        handle_authorisation_error(user_token, error)

def secure_dataset_endpoint(
    security_scopes: SecurityScopes,
    browser_request: bool = Depends(is_browser_request),
    client_token: Optional[str] = Depends(oauth2_scheme),
    user_token: Optional[str] = Depends(oauth2_user_scheme),
    layer: Optional[Layer] = None,
    domain: Optional[str] = None,
    dataset: Optional[str] = None,
):
    try:
        token = validate_token(browser_request, client_token, user_token)
        metadata = process_dataset_metadata(layer, domain, dataset)
        dataset_access_evaluator.can_access_dataset(
            metadata, token.subject, security_scopes.scopes
        )
    except (InvalidTokenError, AuthorisationError) as error:
        handle_authorisation_error(user_token, error)

def process_dataset_metadata(layer, domain, dataset) -> DatasetMetadata:
    if all([layer, domain, dataset]):
        return DatasetMetadata(layer, domain, dataset)
    else:
        AppLogger.info(
            "Cannot retrieve dataset metadata if all values are not provided"
        )
        raise AuthenticationError("You are not authorised to perform this action")

def get_subject_id(request: Request):
    client_token = get_client_token(request)
    user_token = get_user_token(request)
    token = client_token if client_token else user_token
    token = parse_token(token)
    return token.subject

def get_client_token(request: Request) -> Optional[str]:
    authorization: str = request.headers.get("Authorization")
    scheme, jwt_token = get_authorization_scheme_param(authorization)
    if not authorization or scheme.lower() != "bearer":
        return None
    return jwt_token

def get_user_token(request: Request) -> Optional[str]:
    return request.cookies.get(RAPID_ACCESS_TOKEN, None)

def check_credentials_availability(
    browser_request: bool,
    client_token: Optional[str],
    user_token: Optional[str],
) -> None:
    if not have_credentials(client_token, user_token):
        if browser_request:
            AppLogger.info("Cannot retrieve user credentials if no user token provided")
            raise UserCredentialsUnavailableError()
        else:
            AppLogger.info(
                "Cannot retrieve client credentials if no client token provided"
            )
            raise AuthenticationError(
                message="You are not authorised to perform this action"
            )

def have_credentials(client_token: str, user_token: str) -> bool:
    return bool(user_token or client_token)

def check_permissions(subject_id: str, endpoint_scopes: List[Action]):
    """
    Ensure that at least one of the endpoint_scopes exists in the users permissions.
    Raising an AuthorisationError if not.
    """
    if not endpoint_scopes:
        return True

    permissions = permission_service.get_subject_permissions(subject_id)
    if any([permission.type in endpoint_scopes for permission in permissions]):
        return True

    raise AuthorisationError(
        f"User {subject_id} does not have permissions that grant access to the endpoint scopes {endpoint_scopes}"
    )


# Databricks notebook source
# MAGIC %md
# MAGIC ## Load parameters form MLFlow Registry

# COMMAND ----------

import mlflow
from mlflow.tracking import MlflowClient

# set model & registry params
model_name = 'Airbnb_Model'
cmr_uri = 'databricks://mlops_webinar:CMR'
dev_uri = 'databricks://mlops_webinar:dev'

# fetch model from registry
client = MlflowClient(tracking_uri=dev_uri, registry_uri=cmr_uri)
reg_mdl = client.get_registered_model(model_name)

# COMMAND ----------

# find latest production model
prod_mdl = None
for mdl in reg_mdl.latest_versions:
  print(f"{mdl.name} (v. {mdl.version}) in {mdl.current_stage} is {mdl.status}")
  if mdl.current_stage == "Production":
    prod_mdl = mdl

if prod_mdl is None:
  raise RuntimeError("No Production Model Found")

# COMMAND ----------

# get parameters for last production model
run = client.get_run(prod_mdl.run_id)

# extract logged parameters
current_params = run.data.params

print("Current Parameters:")
for param in current_params:
  print(f"{param} = {current_params[param]}")

# COMMAND ----------

# MAGIC %md
# MAGIC ## Load Data

# COMMAND ----------

from sklearn.model_selection import train_test_split

target_col = "price"
all_data_df = spark.table("airbnb.features").drop("id").toPandas()
split_X = all_data_df.drop([target_col], axis=1)
split_y = all_data_df[target_col]

# Split out train data
X_train, split_X_rem, y_train, split_y_rem = train_test_split(split_X, split_y, train_size=0.6, random_state=600418151)

# Split remaining data equally for validation and test
X_val, X_test, y_val, y_test = train_test_split(split_X_rem, split_y_rem, test_size=0.5, random_state=600418151)

# COMMAND ----------

# MAGIC %md
# MAGIC ## Set up Model

# COMMAND ----------

import json
from databricks.automl_runtime.sklearn.column_selector import ColumnSelector

select_cols = json.loads(current_params['column_selector__cols'].replace("'", '"'))
col_selector = ColumnSelector(select_cols)

# COMMAND ----------

from sklearn.compose import ColumnTransformer
from sklearn.impute import SimpleImputer
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import FunctionTransformer, OneHotEncoder

numerical_pipeline = Pipeline(steps=[
    ("converter", FunctionTransformer(lambda df: df.apply(pd.to_numeric, errors="coerce"))),
    ("imputer", SimpleImputer(strategy=current_params['preprocessor__numerical__imputer__strategy']))
])
one_hot_encoder = OneHotEncoder(handle_unknown=current_params['preprocessor__onehot__handle_unknown'])

transformers = []
transformers.append(("numerical", numerical_pipeline, ["availability_365", "calculated_host_listings_count", "minimum_nights", "neighbourhood_idx", "number_of_reviews", "reviews_per_month"]))
transformers.append(("onehot", one_hot_encoder, ["room_type_idx"]))

preprocessor = ColumnTransformer(transformers,
                                 remainder=current_params['preprocessor__remainder'],
                                 sparse_threshold=int(current_params['preprocessor__sparse_threshold']))

# COMMAND ----------

from sklearn.preprocessing import StandardScaler

standardizer = StandardScaler()

# COMMAND ----------

import mlflow
import sklearn
from sklearn import set_config
from sklearn.pipeline import Pipeline
from sklearn.ensemble import RandomForestRegressor

set_config(display='diagram')

skrf_regressor = RandomForestRegressor(
  bootstrap=bool(current_params['regressor__bootstrap']),
  criterion=current_params['regressor__criterion'],
  max_depth=int(current_params['regressor__max_depth']),
  max_features=float(current_params['regressor__max_features']),
  min_samples_leaf=float(current_params['regressor__min_samples_leaf']),
  min_samples_split=float(current_params['regressor__min_samples_split']),
  n_estimators=int(current_params['regressor__n_estimators']),
  random_state=int(current_params['regressor__random_state']),
)

model = Pipeline([
    ("column_selector", col_selector),
    ("preprocessor", preprocessor),
    ("standardizer", standardizer),
    ("regressor", skrf_regressor),
])

model

# COMMAND ----------

# MAGIC %md
# MAGIC ## Train with discovered parameters

# COMMAND ----------

import pandas as pd
import mlflow.sklearn

# Enable automatic logging of input samples, metrics, parameters, and models
mlflow.sklearn.autolog(log_input_examples=True, silent=True)

with mlflow.start_run(run_name="random_forest_regressor") as mlflow_run:
    model.fit(X_train, y_train)

    # Training metrics are logged by MLflow autologging
    # Log metrics for the validation set
    skrf_val_metrics = mlflow.sklearn.eval_and_log_metrics(model, X_val, y_val, prefix="val_")

    # Log metrics for the test set
    skrf_test_metrics = mlflow.sklearn.eval_and_log_metrics(model, X_test, y_test, prefix="test_")

    # Display the logged metrics
    skrf_val_metrics = {k.replace("val_", ""): v for k, v in skrf_val_metrics.items()}
    skrf_test_metrics = {k.replace("test_", ""): v for k, v in skrf_test_metrics.items()}
    display(pd.DataFrame([skrf_val_metrics, skrf_test_metrics], index=["validation", "test"]))

# COMMAND ----------

# MAGIC %md
# MAGIC ## Automatically push model to registry

# COMMAND ----------

import time
from mlflow.entities.model_registry.model_version_status import ModelVersionStatus

artifact_name = "model"
artifact_uri = f"runs:/{mlflow_run.info.run_id}/{artifact_name}"
mlflow.set_registry_uri(cmr_uri)
registered_mdl = mlflow.register_model(artifact_uri, model_name)

# Wait until the model is ready
def wait_until_ready(model_name, model_version):
  client = MlflowClient(registry_uri=cmr_uri)
  for _ in range(20):
    model_version_details = client.get_model_version(
      name=model_name,
      version=model_version,
    )
    status = ModelVersionStatus.from_string(model_version_details.status)
    print("Model status: %s" % ModelVersionStatus.to_string(status))
    if status == ModelVersionStatus.READY:
      break
    time.sleep(5)

wait_until_ready(registered_mdl.name, registered_mdl.version)

# COMMAND ----------

Live Webinars/MLOps Virtual Event - Standardizing MLOps at Scale/Python/09. Automatic Model Retraining.py


# -*- coding: utf-8 -*-
"""
Created on Tue May  7 19:54:42 2019

@author: tt14r
"""
import operator
import torch
from torch.autograd import Variable, Function
import knn_pytorch
import numpy as np

"""
train and test are a list of lists, where each list is a series of float coordinates
i.e. x,y,z,etc. followed by a class (string)
"""
def knn(train,test,k):
    test_y=[]
    Train_class=[]
    Train_coords=[]
    Test_class=[]
    Test_coords=[]
    #take in train data into list of classes
    for i in range(len(train)):
        classification = train[i][len(train[i])-1]
        coords = train[i][:len(train[i])-1]
        Train_class += [classification]
        Train_coords += [coords]
    for i in range(len(test)):
        classification = train[i][len(train[i])-1]
        coords = train[i][:len(train[i])-1]
        Test_class += [classification]
        Test_coords += [coords]
    ref = Train_coords.float().cuda()
    query = Test_coords.float().cuda()

    inds = torch.empty(k, query.shape[1]).long().cuda()
    dists = torch.empty(k, query.shape[1]).float().cuda()

    knn_pytorch.knn(ref, query, inds, dists)

    for y in range(np.shape(inds,0)):
        classVotes = {}
        for x in range(np.shape(inds,1)):
            i=inds[y][x]
            response = Train_class[i]
            if response in classVotes:
                classVotes[response] += 1
            else:
                classVotes[response] = 1
        sortedVotes = sorted(classVotes.iteritems(), key=operator.itemgetter(1), reverse=True)
        test_y[y]=sortedVotes[0][0]
    return test_y


from django.utils import timezone
import braintree

from django.conf import settings
from django.contrib.auth.mixins import LoginRequiredMixin
from django.contrib import messages
from django.http import Http404, JsonResponse
from django.shortcuts import render
from django.views import View
from django.views.generic import ListView

from payments.models import Premium, Receipt

class SelectPremiumView(ListView):
    model = Premium
    context_object_name = 'premiums'
    template_name = 'payments/select_premium.html'

class PaymentsView(LoginRequiredMixin, View):
    template_name = 'payments/payments.html'

    def get_context_data(self, **kwargs):
        try:
            braintree_client_token = braintree.ClientToken.generate({'cutomer_id': self.request.user.id})
        except:
            braintree_client_token = braintree.ClientToken.generate({})

        try:
            premium = Premium.objects.get(pk=kwargs['premium_pk'])
        except:
            raise Http404('Данный премиум не найден!')

        context = {'braintree_client_token': braintree_client_token,
                   'premium': premium}
        context.update(kwargs)
        return context

    def get(self, request, **kwargs):
        if settings.BRAINTREE_PRODUCTION:
            braintree_env = braintree.Environment.Production
        else:
            braintree_env = braintree.Environment.Sandbox

        braintree.Configuration.configure(environment=braintree_env,
                                          merchant_id=settings.BRAINTREE_MERCHANT_ID,
                                          public_key=settings.BRAINTREE_PUBLIC_KEY,
                                          private_key=settings.BRAINTREE_PRIVATE_KEY)

        return render(request,
                      self.template_name,
                      self.get_context_data(premium_pk=kwargs['premium_pk']))

    def post(self, request, **kwargs):
        if 'done' in request.POST:
            if request.POST['done']:
                messages.add_message(request, messages.SUCCESS, 'Оплата прошла успешно!')
                premium = Premium.objects.get(pk=kwargs['premium_pk'])
                receipt = Receipt(premium=premium,
                                  price=premium.price,
                                  discount=premium.discount,
                                  owner=request.user,
                                  premium_end_date=timezone.timedelta(days=30 * premium.months) + timezone.now())
                receipt.save()
            else:
                messages.add_message(request, messages.ERROR, 'При оплате возникли ошибки, попробуйте ещё раз!')
        else:
            payment_method_nonce = request.POST['payment_method_nonce']
            customer = braintree.Customer.create({'first_name': None,
                                                  'last_name': request.user.username,
                                                  'email': request.user.email})
            context = self.get_context_data(premium_pk=kwargs['premium_pk'])
            result = braintree.Transaction.sale({
                'amount': Premium.objects.get(premium_pk=kwargs['premium_pk']).get_result_price(),
                'payment_method_nonce': payment_method_nonce,
                'merchant_account_id': settings.BRAINTREE_MERCHANT_ACCOUNT_ID,
                'options': {
                    'submit_for_settlement': True,
                },
            })
        return JsonResponse(dict())


import os
os.environ["XLA_PYTHON_CLIENT_PREALLOCATE"]="false"
os.environ["XLA_PYTHON_CLIENT_ALLOCATOR"]="platform"
import jax
jax.config.update("jax_enable_x64", True)
import jax.numpy as jnp
import numpy as np
from jax.scipy.special import digamma, gammaln
from .utils import get_s_cp, get_s_cp_missing, get_s_cp_notmissing, lbeta_matrix, lbeta_scalar, rmse, mape

def vb(X: jnp.ndarray,
           R: int,
           a: jnp.ndarray,
           b: jnp.ndarray = jnp.nan,
           num_epochs: int = 1,
           seed: int = 0,
           dir_smooth: jnp.ndarray = 0.1,
           print_freq=0
           ):
   if jnp.isnan(b):
      b = a / jnp.nansum(X)
   I = X.shape
   d = len(I)
   alpha_I = jnp.array([a/(R*I_n) for I_n in I])
   alpha_R = a / R

   S_sum = jnp.nansum(X)

   keys = jax.random.split(jax.random.PRNGKey(seed), d + 2)
   lkeys, key, rkey = keys[:-2], keys[-2], keys[-1]

   log_lam = digamma(S_sum + a) - jnp.log(b + 1)
   log_theta_R = jnp.log(jax.random.dirichlet(rkey, jnp.ones(R) * alpha_R + dir_smooth))
   log_theta_I = [jnp.log(jax.random.dirichlet(lkeys[n], jnp.ones(I_n) * alpha_I[n] + dir_smooth, [R])).T for n, I_n in enumerate(I)]
   log_C = a*jnp.log(b) - (a + S_sum)*jnp.log(b + 1) + gammaln(a + S_sum) - gammaln(a) - gammaln(X + 1).sum()
   ELBO = (jnp.ones(num_epochs) * log_C).tolist()

   all_idx = jnp.array(jnp.where(X!=0)).T
   for i in range(num_epochs):
      s_all, sp_all = get_s_cp(all_idx, log_theta_R, log_theta_I, X)
      S_I = [jax.ops.segment_sum(s_all, all_idx[:, n]).T for n in range(d)]
      S_R = s_all.sum(0)
      ELBO[i] -= sp_all.sum()

      for n, I_n in enumerate(I):
         log_theta_I[n] = digamma(S_I[n] + alpha_I[n]).T - digamma(S_R + alpha_R).T
         ELBO[i] += jnp.apply_along_axis(lbeta_matrix, -1, alpha_I[n] + S_I[n]).sum() - R * lbeta_scalar(alpha_I[n],I_n)
      log_theta_R = digamma(S_R + alpha_R) - digamma(S_sum + R * alpha_R)
      ELBO[i] += lbeta_matrix(alpha_R + S_R) - lbeta_scalar(alpha_R,R)
      if i and print_freq:
         if (i+1) % print_freq == 0:
            print(f"Epoch: {i+1}, ELBO: {ELBO[i]: .0f}")
   return jnp.array(ELBO), [log_lam, log_theta_R, *log_theta_I]

def vb_batch(X: jnp.ndarray,
           R: int,
           a: jnp.ndarray,
           b: jnp.ndarray = jnp.nan,
           num_epochs: int = 1,
           seed: int = 0,
           dir_smooth: jnp.ndarray = 0.1,
           num_batches=0,
           print_freq=0):
   if jnp.isnan(b):
      b = a / jnp.nansum(X)
   I = X.shape
   d = len(I)
   alpha_I = jnp.array([a/(R*I_n) for I_n in I])
   alpha_R = a / R

   S_sum = jnp.nansum(X)

   keys = jax.random.split(jax.random.PRNGKey(seed), d + 2)
   lkeys, key, rkey = keys[:-2], keys[-2], keys[-1]

   log_lam = digamma(S_sum + a) - jnp.log(b + 1)
   log_theta_R = jnp.log(jax.random.dirichlet(rkey, jnp.ones(R) * alpha_R + dir_smooth))
   log_theta_I = [jnp.log(jax.random.dirichlet(lkeys[n], jnp.ones(I_n) * alpha_I[n] + dir_smooth, [R])).T for n, I_n in enumerate(I)]
   log_C = a*jnp.log(b) - (a + S_sum)*jnp.log(b + 1) + gammaln(a + S_sum) - gammaln(a) - gammaln(X + 1).sum()
   ELBO = (jnp.ones(num_epochs) * log_C).tolist()

   all_idx = jnp.array(jnp.where(X!=0)).T
   for i in range(num_epochs):
      if num_batches:
         S_R = jnp.zeros(R)
         S_I = [jnp.zeros_like(log_theta_I_n).T for log_theta_I_n in log_theta_I]
         sp_all = 0
         b = int((len(all_idx) / num_batches)+1)
         for j in range(num_batches):
            s_all, sp_all_  = get_s_cp(all_idx[j*b:(j+1)*b], log_theta_R, log_theta_I, X)
            S_R += s_all.sum(0)
            sp_all += sp_all_.sum()
            S_I = [S_I[n] + jax.ops.segment_sum(s_all, all_idx[j*b:(j+1)*b, n], num_segments=I[n]).T for n in range(d)]
      else:
         s_all, sp_all = get_s_cp(all_idx, log_theta_R, log_theta_I, X)
         S_I = [jax.ops.segment_sum(s_all, all_idx[:, n]).T for n in range(d)]
         S_R = s_all.sum(0)

      ELBO[i] -= sp_all.sum()

      for n, I_n in enumerate(I):
         log_theta_I[n] = digamma(S_I[n] + alpha_I[n]).T - digamma(S_R + alpha_R).T
         ELBO[i] += jnp.apply_along_axis(lbeta_matrix, -1, alpha_I[n] + S_I[n]).sum() - R * lbeta_scalar(alpha_I[n],I_n)
      log_theta_R = digamma(S_R + alpha_R) - digamma(S_sum + R * alpha_R)
      ELBO[i] += lbeta_matrix(alpha_R + S_R) - lbeta_scalar(alpha_R,R),
      if i and print_freq:
         if (i+1) % print_freq == 0:
            print(f"Epoch: {i+1}, ELBO: {ELBO[i]: .0f}")
   return jnp.array(ELBO), [log_lam, log_theta_R, *log_theta_I]

def vb_missing(X: jnp.ndarray,
           R: int,
           a: jnp.ndarray,
           b: jnp.ndarray = jnp.nan,
           num_epochs: int = 1,
           seed: int = 0,
           print_freq=0,
           es_freq=100,
           es_epochs=500,
           X_gt=None,
           trend=None,
           es_idx=None,
           einsum=None,
           b_correction=False,
           dir_smooth: jnp.ndarray = 0.1,
           ):
   jax.config.update("jax_enable_x64", False)
   if jnp.isnan(b):
      b = a / (jnp.nansum(X)/(1-jnp.isnan(X).mean())) if b_correction else a / jnp.nansum(X)
   I = X.shape
   d = len(I)

   alpha_R = a / jnp.prod(R)
   alpha_I = jnp.array([(a/(R*I_n)) for n, I_n in enumerate(I)])
   Alpha_R = jnp.ones(R) * alpha_R
   Alpha_I = [jnp.ones((R, I[n])) * alpha_I_n for n, alpha_I_n in enumerate(alpha_I)]

   keys = jax.random.split(jax.random.PRNGKey(seed), d + 3)
   lkeys, key, rkey, rrkey = keys[:-3], keys[-3], keys[-2], keys[-1]
   S_sum = jnp.nansum(X)

   log_lam = jnp.log(jax.random.gamma(rrkey, a)/b) #digamma(S_sum + a) - jnp.log(b + 1)
   log_theta_R = jnp.log(jax.random.dirichlet(rkey, jnp.ones(R) * alpha_R + dir_smooth))
   log_theta_I = [jnp.log(jax.random.dirichlet(lkeys[n], jnp.ones(I_n) * alpha_I[n] + dir_smooth, [R])).T for n, I_n in enumerate(I)]

   ELBO = jnp.zeros(num_epochs).tolist()
   all_idx_missing = jnp.array(jnp.where(jnp.isnan(X))).T
   all_idx_nonmissing = jnp.array(jnp.where(~jnp.isnan(X))).T
   all_idx = jnp.vstack((all_idx_missing, all_idx_nonmissing))
   best_mape, best_rmse, best_epoch, best_params, es_evals = 1e6, 1e6, 0, [], []

   for i in range(num_epochs):

      X_idx_m, s_all_m = get_s_cp_missing(all_idx_missing, log_lam, log_theta_R, log_theta_I, X)
      s_all, X_idx_elbo_nm = get_s_cp_notmissing(all_idx_nonmissing, log_lam, log_theta_R, log_theta_I, X)
      s_all = jnp.vstack((s_all_m, s_all))
      del s_all_m
      S_I = [jax.ops.segment_sum(s_all, all_idx[:, n]).T for n in range(d)]
      S_R = s_all.sum(0)
      S_sum = jnp.nansum(X) + X_idx_m.sum()
      ELBO[i] += X_idx_m.sum() + X_idx_elbo_nm.sum()
      ELBO[i] += a * jnp.log(b) - (S_sum + a) * jnp.log(b+1)
      for n in range(d):
         ELBO[i] +=  gammaln(Alpha_I[n] + S_I[n]).sum() - gammaln(Alpha_I[n]).sum()
      ELBO[i] += 2*gammaln(Alpha_R).sum() - 2*gammaln(Alpha_R + S_R).sum()
      ELBO[i] -= S_sum * log_lam + (S_R * log_theta_R).sum()
      for n in range(d):
         ELBO[i] -= (S_I[n] * log_theta_I[n].T).sum()

      log_lam = digamma(S_sum+a) - jnp.log(b+1.0)
      log_theta_R = digamma(S_R+Alpha_R) - digamma(S_sum+a)
      for n in range(d):
         log_theta_I[n] = digamma(S_I[n] + Alpha_I[n]).T - digamma(S_R + Alpha_R).T

      if i and es_freq and (i%es_freq==0):
         X_hat = np.einsum(einsum, *[jnp.exp(par) for par in [log_theta_R, *log_theta_I]])  * jnp.exp(log_lam)
         cur_rmse = rmse((X_hat + trend)[es_idx], X_gt[es_idx])
         cur_mape = mape((X_hat + trend)[es_idx], X_gt[es_idx])
         es_evals.append((i, cur_mape, cur_rmse, ELBO[i-1]))
         if i % print_freq == 0:
            print(f"Epoch: {i}, ELBO: {ELBO[i]: .0f}, Val MAPE: {es_evals[-1][1]: .6f}, Val RMSE: {es_evals[-1][2]: .6f}")
         if (cur_rmse < best_rmse) and (cur_mape < best_mape):
            best_rmse = cur_rmse
            best_mape = cur_mape
            best_epoch = i
            best_params = [arr.copy() for arr in [log_lam, log_theta_R, *log_theta_I]]
         else:
            if i - best_epoch >= es_epochs:
               print(f"Early stopping at epoch {i}, reverting to the best parameters obtained at epoch {best_epoch}.")
               log_lam, log_theta_R, *log_theta_I = best_params
               ELBO = ELBO[:best_epoch+1]
               break
   return jnp.array(ELBO), es_evals, [log_lam, log_theta_R, *log_theta_I]


from typing import Dict, Tuple, Set, Union, Optional
import qubovert as qv
from icontract import require
from qcware.types.optimization.problem_spec.utils import (
    polynomial_validation as validator,
)
from qcware.types.optimization.variable_types import Domain

class PolynomialObjective:
    """Integer-valued polynomial of binary variables with int coefficients.

    Objects of this class specify polynomials of some number of
    binary variables with integer coefficients. "Binary variables" can either
    mean variables taking on the values 0 and 1 or variables taking on the
    values 1 and -1. The former a referred to as boolean variables and the
    latter are referred to as spin variables

    Objects of this class are meant to be treated as objective functions
    for optimization. They do not know about constraints. Constraints
    can be specified separately using a Constraints object.

    Polynomials are specified with a dict that specifies the coefficients
    for the polynomial. For example, suppose that we are interested
    in the polynomial of three boolean variables defined by::

        p(x, y, z) = 12 x - 2 y z + x y z - 50

    The three variables should be associated with the integers 0, 1, and 2.
    We choose the association x ~ 0, y ~ 1, and z ~ 2.

    There are four terms in p. Consider the term -2yz. We can specify this
    term by matching the tuple (1, 2), which represents yz, with the
    coefficient -2. This can be encoded with an entry in a dict (1, 2): -2.
    Overall, p can be defined by::

        {
            (): -50,
            (0,): 12,
            (1, 2): -2,
            (0, 1, 2): -50
        }

    The number of variables must be specified explicitly, even if it seems
    obvious how many variables there are. The reason for this is to allow
    for the possibility that there are more variables than appear explicitly
    in the polynomial. For example, the polynomial p(a, b) = 12 b might be
    mistaken for q(b) = 12 b.

    Attributes:
        polynomial: The polynomial is specified by a dict as described above.
            We only use tuples of int as keys and the range of ints must
            be from 0 to `num_variables - 1`. Values for the dict must be
            type int. This is because we are only treating integer-coefficient
            polynomials.

        variables: Set of ints representing the variables.

        num_variables: The number of variables for the polynomial. This number
            can be larger than the actual number of variables that appear
            in `polynomial` but it cannot be smaller.

        degree: The degree of the polynomial. This is not the mathematically
            correct degree. It is instead the length of the longest key in
            the polynomial dict or -inf in the case when the dict is {}.
            For example, the boolean PolynomialObjective {(1,): 12, (0, 1): 0}
            has mathematical degree 1 but the attribute `degree` is 2.

        domain: Specifies if variables take on boolean (0, 1) or spin (1, -1)
            values.
    """

    polynomial: Dict[Tuple[int, ...], int]
    variables: Set[int]
    num_variables: int
    degree: Union[int, float]
    domain: Domain
    variable_name_mapping: Dict[int, str]

    #    @require(lambda polynomial: len(polynomial) > 0)
    def __init__(
        self,
        polynomial: Dict[Tuple[int, ...], int],
        num_variables: int,
        domain: Union[Domain, str] = Domain.BOOLEAN,
        variable_name_mapping: Optional[Dict[int, str]] = None,
        validate_types: bool = True,
    ):
        # TODO: introduce mapping structure to guarantee consistent variable
        #   reduction. This is also critical because we don't want
        #   reduction to annoyingly re-order variables when it's not necessary.
        self.num_variables = num_variables
        self.domain = Domain(domain.lower())
        polynomial = simplify_polynomial(polynomial, self.domain)

        parsed_polynomial = validator.polynomial_validation(
            polynomial=polynomial,
            num_variables=num_variables,
            validate_types=validate_types,
        )

        self.polynomial = parsed_polynomial.poly
        self.active_variables = parsed_polynomial.variables
        self.num_active_variables = len(self.active_variables)
        self.degree = parsed_polynomial.deg
        if self.degree < 0:
            self.degree = float("-inf")

        self.variable_name_mapping = variable_name_mapping

        def default_symbol(variable_type: Domain):
            if variable_type is Domain.BOOLEAN:
                return "x"
            elif variable_type is Domain.SPIN:
                return "z"

        if variable_name_mapping is None:
            symbol = default_symbol(self.domain)
            self.variable_name_mapping = {
                i: f"{symbol}_{i}" for i in range(num_variables)
            }

        # We use qubovert to compute function values. Since we don't want
        # to reconstruct a qubovert object every time we use it, we keep
        # these private attributes around to use as a cache.
        self._qv_polynomial = None
        self._qv_polynomial_named = None

    def keys(self):
        return self.polynomial.keys()

    def values(self):
        return self.polynomial.values()

    def items(self):
        return self.polynomial.items()

    def __iter__(self):
        return self.polynomial.__iter__()

    def __repr__(self):
        out = "PolynomialObjective(\n"
        out += "    polynomial=" + self.polynomial.__repr__() + "\n"
        out += "    num_variables=" + str(self.num_variables) + "\n"
        out += "    domain=" + repr(self.domain) + "\n"
        out += "    variable_mapping=" + repr(self.variable_name_mapping)
        out += "\n)"
        return out

    def __str__(self):
        return self.pretty_str()

    def __getitem__(self, item):
        return self.polynomial.__getitem__(item)

    def clone(self):
        """Make a copy of this PolynomialObjective."""
        return PolynomialObjective(
            polynomial=self.polynomial,
            num_variables=self.num_variables,
            domain=self.domain,
            variable_name_mapping=self.variable_name_mapping.copy(),
            validate_types=False,
        )

    def qubovert(self, use_variable_names: bool = False):
        """Get a qubovert model describing this polynomial.

        This method will return a qubovert PUBO or PUSO depending on
        the domain of the variables.

        This method creates a cached qubovert object once it is called.
        TODO: This fact makes it particularly important that
            PolynomialObjective is immutable. We should try to enforce this.

        Args:
            use_variable_names: When True, the variables in the qubovert
                object will use the same string names as appear in
                the attribute variable_name_mapping.
        """
        if not use_variable_names:
            if self._qv_polynomial is None:
                self._qv_polynomial = self._qubovert(use_variable_names=False)
            return self._qv_polynomial

        else:
            if self._qv_polynomial_named is None:
                self._qv_polynomial_named = self._qubovert(use_variable_names=True)
            return self._qv_polynomial_named

    def _qubovert(self, use_variable_names: bool = False):
        """Get a qubovert model describing this polynomial.

        This method will return a qubovert PUBO or PUSO depending on
        the domain of the variables.

        Args:
            use_variable_names: When True, the variables in the qubovert
                object will use the same string names as appear in
                the attribute variable_name_mapping.
        """
        if use_variable_names:
            polynomial = dict()
            for k, v in self.polynomial.items():
                new_key = tuple(self.variable_name_mapping[i] for i in k)
                polynomial[new_key] = v
        else:
            polynomial = self.polynomial

        if self.domain is Domain.BOOLEAN:
            model = qv.PUBO(polynomial)
        elif self.domain is Domain.SPIN:
            model = qv.PUSO(polynomial)
        else:
            raise RuntimeError("Domain is not valid.")
        if use_variable_names:
            model.set_reverse_mapping(self.variable_name_mapping)
        return model

    def qubovert_boolean(self):
        """Get a boolean qubovert model equivalent to this polynomial.

        This produces a PUBOMatrix which is qubovert's enumerated form of a
        PUBO (polynomial unconstrained boolean optimization). This
        transformation may change the variable ordering and for that
        reason we also return a mapping that can be used to recover
        the original variable identifiers.
        """
        qv_model = self.qubovert(use_variable_names=False)
        return {"polynomial": qv_model.to_pubo(), "mapping": qv_model.mapping}

    def qubovert_spin(self):
        """Get a spin qubovert model equivalent to this polynomial.

        This produces a PUSOMatrix which is qubovert's enumerated form of a
        PUSO (polynomial unconstrained spin optimization). This transformation
        may change the variable ordering and for that
        reason we also return a mapping that can be used to recover
        the original variable identifiers.
        """
        qv_model = self.qubovert(use_variable_names=False)
        return {"polynomial": qv_model.to_puso(), "mapping": qv_model.mapping}

    def reduce_variables(self):
        """Return a PolynomialObjective with trivial variables removed.

        As an example, suppose that we have a PolynomialObjective defined by

        PolynomialObjective(
            polynomial={(1,): 1},
            num_variables=3
        )

        which essentially means the polynomial p defined by p(x, y, z) = y.
        This has two variables that p doesn't actually depend on. In this
        case, the method `reduce_variables` will return a polynomial of one
        variable along with a mapping to identify variables appropriately.
        Specifically, in this case the return would be

        {
            'polynomial': PolynomialObjective(polynomial={(0,): 1},
                                              num_variables=1
                                              )
            'mapping': {1: 0}
        }.
        """
        qv_form = self.qubovert(use_variable_names=False)
        reduced_qv = qv_form.to_enumerated()
        mapping = qv_form.mapping
        num_vars = reduced_qv.num_binary_variables

        return {
            "polynomial": PolynomialObjective(
                polynomial=reduced_qv,
                num_variables=num_vars,
                domain=self.domain,
                variable_name_mapping=None,
                validate_types=False,
            ),
            "mapping": mapping,
        }

    def compute_value(self, variable_values: dict, use_variable_names: bool = False):
        """Compute the value of this polynomial at a specified input."""
        qv_polynomial = self.qubovert(use_variable_names=use_variable_names)
        return qv_polynomial.value(variable_values)

    @classmethod
    def __get_validators__(cls):
        yield cls.validate_type

    @classmethod
    def validate_type(cls, v):
        """
        This validator only confirms that we are dealing with an object
        of the expected type. This does not validate internal consistency
        of attributes for the Objective function because that is done
        by __init__.

        """
        if not isinstance(v, PolynomialObjective):
            raise TypeError(
                "Expected an object of class PolynomialObjective, but found "
                f"{type(v)}."
            )
        return v

    def pretty_str(self, include_domain: bool = True):
        """Make a string that presents the polynomial algebraically.

        The names of variables can be controlled with the attribute
        variable_name_mapping.

        Adapted from qubovert--thanks to Joseph T. Iosue.
        """
        if self.domain is Domain.BOOLEAN:
            domain_label = f"({self.num_variables} boolean variables)"
        elif self.domain is Domain.SPIN:
            domain_label = f"({self.num_variables} spin variables)"
        else:
            raise RuntimeError("Variable domain type seems to be invalid.")

        if self.polynomial == {}:
            if include_domain:
                return "0  " + domain_label
            else:
                return "0"

        res = ""
        first = True
        for term, coef in self.items():
            if coef >= 0 and (coef != 1 or not term):
                res += f"{coef} "
            elif coef < 0:
                if coef == -1:
                    if first:
                        res += "-" if term else "-1 "
                    else:
                        res = res[:-2] + ("- " if term else "- 1 ")
                else:
                    if first:
                        res += f"{coef} "
                    else:
                        res = res[:-2] + f"- {abs(coef)} "

            for x in term:

                res += self.variable_name_mapping[x] + " "
            res += "+ "
            first = False
        res = res[:-2].strip()

        if include_domain:
            return res + "  " + domain_label
        else:
            return res

    def dict(self):
        return {
            "polynomial": self.polynomial,
            "num_variables": self.num_variables,
            "domain": self.domain.lower(),
            "variable_name_mapping": self.variable_name_mapping,
        }

def simplify_polynomial(polynomial: dict, domain: Domain) -> dict:
    """Simplify given polynomial dict."""
    domain = Domain(domain.lower())
    if domain is Domain.BOOLEAN:
        simplified_qv = qv.utils.PUBOMatrix(polynomial)
    elif domain is Domain.SPIN:
        simplified_qv = qv.utils.PUSOMatrix(polynomial)
    else:
        raise TypeError(f"Expected a Domain but found {type(domain)}.")

    return dict(simplified_qv)


from django.shortcuts import render, redirect, get_object_or_404
from .models import Blog
from .forms import BlogForm

def list_view(request):
    context = {
        "blogs": Blog.objects.all(),
    }
    return render(request, "list.html", context)

def create_view(request):

    form = BlogForm()

    if request.method == "POST":
        form = BlogForm(request.POST)

        if form.is_valid():
            form.save()
            return redirect("blog:list")

    context = {
        "form": form,
    }
    return render(request, "create.html", context)

def update_blog_view(request, id):
    blog = get_object_or_404(Blog, id=id)
    form = BlogForm(instance=blog)

    if request.method == "POST":
        form = BlogForm(request.POST, instance=blog)
        if form.is_valid():
            form.save()

            return redirect("blog:list")
    context = {
        "form": form
    }
    return render(request, "update.html", context)

def detail_view(request, id):
    blogs = Blog.objects.get(id=id)
    context = {
        "id": id,
        "blogs": blogs
    }
    return render(request, "detail.html", context)


# Program to find the max depth of the tree
# =======================================================
# Technique : Iterative BFS using queue

from collections import deque
from itertools import count
from btree import TreeNode

def maxDepth(root):
    if root is None:
        return 0
    myqueue = deque([root])
    level = 0

    while myqueue:
        for i in range(len(myqueue)):
            node = myqueue.popleft()
            if node.left:
                myqueue.append(node.left)

            if node.right:
                myqueue.append(node.right)
        level += 1

    return level

def main():
    root = TreeNode(1)
    root.left = TreeNode(2)
    root.right = TreeNode(6)
    root.left.left = TreeNode(4)
    root.left.right = TreeNode(9)
    root.left.left.right = TreeNode(11)

    print("The size of the tree is {}".format(maxDepth(root)))

if __name__ == "__main__":
    main()


import requests
import logging
from bs4 import BeautifulSoup
import re
import sys
import os
from lxml import etree

logging.basicConfig(
    format='%(asctime)s %(levelname)-8s %(message)s',
    level=logging.INFO,
    # level=logging.INFO,
    datefmt='%Y-%m-%d %H:%M:%S'
)
logger = logging.getLogger(__name__)

def get_via_clerk():
    URL = 'http://clerk.house.gov/member_info/ttd.aspx'
    html_doc = requests.get(URL).text

    soup = BeautifulSoup(html_doc, 'html.parser')

    table = soup.find("table")
    yml_data = ''
    for row in table.findAll("tr"):
        # logger.debug(row)
        tds = row.findAll("td")
        if len(tds) != 0:
            if len(row.findAll("strong")) == 1 and len(row.findAll("em")) == 1:
                party = 'Commissioner/Delegate'
            elif len(row.findAll("em")) == 1:
                party = 'D'
            else:
                party = 'R'
            name = tds[0].text
            state = tds[1].text
            district = tds[2].text

            # logger.debug(district)
            if district not in ['At Large', 'Delegate', 'Resident Commissioner']:
                district = re.sub('[^0-9]','', tds[2].text)

            phone = '(202)' + tds[3].text
            room_num = tds[4].text
            if len(room_num) == 3:
                room = '{} Cannon'.format(room_num)
            elif len(room_num) == 4 and room_num[0] == '1':
                room = '{} Longworth'.format(room_num)
            elif len(room_num) == 4 and room_num[0] == '2':
                room = '{} Rayburn'.format(room_num)
            else:
                room = 'NO ROOM'

            info = """
      - name: {}
        state: {}
        district: {}
        party: {}
        room: {}
        phone: {}
        committee: """.format(name, state, district, party, room, phone)
            yml_data += info
            # logger.debug('{} ems = {}'.format(name, len(ems)))
            # logger.debug('{}\t{}\t{}\t{}'.format(name, state, district, phone))
    # logger.debug(yml_data)
    with open(os.path.join(sys.path[0], "hor.yml"), 'w', encoding='utf-8') as f:
        print(yml_data, file=f)

    # with open('house.yml', 'w') as f:
    #     f.write('Hello\n')
    logger.info("saved!")

def get_via_house_gov():
    URL = 'https://www.house.gov/representatives'
    html_doc = requests.get(URL).text

    soup = BeautifulSoup(html_doc, 'html.parser')
    tables = soup.findAll("table")

    rows = []
    for table in tables[56:81]:
        for row in table.findAll("tr"):
            # logger.debug(row.prettify())
            tds = row.findAll("td")
            if len(tds) != 0:
                rows.append(row)
    # logger.debug(rows)
    yml_data = '---\n'
    name_fixer = re.compile(r'(\W|^)(.+)\2')
    for row in rows:
        tds = row.findAll("td")
        name = tds[0].find('a').text # returns names repeated 2x (i.e. Abraham, RalphAbraham, Ralph)
        if name_fixer.match(name):  # if name exists
            name = name_fixer.match(name).group(2) # fix the repeeated name
        # else:
            # continue # don't save if the name isn't right

        state_district = tds[1].text.split(' ')
        state_district = list(filter(None, state_district))
        if state_district[-2] == 'At' and state_district[-1] == 'Large':
            district = 'At Large'
            state_district.pop()
            state_district.pop()
        elif state_district[-2] == 'Resident' and state_district[-1] == 'Commissioner':
            district = 'Resident Commissioner'
            state_district.pop()
            state_district.pop()
        else:
            district = state_district[-1]
            state_district.pop()
        state = ' '.join(state_district)
        # logger.debug('{}     {}'.format(district, state))
        if district not in ['At Large', 'Delegate', 'Resident Commissioner']:
            district = re.sub('[^0-9]','', district)

        party = tds[2].text

        room = tds[3].text
        room_num = re.sub('[^0-9]','', room)
        if len(room_num) == 3:
            room = '{} Cannon'.format(room_num)
        elif len(room_num) == 4 and room_num[0] == '1':
            room = '{} Longworth'.format(room_num)
        elif len(room_num) == 4 and room_num[0] == '2':
            room = '{} Rayburn'.format(room_num)
        else:
            room = 'NO ROOM'

        phone = tds[4].text

        committee = tds[5].findAll('li')
        committee_list = [c.text for c in committee]
        committee_info = ', '.join(committee_list)
        # logger.debug(committee_info)

        info = """
  - name: {}
    state: {}
    district: {}
    party: {}
    room: {}
    phone: {}
    committee: {}""".format(name.strip(), state.strip(), district.strip(), party.strip(), room.strip(), phone.strip(), committee_info.strip())
        yml_data += info
    logger.debug(yml_data)
    file_name = "hor.yml"
    with open(os.path.join(sys.path[0], file_name), 'w', encoding='utf-8') as f:
        print(yml_data, file=f)
    logger.info('saved to {}'.format(file_name))

def get_senate_info():
    headers = {
        'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Mobile Safari/537.36',
    }

    xml_doc = requests.get('https://www.senate.gov/general/contact_information/senators_cfm.xml', headers=headers)
    # logger.debug(xml_doc.text)
    root = etree.fromstring(xml_doc.text.replace('<?xml version="1.0" encoding="UTF-8"?>', ''))

    yml_info = '---\n'
    for s in root.findall('member'):
        full = s.find('member_full').text
        last = s.find('last_name').text
        first = s.find('first_name').text
        party = s.find('party').text
        state = s.find('state').text
        address = s.find('address').text
        address = address.replace('\n', ' ')
        address = re.sub(' +', ' ', address)
        address = address.replace('Senate Office Building Washington DC 20510', '')
        phone = s.find('phone').text
        email = s.find('email').text
        website = s.find('website').text

        info = """
  - name: {}
    last: {}
    firstname: {}
    party: {}
    location: {}
    address: {}
    phone: {}
    contact: {}
    website: {}""".format(full, last, first, party, state, address.rstrip(), phone, email, website)
        yml_info += info

    # logger.debug(yml_info)
    file_name = 'senators.yml'
    with open(os.path.join(sys.path[0], file_name), 'w', encoding='utf-8') as f:
        print(yml_info, file=f)
    logger.info('saved to {}'.format(file_name))

get_via_house_gov()
# get_senate_info()


from aiogram.types import Message
from aiogram.filters.command import CommandStart

async def command_start(m: Message):
    await m.answer(
        text='Hi'
    )

def register_message_handlers(dp):
    dp.message.register(command_start, CommandStart())


"""MPD Controller"""
import mpd.asyncio
import datetime
import logging
import yaml

class NoKnownActionException(Exception):
    pass

class CardIDNotKnownException(Exception):
    pass

class MusicController:

    def __init__(self, cfg, btn_queue):
        self.mpc = mpd.asyncio.MPDClient()
        self.rfid_map = cfg["RFID"]["mapping"]
        self.swipe_timeout = datetime.timedelta(
            seconds=int(cfg["RFID"]["reswipe_time"]))
        self._swipe_ts = datetime.datetime(2020, 1, 1)
        self.host = cfg["MPD"]["host"]
        self.port = cfg["MPD"].get("port", None)
        self.vol_max = cfg["MPD"].getint("volume_max", 30)
        self.vol_default = cfg["MPD"].getint("volume_default", 9)
        self.btn_queue = btn_queue
        self.last_swiped_id = 0
        self.last_swiped_id_count = 0

        with open(self.rfid_map, "r") as stream:
            self._yaml = yaml.safe_load(stream)

        logging.debug(self._yaml)

    async def connect_mpd(self):
        """ Connect to MPD Server and start GPIO listen queue
        """
        await self.mpc.connect(self.host, self.port)
        await self.mpc.setvol(self.vol_default)
        await self._run_listen_btn()

    async def _run_listen_btn(self):
        while True:
            logging.info("Run queue.get")
            cmd = await self.btn_queue.get()
            logging.info("Received Button Event: %s", cmd)
            await self.command(cmd)
            self.btn_queue.task_done()

    async def play_card(self, cardid):
        current_ts = datetime.datetime.now()
        td = current_ts - self._swipe_ts

        if cardid == self.last_swiped_id:
            self.last_swiped_id_count += 1
        else:
            self.last_swiped_id = cardid
            self.last_swiped_id_count = 0

        if self.last_swiped_id_count > 0:
            logging.info("Card already playing, skipping swipe!")
            return

        if td > self.swipe_timeout:
            try:
                (is_command, data) = self.retrieve_card(cardid)
            except CardIDNotKnownException as e:
                logging.warning(e)
                return
            except NoKnownActionException as e:
                logging.warning(e)
                return
            logging.info("Received CardID: %s with data: %s", cardid, data)
            if is_command:
                await self.command(data)
            else:
                await self.play_pl(data)
            self._swipe_ts = current_ts
        else:
            logging.info("Swipe Timeout (%f) not reached: %f",
                         self.swipe_timeout.total_seconds(),
                         td.total_seconds())

    def retrieve_card(self, cardid):
        if cardid in self._yaml:
            if "playlist" in self._yaml[cardid]:
                return (False, self._yaml[cardid]["playlist"])
            elif "command" in self._yaml[cardid]:
                return (True, self._yaml[cardid]["command"])
            else:
                raise NoKnownActionException()
        else:
            raise CardIDNotKnownException("Could not find CardID {0}"
                                          .format(cardid))

    async def command(self, cmd):
        ucmd = cmd.lower()
        if ucmd.startswith("play"):
            await self.play()
        elif ucmd.startswith("sto"):
            await self.stop()
        elif ucmd.startswith("pre"):
            await self.previous()
        elif ucmd.startswith("nex"):
            await self.next()
        elif ucmd.startswith("vol+"):
            await self.volumeup()
        elif ucmd.startswith("vol-"):
            await self.volumedown()

    async def play(self):
        status = await self.mpc.status()
        if status["state"] == "play":
            await self.mpc.pause()
        else:
            await self.mpc.play()

    async def stop(self):
        await self.mpc.stop()

    async def next(self):
        status = await self.mpc.status()
        if not status["state"] == "stop":
            await self.mpc.next()

    async def previous(self):
        status = await self.mpc.status()
        if not status["state"] == "stop":
            await self.mpc.previous()

    async def volumeup(self):
        vol = await self.get_volume() + 1
        if vol > self.vol_max:
            vol = self.vol_max
        await self.mpc.setvol(vol)

    async def volumedown(self):
        vol = await self.get_volume() - 1
        if vol < 0:
            vol = 0
        await self.mpc.setvol(vol)

    async def get_volume(self):
        status = await self.mpc.status()
        vol = int(status["volume"])
        return vol

    async def play_pl(self, playlist):
        await self.mpc.stop()
        await self.mpc.clear()
        try:
            await self.mpc.load(playlist)
        except mpd.base.CommandError as e:
            logging.warning(e)
            return
        await self.mpc.play()

    async def status(self):
        return await self.mpc.status()


# 나의 풀이
from collections import deque

def solution(bridge_length, weight, truck_weights):
    truck_weights = deque(truck_weights)
    bridge = deque([0] * bridge_length)  # 남은 트럭들이 가야하는 거리
    time = 0  # 소요시간
    bridge_weight = 0  # 다리 위의 무게

    while len(bridge) != 0:
        out = bridge.popleft()
        bridge_weight -= out
        time += 1

        if truck_weights:
            if bridge_weight + truck_weights[0] <= weight:
                left = truck_weights.popleft()
                bridge_weight += left
                bridge.append(left)
            else:
                bridge.append(0)

    return time

# bridge = deque([0] * bridge_length) -> 이 부분 사용하는거 손에 익혀두기
# 거리나 그런 계산에 유용하게 쓸 수 있는 것 같다. (deque부분은 필수가 아니라 유동적으로 사용하는 거고)

17_다리를_지나는_트럭.py


import logging as _logging

from simbak.agent.base import BaseAgent as _BaseAgent

_logger = _logging.getLogger(__name__)

class NormalAgent(_BaseAgent):
    def __init__(self, sources: list, destinations: list, name: str,
                 compression_level: int = 6):
        """
        Args:
            sources (list of str): Paths to the files that you are
                backing up.
            destinations (list of str): Paths of where you want the
                backup to be stored.
            name (str): Name of the backup, this will name the backup
                files.
            compression_level (int, optional): The gzip compression
                level that you want to use for the backup. Default to 6.
        """
        super().__init__(sources, destinations, name, compression_level)

    def backup(self):
        """Standard simbak backup.

        This will backup all the files defined in the sources and store
        them in a gzip'd file in each of the destinations. The name of
        the gzip file will be the name parameter suffixed with a time
        stamp, the format of the timestamp is YYYY-MM-DD--hh-mm-ss
        """
        _logger.info(f'Starting backup [{self._name}]')
        super()._backup()


"""Напишіть реалізацію функції factorize, яка приймає список чисел та повертає список чисел, на які числа з вхідного списку поділяються без залишку.
Реалізуйте синхронну версію та виміряйте час виконання.
Потім покращіть продуктивність вашої функції, реалізувавши використання кількох ядер процесора для паралельних обчислень і замірьте час виконання знову.
Для визначення кількості ядер на машині використовуйте функцію cpu_count() з пакета multiprocessing"""

from multiprocessing import Process, RLock as PRLock
from multiprocessing.dummy import Pool  # Thread с оболонкою Process
from threading import Thread, RLock as TRlock
from time import time
import logging

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("factorize")

def factorize(num, filename, lock):
    divisors = [i for i in range(1, num + 1) if num % i == 0]
    logger.info(f"{num} == {divisors}")
    with lock:
        with open(filename, "a") as f:
            f.write(f"{num} == {divisors}\n")

# Синхронна версія, треди
def synchronous_version(numbers):
    th_filename = "th_factorize.txt"
    th_lock = TRlock()
    threads = []
    for num in numbers:
        thread = Thread(target=factorize, args=(num, th_filename, th_lock))
        threads.append(thread)

    timer = time()
    [thread.start() for thread in threads]
    [thread.join() for thread in threads]
    elapsed_time = round(time() - timer, 4)  # Обчислюємо час, що пройшов
    done_threads = [
        num + 1 for num in numbers if not thread.is_alive()
    ]  # Знаходимо завершені потоки
    print(f"Done by {len(done_threads)} threads: {elapsed_time} seconds")

# Процеси
def processs_version(numbers):
    pr_filename = "pr_factorize.txt"
    p_lock = PRLock()
    processes = []
    for num in numbers:
        process = Process(target=factorize, args=(num, pr_filename, p_lock))
        processes.append(process)

    timer = time()
    [process.start() for process in processes]
    [process.join() for process in processes]
    [process.close() for process in processes]
    elapsed_time = round(time() - timer, 4)  # Обчислюємо час, що пройшов
    done_process = [num + 1 for num in numbers]
    print(f"Done by {len(done_process)} processes: {elapsed_time} seconds")

# 1 процес
def one_process_version(numbers):
    lock = TRlock()
    timer = time()
    for num in numbers:
        factorize(num, "factorize.txt", lock)
    elapsed_time = round(time() - timer, 4)
    print(f"Done by 1 main processes: {elapsed_time} seconds")

# Пул - треди в оболонці процесів
def pool_version(numbers):
    pl_filename = "pl_factorize.txt"
    timer = time()
    th_lock = TRlock()
    args_list = [(num, pl_filename, th_lock) for num in numbers]
    with Pool(len(numbers)) as pool:
        results = pool.starmap(factorize, args_list)
    elapsed_time = round(time() - timer, 4)
    done_process = [num + 1 for num in numbers]
    print(f"Done by {len(done_process)} processes: {elapsed_time} seconds")

if __name__ == "__main__":
    numbers = [128, 255, 99999, 10651060]
    # Синхронна версія, треди
    synchronous_version(numbers)

    # Процеси
    processs_version(numbers)

    # 1 процес
    one_process_version(numbers)

    # Пул - треди в оболонці процесів
    pool_version(numbers)


from setuptools import setup, find_packages

version = "0.1"
setup(name='kinoknecht',
      version=version,
      description=("A media management server with the ability to locally ",
                   "play back files via mplayer, an RPC-API to allow the ",
                   "implementation of MPD-like clients and a Flask-backed",
                   "webinterface for ease of administration"),
      classifiers=[
          "Programming Language :: Python",
        ],
      author="Johannes Baiter",
      author_email="johannes.baiter@gmail.com",
      license="MIT",
      packages=find_packages(),
      include_package_data=True,
      scripts = ['kinoknecht/kinoknecht'],
      install_requires=[
          'Flask>=0.6.1',
          'SQLAlchemy>=0.6',
          'Flask-SQLAlchemy>=0.11',
          'IMDbPy>=4.7',
          'FFVideo>=0.0.9',
          'PyMPlayer>=0.6.0',
          'setuptools-git'
      ]
)


# codeing: utf-8

import math
import os
import numpy as np
import logging

import mxnet as mx
from mxnet import gluon, autograd as ag
from mxnet.gluon import nn, rnn
from mxnet.gluon.loss import L2Loss

from mxnet.gluon.block import HybridBlock, Block
from gluonnlp.model import AttentionCell, MLPAttentionCell, DotProductAttentionCell, MultiHeadAttentionCell

class QuestionAnsweringClassifier(HybridBlock):
    """
    Primary model block for Bi-LSTM and attention model
    """
    def __init__(self, emb_input_dim, emb_output_dim, num_classes,
                 max_seq_len=32, dropout=.2, attn_cell='multi_head'):
        super(QuestionAnsweringClassifier, self).__init__()
        with self.name_scope():
            self.embedding = nn.Embedding(emb_input_dim, emb_output_dim)
            self.bilstm_question = rnn.LSTM(hidden_size=2048,
                                            dropout=dropout,
                                            bidirectional=True)
            self.bilstm_context = rnn.LSTM(hidden_size=2048,
                                           dropout=dropout,
                                           bidirectional=True)
            self.attention_transform = BaseEncoder(attention_cell=attn_cell,
                                                   units=emb_output_dim,
                                                   hidden_size=2048,
                                                   max_length=max_seq_len,
                                                   num_heads=4,
                                                   scaled=True,
                                                   dropout=dropout,
                                                   use_residual=True,
                                                   output_attention=False,
                                                   weight_initializer=None,
                                                   bias_initializer='zeros',
                                                   prefix=None,
                                                   params=None)
            self.output = nn.Dense(num_classes)

    def hybrid_forward(self, F, question, context):
        # embedding layers for question and context
        question_embedded = self.embedding(question)  ## shape (batch_size, question_len, emb_dim)
        context_embedded = self.embedding(context)
        logging.debug(f"shape of question after embedding {question_embedded.shape}")
        logging.debug(f"shape of context after embedding {context_embedded.shape}")

        # bilstm layers for question and context
        question_after_lstm = self.bilstm_question(question_embedded)  # shape (batch_size, question_len,
        context_after_lstm = self.bilstm_context(context_embedded)
        logging.debug(f"shape of question after lstm {question_after_lstm.shape}")
        logging.debug(f"shape of context after lstm {context_after_lstm.shape}")

        # attention layer
        q_after_attn = self.attention_transform(question_after_lstm, context_after_lstm)
        c_after_attn = self.attention_transform(context_after_lstm, question_after_lstm)
        logging.debug(f"shape of q_after_attn {q_after_attn.shape}")
        logging.debug(f"shape of c_after_attn {c_after_attn.shape}")

        # combine q_after_attn and c_after_attn in some way
        encoded = mx.ndarray.concat(q_after_attn, c_after_attn, dim=1)
        logging.debug(f"shape of encoded {encoded.shape}")

        # output layer
        outputs = self.output(encoded)
        logging.debug(f"shape of outputs after output layer {outputs.shape}")
        return outputs

class PositionwiseFFN(HybridBlock):
    """
    Taken from the gluon-nlp library.
    """

    def __init__(self, units=512, hidden_size=1024, dropout=0.0, use_residual=True,
                 weight_initializer=None, bias_initializer='zeros', activation='relu',
                 prefix=None, params=None):
        super(PositionwiseFFN, self).__init__(prefix=prefix, params=params)
        self._hidden_size = hidden_size
        self._units = units
        self._use_residual = use_residual
        with self.name_scope():
            self.ffn_1 = nn.Dense(units=hidden_size, flatten=False,
                                  activation=activation,
                                  weight_initializer=weight_initializer,
                                  bias_initializer=bias_initializer,
                                  prefix='ffn_1_')
            self.ffn_2 = nn.Dense(units=units, flatten=False,
                                  weight_initializer=weight_initializer,
                                  bias_initializer=bias_initializer,
                                  prefix='ffn_2_')
            self.dropout_layer = nn.Dropout(dropout)
            self.layer_norm = nn.LayerNorm()

    def hybrid_forward(self, F, inputs):  # pylint: disable=arguments-differ
        # pylint: disable=unused-argument
        outputs = self.ffn_1(inputs)
        outputs = self.ffn_2(outputs)
        outputs = self.dropout_layer(outputs)
        if self._use_residual:
            outputs = outputs + inputs
        outputs = self.layer_norm(outputs)
        return outputs

def _get_attention_cell(attention_cell, units=None,
                        scaled=True, num_heads=None,
                        use_bias=False, dropout=0.0):
    """
    Parameters
    ----------
    attention_cell : AttentionCell or str
    units : int or None

    Returns
    -------
    attention_cell : AttentionCell
    """
    if isinstance(attention_cell, str):
        if attention_cell == 'scaled_luong':
            return DotProductAttentionCell(units=units, scaled=True, normalized=False,
                                           use_bias=use_bias, dropout=dropout, luong_style=True)
        elif attention_cell == 'scaled_dot':
            return DotProductAttentionCell(units=units, scaled=True, normalized=False,
                                           use_bias=use_bias, dropout=dropout, luong_style=False)
        elif attention_cell == 'dot':
            return DotProductAttentionCell(units=units, scaled=False, normalized=False,
                                           use_bias=use_bias, dropout=dropout, luong_style=False)
        elif attention_cell == 'cosine':
            return DotProductAttentionCell(units=units, scaled=False, use_bias=use_bias,
                                           dropout=dropout, normalized=True)
        elif attention_cell == 'mlp':
            return MLPAttentionCell(units=units, normalized=False)
        elif attention_cell == 'normed_mlp':
            return MLPAttentionCell(units=units, normalized=True)
        elif attention_cell == 'multi_head':
            base_cell = DotProductAttentionCell(scaled=scaled, dropout=dropout)
            return MultiHeadAttentionCell(base_cell=base_cell, query_units=units, use_bias=use_bias,
                                          key_units=units, value_units=units, num_heads=num_heads)
        else:
            raise NotImplementedError
    else:
        assert isinstance(attention_cell, AttentionCell), \
            f'attention_cell must be either string or AttentionCell. Received attention_cell={attention_cell}'
        return attention_cell

class BaseEncoderCell(HybridBlock):
    """Structure of the Transformer Encoder Cell.

    Parameters
    ----------
    attention_cell : AttentionCell or str, default 'multi_head'
        Arguments of the attention cell.
        Can be 'multi_head', 'scaled_luong', 'scaled_dot', 'dot', 'cosine', 'normed_mlp', 'mlp'
    units : int
    hidden_size : int
        number of units in the hidden layer of position-wise feed-forward networks
    num_heads : int
        Number of heads in multi-head attention
    scaled : bool
        Whether to scale the softmax input by the sqrt of the input dimension
        in multi-head attention
    dropout : float
    use_residual : bool
    output_attention: bool
        Whether to output the attention weights
    weight_initializer : str or Initializer
        Initializer for the input weights matrix, used for the linear
        transformation of the inputs.
    bias_initializer : str or Initializer
        Initializer for the bias vector.
    prefix : str, default 'rnn_'
        Prefix for name of `Block`s
        (and name of weight if params is `None`).
    params : Parameter or None
        Container for weight sharing between cells.
        Created if `None`.
    """

    def __init__(self, attention_cell='multi_head', units=128,
                 hidden_size=512, num_heads=4, scaled=True,
                 dropout=0.0, use_residual=True, output_attention=False,
                 weight_initializer=None, bias_initializer='zeros',
                 prefix=None, params=None):
        super(BaseEncoderCell, self).__init__(prefix=prefix, params=params)
        self._units = units
        self._num_heads = num_heads
        self._dropout = dropout
        self._use_residual = use_residual
        self._output_attention = output_attention
        with self.name_scope():
            self.dropout_layer = nn.Dropout(dropout)
            self.attention_cell = _get_attention_cell(attention_cell,
                                                      units=units,
                                                      num_heads=num_heads,
                                                      scaled=scaled,
                                                      dropout=dropout)
            self.proj = nn.Dense(units=units, flatten=False, use_bias=False,
                                 weight_initializer=weight_initializer,
                                 bias_initializer=bias_initializer,
                                 prefix='proj_')
            self.ffn = PositionwiseFFN(hidden_size=hidden_size, units=units,
                                       use_residual=use_residual, dropout=dropout,
                                       weight_initializer=weight_initializer,
                                       bias_initializer=bias_initializer)
            self.layer_norm = nn.LayerNorm()

    def hybrid_forward(self, F, query, key):  # pylint: disable=arguments-differ
        # outputs has shape (batch_size, max_seq_len, units)
        # attention_weights has shape  (batch_size, 4, max_seq_len, max_seq_len)
        outputs, attention_weights = self.attention_cell(query, key, None, None)
        outputs = self.proj(outputs)  # shape (batch_size, max_seq_len, units)
        outputs = self.dropout_layer(outputs)  # shape (batch_size, max_seq_len, units)
        outputs = self.layer_norm(outputs)
        outputs = self.ffn(outputs)
        return outputs

class BaseEncoder(HybridBlock):

    def __init__(self, attention_cell='multi_head',
                 units=512, hidden_size=2048, max_length=64,
                 num_heads=4, scaled=True, dropout=0.0,
                 use_residual=True, output_attention=False,
                 weight_initializer=None, bias_initializer='zeros',
                 prefix=None, params=None):
        super(BaseEncoder, self).__init__(prefix=prefix, params=params)
        assert units % num_heads == 0, \
            'In TransformerEncoder, The units should be divided exactly ' \
                f'by the number of heads. Received units={units}, num_heads={num_heads}'
        self._max_length = max_length
        self._num_heads = num_heads
        self._units = units
        self._hidden_size = hidden_size
        self._output_attention = output_attention
        self._dropout = dropout
        self._use_residual = use_residual
        self._scaled = scaled
        with self.name_scope():
            # self.dropout_layer = nn.Dropout(dropout)
            # self.layer_norm = nn.LayerNorm()
            self.transformer_cells = BaseEncoderCell(units=units,
                                                     hidden_size=hidden_size,
                                                     num_heads=num_heads,
                                                     attention_cell=attention_cell,
                                                     weight_initializer=weight_initializer,
                                                     bias_initializer=bias_initializer,
                                                     dropout=dropout,
                                                     use_residual=use_residual,
                                                     scaled=scaled,
                                                     output_attention=output_attention,
                                                     prefix=f'transformer')

    def __call__(self, query, key):  # pylint: disable=arguments-differ
        return super(BaseEncoder, self).__call__(query, key)

    def hybrid_forward(self, F, query, key):  # pylint: disable=arguments-differ
        outputs = self.transformer_cells(query, key)
        return outputs


# -*- coding: utf-8 -*-
"""
Created on Mon Feb 18 16:18:30 2019

@author: mluci
"""

from flask import Flask, render_template, request, jsonify, flash, url_for, redirect
from flask_cors import CORS
#from forms import LoginForm
#from forms import RegistrationForm

from engine import *
from engine import anotherTask, deleteTask, resultsDictionary

app=Flask (__name__)
app.config['SECRET_KEY'] = 'you-will-never-guess'
cors = CORS(app)

########overview of every task######## includes delete####
@app.route('/tasks', methods=['GET', 'DELETE'])
def show_tasks():
    if request.method == 'GET':
        tasks = resultsDictionary()
    if request.method == 'DELETE': #delete A TASK
        delete_task = deleteTask(taskId)
    return render_template("tasks.html", title="Overview of tasks", **locals())

###### ############add task ###############
@app.route('/addtasks')
def add_new_tasks():
    form_data = request.form
    taskname = form_data['task']
    result = anotherTask(taskname)
  #  result = "Task added successfully."
    return redirect('/tasks')

################overview of all tasks and their information includes delete #######################
@app.route('/taksoverview', methods=['GET', 'DELETE'])
def task_overview():
    if request.method == 'GET':
        single_task = resultsDictionary()

    if request.method == 'DELETE': #delete A TASK
        delete_task = deleteTask(taskId)
    return render_template("tasksoverview.html", title="Overview of task information", **locals())

if __name__ == '__main__':
   app.run(debug=True)


#!/usr/bin/env python

#import necessary libraries
#pip install flask
from flask import Flask, json,render_template,redirect,url_for
from bs4 import BeautifulSoup as bs
import requests
import pandas as pd

app  = Flask(__name__)

#decorator
@app.route("/")
def echo_hello():
    return_text = """
Web Scraping 50 restaurant near me from yelp!

/scrape endpoint will call the function to scrape yelp and store data in CSV file

/all endpoint will display scrapped data

                      """
    return return_text

@app.route("/scrape")
def web_scrape():
    #Call function
    scrape_restaurant_details()
    return redirect(url_for("show_data"))

# Scrape 50 yelp restaurants near you
def scrape_restaurant_details():
    i=0
    restaurant_list = []
    #each page has 10 restaurants. To get 50 restaurants loop thourgh 5 times
    while i<=40:
        yurl = f'https://www.yelp.com/search?find_desc=Restaurants&find_loc=Ballwin%2C+MO+63021&start={i}'
        response = requests.get(yurl)
        print(yurl)
        page_soup = bs(response.text, 'html.parser')

        #Get all the restaurant names in each page. One page has 10 restaurants
        business = page_soup.find_all("div",class_="container__09f24__sxa9- hoverable__09f24__3HkY2 margin-t3__09f24__hHZsm margin-b3__09f24__3h89A padding-t3__09f24__1VTAn padding-r3__09f24__11Xv2 padding-b3__09f24__2I83c padding-l3__09f24__1JEx9 border--top__09f24__37VAs border--right__09f24__Z9jGU border--bottom__09f24__3lElq border--left__09f24__akfOa border-color--default__09f24__3Epto")
        try:
            for b in business:
                #Restaurant Name
                rest_name = b.find("div",class_="businessName__09f24__3Ml2X display--inline-block__09f24__3SvIn border-color--default__09f24__3Epto").find("a").text

                #Price range
                price = b.find("span",class_="priceRange__09f24__2GspP css-xtpg8e")
                if price!=None:
                    price = price.text
                else:
                    price = "Not Specified"

                #Facilities like - takeout, dinein, delivery etc
                #facility = b.find_all("span",class_="raw__09f24__3Azrj")
                #f_list=[]
                #if facility!=None:
                #    for f in facility:
                #        f_list.append(f.text)
                #else:
                #    f_list.append("Not Specified")

                #Facilities like - takeout, dinein, delivery etc
                facility =  b.find_all("ul",class_="undefined list__09f24__36n6_")
                #print(len(facility))
                f_list=[]
                if facility!=None:
                    for f in facility:
                        f_list.append(f.find("span",class_="raw__09f24__3Azrj").text)
                else:
                    f_list.append("Not Specified")

                #First review
                first_review = b.find("p",class_="css-e81eai").text

                #rating
                ratings = b.find("span", class_="display--inline__09f24__nqZ_W border-color--default__09f24__3Epto").div.get('aria-label')

                #Review Count
                review_count = b.find("span",class_="reviewCount__09f24__3GsGY css-e81eai")
                if review_count!=None:
                    review_count = review_count.text
                else:
                    review_count = "Not specified"

                #Get list of food category
                food_category = b.find_all("p",class_="css-1j7sdmt")
                #print(len(food_category))
                fc_list=[]
                if food_category!=None:
                    for fc in food_category:
                        #Find text for each food category
                        fcat = fc.find_all("p",class_="text__09f24__2NHRu css-1hx6l2b")
                        if(fcat!=None):
                            for c in fcat:
                                if(c!=None):
                                    fc_list.append(c.text)
                                else:
                                    fc_list.append("Not Specified")
                        else:
                            fc_list.append("Not Specified")
                else:
                    fc_list.append("Not Specified")

                #Phone number and address:
                address = b.find_all("p" ,class_="css-8jxw1i")
                cnt = 0
                phone_number="Not Specified"
                addr="Not Specified"
                if address!=None:
                    for a in address:
                        if cnt==0:
                            phone_number = a.text
                        if cnt==1:
                            addr = a.find("span",class_="raw__09f24__3Azrj").text
                        cnt = cnt + 1
                    #print("Phone and address",phone_number,addr)

                #Dictionary of restaurant name, rating etc.
                rest_dict = {"Name":rest_name,
                            "Phone Number":phone_number,
                            "Address":addr,
                            "Price range":price,
                            "Facility":f_list,
                            "1st Review": first_review,
                            "Ratings":ratings,
                            "Review Count":review_count,
                            "Food Category":fc_list
                            }

                restaurant_list.append(rest_dict)
        except AttributeError as e:
                print(e)

        #Go to next 10 restaurants
        i= i + 10

    rest_df = pd.DataFrame(restaurant_list)
    rest_df.to_csv("./static/yelp_ballwin_rest.csv")
    #return render_template('index.html',  tables=[rest_df.to_html(classes='data', header="true")])
    #render template is always looking in tempate folder
    #return "
Web Scraping 50 restaurants near me from yelp and storing them in CSV file!
"
    #return redirect(url_for("show_data"))

@app.route("/all")
def show_data():
    rest_df = pd.read_csv("./static/yelp_ballwin_rest.csv")
    #render template is always looking in tempate folder
    return render_template('index.html',  tables=[rest_df.to_html(classes='data', header="true")])

if __name__ == "__main__":
    app.run(debug=True)


"""

This class provides methods for speaking to the humans.
"""
from engagement_service import paths
import os
import time
import subprocess
import random
import logging

logger = logging.getLogger(__name__)

ESPEAK_CMD = "espeak -v en+f2"
ESPEAK_AFFIX = "--stdout | aplay"

GREETINGS = [
    ["Hello", None],
    ["Yo!", "what up?"],
    ["Hi", "good to see you"],
    ["What are you doing, ", None],
    [None, "...my huuman.  What up?"],
    ["Yo, ", "...looking good, darling."]
]
INTRODUCTIONS = [
    "Oh, hello.  What is your name?",
    "Hello, I did not see you there. What is your name?",
    "Hi, my name is Shelly bot, what is your name?"
]

POSES = [
    "Pose for me.  Let me behold your beauty.",
    "You are fierce.",
    "Yaaaas, the camera loves you darling.",
    "Make love to the camera.",
    "You're a tiger. Yaaaas."
    "You're a lemur; cute and cuddly with sharp fierce claws",
    "You're happy.  You're sad.  You couldn't give a rats ass.",
    "Make love to my cold electronic eye.  Yaaaassss.",
    "Strike a pose, let's get down to it.",
    "Give me fierce, yaaaas."
    "Hey!  Eyes up here, buddy"
]

REJECTIONS = [
    "Well FINE.  Be that way then.",
    "I didn't want to be her friend anyway.",
    "Ok, well see if I remember you.  not."
]

BOREDOM = [
    "Yawn, I'm bored.",
    "Well this is fun isn't it?",
    "I might as well take a nap",
    "If only someone would be my friend"
]

MIN_SECONDS_BETWEEN_GREETINGS = 120

def async_cmd(cmd):
    subprocess.Popen(
        [cmd],
        shell=True,
        stdin=None,
        stdout=None,
        stderr=None,
        close_fds=True
    )

def name_wav_file_path(name):
    return f"{paths.NAMES_DATA_DIR}/{name}/name.mp3"

def play_name_cmd(name):
    return f"mpg123 {name_wav_file_path(name)}"

def play_name(name):
    logger.info(f"play_name: {name}")
    os.system(play_name_cmd(name))

def play_camera_snap():
    async_cmd(f"aplay media/sounds/camera_snap.wav")

def espeak_cmd(text):
    return f"{ESPEAK_CMD} \"{text}\" {ESPEAK_AFFIX}"

def say(text):
    logger.info(f"say: {text}")
    os.system(espeak_cmd(text))

def say_async(text):
    logger.info(f"say_async: {text}")
    async_cmd(espeak_cmd(text))

def say_random_from(collection, runAsync=False):
    msg = collection[random.randrange(
        len(collection))]
    if runAsync:
        say_async(msg)
    else:
        say(msg)

# kv of "name": timeLastGreeted
_lastTimeGreeted = {}

def say_hello(names):
    greeting = GREETINGS[random.randrange(
        len(GREETINGS))]

    names_to_greet = []
    for name in names:
        if name in _lastTimeGreeted and \
           time.time() - _lastTimeGreeted[name] < MIN_SECONDS_BETWEEN_GREETINGS:
            continue
        elif name not in names_to_greet:
            names_to_greet.append(name)

    if len(names_to_greet) == 0:
        return

    if greeting[0]:
        say(greeting[0])

    for name in names_to_greet:
        play_name(name)
        _lastTimeGreeted[name] = time.time()

    if greeting[1]:
        say(greeting[1])

    logger.info(f"greeted {names_to_greet}")

def introduce_yourself():
    say_random_from(INTRODUCTIONS)

def request_name_again():
    say("I'm sorry, I didn't hear that. Please say your name.")

def let_me_get_a_look_at_you():
    say("Let me get a good look at you")

def where_did_you_go():
    say("Where did you go?  Let me get a good look at you.")

def pose_for_me():
    say_random_from(POSES)

def and_im_spent():
    say("...and I'm spent.")

def nice_to_meet_you(name):
    say("It was nice to meet you...")
    play_name(name)
    say("... I will try and remember your name.")

def rejection():
    say_random_from(REJECTIONS)

def i_need_a_nap():
    say("I'm tired and need to nap now.")

_last_time_malfunction = time.time()
MIN_MALFUNCTION_DELAY = 20

def brain_malfunction():
    global _last_time_malfunction
    current_time = time.time()
    if current_time - _last_time_malfunction > MIN_MALFUNCTION_DELAY:
        _last_time_malfunction = current_time
        say("I seem to be having difficulty talking to my brain. Need Reboot. Need Reboot.")

_last_time_bored = time.time()
MIN_BORED_DELAY = 40

def im_bored():
    global _last_time_bored
    current_time = time.time()
    if current_time - _last_time_bored > MIN_BORED_DELAY:
        _last_time_bored = current_time
        say_random_from(BOREDOM)


import json

class Budget:
    _transactions = {}

    def __init__(cls, patch):
        with open(patch) as f:
            field = json.load(f)
            for transaction in field["transactions"]:
                try:
                    Budget._transactions[transaction["category"]].append(transaction["value"])
                except:
                    Budget._transactions[transaction["category"]] = [transaction["value"]]

    def get_category(cls):
        category = []
        for cate in Budget._transactions:
            category.append(cate)
        return(category)

    def show_transactions(cls, category):
        for value in Budget._transactions[category]:
            print(value)

wallet = Budget("./data.json")
for category in wallet.get_category():
    print(category)
    wallet.show_transactions(category)