import numpy as np
from sklearn.metrics import accuracy_score

from utils import baseline, compute_cost, load_data, sigmoid, sigmoid_prime

np.random.seed(1)

def init_parameters(n_x: int, n_h: int, n_y: int = 1) -> dict:
    """
    Initialise parameters (weights/biases) to be learnt by the neural network.

    Parameters
    ----------
    n_x : size of input layer (number of features)
    n_h : size of hidden layer (number of hidden neurons)
    n_y : size of output layer (number of output neurons)

    Returns
    ----------
    a dict of parameters W & b
    """
    return {
        "W1": np.random.randn(n_h, n_x) * 0.01,  # one weight per feature,
        "W2": np.random.randn(n_y, n_h) * 0.01,
        "b1": np.zeros((n_h, 1)),  # one bias
        "b2": np.zeros((n_y, 1)),
    }

def forward_prop(X: np.ndarray, params: dict) -> [np.ndarray, dict]:
    """
    Forward propagation step

    Parameters
    ----------
    X: Input matrix. [n,m] dimension where n is number of features and m is the number of samples.
    params: Dictionary of parameters containing weights and biases

    Returns
    --------
    a tuple (Prediction of the neural network, Dictionary containing intermediate variables for use in backpropagation)
    """
    W1 = params["W1"]  # [n_h,n_x]
    b1 = params["b1"]  # [n_h,1]

    W2 = params["W2"]  # [n_y,n_h]
    b2 = params["b2"]  # [n_y,1]

    Z1 = np.dot(W1, X) + b1  # [n_h,m] = [n_h,n_x] . [n_x,m] + [1,m]
    A1 = sigmoid(Z1)  # [n_h,m]
    Z2 = np.dot(W2, A1) + b2  # [n_y,m] = [n_y,n_h] . [n_h,m] + [n_y,1]
    A2 = sigmoid(Z2)  # [n_y,m]

    cache = dict(A1=A1, A2=A2)

    return A2, cache

def compute_grads(X: np.ndarray, Y: np.ndarray, cache: dict, params: dict) -> dict:
    """
    Compute gradients using backpropagation algorithm

    Parameters
    ----------
    X: [n,m] matrix of training examples
    Y: [1,m] matrix of output labels/values
    cache: Dictionary of intermediate values from forward-propagation step
    params: Dictionary of weights & biases

    Returns
    ---------
    a dictionary containing the gradients
    """

    m = X.shape[0]
    A1 = cache["A1"]  # [n_h,m]
    A2 = cache["A2"]  # [n_y,m]
    W2 = params["W2"]  # [n_y,n_h]

    dA2 = -(Y / A2) + (1 - Y) / (1 - A2)  # [n_y,m]
    dZ2 = dA2 * sigmoid_prime(A2)  # [n_y,m]

    dW2 = np.dot(dZ2, A1.T) / m  #  [n_y,n_h] = [n_y,m] . [m,n_h]
    db2 = np.mean(dZ2, axis=1, keepdims=True)  # [n_y,1]

    dA1 = np.dot(W2.T, dZ2)  # [n_h,m] =  [n_h,n_y] . [n_y,m]
    dZ1 = dA1 * sigmoid_prime(A1)  # [n_h,m]

    dW1 = np.dot(dZ1, X.T) / m  # [n_h,n] = [n_h,m] . [ m,n]
    db1 = np.mean(dZ1, axis=1, keepdims=True)  # [n_h,1]

    return dict(dW1=dW1, db1=db1, dW2=dW2, db2=db2)

def update_parameters(params: dict, grads: dict, learning_rate: float) -> dict:
    """
    Update weights & biases using gradient descent.

    Parameters
    ----------
    params: dictionary of weights & biases for each layer
    grads: dictionary of gradients of weights & biases computed from back-propagation
    learning_rate: step-size for gradient descent update

    Returns
    --------
    a dictionary of updated parameters (weights & biases)
    """

    W1, W2 = params["W1"], params["W2"]
    b1, b2 = params["b1"], params["b2"]

    dW1, dW2 = grads["dW1"], grads["dW2"]
    db1, db2 = grads["db1"], grads["db2"]

    # update weights
    W1 = W1 - learning_rate * dW1
    W2 = W2 - learning_rate * dW2

    # update bias
    b1 = b1 - learning_rate * db1
    b2 = b2 - learning_rate * db2

    params["W1"] = W1
    params["W2"] = W2
    params["b1"] = b1
    params["b2"] = b2

    return params

def train(
    X: np.ndarray,
    y: np.ndarray,
    n_h: int = 3,
    epoch: int = 100,
    learning_rate: float = 0.01,
) -> dict:
    """
    Train a neural network

    Parameters
    ----------
    X: [n,m] matrix of training examples
    Y: [1,m] matrix of output labels/values
    n_h: size of hidden layer
    epoch: number of iterations to perform
    learning_rate: step-size for gradient descent update

    Returns
    ---------
    a dictionary of learnt parameters (weights & biases)
    """
    n_x = X.shape[0]
    n_y = y.shape[0]
    params = init_parameters(n_x=n_x, n_h=n_h, n_y=n_y)

    print(f"X.shape = {X.shape}, Y.shape = {Y.shape}")
    print(f"n_x = {n_x}, n_h = {n_h}, n_y = {n_y}")
    print(f"initial params: {params}")
    for i in range(epoch):
        A, cache = forward_prop(X=X, params=params)  # forward prop to get prediction
        cost = compute_cost(Y=Y, Y_hat=A)  # compute cost
        grads = compute_grads(X=X, Y=Y, cache=cache, params=params)  # compute gradient
        params = update_parameters(
            params=params, grads=grads, learning_rate=learning_rate
        )  # update parameters using gradient descent
        if i % 100 == 0:
            print(f"epoch={i}\tcost={cost}")
    print(f"learnt params: {params}")

    return params

if __name__ == "__main__":

    X, Y = load_data()
    params = train(X=X, y=Y, n_h=10, epoch=10000, learning_rate=0.001)

    y_pred, _ = forward_prop(X, params)
    y_pred = y_pred.round()

    acc = np.round(accuracy_score(y_true=np.ravel(Y), y_pred=np.ravel(y_pred)), 3) * 100
    acc_lib = baseline(X.T, np.ravel(Y))
    print(f"accuracy [our implementation] = {acc}%")
    print(f"accuracy [sklearn] = {acc_lib}%")


__all__ = ["SqlConfigClient"]

from typing import Dict, Any
from dbacademy.clients.rest.common import ApiClient, ApiContainer

DATA_ACCESS_CONTROL = "DATA_ACCESS_CONTROL"
SECURITY_POLICIES = [DATA_ACCESS_CONTROL]

class SqlConfigClient(ApiContainer):

    def __init__(self, client: ApiClient):
        self.client = client

    def get(self):
        return self.client.api("GET", f"{self.client.endpoint}/api/2.0/sql/config/endpoints")

    def update_all(self, settings: Dict[str, Any]):
        self.client.api("PUT", f"{self.client.endpoint}/api/2.0/sql/config/endpoints", _data=settings)
        return None

    def update(self, property_name: str, property_value: Any):
        settings = self.get()
        settings[property_name] = property_value

        return self.update_all(settings)

    def edit(self,
             security_policy: str,
             instance_profile_arn: str,
             data_access_config: Dict[str, Any],
             sql_configuration_parameters: Dict[str, Any]):

        assert security_policy in SECURITY_POLICIES, f"Expected security_policy to be one of {SECURITY_POLICIES}, found {security_policy}"

        params = {
            "security_policy": security_policy,
            "instance_profile_arn": instance_profile_arn,
            "data_access_config": [],
            "sql_configuration_parameters": {"configuration_pairs": []}
        }

        for key in data_access_config:
            value = data_access_config[key]
            params.get("data_access_config").append({
                "key": key,
                "value": value
            })

        for key in sql_configuration_parameters:
            value = sql_configuration_parameters[key]
            params.get("sql_configuration_parameters").get("configuration_pairs").append({
                "key": key,
                "value": value
            })

        return self.client.api("POST", f"{self.client.endpoint}/api/2.0/sql/config/endpoints", params)


#Imported modules
from tkinter import messagebox, font
import tkinter as tk
import requests

#Class called GUI
class GUI:
    #initializes the instance of the class
    def __init__(self):

        #Constants and our root/main_window
        self.SCREEN_WIDTH = "600"
        self.SCREEN_HEIGHT = "500"
        self.root = tk.Tk()
        #Specifies the dimensions of the GUI
        self.root.geometry(self.SCREEN_WIDTH + 'x' + self.SCREEN_HEIGHT)

        #Background image that covers the GUI
        self.background_image = tk.PhotoImage(file="landscape.png")
        self.background_label = tk.Label(self.root, image=self.background_image)
        self.background_label.place(x=0, y=0, relwidth=1, relheight=1)

        #Our upper-frame, that contains an entry and a button
        self.frame = tk.Frame(self.root, bg="#80c1ff", bd=3)
        self.frame.place(relx=0.5, rely=0.1, relwidth=0.75, relheight=0.1, anchor="n")

        #entry field
        self.entry = tk.Entry(self.frame, font=40)
        self.entry.place(relwidth=0.65, relheight=1)

        #button -> Has a lambda function that passes the entry data on click
        self.button = tk.Button(self.frame, text="Check Weather", font=40, command=lambda: self.get_weather(self.entry.get()))
        self.button.place(relx = 0.65, relwidth=0.35, relheight=1)

        #Lower_frame that contains a label
        self.lower_frame = tk.Frame(self.root, bg="#80c1ff", bd=3)
        self.lower_frame.place(relx = 0.5, rely=0.25, relwidth=0.75, relheight=0.6, anchor="n")

        #The label we output information into
        self.label = tk.Label(self.lower_frame, font=("Verdana", 15))
        self.label.place(relwidth=1, relheight=1)

        #Loop that keeps the GUI going
        self.root.mainloop()

    #Function that takes a city as parameter, and converts it to
    #latitude and longitude, then returns the data
    def converter(self, city):
        url = "https://us1.locationiq.com/v1/search.php"
        data = {
            'key': "27dde2a09574d6",
            'q': f"{city}",
            'format': "json"
        }

        response = requests.get(url, params=data)
        lat, lon = response.json()[0]["lat"], response.json()[0]["lon"]
        return lat, lon

    #Function that formats the data from the weather api and
    #returns it
    def format_response(self, weather):
        temp = weather["properties"]["timeseries"][0]["data"]["instant"]["details"]["air_temperature"]
        wind = weather["properties"]["timeseries"][0]["data"]["instant"]["details"]["wind_speed"]
        summary = weather["properties"]["timeseries"][0]["data"]["next_1_hours"]["summary"]["symbol_code"]

        return "Wind speed: %s m/s\n Forecast: %s\nTemperature: %s degrees Celsius" % (wind, summary, temp)

    #Function that graps weather data, calls the converter function
    #to get the latitude and longitude and puts everything inside the label
    def get_weather(self, city):
        try:
            if city.isalpha():
                lat, lon = self.converter(f"{city}")
                url = f"https://api.met.no/weatherapi/locationforecast/2.0/compact?lat={lat}&lon={lon}"
                headers = requests.utils.default_headers()
                headers.update(
                    {
                        'User-Agent': 'My User Agent 1.0',
                    }
                )
                response = requests.get(url, headers=headers)
                weather = response.json()
                self.label["text"] = self.format_response(weather)
            else:
                messagebox.showinfo("ERROR", "It has to be alphabetical!")
        #Incase we do not find a city
        except KeyError:
            messagebox.showinfo("ERROR", "You did not specify a known city!")
#Only creates an instance of the GUI for this file.
if __name__ == "__main__":
    weather_GUI = GUI()


import matplotlib.pyplot as plt
import matplotlib as mp
import numpy as np

def f(x,y):
    return (1- x/2 + x**5 + y**3) * np.exp(-x**2-y**2)

n = 256
x = np.linspace(-3,3,n)
y = np.linspace(-3,3,n)

# 用meshgrid在二维平面中将每一个x和每一个y分别对应起来，编织成网格
X,Y = np.meshgrid(x,y)

# 使用plt.contourf进行颜色填充，从cmap中寻找f(X,y)对应的颜色，8代表将等高线分成10部分
plt.contourf(X, Y, f(X,Y), 8, alpha=.75, cmap=plt.cm.hot)

# 进行等高线绘制
C = plt.contour(X, Y, f(X,Y), 8, colors='black', linewidth=.5)

# 添加高度数字
plt.clabel(C, inline=True, fontsize=10)
plt.xticks(())
plt.yticks(())

plt.show()


import os

from dotenv import load_dotenv, find_dotenv

ENV_FILE = find_dotenv()
if ENV_FILE:
    load_dotenv(ENV_FILE)

ETHERSCAN_API_KEY = os.environ.get('ETHERSCAN_API_KEY')
CONTRACT_ADDRESS = os.environ.get('CONTRACT_ADDRESS')
START_BLOCK = int(os.environ.get('START_BLOCK'))

NUMBER_OF_LAST_BLOCKS = 10

BLOCKS_PROCESSED = 300
TRANSACTIONS_PROCESSED = 200
WALLETS_PROCESSED = 300

BLOCKS_QUEUE = 'BlocksToProcessQueue'
TRANSACTIONS_QUEUE = 'TransactionsToProcessQueue'
WALLETS_QUEUE = 'WalletBalanceToUpdateQueue'

TRANSACTIONS_TABLE_NAME = 'TokenTransactions'
WALLETS_TABLE_NAME = 'WalletAddresses'
TRANSACTIONS_RCU = 5
TRANSACTIONS_WCU = 10
WALLETS_RCU = 5
WALLETS_WCU = 10


from model.db_connect import pyodbc_connect as odbc
import logging
from datetime import datetime
import pandas as pd
import numpy as np
from datetime import datetime
from azure.logging_setup import setup_azure_logger
logger = setup_azure_logger(__name__)

def select_dantai_nm(dantai_cd):
    query = "SELECT dantai_nm " \
            "FROM dantai " \
            "WHERE dantai_cd ={}".format(dantai_cd)
    try:
        return odbc.exec_query(query)
    except Exception as e:
        logging.error('An exception occurred: {}'.format(e))
        raise e

def select_file_ids(dantai_cd):
    query = "SELECT 不動産登記簿ファイルID "\
            "FROM v_toukibo_bukken "\
            "WHERE 全国地方公共団体コード = {} " \
            "GROUP BY 不動産登記簿ファイルID".format(dantai_cd)
    try:
        return odbc.exec_query(query)
    except Exception as e:
        logging.error('An exception occurred: {}'.format(e))
        raise e

def select_chiban_rireki_by_file_id(file_id):
    query = "SELECT 不動産登記簿ファイルID "\
            "FROM toukibo_chiban_rireki "\
            "WHERE 不動産登記簿ファイルID = {}".format(file_id)
    try:
        return odbc.exec_query(query)
    except Exception as e:
        logging.error('An exception occurred: {}'.format(e))
        raise e

def delete_chiban_rireki_by_file_id(file_id, cursor):
    query = "DELETE FROM toukibo_chiban_rireki "\
            "WHERE 不動産登記簿ファイルID = {}".format(file_id)
    try:
        return cursor.execute(query)
    except Exception as e:
        logging.error('An exception occurred: {}'.format(e))
        raise e

def select_bukken_view(dantai_cd, file_id):
    query = "SELECT * " \
            "FROM v_toukibo_bukken " \
            "WHERE 全国地方公共団体コード ={0} "\
            "AND 不動産登記簿ファイルID = {1}".format(dantai_cd, file_id)
    try:
        return odbc.exec_query(query)
    except Exception as e:
        logging.error('An exception occurred: {}'.format(e))
        raise e

def select_hyoujirireki_view(dantai_cd, file_id):
    query = "SELECT * " \
            "FROM v_toukibo_hyouji_rireki " \
            "WHERE 全国地方公共団体コード ={0}"\
            "AND 不動産登記簿ファイルID = {1} "\
            "ORDER BY 全国地方公共団体コード, 不動産登記簿ファイルID, 項番, 表示履歴番号".format(dantai_cd, file_id)
    try:
        return odbc.exec_query(query)
    except Exception as e:
        logging.error('An exception occurred: {}'.format(e))
        raise e

def select_shoyusha_view(dantai_cd, file_id):
    query = "SELECT * " \
            "FROM v_toukibo_shoyuken " \
            "WHERE 全国地方公共団体コード ={0}"\
            "AND 不動産登記簿ファイルID = {1}".format(dantai_cd, file_id)
    try:
        return odbc.exec_query(query)
    except Exception as e:
        logging.error('An exception occurred: {}'.format(e))
        raise e

def select_chibanrireki_view(dantai_cd):
    query = "SELECT * " \
            "FROM v_chiban_rireki " \
            "WHERE 全国地方公共団体コード ='{}'".format(dantai_cd)
    try:
        return odbc.exec_query(query)
    except Exception as e:
        logging.error('An exception occurred: {}'.format(e))
        raise e

def select_v_jusho_code_meishou(dantai_cd):

    query = "SELECT DISTINCT " \
            "市区町村コード " \
            ",大字コード" \
            ",大字名 " \
            "FROM dbo.v_juusho_code_meishou " \
           "WHERE 市区町村コード = '{}'".format(dantai_cd)
    try:
        return odbc.exec_query(query)
    except Exception as e:
        logging.error('An exception occurred: {}'.format(e))
        raise e

def select_sikuchouson_codes(str_sikuchoson_code):
    query = "SELECT " \
            "DISTINCT 市区町村コード " \
            "FROM dbo.v_toukibo_bukken " \
           f"WHERE 全国地方公共団体コード IN ({str_sikuchoson_code})"
    try:
        return odbc.exec_query(query)
    except Exception as e:
        logging.error('An exception occurred: {}'.format(e))
        raise e

def change_ooaza_kokyo_ninni_view(dantai_cd):
    query = "SELECT * " \
            "FROM v_change_ooaza_koukyou_ninni " \
            "WHERE 全国地方公共団体コード ='{}'".format(dantai_cd)
    try:
        return odbc.exec_query(query)
    except Exception as e:
        logging.error('An exception occurred: {}'.format(e))
        raise e

def query_insert_chibanrireki(items, cursor):
    values=""
    file_id = items.at[0, "不動産登記簿ファイルID"]
    logger.info(f"紐づけ管理Function:HB002:insert_chibanrireki:file_id:{file_id}, length:{len(items)}")
    for index,colums in items.iterrows():
        hudousan_file_id=colums["不動産登記簿ファイルID"]
        koban=colums["項番"]
        henkou_rireki=colums["変更履歴"]
        kousingappi=colums["更新年月日"]
        if colums["更新年月日"] is np.nan or colums["更新年月日"]=='':
            kousingappi='NULL'
        if kousingappi!='NULL':
            kousingappi=f"'{kousingappi}'"
        sakujyo_flag=colums["削除フラグ"]
        henkou_go=colums["変更後"]
        ooaza=colums["大字名"]
        koaza=colums["小字名"]
        if colums["小字名"] is np.nan:
            koaza='NULL'
        else:
            koaza_tmp=colums["小字名"]
            koaza=f"'{koaza_tmp}'"
        if colums["小字名"] =="":
            koaza='NULL'
        if colums["小字名"] == "nan":
            koaza='NULL'
        chiban=colums["地番"]
        chimoku=colums["地目"]

        if index==0:
            values+=f"({hudousan_file_id},{koban},N'{henkou_rireki}',{kousingappi},'{sakujyo_flag}',N'{henkou_go}',N'{ooaza}',{koaza},N'{chiban}',N'{chimoku}') "
        if index>0 and values!="":
            values+=f",({hudousan_file_id},{koban},N'{henkou_rireki}',{kousingappi},'{sakujyo_flag}',N'{henkou_go}',N'{ooaza}',{koaza},N'{chiban}',N'{chimoku}') "
        if values=="":
            values+=f"({hudousan_file_id},{koban},N'{henkou_rireki}',{kousingappi},'{sakujyo_flag}',N'{henkou_go}',N'{ooaza}',{koaza},N'{chiban}',N'{chimoku}') "

        if index%500==0 and index!=0:
            query = "INSERT INTO toukibo_chiban_rireki ("\
                    "不動産登記簿ファイルID, 項番, 変更履歴, 更新年月日, 削除フラグ, 変更後, 大字名, 小字名, 地番, 地目) "\
                    f"VALUES {values}"

            cursor.execute(query)
            values=""

    if(len(values) > 0):
        query = "INSERT INTO toukibo_chiban_rireki ("\
                "不動産登記簿ファイルID, 項番, 変更履歴, 更新年月日, 削除フラグ, 変更後, 大字名, 小字名, 地番, 地目) "\
                f"VALUES {values}"
        # logger.info(f"query002:{query}")
        cursor.execute(query)

def query_delete_chibanrireki(toukibo_id_list,cursor):
    query = "DELETE FROM toukibo_chiban_rireki "\
            f"WHERE 不動産登記簿ファイルID IN ({toukibo_id_list})"
    cursor.execute(query)

def query_update_chibanrireki(merged_toukibo,cursor):
    values=""
    for index,columns in merged_toukibo.iterrows():
        chiban_rireki_id = columns["地番履歴ID"]
        ooaza = columns["大字名"]
        values+=f"WHEN 地番履歴ID ={chiban_rireki_id} "\
        f"THEN '{ooaza}' "

    query = "UPDATE toukibo_chiban_rireki SET "\
            f"大字名= CASE {values} "\
            "END;"
    #logger.info(f"query002:{query}")
    cursor.execute(query)


#!/usr/bin/env python3

import sys
import numpy as np
import scipy.interpolate
import matplotlib
import matplotlib.pyplot as plt

vmin = -0.03
vmax =  0.03
# 0.3
# 0.1
# 0.1

alnmethods = ["aligned_clustalw",
              "aligned_mafft",
              "aligned_msaprobs",
              "aligned_muscle",
              "aligned_probalign",
              "aligned_probcons",
              "aligned_tcoffee",
              "correctaligned",
              "integratedaln"]

startindex = 2

def getData(fname):

    correct_index = 9

    dx = 0.1
    dy = 0.01

    XY = []
    C_aln = []
    C_int = []

    handle = open(fname, "r")
    handle.readline()
    for line in handle:
        linearr = [ float(x) for x in line.strip().split(",") ]
        brlen  = linearr[0]
        indelr = linearr[1]
        cor_d  = linearr[correct_index]
        aln_d  = min(linearr[2:9])
        int_d  = linearr[10]
        XY.append([brlen,indelr])
        C_aln.append(cor_d - aln_d)
        C_int.append(cor_d - int_d)
    handle.close()
    return (XY,C_aln,C_int)

fig, axs = plt.subplots(nrows=4,ncols=2,sharex=True,sharey=True,figsize=(3.5,4.5))

fnameparts = ["all", "res", "ins", "del"]
err_types  = ["Total Errors", "Residue Errors", "Insertion Errors", "Deletion Errors"]
for row in range(len(fnameparts)):
    fname = "err_report.%s.csv" % fnameparts[row]

    # get results #
    (xy,c_aln,c_int) = getData(fname)
    xx,yy = np.meshgrid(np.arange(0.1,0.81,0.01),np.arange(0.01,0.081,0.001))
    grid_x, grid_y = np.mgrid[0:1:100j, 0:1:100j]
    zz_aln = scipy.interpolate.griddata(xy,c_aln, (xx,yy), method='cubic')
    zz_int = scipy.interpolate.griddata(xy,c_int, (xx,yy), method='cubic')
    results = [zz_aln, zz_int]

    for i in range(2):
        # get axes #
        col = i
        ax  = axs[row,col]
        ret = ax.pcolormesh(xx,yy,results[i], vmin=vmin, vmax=vmax, shading='gouraud', antialiased=True, cmap='bwr_r')
        fig.colorbar(ret,ax=ax)

        if row == 3:
            ax.set_xlabel("Branch Length")

        if col == 0:
            ax.set_ylabel("%s\nIndel Rate" % err_types[row])

        if row == 0 and col == 0:
            ax.set_title("Sequence Alignment")
        elif row == 0 and col == 1:
            ax.set_title("Integrated Alignment")

#plt.savefig("FIG.pdf", dpi=300)
plt.show()


from __future__ import division
__author__ = 'Perkel'

from scripts.utils.graphics_sound_handling import load_image
from scripts.utils.graphics_sound_handling import CreateSprite
import pygame

def sprite_sheet(name, nr_of_frames):
    """
    :param name: Takes spritesheet name without whole path just spritesheet.jpg
    :param nr_of_frames: how many strips sprite has
    :return: list of sprites from spritesheet
    """
    image = load_image(name)
    strips = []
    count = 0

    while count < nr_of_frames:
        sprite = pygame.Surface((32, 32))
        sprite.blit(image, (0, 0), (count*32, 0, 32, 32))
        sprite.convert_alpha()
        sprite.get_rect()
        strips.append(sprite)
        count += 1
    return strips

def sprite_animation(spritesheet, seconds):
    import storage as st
    time_future = st.System.currentTime + seconds
    nr_frames = seconds / len(spritesheet)
    pass

class SpriteAnim():
    def __init__(self, sprite_list,  seconds):
        self.timeDuration = seconds
        self.timeStartOfAnim = None
        self.timeCurrentTime = None
        self.timeCurrentAnimTime = 0
        self.timeEndOfAnim = None
        self.listOfFrames = []
        self.animationStarted = False
        self.animationLoop = True
        self.sprite_list = sprite_list
        self.image = self.sprite_list[0]
        self.animShowOffset = self.timeDuration / len(sprite_list)
        self.lastFrameTime = None

    def update(self):
        import time
        self.timeCurrentTime = time.time()
        SpriteAnim.anim_start(self)
        if self.animationStarted is True:
            for x in self.listOfFrames:
                if x[1] > self.timeCurrentTime:
                    self.lastFrameTime = x[1]
                    self.image = x[0]
                    break
            if self.timeEndOfAnim < self.timeCurrentTime:
                self.animationStarted = False

    def anim_start(self):
        import time
        if self.animationLoop is True:
            if self.animationStarted is False:
                self.timeCurrentTime = time.time()
                self.timeStartOfAnim = self.timeCurrentTime
                self.timeEndOfAnim = self.timeCurrentTime + self.timeDuration
                self.animationStarted = True
                self.lastFrameTime = self.timeStartOfAnim

                count = 1
                for sprite in self.sprite_list:
                    self.listOfFrames.append(
                        [sprite, self.timeStartOfAnim+count*self.animShowOffset])
                    count += 1

    def anim_reset(self):
        pass


import json
import os
import pickle
import re
import string

import requests
import zhconv
import jieba
from keras.preprocessing import sequence
from config import config

class Result:
    label2zh = {'location_traffic_convenience': '交通是否便利',
                'location_distance_from_business_district': '距离商圈远近',
                'location_easy_to_find': '是否容易寻找',
                'service_wait_time': '排队等候时间',
                'service_waiters_attitude': '服务人员态度',
                'service_parking_convenience': '是否容易停车',
                'service_serving_speed': '点菜/上菜速度', 'price_level': '价格水平',
                'price_cost_effective': '性价比', 'price_discount': '折扣力度',
                'environment_decoration': '装修情况', 'environment_noise': '嘈杂情况',
                'environment_space': '就餐空间', 'environment_cleaness': '卫生情况',
                'dish_portion': '分量', 'dish_taste': '口感', 'dish_look': '外观',
                'dish_recommendation': '推荐程度',
                'others_overall_experience': '本次消费感受',
                'others_willing_to_consume_again': '再次消费的意愿', 'location': '位置',
                'service': '服务', 'price': '价格', 'environment': '环境',
                'dish': '菜品', 'others': '其它'}

    label_layers = {'location': ['location_traffic_convenience',
                                 'location_distance_from_business_district',
                                 'location_easy_to_find'],
                    'service': ['service_wait_time', 'service_waiters_attitude',
                                'service_parking_convenience',
                                'service_serving_speed'],
                    'price': ['price_level', 'price_cost_effective',
                              'price_discount'],
                    'environment': ['environment_decoration',
                                    'environment_noise', 'environment_space',
                                    'environment_cleaness'],
                    'dish': ['dish_portion', 'dish_taste', 'dish_look',
                             'dish_recommendation'],
                    'others': ['others_overall_experience',
                               'others_willing_to_consume_again']}

    sentiment = ['未提及', '负向', '中性', '正向']

    def render(self, prob_dict, mode='tree'):
        if mode == 'tree':
            results = {'name': '点评', 'children': []}
            for layer1 in self.label_layers.keys():
                layer1_zh = self.label2zh[layer1]
                layer1_dict = {'name': layer1_zh, 'children': []}
                for layer2 in self.label_layers[layer1]:
                    probs = prob_dict.get(
                        layer2,{"predictions": [[0, 0, 0, 0]]})["predictions"][0]
                    label = ['未提及', '负向', '中性', '正向']
                    layer2_zh = self.label2zh[layer2]
                    prob_with_label = [{'name': l, 'value': str(round(v, 2))}
                                       for l, v in zip(label, probs)]
                    # 如何可视化这结果？？？
                    layer2_dict = {'name': layer2_zh,
                                   'children': prob_with_label}
                    layer1_dict['children'].append(layer2_dict)
                results['children'].append(layer1_dict)
            return results

class SentimentClassifier:
    def __init__(self):
        clfs_dir = config['production'].CLASSIFIER_DIR
        clfs_path = {p: os.path.join(clfs_dir, p) for p in os.listdir(clfs_dir)
                     if os.path.isdir(os.path.join(clfs_dir, p))}
        base_api = 'http://localhost:8012/v1/models/{}:predict'
        clfs_api = {cate: base_api.format(cate) for cate in clfs_path}

        with open(config['production'].TOKENIZER, 'rb') as f:
            tokenizer = pickle.load(f)

        self.tokenizer = tokenizer
        self.clfs_api = clfs_api

    def preprocess(self, text):
        punct = r"，。！？、；：“”\n＂＃＄％＆＇（）＊＋－／＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､" \
                r"〃《》「」『』【】〔〕〖〗〘〙〚〛〜〝〟〰〾〿–—‛„‟…‧﹏★☆•→▽"

        def clean_special_chars(text):
            text = text.encode('utf-8').decode('utf-8')
            re_tok = re.compile(f'([{string.punctuation}{punct}])')
            return re_tok.sub(r' ', text)

        def simplify(text):
            return zhconv.convert(text, 'zh-cn')

        def cut_join(text):
            space = ' '
            words = jieba.cut(text)
            return space.join([w.lower() for w in words if not w.isspace()])

        sents = re.split('[.!?。！？]', text.strip())
        results = []
        for sent in sents:
            if sent:
                sent = clean_special_chars(sent)
                sent = simplify(sent)
                sent = cut_join(sent)
                results.append(sent)
        return ' '.join(results)

    def text2vector(self, text):
        text = self.preprocess(text)
        text_seq = self.tokenizer.texts_to_sequences([text])
        text_seq = sequence.pad_sequences(text_seq, maxlen=250)
        return text_seq

    def predict(self, text):
        results = {}
        X_input = self.text2vector(text)

        X_input = X_input.astype('int').tolist()[0]
        print(X_input)
        payload = {"instances": [{'input_text': X_input}]}
        for cate in self.clfs_api.keys():
            r = self.single_predict(cate, payload)
            results[cate] = r
        return self.format_probs(results)

    def single_predict(self, cate, payload):
        api = self.clfs_api[cate]
        r = requests.post(api, json=payload)
        if r.ok:
            print(cate, r.content.decode('utf-8'))
            result = json.loads(r.content.decode('utf-8'))
        else:
            result = {"predictions": [[0.0, 0.0, 0.0, 0.0]]}
        return result

    def format_probs(self, probs):
        r = Result()
        results = r.render(probs, mode='tree')

        return results


import gc
import time, datetime
import os, sys
import numpy as np
import pandas as pd
import math
import random
import argparse
from argparse import RawTextHelpFormatter

from pyspark import SparkContext, StorageLevel
from pyspark.sql import SparkSession
import pyspark_cassandra

sys.path.append(
    os.path.abspath(os.path.join(os.path.realpath(__file__),
            os.pardir))+'/utils/')
import comm
from bootstrapper import createRDDs, loadFromFiles, createTopR
from updater import update
from helpers import prediction

if __name__ == "__main__":
    parser = argparse.ArgumentParser(formatter_class=RawTextHelpFormatter, \
            description='SwIFT backend\n \
            Example Usage: \n \
                spark-submit --master spark://$(hostname):7077 \n \
                --packages anguenot:pyspark-cassandra:2.4.0 \n \
                --conf spark.cassandra.connection.host=$(hostname) \n \
                --conf spark.driver.memory=100G --executor-memory 100G \n \
                --total-executor-cores 96 \n \
                fast_swift.py <args>')
    parser.add_argument("--training_set", help="file://path/to/trainingSet")
    parser.add_argument("--test_set", help="file://path/to/testSet")
    parser.add_argument("--k", default=5, help="model size for kNN")
    parser.add_argument("--alpha", default=0, help="ISIM temporal param")
    parser.add_argument("--max_mbatch_size", default=1)
    parser.add_argument("--p", default=100, help="Size of most popular items to choose from for cold-start recommendations")
    parser.add_argument('--listR', dest='listR', nargs='+', type=int, help="List of number \
            of recommendations. A separate output for each value.")
    parser.add_argument('--VIEWSIM', dest='viewsim', action='store_true', \
            help='If set => calculate average view similarity after every batch.')
    parser.add_argument('--RMSE', dest='rmse', action='store_true', \
            help='If set => calculate RMSE for the last rating in \
            every batch.')
    parser.add_argument('--no-bootstrap', dest='bootstrap', \
            action='store_false',
            help='Skip bootstrap phase. The values are obtained from cassandra')
    parser.add_argument('--loadDir', type=str, default=None, \
            help='file://folder/to/load/key-value/relations/ \n \
            k must be <= stored k! \n \
            p must be <= stored p! \n \
            bootstrap must be True to derive topR!')
    parser.add_argument('--storeDir', type=str, default=None, \
            help='file://folder/to/store/key-value/relations/ \n \
            Must empty this directory before! \n \
            bootstrap must be True')
    parser.add_argument('--listItems', dest='listI', nargs='+', type=int, \
            help="List of items to print the processing timestamps. Usefull \
            for energy consumption experiments.")
    parser.add_argument('--seed', type=int, default=42)
    parser.add_argument("--backend_port", type=int, default=9995,
                        help='Port to listen for client messages')
    parser.add_argument("--backend_ip", type=str,
                        default="127.0.0.1",
                        help='IP address to listen to')

    parser.set_defaults(bootstrap=True)
    args = parser.parse_args()

    if sys.version_info[0] != 3:
      print("Requires Python 3")
      sys.exit(1)

    random.seed(args.seed)

    assert args.storeDir is None or not os.listdir(args.storeDir[6:]), \
        "storeDir directory not empty!"

    sess = SparkSession\
        .builder\
        .appName("SwIFT")\
        .getOrCreate()
    conf = sess.sparkContext._conf
    sess.stop()

    sc = pyspark_cassandra.CassandraSparkContext(conf=conf)

    part = int(conf.get("spark.cores.max"))
    sqrtpart = int(math.ceil(math.sqrt(part)))

    # define cassandra's keyspace
    keyspace = "mykeyspace"

    # define whether to create RDDs in cassandra
        # if false => bootstrap phase is skipped
    bootstrap = args.bootstrap

    # define whether to calculate RMSE
    rmse = args.rmse

    # mostPopular size for prediction
    popularSize = int(args.p)
    print("mostPopular: %d" % popularSize)

    # define dataset length for scanrate computation
    size = 1700 # ML100K

    # define whether to include key_item in neighbors
    checkNeighbors = False

    # define if duplicate ratings (same user + same item)
    # are provided in the incremental update phase
    containDuplicates = False

    # define whether to ignore new users and new items in testPhase
    ignoreNew = False

    # top k neighbors param
    k = int(args.k)
    print("topK: %d" % k)

    # maximum number of provided recommendations param
    R = max(args.listR) # N is denoted with R
    listR = args.listR
    print("topN: %s" % R)

    # maximum number for each item's recommendations (set <= 0 to deactivate)
    maxR = -1
    print("maxR: %d" % maxR)

    # isim alpha param
    alpha = float(args.alpha)
    print("alpha: %g" % alpha)

    print("Batch size: %d" % args.max_mbatch_size)

    # total number of computed similarities
    totalSimEval = sc.accumulator(0)

    new_lines = sc.textFile(args.test_set, part)

    print("BOOTSTRAP")
    """PHASE1: BOOTSTRAP"""
    start = time.time()
    if args.loadDir is not None:
        loadFromFiles(args.loadDir, popularSize, k, R, sc, conf, keyspace, part)

    elif bootstrap:
        createRDDs(args.training_set, popularSize, k, R, alpha,
                   sc, conf, keyspace, part, args.storeDir)
    else:
        print("Data will be read from the previous state of Cassandra.")

    # derive topR (that depends on config parameters and not just on the dataset)
    # and fetch most popular items
    popularHeap = createTopR(R, args.max_mbatch_size, popularSize, sc, keyspace)

    allItems = set(sc.cassandraTable(keyspace, "topk") \
            .select("item_id").map(lambda p: p.item_id).collect())

    end = time.time()
    print("Total training latency (sec): %g" % (end-start))

    print("INCREMENTAL UPDATES")
    """PHASE2: INCREMENTAL UPDATES"""

    # dictionary for batching observations
    # (user, iid) -> (rating, timestep)
    fresh_ratings = {}

    # temp timestep for fresh_ratings
    # user -> count
    tempTimestep = {}

    firstIteration = True

    recommendations = {} # R -> (user_id, recommended_items_list)
    for val in args.listR:
        recommendations[val] = []
    rec = {}
    batch_latency = [] # contain the latency for each batch event
    update_latency = [] # contain the latency for each update
    RMSEcount = 0
    MAEsum = 0
    RMSEsum = 0
    RMSErbarsum = 0
    it = 0

    conn = comm.openServerConn(args.backend_port, args.backend_ip)

    batch_id = 0
    while True:
        batch_id += 1
        print("Waiting for new batch...")
        try:
            comm.sendMessage(conn, "ACK")
        except BrokenPipeError:
            print("Connection with frontend terminated!")
            break
        msg = comm.getMessage(conn, isJSON=True) # fetch data from frontend
        mbatch = pd.read_json(msg)

        """Batching phase"""
        start = time.time()
        for index, row in mbatch.iterrows():

            '''batch1: Get new observation (user, item, rating, timestamp)'''
            user = int(row['user'])
            item_id = int(row['item'])
            rating = float(row['rating'])
            timestamp = int(row['unixtimestamp'])

            '''batch2: Add it to fresh_ratings dictionary'''
            if user in tempTimestep: tempTimestep[user] += 1
            else: tempTimestep[user] = 1
            fresh_ratings[(user, item_id)] = (rating, tempTimestep[user])

            if rmse:
                '''batch3a: Predict'''
                # get user_items
                profile = sc.cassandraTable(keyspace, "userprofiles") \
                        .select("profile").where("user_id=" + str(user)) \
                        .map(lambda p: p.profile).collect()

                if profile:
                    profile = profile[0]
                    # derive rbar
                    rbar = 0
                    user_items = {} # already rated items
                    for value in profile:
                        iid = value[0]
                        rating = value[1]
                        user_items[iid] = rating
                        rbar += rating

                    rbar = rbar / len(user_items.keys())

                    # get topk
                    knn = sc.cassandraTable(keyspace, "topk") \
                            .select("neighbors").where("item_id=" + str(item_id)) \
                            .map(lambda p: p.neighbors).collect()

                    if knn:
                        knn = knn[0]

                    # calculate prediction and rmse
                    pred = prediction(knn, user_items, rbar)
                    if pred:
                        RMSEsum += (pred - rating)**2
                        RMSErbarsum += (rbar - rating)**2
                        MAEsum += abs(pred - rating)
                        RMSEcount += 1

            '''batch3b: Recommend'''
            recom = sc.cassandraTable(keyspace, "topr").select("recommendations") \
                    .where("user_id=" + str(user)) \
                    .map(lambda p: p.recommendations).collect()

            new_recom = {} # R -> new recommendations list
            for val in listR:
                new_recom[val] = []
            if recom != []: # if user exists
                if recom != [None]: # if recommendations for the user exist
                    recom = recom[0]
                    count = 0 # counter to contain the number of recoms
                    for x in recom:
                        # if x is recommended more than maxR times => skip
                        if maxR > 0:
                            if user in rec:
                                if rec[user].count(x[0]) == maxR:
                                    continue
                        # if x is not-yet-rated item
                        if (user, x[0]) not in fresh_ratings:
                            for val in listR:
                                if count < val:
                                    new_recom[val].append(int(x[0]))
                            count += 1
                            if count == R: break
                    for val in listR:
                        # first int is user and the rest are recoms
                        msg = [user] + new_recom[val]
                        comm.sendMessage(conn, msg, isJSON=False)
                        recommendations[val].append((user, new_recom[val]))

                    # TODO make rec update more efficient
                    if maxR > 0 :
                        if user in rec:
                            old = rec[user]
                            rec[user] = old + new_recom
                        else:
                            rec[user] = new_recom

        end = time.time()
        batch_latency.append(end-start)
        print("Batching latency for batch %d: %g" % (batch_id, end-start))
        """End of batching phase"""

        start = time.time()
        """Update phase"""
        popularHeap = update(fresh_ratings, allItems, k, alpha, R + len(mbatch),
                             popularHeap, containDuplicates, ignoreNew,
                             sc, keyspace, part)
        # reset for next micro-batch
        fresh_ratings = {}
        tempTimestep = {}

        """End of Update phase"""
        end = time.time()
        update_latency.append(end - start)
        print("update latency for batch %d (sec): %g" % (batch_id, end-start))

        if args.viewsim:
            """Calculate average view similarity
            get results: cat out | awk -F: '/Average view similarity:/ {getline; print $0}'
            """
            simAvg = sc.cassandraTable(keyspace, "topk") \
                    .select("item_id", "neighbors") \
                    .map(lambda p: np.average(list(map(lambda x: x[1], p.neighbors)))) \
                    .reduce(add)
            print("Average view similarity: %g" % (simAvg / float(len(allItems))))

        # popularity energy consumption
        if args.listI and item_id in args.listI:
            print("start and end datetime for item: %s" % item_id)
            print(datetime.datetime.fromtimestamp(start)).strftime(
                    '%Y-%m-%d %H:%M:%S')
            print(start)
            print(datetime.datetime.fromtimestamp(end)).strftime(
                    '%Y-%m-%d %H:%M:%S')
            print(end)

    if rmse:
        print("phase3a: Calculate RMSE")
        print("RMSE count: %g" % RMSEcount)
        RMSEfinal = np.sqrt(RMSEsum / RMSEcount)
        RMSErbarfinal = np.sqrt(RMSErbarsum / RMSEcount)
        MAEfinal = MAEsum / RMSEcount
        print("RMSE rbar: %g" % RMSErbarfinal)
        print("MAE: %g" % MAEfinal)
        print("RMSE: %g" % RMSEfinal)

    print("Iterations: %d" % batch_id)
    print("Average update latency: %g" % np.average(update_latency))
    print("Min update latency: %g" % min(update_latency))
    print("50th percentile update latency: %g" % np.percentile(update_latency, 50))
    print("90th percentile update latency: %g" % np.percentile(update_latency, 90))
    print("99th percentile update latency: %g" % np.percentile(update_latency, 99))
    print("Max update latency: %g" % max(update_latency))

    print("Average batch latency: %g" % np.average(batch_latency))
    print("Min batch latency: %g" % min(batch_latency))
    print("50th percentile batch latency: %g" % np.percentile(batch_latency, 50))
    print("90th percentile batch latency: %g" % np.percentile(batch_latency, 90))
    print("99th percentile batch latency: %g" % np.percentile(batch_latency, 99))
    print("Max batch latency: %g" % max(batch_latency))

    sc.stop()


from pyspark import SparkConf, SparkContext
import sys
import random
import time
import json
import math
import csv
import itertools
from pyspark.sql import SparkSession

# print('System arguments --> ',sys.argv)
# Initializing input variables
stime = time.time()
out_dict = dict()
delm = ','
#ct_dict = {}
#case = int(sys.argv[1])
#support = int(sys.argv[2])
i_file_loc = 'task2.model'
#o_file = sys.argv[4]
t_file_loc = 'test_review.json'
o_file = 'task2.predict'

user_dict = dict()
bus_dict = dict()

def compute_sim(line):
    user_words = set(user_dict.get(line[0],''))
    bus_words = set(bus_dict.get(line[1],''))
    num = len(user_words.intersection(bus_words))
    den = math.sqrt(len(user_words))*math.sqrt(len(bus_words))
    if den ==0:
        sim = 0
    else:
        sim = num/den
    return('{"user_id": "'+line[0] + '", "business_id": "' + line[0] + '", "sim": ' + str(sim) + '}')

#setting Spark session
spark = SparkSession.builder.getOrCreate()
sc = spark.sparkContext

# load input file
r_file = sc.textFile(i_file_loc)
#print(r_file.collect())

#create user dict and bus dict
a = r_file.map(lambda x:x.split('\x01')).persist()
#print(a.collect())
#sys.exit(1)
#bus_dictt = a.filter(lambda x: x[0] == 'business').map(lambda x:(x[1],x[2].strip()))
#print('a -->',bus_dictt.collect())
bus_dict = a.filter(lambda x: x[0] == 'business').map(lambda x:(x[1],eval(x[2]))).collectAsMap()
user_dict = a.filter(lambda x: x[0] == 'user').map(lambda x:(x[1],eval(x[2]))).collectAsMap()

#b = r_file.map(lambda x:for i in x.split('|'))
#print(b.collect())

t_file = sc.textFile(t_file_loc)
t_file_json = t_file.map(lambda x: (json.loads(x)["user_id"],json.loads(x)["business_id"]))

out_tup = t_file_json.map(compute_sim).filter(lambda x:eval(x)["sim"] > 0.01)
out_str = out_tup.collect()
#print('out_str --> ',out_str)

with open(o_file,'w',newline='') as out_file:
    #out_writer = csv.writer(out_file,delimiter='|',quoting=csv.QUOTE_MINIMAL)
    #out_writer.writerow(['user_id','business_id'])
    for line in out_str:
        out_file.write(line+'\n')

print('Processing time  : ',time.time() - stime)


from django.db import models
from django.contrib.auth.signals import user_logged_in, user_logged_out
from django.dispatch import receiver

class AuditLog(models.Model):
    timestamp = models.DateTimeField(auto_now=True)
    user = models.CharField(max_length=250, null=False, blank=False)
    user_ip = models.CharField(max_length=100, null=False, blank=False)
    device_info = models.CharField(max_length=500, null=True, blank=True)
    action_name = models.CharField(max_length=50, null=False, blank=False)
    table_name = models.CharField(max_length=250, null=True, blank=True)
    task_name = models.CharField(max_length=50, null=True, blank=True)
    action_details = models.CharField(max_length=200, null=True, blank=True)
    data = models.TextField(null=True, blank=True)

    def __str__(self):
        return str(self.timestamp)+'_'+self.user

    class Meta:
        db_table = 'AUTH_LogWithAudit_AuditLog'

@receiver(user_logged_in)
def user_logged_in_callback(sender, request, user, **kwargs):
    us = user.username
    x_f = request.META.get('HTTP_X_FORWARDED_FOR')
    if x_f:
        uip = x_f.split(',')[0]
    else:
        uip = request.META.get('REMOTE_ADDR')
    try:
        deviceinfo = request.META['HTTP_USER_AGENT']
    except:
        deviceinfo = ''
    AuditLog.objects.create(user=us, user_ip=uip, action_name="login", device_info=deviceinfo, action_details=us+" logged into the system", table_name="user")

@receiver(user_logged_out)
def user_logged_out_callback(sender, request, user, **kwargs):
    us = user.username
    x_f = request.META.get('HTTP_X_FORWARDED_FOR')
    if x_f:
        uip = x_f.split(',')[0]
    else:
        uip = request.META.get('REMOTE_ADDR')
    try:
        deviceinfo = request.META['HTTP_USER_AGENT']
    except:
        deviceinfo = ''
    AuditLog.objects.create(user=us, user_ip=uip, device_info=deviceinfo, action_name="logout", action_details=us+" logged out from the system", table_name="user")


# -*- coding: utf-8 -*-
import logging
import phonenumbers
from datetime import datetime, timedelta
from dateutil import tz
from django.conf import settings
from django.contrib.auth.models import Group
from django.contrib.auth import authenticate
from django.db import transaction
from django.db.models import Q, Prefetch
from django.utils.translation import ugettext_lazy as _
from django.utils import timezone
from django.utils.http import urlquote
from rest_framework import exceptions, serializers
from rest_framework.response import Response
from rest_framework.validators import UniqueValidator

from shared_foundation.custom.drf.fields import PhoneNumberField
from shared_foundation.constants import *
from shared_foundation.custom.drf.validation import MatchingDuelFieldsValidator, EnhancedPasswordStrengthFieldValidator
from shared_foundation.models import SharedUser
# from tenant_api.serializers.associate_comment import AssociateCommentSerializer
from tenant_api.serializers.skill_set import SkillSetListCreateSerializer
from tenant_api.serializers.tag import TagListCreateSerializer
from tenant_api.serializers.insurance_requirement import InsuranceRequirementListCreateSerializer
from tenant_api.serializers.vehicle_type import VehicleTypeListCreateSerializer
from tenant_foundation.models import (
    AssociateComment,
    Associate,
    Comment,
    InsuranceRequirement,
    SkillSet,
    Organization,
    VehicleType,
    HowHearAboutUsItem,
    TaskItem
)

logger = logging.getLogger(__name__)

class TaskItemAvailableAssociateListCreateSerializer(serializers.ModelSerializer):
    full_name = serializers.SerializerMethodField()
    telephone = PhoneNumberField()
    e164_telephone = serializers.SerializerMethodField()
    pretty_tags = serializers.SerializerMethodField()

    # Meta Information.
    class Meta:
        model = Associate
        fields = (
            'id',
            'type_of',
            'full_name',
            'past_30_days_activity_sheet_count',
            'telephone',
            'e164_telephone',
            'email',
            'wsib_number',
            'hourly_salary_desired',
            'score',
            'tags',
            'pretty_tags',
        )
        extra_kwargs = {
            # "is_ok_to_email": {
            #     "error_messages": {
            #         "invalid": _("Please pick either 'Yes' or 'No' choice.")
            #     }
            # },
            # "is_ok_to_text": {
            #     "error_messages": {
            #         "invalid": _("Please pick either 'Yes' or 'No' choice.")
            #     }
            # },
            # "hourly_salary_desired": {
            #     "error_messages": {
            #         "min_value": _("Ensure this value is greater than or equal to 0."),
            #         "invalid": _("Please enter a value with no $, such as 20")
            #     }
            # }
        }

    def setup_eager_loading(cls, queryset):
        """ Perform necessary eager loading of data. """
        queryset = queryset.prefetch_related(
            'owner',
            'created_by',
            'last_modified_by',
            'tags',
            'skill_sets',
            'vehicle_types',
            'comments',
            'insurance_requirements',
        )
        return queryset

    def get_full_name(self, obj):
        try:
            return str(obj)
        except Exception as e:
            return None

    def get_e164_telephone(self, obj):
        """
        Converts the "PhoneNumber" object into a "NATIONAL" format.
        See: https://github.com/daviddrysdale/python-phonenumbers
        """
        try:
            if obj.telephone:
                return phonenumbers.format_number(obj.telephone, phonenumbers.PhoneNumberFormat.E164)
            else:
                return "-"
        except Exception as e:
            return None

    def get_pretty_tags(self, obj):
        try:
            s = TagListCreateSerializer(obj.tags.all(), many=True)
            return s.data
        except Exception as e:
            return None

    @transaction.atomic
    def create(self, validated_data):
        """
        Override the `create` function to add extra functinality:
        """
        return validated_data


import cv2 as cv
import numpy as np

img = cv.imread('../photos/big-bang.jpg')
cv.imshow('Bang',img)

blank = np.zeros(img.shape[:2], dtype='uint8')

mask = cv.circle(blank, (img.shape[1]//2,img.shape[0]//2),100,255,-1)
cv.imshow('Mask',mask)

masked = cv.bitwise_and(img,img,mask=mask)
cv.imshow('Masked image',masked)

cv.waitKey(0)


from flask import Flask, render_template, request, Response, make_response, jsonify
from functools import wraps
from datetime import datetime, timedelta

app = Flask(__name__)

SECRET="redteam rocks!"

# Decorator function to check if the request's host matches the required host
def check_host_decorator(required_host):
    def actual_decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            if request.headers["Host"].split(":")[0] == required_host:
                return func(*args, **kwargs)
            else:
                return "ok"
        return wrapper
    return actual_decorator

# Decorator function that validates a cookie value and sets the response header using a callback function
def validate_cookie_decorator(cookie_name, required_value, set_header_callback):
    def actual_decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            secret_cookie = request.cookies.get(cookie_name, "")

            if secret_cookie == required_value:
                response = func(*args, **kwargs)
            else:
                response = Response("Unauthorized", status=401)

            set_header_callback(response)
            return response

        return wrapper
    return actual_decorator

# Callback function to set the Access-Control-Allow-Origin header to *
def set_acao_header_star(response):
    response.headers["Access-Control-Allow-Origin"] = "*"

# Callback function to set the Access-Control-Allow-Origin header based on the request Origin
def set_acao_header_origin(response):
    response.headers["Access-Control-Allow-Origin"] = request.headers.get(
        "Origin", "http://cors-lab.com:8000/"
    )
    """
    a better approach
    whitelist = ['asdf.com','cors-lab.com']
    for i in whitelist:
        if request.headers.get("Origin", "http://cors-lab.com:8000/") == i:
            response.headers["Access-Control-Allow-Origin"] = i
            break
    """

# Callback function to set the Access-Control-Allow-Origin and Access-Control-Allow-Credentials headers based on the request Origin
def set_acao_acac_headers_origin(response):
    response.headers["Access-Control-Allow-Origin"] = request.headers.get(
        "Origin", "http://cors-lab.com:8000/"
    )
    response.headers["Access-Control-Allow-Credentials"] = "true"

@app.route("/")
@check_host_decorator("cors-lab.com")
def hello():
    return render_template("index.html")

# exploit the SameSite=None and Secure=True
@app.route("/exploit.html")
@check_host_decorator("cors-exp.com")
def exploit():
    return render_template("exploit.html")

# exploit the subdomain takeover or XSS on subdomains
@app.route("/sub-exploit.html")
@check_host_decorator("sub.cors-lab.com")
def sub():
    return render_template("exploit.html")

# weird behaviour for firefox
@app.route("/iframe.html")
@check_host_decorator("cors-lab.com")
def iframe():
    return render_template("iframe.html")

# public api, no need auth
@app.route("/notime")
@check_host_decorator("api.cors-lab.com")
def notime():
    current_time = (datetime.utcnow() + timedelta(hours=8)).strftime("%H:%M:%S")
    response = Response(current_time)
    return response

@app.route("/local-secret")
@check_host_decorator("127.0.0.1")
def local_secret():
    response = Response("this is a top secret!!!")
    response.headers["Access-Control-Allow-Origin"] = "*"
    return response

# public api, no need auth
@app.route("/time")
@check_host_decorator("api.cors-lab.com")
def time():
    current_time = (datetime.utcnow() + timedelta(hours=8)).strftime("%H:%M:%S")
    response = Response(current_time)
    response.headers["Access-Control-Allow-Origin"] = "*"
    return response

@app.route("/cookie-auth")
@check_host_decorator("api.cors-lab.com")
def cookie_auth():
    response = make_response("Cookie has been set.")

    # set the secret cookie
    response.set_cookie("secret", "asdf1234")

    # Set the Access-Control-Allow-Origin headers and Access-Control-Allow-Credentials to the specific origin
    response.headers["Access-Control-Allow-Origin"] = request.headers.get("Origin", "*")
    response.headers["Access-Control-Allow-Credentials"] = "true"
    return response

@app.route("/cookie-auth-2")
@check_host_decorator("api.cors-lab.com")
def cookie_auth_2():
    response = make_response("Cookie has been set.")

    # set the secret cookie
    response.set_cookie("secret", "asdf1234", samesite="None", secure=True)

    # Set the Access-Control-Allow-Origin headers and Access-Control-Allow-Credentials to the specific origin
    response.headers["Access-Control-Allow-Origin"] = request.headers.get("Origin", "*")
    response.headers["Access-Control-Allow-Credentials"] = "true"
    return response

@app.route("/token-auth")
@check_host_decorator("api.cors-lab.com")
def token_auth():
    response = make_response("Custom header has been set.")

    # Set the x-auth-token header
    response.headers["x-auth-token"] = "asdf1234"

    # Set the Access-Control-Allow-Origin headers and Access-Control-Expose-Headers to the specific origin
    response.headers["Access-Control-Allow-Origin"] = request.headers.get("Origin", "*")
    response.headers["Access-Control-Expose-Headers"] = "x-auth-token"
    return response

# won't work. `Access-Control-Allow-Origin: *` does not work for cookie-authenticated requests
@app.route("/cookie-secret-1")
@check_host_decorator("api.cors-lab.com")
@validate_cookie_decorator('secret','asdf1234',set_acao_header_star)
def cookie_secret_1():
    response = Response(SECRET)
    return response

# won't work. cookie-authenticated requests must have `Access-Control-Allow-Credentials` set to `true`
@app.route("/cookie-secret-2")
@check_host_decorator("api.cors-lab.com")
@validate_cookie_decorator('secret','asdf1234',set_acao_header_origin)
def cookie_secret_2():
    response = Response(SECRET)
    return response

@app.route("/cookie-secret-3")
@check_host_decorator("api.cors-lab.com")
@validate_cookie_decorator('secret','asdf1234',set_acao_acac_headers_origin)
def cookie_secret_3():
    response = Response(SECRET)
    return response

@app.route("/edit-secret", methods=["OPTIONS"])
@check_host_decorator("api.cors-lab.com")
def edit_secret_preflight():
    origin = request.headers.get('Origin', "https://cors-lab.com:8000/")
    headers = {
        'Access-Control-Allow-Origin': origin,
        'Access-Control-Allow-Credentials': "true",
        'Access-Control-Allow-Methods': 'GET, PATCH, OPTIONS',
        'Access-Control-Allow-Headers': 'content-type'
    }
    return Response('', status=204, headers=headers)

@app.route("/edit-secret", methods=["PATCH"])
@check_host_decorator("api.cors-lab.com")
@validate_cookie_decorator('secret','asdf1234',set_acao_acac_headers_origin)
def edit_secret():
    print('asdfasdfasdfasdfasdf')
    global SECRET
    data = request.json
    SECRET = data.get('secret', SECRET)
    print(SECRET)
    return Response("Secret has been updated to: "+SECRET)

@app.route("/token-secret-1", methods=['GET'])
@check_host_decorator("api.cors-lab.com")
def token_secret_1():
    secret_token = request.headers.get("x-auth-token", "")

    if secret_token == "asdf1234":
        response = Response(SECRET)
    else:
        response = Response("Unauthorized", status=401)

    response.headers["Access-Control-Allow-Origin"] = request.headers.get(
        "Origin", "http://cors-lab.com:8000/"
    )
    response.headers["Access-Control-Allow-Credentials"] = "true"
    return response

@app.route('/token-secret-2', methods=['OPTIONS'])
@check_host_decorator("api.cors-lab.com")
def token_secret_2_preflight():
    origin = request.headers.get('Origin', "https://cors-lab.com:8000/")
    headers = {
        'Access-Control-Allow-Origin': origin,
        'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',
        'Access-Control-Allow-Headers': 'x-auth-token'
    }

    return Response('', status=204, headers=headers)

@app.route("/token-secret-2", methods=['GET'])
@check_host_decorator("api.cors-lab.com")
def token_secret_2():
    secret_token = request.headers.get("x-auth-token", "")

    if secret_token == "asdf1234":
        response = Response(SECRET)
    else:
        response = Response("Unauthorized", status=401)

    response.headers["Access-Control-Allow-Origin"] = request.headers.get(
        "Origin", "http://cors-lab.com:8000/"
    )
    test = 1
    return response

if __name__ == "__main__":
    app.run(host="0.0.0.0", port=8000, debug=True, ssl_context='adhoc')


#	File Created:
#	Modifications and additions to code by brooksandrew

import itertools
import networkx as nx
import matplotlib.pyplot as plt
from postman_problems.turn_graph import turn_weight_function_distance, create_turn_weight_edge_attr
from postman_problems.graph import get_shortest_paths_distances, create_complete_graph, add_augmenting_path_to_graph

def create_pos_and_add_to_graph(g_full, complete_g):
    """
    Since g_full does not contain any position data,
    uses complete_g to create a position dictionary,
    and adds this data to g_full as node attributes
    Args:
        g_full (NetworkX MultiDiGraph): reconstituted graph of required and optional edges
            generated by postman_problems.graph.create_networkx_graph_from_edgelist
        complete_g (NetworkX MultiDiGraph): complete graph with all edge and node data
            generated by initialize_rpp.InnerAndOuterToEdgeListFile
    Returns:
        g_full (NetworkX MultiDiGraph): g_full with position data added to nodes
        pos (dict): node position dictionary
    """
    pos = {}
    g_full_copy = g_full.copy()
    for k in g_full.nodes():
        exec("pos[k] = (complete_g.nodes["+k+"].get('x'), complete_g.nodes["+k+"].get('y'))")
        g_full_copy.add_node(k, x = pos[k][0], y = pos[k][1])
    return g_full_copy, pos

def visualize_g_req(g_req, pos):
    """
    Plots g_req
    Args:
        g_req (NetworkX MultiDiGraph): required edge graph
            generated by postman_problems.graph.create_required_graph
        pos (dict): node position dictionary
            generated by create_pos_and_add_to_graph
    """
    fig, ax = plt.subplots()
    el = [e for e in g_req.edges(data=True)]
    nx.draw_networkx_edges(g_req, pos, edgelist=el, ax = ax)
    ax.set_xlim(left = min([pos[node][0] for node in g_req.nodes()]), right = max([pos[node][0] for node in g_req.nodes()]))
    ax.set_ylim(bottom = min([pos[node][1] for node in g_req.nodes()]), top = max([pos[node][1] for node in g_req.nodes()]))
    print('\nReconstituted Directed Graph in Solver, with Contracted Connector Edges')
    plt.show()

def make_graph_eulerian(g_req, g_full):
    """
    Directed graphs must satisfy 2 criteria to be eulerian
        strong connectivity
        in degree equals out degree for each node
    In a directed graph, the total in degree equals the total out degree
    This function augments g_req into an eulerian graph by adding edges that
    balance in degree and out degree of each node
    Args:
        g_req (NetworkX MultiDiGraph): required edge graph
            generated by postman_problems.graph.create_required_graph
        g_full (NetworkX MultiDiGraph): reconstituted graph of required and optional edges
            generated by postman_problems.graph.create_networkx_graph_from_edgelist
            augmented by create_pos_and_add_to_graph
    """
    asymmetric_nodes = []
    asymmetric_nodes_list = []
    for n in g_req.nodes():
        if g_req.in_degree(n) != g_req.out_degree(n):
            asymmetric_nodes += [(n, {'in_degree': g_req.in_degree(n), 'out_degree': g_req.out_degree(n)})]
            asymmetric_nodes_list += [n]
    nodes_to_add_out_edges = {}
    nodes_to_add_out_edges_list = []
    nodes_to_add_in_edges = {}
    nodes_to_add_in_edges_list = []
    for n in asymmetric_nodes:
        if n[1]['in_degree'] > n[1]['out_degree']:
            nodes_to_add_out_edges[n[0]] = {'edges_to_add': n[1]['in_degree']-n[1]['out_degree']}
            nodes_to_add_out_edges_list += [n[0]]
        else:
            nodes_to_add_in_edges[n[0]] = {'edges_to_add': n[1]['out_degree']-n[1]['in_degree']}
            nodes_to_add_in_edges_list += [n[0]]
    degree_matching = []
    degree_matching_depth = 1
    while len(asymmetric_nodes_list) > 0:
        degree_matching_depth += 1
        possible_degree_matching_node_pairs = []
        for (n1, n2) in itertools.combinations(asymmetric_nodes_list, 2):
            if (n1 in nodes_to_add_out_edges_list and n2 in nodes_to_add_in_edges_list) or (n1 in nodes_to_add_in_edges_list and n2 in nodes_to_add_out_edges_list):
                possible_degree_matching_node_pairs += [(n1, n2)]

        degree_matching_pairs_shortest_paths = get_shortest_paths_distances(g_full, possible_degree_matching_node_pairs, edge_weight_name=turn_weight_function_distance)
        g_degree_matching = create_complete_graph(degree_matching_pairs_shortest_paths, flip_weights=True)
        degree_matching_generator = list(nx.algorithms.max_weight_matching(g_degree_matching, True))
        for (n1, n2) in degree_matching_generator:
            if n1 in nodes_to_add_out_edges_list:
                degree_matching += [(n1, n2)]
                nodes_to_add_out_edges[n1]['edges_to_add']-=1
                nodes_to_add_in_edges[n2]['edges_to_add']-=1
            else:
                degree_matching += [(n2, n1)]
                nodes_to_add_out_edges[n2]['edges_to_add']-=1
                nodes_to_add_in_edges[n1]['edges_to_add']-=1
        print()
        for n in nodes_to_add_out_edges:
            if nodes_to_add_out_edges[n]['edges_to_add'] == 0 and n in nodes_to_add_out_edges_list:
                nodes_to_add_out_edges_list.remove(n)
                asymmetric_nodes_list.remove(n)
        for n in nodes_to_add_in_edges:
            if nodes_to_add_in_edges[n]['edges_to_add'] == 0 and n in nodes_to_add_in_edges_list:
                nodes_to_add_in_edges_list.remove(n)
                asymmetric_nodes_list.remove(n)
    print('Degree Matching Depth = {}'.format(degree_matching_depth))
    g_aug = add_augmenting_path_to_graph(g_req, g_full, degree_matching, edge_weight_name=turn_weight_function_distance)
    return g_aug

def is_graph_eulerian(g_aug):
    if nx.is_eulerian(g_aug):
        print('\nAugmented Graph is Eulerian\n')
    else:
        print('\nAugmented Graph is NOT Eulerian\n')


from __future__ import print_function, division
import os
import torch
import pandas as pd
from skimage import io, transform
import numpy as np
import skimage
import glob
import matplotlib.pyplot as plt
from torch.utils.data import Dataset, DataLoader
from torchvision import transforms, utils

import torch.utils.data
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torchvision
# import torchvision.transforms as transforms
from sklearn.model_selection import train_test_split
# from torchsummary import summary
S_PATH = '/home/shasvatmukes/project/audio_classification/All_Spectrograms/Mel_Spectrograms/Recording_'

def fit(train_loader, val_loader, model, loss_fn, optimizer, n_epochs, cuda, train_mode):

    if train_mode:
        train(train_loader, model, loss_fn, optimizer, n_epochs, cuda)
    else:
        test(val_loader, model, cuda)

def train(train_loader, model, loss_fn, optimizer, num_epochs, cuda):

    total_step = len(train_loader)
    for epoch in range(num_epochs):
        for i, (images, labels) in enumerate(train_loader):
            images = np.swapaxes(images, 1, -1)
            images = images.float()
            labels = labels.long()
            #print(epoch,i)
            #print(labels)
            if cuda:
                images = images.cuda()  # to(device)
                labels = labels.cuda()  # to(device)
            #print(labels)
            outputs = model(images)
            loss = loss_fn(outputs, labels)
            # Backward and optimize
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            if (i + 1) % 1 == 0:
                print ('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}' .format(
                    epoch + 1, num_epochs, i + 1, total_step, loss.item()))

def test(test_loader, model, cuda):
    model.eval()  # eval mode (batchnorm uses moving mean/variance instead of mini-batch mean/variance)
    with torch.no_grad():
        correct = 0
        total = 0
        for images, labels in test_loader:
            images = np.swapaxes(images, 1, -1)
            images = images.float()
            # labels = labels.float()
            # print(images.shape, 'IMAGES')
            # print(labels.shape,'LABELS')
            if cuda:
                images = images.cuda()  # to(device)
                labels = labels.cuda()  # to(device)

            outputs = model(images)
            _, predicted = torch.max(outputs.data, 1)
            total += labels.size(0)
            correct += (predicted == labels).sum().item()

        print('Test Accuracy of the model: {} %'.format(100 * correct / total))


import ast
import dash
import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Output, State
import dash_table_experiments as dt
import io
import json
from lmfit.model import load_modelresult
from lmfit.model import save_modelresult
import numpy as np
import os
import pandas as pd
import plotly
import urllib
import urllib.parse

from diffcapanalyzer.databasewrappers import get_filename_pref
from diffcapanalyzer.databasewrappers import if_file_exists_in_db
from diffcapanalyzer.databasewrappers import process_data
from diffcapanalyzer.databasewrappers import macc_chardis
from diffcapanalyzer.databasefuncs import init_master_table
from diffcapanalyzer.databasefuncs import get_file_from_database
from diffcapanalyzer.chachifuncs import col_variables
from diffcapanalyzer.descriptors import generate_model

def parse_contents(
        decoded,
        filename,
        datatype,
        database,
        windowlength=9,
        polyorder=3):
    """Checks if the uploaded file exists in the database yet. Will
    process and add that file to the database if it doesn't appear in
    the master table yet. Otherwise will return html.Div that the
    file already exists in the database. """

    cleanset_name = get_filename_pref(filename) + 'CleanSet'
    # this gets rid of any filepath in the filename and just leaves the
    # clean set name as it appears in the database check to see if the
    # database exists, and if it does, check if the file exists.
    ans_p = if_file_exists_in_db(database, filename)
    if ans_p:
        df_clean = get_file_from_database(cleanset_name, database)
        new_peak_thresh = 0.7
        feedback = generate_model(
            df_clean, filename, new_peak_thresh, database)
        return 'That file exists in the database: ' + \
            str(get_filename_pref(filename))
    else:
        try:
            decoded_dataframe = decoded_to_dataframe(
                decoded, datatype, filename)
            process_data(
                filename,
                database,
                decoded_dataframe,
                datatype,
                windowlength,
                polyorder)
            df_clean = get_file_from_database(cleanset_name, database)
            new_peak_thresh = 0.7
            feedback = generate_model(
                df_clean, filename, new_peak_thresh, database)
            return 'New file has been processed: ' + \
                str(get_filename_pref(filename))
        except Exception as e:
            return 'There was a problem uploading that file. ' + \
                'Check the format of the upload file is as expected.' + \
                str(e)

def decoded_to_dataframe(decoded, datatype, file_name):
    """Decodes the contents uploaded via the app. Returns
    contents as a dataframe. Accepts only csv files for
    both Arbin and MACCOR type data."""
    if decoded is None:
        try:
            data1 = pd.read_csv(file_name, index_col=False)
        except BaseException:
            data1 = pd.read_csv(file_name, delimiter='\t', index_col=False)
    else:
        try:
            contents = io.StringIO(decoded.decode('utf-8'))
        except BaseException:
            contents = io.StringIO(decoded.decode('utf-16'))
        try:
            data1 = pd.read_csv(contents, index_col=False)
        except BaseException:
            data1 = pd.read_csv(contents, delimiter='\t', index_col=False)
    if datatype == 'ARBIN':
        data1['datatype'] = 'ARBIN'
        expected_cols = [
            'Cycle_Index',
            'Voltage(V)',
            'Current(A)',
            'Charge_Capacity(Ah)',
            'Discharge_Capacity(Ah)',
            'Step_Index']
        assert all(item in list(data1.columns) for item in expected_cols)
    elif datatype == 'MACCOR':
        data1['MaccCharLab'] = data1.apply(
            lambda row: macc_chardis(row), axis=1)
        data1['Current(A)'] = data1['Current [A]'] * data1['MaccCharLab']
        data1['datatype'] = 'MACCOR'
        expected_cols = [
            'Cycle C',
            'Voltage [V]',
            'Current [A]',
            'Cap. [Ah]',
            'Md']
        assert all(item in list(data1.columns) for item in expected_cols)
        data1.rename(
            columns={
                'Cycle C': 'Cycle_Index',
                'Voltage [V]': 'Voltage(V)',
                'Current [A]': 'Abs_Current(A)',
                'Cap. [Ah]': 'Cap(Ah)'},
            inplace=True)
    else:
        None
    return data1

def pop_with_db(filename, database):
    """Returns dataframes that can be used to populate the app graphs.
    Finds the already existing file in the database and returns
    the cleaned version (as a dataframe) and the raw version
    (also as a dataframe)."""
    cleanset_name = get_filename_pref(filename) + 'CleanSet'
    rawset_name = get_filename_pref(filename) + 'Raw'
    if if_file_exists_in_db(database, filename):
        # then the file exists in the database and we can just read it
        df_clean = get_file_from_database(cleanset_name, database)
        df_raw = get_file_from_database(rawset_name, database)
        datatype = df_clean['datatype'].iloc[0]
        (cycle_ind_col, data_point_col, volt_col, curr_col, dis_cap_col,
         char_cap_col, charge_or_discharge) = col_variables(datatype)

    else:
        df_clean = None
        df_raw = None
        peakloc_dict = {}

    return df_clean, df_raw


# crawling용 모듈
from urllib.request import urlopen
from bs4 import BeautifulSoup
import urllib
import requests

def musicChart():
	RANK = 10 ## 멜론 차트 순위가 1 ~ 100위까지 있음

	header = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko'}
	req = requests.get('https://www.melon.com/chart/week/index.htm', headers = header) ## 주간 차트를 크롤링 할 것임
	html = req.text
	parse = BeautifulSoup(html, 'html.parser')

	titles = parse.find_all("div", {"class": "ellipsis rank01"})
	singers = parse.find_all("div", {"class": "ellipsis rank02"})
	albums = parse.find_all("div",{"class": "ellipsis rank03"})

	title = []
	singer = []
	album = []
	answer = '음악 주간 차트 TOP 10 입니다.\n\n'

	for t in titles:
		title.append(t.find('a').text)

	for s in singers:
		singer.append(s.find('span', {"class": "checkEllipsis"}).text)

	for a in albums:
		album.append(a.find('a').text)

	for i in range(RANK):
		answer += '%d위:  %s - %s\n'%(i+1, title[i], singer[i])

	res = {
		"version": "2.0",
		"template": {
			"outputs": [
				{
					"simpleText": {
						"text": answer
				  }
				}
			]
		}
	  }

	return res


from numbers import Real
from typing import Tuple

from . import bounds
from .hints import (Expansion,
                    Point)

def fast_two_sum(left: Real, right: Real) -> Tuple[Real, Real]:
    estimation = left + right
    right_virtual = estimation - left
    tail = right - right_virtual
    return tail, estimation

def two_sum(left: Real, right: Real) -> Tuple[Real, Real]:
    estimation = left + right
    right_virtual = estimation - left
    left_virtual = estimation - right_virtual
    right_tail = right - right_virtual
    left_tail = left - left_virtual
    tail = left_tail + right_tail
    return tail, estimation

def split(value: Real,
          *,
          splitter: Real = bounds.splitter) -> Tuple[Real, Real]:
    base = splitter * value
    high = base - (base - value)
    low = value - high
    return low, high

def two_product_presplit(left: Real, right: Real, right_low: Real,
                         right_high: Real) -> Tuple[Real, Real]:
    estimation = left * right
    left_low, left_high = split(left)
    first_error = estimation - left_high * right_high
    second_error = first_error - left_low * right_high
    third_error = second_error - left_high * right_low
    tail = left_low * right_low - third_error
    return tail, estimation

def two_product(left: Real, right: Real) -> Tuple[Real, Real]:
    estimation = left * right
    left_low, left_high = split(left)
    right_low, right_high = split(right)
    first_error = estimation - left_high * right_high
    second_error = first_error - left_low * right_high
    third_error = second_error - left_high * right_low
    tail = left_low * right_low - third_error
    return tail, estimation

def two_two_diff(left_tail: Real, left: Real, right_tail: Real,
                 right: Real) -> Tuple[Real, Real, Real, Real]:
    third_tail, interim_tail, interim = two_one_diff(left_tail, left,
                                                     right_tail)
    second_tail, first_tail, estimation = two_one_diff(interim_tail, interim,
                                                       right)
    return third_tail, second_tail, first_tail, estimation

def two_two_sum(left_tail: Real, left: Real, right_tail: Real,
                right: Real) -> Tuple[Real, Real, Real, Real]:
    third_tail, interim_tail, interim = two_one_sum(left_tail, left,
                                                    right_tail)
    second_tail, first_tail, estimation = two_one_sum(interim_tail, interim,
                                                      right)
    return third_tail, second_tail, first_tail, estimation

def two_one_sum(left_tail: Real, left: Real,
                right: Real) -> Tuple[Real, Real, Real]:
    second_tail, interim = two_sum(left_tail, right)
    first_tail, estimation = two_sum(left, interim)
    return second_tail, first_tail, estimation

def two_one_diff(left_tail: Real, left: Real,
                 right: Real) -> Tuple[Real, Real, Real]:
    second_tail, interim = two_diff(left_tail, right)
    first_tail, estimation = two_sum(left, interim)
    return second_tail, first_tail, estimation

def two_diff(left: Real, right: Real) -> Tuple[Real, Real]:
    estimation = left - right
    return two_diff_tail(left, right, estimation), estimation

def two_diff_tail(left: Real, right: Real, estimation: Real) -> Real:
    right_virtual = left - estimation
    left_virtual = estimation + right_virtual
    right_error = right_virtual - right
    left_error = left - left_virtual
    return left_error + right_error

def square(value: Real) -> Tuple[Real, Real]:
    estimation = value * value
    value_low, value_high = split(value)
    first_error = estimation - value_high * value_high
    second_error = first_error - (value_high + value_high) * value_low
    tail = value_low * value_low - second_error
    return tail, estimation

def sum_expansions(left_expansion: Expansion,
                   right_expansion: Expansion) -> Expansion:
    """
    Sums two expansions with zero components elimination.
    """
    left_length, right_length = len(left_expansion), len(right_expansion)
    left_element, right_element = left_expansion[0], right_expansion[0]
    left_index = right_index = 0
    if (right_element > left_element) is (right_element > -left_element):
        accumulator = left_element
        left_index += 1
        try:
            left_element = left_expansion[left_index]
        except IndexError:
            pass
    else:
        accumulator = right_element
        right_index += 1
        try:
            right_element = right_expansion[right_index]
        except IndexError:
            pass
    result = []
    if (left_index < left_length) and (right_index < right_length):
        if (right_element > left_element) is (right_element > -left_element):
            tail, accumulator = fast_two_sum(left_element, accumulator)
            left_index += 1
            try:
                left_element = left_expansion[left_index]
            except IndexError:
                pass
        else:
            tail, accumulator = fast_two_sum(right_element, accumulator)
            right_index += 1
            try:
                right_element = right_expansion[right_index]
            except IndexError:
                pass
        if tail:
            result.append(tail)
        while (left_index < left_length) and (right_index < right_length):
            if ((right_element > left_element)
                    is (right_element > -left_element)):
                tail, accumulator = two_sum(accumulator, left_element)
                left_index += 1
                try:
                    left_element = left_expansion[left_index]
                except IndexError:
                    pass
            else:
                tail, accumulator = two_sum(accumulator, right_element)
                right_index += 1
                try:
                    right_element = right_expansion[right_index]
                except IndexError:
                    pass
            if tail:
                result.append(tail)
    while left_index < left_length:
        tail, accumulator = two_sum(accumulator, left_element)
        left_index += 1
        try:
            left_element = left_expansion[left_index]
        except IndexError:
            pass
        if tail:
            result.append(tail)
    while right_index < right_length:
        tail, accumulator = two_sum(accumulator, right_element)
        right_index += 1
        try:
            right_element = right_expansion[right_index]
        except IndexError:
            pass
        if tail:
            result.append(tail)
    if accumulator or not result:
        result.append(accumulator)
    return result

def scale_expansion(expansion: Expansion, scalar: Real) -> Expansion:
    """
    Multiplies an expansion by a scalar with zero components elimination.
    """
    expansion = iter(expansion)
    scalar_low, scalar_high = split(scalar)
    tail, accumulator = two_product_presplit(next(expansion), scalar,
                                             scalar_low, scalar_high)
    result = []
    if tail:
        result.append(tail)
    for element in expansion:
        product_tail, product = two_product_presplit(element, scalar,
                                                     scalar_low, scalar_high)
        tail, interim = two_sum(accumulator, product_tail)
        if tail:
            result.append(tail)
        tail, accumulator = fast_two_sum(product, interim)
        if tail:
            result.append(tail)
    if accumulator or not result:
        result.append(accumulator)
    return result

def to_cross_product(minuend_multiplier_x: Real,
                     minuend_multiplier_y: Real,
                     subtrahend_multiplier_x: Real,
                     subtrahend_multiplier_y: Real) -> Expansion:
    """
    Returns expansion of vectors' planar cross product.
    """
    minuend_tail, minuend = two_product(minuend_multiplier_x,
                                        minuend_multiplier_y)
    subtrahend_tail, subtrahend = two_product(subtrahend_multiplier_y,
                                              subtrahend_multiplier_x)
    return two_two_diff(minuend_tail, minuend, subtrahend_tail, subtrahend)

def to_perpendicular_point(point: Point) -> Point:
    x, y = point
    return -y, x

def to_sign(value: Real) -> int:
    if value > 0:
        return 1
    elif value < 0:
        return -1
    else:
        return 0


import os
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
import numpy as np
import matplotlib.pyplot as plt
# local model
import sys
sys.path.append("../network")
from lstm import LSTMHardSigmoid

class cnnblstm(nn.Module):

	def __init__(self, time_steps = 800, n_features = 3, n_outputs = 10, params_file = "./params.pkl", use_cuda = False):
		super(cnnblstm, self).__init__()

		self.time_steps = time_steps
		self.n_features = n_features
		self.n_outputs = n_outputs
		self.params_file = params_file
		self.use_cuda = use_cuda

		self.n_filters = 128
		self.kernel_size = 15
		self.n_hidden = 150	# 150
		self.n_layers = 1
		self.bidirectional = True

		# build net1 cnn
		self.net1 = nn.Sequential(
			nn.Conv1d(in_channels = self.n_features, out_channels = self.n_filters, kernel_size = self.kernel_size),
			nn.ReLU(),
			# nn.Sigmoid(),
			nn.Dropout(p = 0.5),
			nn.MaxPool1d(kernel_size = 2)
		)

		# build net2 features.blstm
		# self.net2 = nn.LSTM(input_size = 128, hidden_size = self.n_hidden, num_layers = self.n_layers, dropout = 0.2, batch_first = True, bidirectional = self.bidirectional, bias = True)
		self.net2 = LSTMHardSigmoid(input_size = self.n_filters, hidden_size = self.n_hidden, num_layers = self.n_layers, dropout = 0.2, batch_first = True, bidirectional = self.bidirectional, bias = True)
		if self.bidirectional:
			n_blstm_output = self.n_hidden * 2
		else:
			n_blstm_output = self.n_hidden

		# build net3 classifier(fc->fc)
		self.net3 = nn.Sequential(
			# nn.Tanh(),
			nn.Linear(n_blstm_output, 50, bias = True),
			nn.ReLU(),
			# nn.Sigmoid(),
			nn.Dropout(p = 0.2),
			nn.Linear(50, self.n_outputs, bias = True),
			nn.Softmax(dim = 1)
		)

	def reset_parameters(self):
		"""
		temp useless
		Here we reproduce Keras default initialization weights for consistency with Keras version
		"""
		# get weights & bias set
		net1_weights = ((name, param.data) for name, param in self.named_parameters() if (("weight" in name) and (("net1" in name) and ("net1_adabn" not in name))))
		net1_biases = ((name, param.data) for name, param in self.named_parameters() if (("bias" in name) and (("net1" in name) and ("net1_adabn" not in name))))
		# net2_weights = ((name, param.data) for name, param in self.named_parameters() if (("weight" in name) and (("net2" in name) and ("net2_adabn" not in name))))
		# net2_biases = ((name, param.data) for name, param in self.named_parameters() if (("bias" in name) and (("net2" in name) and ("net2_adabn" not in name))))
		net3_weights = ((name, param.data) for name, param in self.named_parameters() if (("weight" in name) and (("net3" in name) and ("net3_adabn" not in name))))
		net3_biases = ((name, param.data) for name, param in self.named_parameters() if (("bias" in name) and (("net3" in name) and ("net3_adabn" not in name))))
		# init weights & bias
		for name, params_data in net1_weights:
			nn.init.xavier_uniform_(params_data)
		for name, params_data in net1_biases:
			nn.init.constant_(params_data, 0)
		self.net2.reset_parameters()		# lstm reset parameters
		for name, params_data in net3_weights:
			nn.init.xavier_uniform_(params_data)
		for name, params_data in net3_biases:
			nn.init.constant_(params_data, 0)

	def forward(self, input):
		"""
		compute the output of input according to the entire network model
		"""
		# MaxPool1d
		maxPool1d_output = self.net1(input)
		maxPool1d_t_output = maxPool1d_output.permute(0, 2, 1).contiguous()
		# BiLSTM
		bilstm_output, _ = self.net2(maxPool1d_t_output, None)
		# MaxPooling1D time_steps
		# print(bilstm_output.size())
		bilstm_output = bilstm_output.permute(0, 2, 1)
		# print(bilstm_output.size())
		maxPooling_output = F.max_pool1d(bilstm_output, kernel_size = bilstm_output.size(2)).squeeze(2)
		# maxPooling_output = bilstm_output[:, :, -1].squeeze(-1)
		# print(maxPooling_output.size())
		# get classifier
		linear2_softmax_output = self.net3(maxPooling_output)

		return linear2_softmax_output

	def trainAllLayers(self, train_x, train_y, learning_rate = 0.001, n_epoches = 20, batch_size = 10, shuffle = True):
		"""
		train all layers of network model
		"""
		# get train_data
		train_data = torch.utils.data.TensorDataset(train_x, train_y)
		# Data Loader for easy mini-batch return in training
		train_loader = torch.utils.data.DataLoader(dataset = train_data, batch_size = batch_size, shuffle = shuffle)
		# optimize all cnn parameters
		optimizer = torch.optim.Adam(self.parameters(), lr = learning_rate)
		# the target label is not one-hotted
		loss_func = nn.CrossEntropyLoss()

		# init params
		self.reset_parameters()

		# load params
		self.load_params()

		# set train mode True
		self.train()

		# training and testing
		for epoch in range(n_epoches):
			# init loss & acc
			train_loss = 0
			train_acc = 0
			for step, (b_x, b_y) in enumerate(train_loader):		# gives batch data
				b_x = b_x.view(-1, self.n_features, self.time_steps)	# reshape x to (batch, n_features, time_step)
				if self.use_cuda:
					b_x, b_y = Variable(b_x).cuda(), Variable(b_y).cuda()
				else:
					b_x, b_y = Variable(b_x), Variable(b_y)
				# get output
				output = self(b_x)									# CNN_BLSTM output
				# get loss
				loss = loss_func(output, b_y)						# cross entropy loss
				train_loss += loss.item() * len(b_y)
				_, pre = torch.max(output, 1)
				num_acc = (pre == b_y).sum()
				train_acc += num_acc.item()
				# backward
				optimizer.zero_grad()								# clear gradients for this training step
				loss.backward()										# backpropagation, compute gradients
				optimizer.step()									# apply gradients

				# print loss
				# if (step + 1) % 10 == 0:
				# 	print("[{}/{}], train loss is: {:.6f}, train acc is: {:.6f}".format(step, len(train_loader), train_loss / ((step + 1) * batch_size), train_acc / ((step + 1) * batch_size)))
			print("[{}/{}], train loss is: {:.6f}, train acc is: {:.6f}".format(len(train_loader), len(train_loader), train_loss / (len(train_loader) * batch_size), train_acc / (len(train_loader) * batch_size)))

			# save params
			self.save_params()

		print("train finish!")

	def getTestAccuracy(self, test_x, test_y):
		"""
		test network model with test set
		"""
		# init params
		self.reset_parameters()

		# load params
		self.load_params()

		# set eval
		self.eval()

		with torch.no_grad():
			if self.use_cuda:
				test_x, test_y = Variable(test_x).cuda(), Variable(test_y).cuda()
			else:
				test_x, test_y = Variable(test_x), Variable(test_y)
		# get output
		output = self(test_x)
		# print(output)
		prediction = torch.max(output, 1)[1]
		pred_y = prediction.cpu().data.numpy()
		# print(pred_y)
		target_y = test_y.cpu().data.numpy()
		# print(test_y)

		accuracy = float((pred_y == target_y).astype(int).sum()) / float(target_y.size)
		print("Accuracy: ", str(accuracy))

	def save_params(self):
		"""
		save params
		"""
		torch.save(self.state_dict(), self.params_file)
		print("save_params success!")

	def load_params(self):
		"""
		load params
		"""
		if os.path.exists(self.params_file):
			if self.use_cuda:
				self.load_state_dict(torch.load(self.params_file, map_location = torch.device('cuda')))
			else:
				self.load_state_dict(torch.load(self.params_file, map_location = torch.device('cpu')))
			print("load_params success!")

	def get_model(self, pre_trained = False):
		"""
		get pretrained model
		"""
		if pre_trained:
			self.load_params()
		return self

if __name__ == '__main__':
	use_cuda = torch.cuda.is_available()
	if use_cuda:
		cnnblstm = cnnblstm(use_cuda = use_cuda).cuda()
	else:
		cnnblstm = cnnblstm(use_cuda = use_cuda)
	print(cnnblstm)
	# get train_x, train_y
	train_x = torch.rand(20, 3, 800, dtype = torch.float32)
	train_y = torch.randint(10, (20, ), dtype = torch.int64)
	# train_y = torch.LongTensor(20, 1).random_() % 10
	print(train_x.type())
	# train_y = torch.zeros(20, 10).scatter_(1, train_y, 1)
	print(train_y)
	train_data = torch.utils.data.TensorDataset(train_x, train_y)
	cnnblstm.trainAllLayers(train_data)