import sys, time, json, threading, queue, requests, io, base64, picamera, logging, datetime, configparser
import numpy as np
import RPi.GPIO as GPIO
from PIL import Image

try:
        config = configparser.ConfigParser()
        config.read('config.ini')
except:
        raise Exception("config.ini file missing.")

class DispenserClient:
    def __init__(self, ):

        self.route = config.get('SERVER', 'Route')
        self.init_logger()
        self.init_camera()
        self.init_ir_sensor()

        # temporary data structure for storage
        self.payload_queue =  queue.Queue() # If length no supply for queue, it's dynamics
        self.logger.info('dispenser client payload queue initialized')

        # http thread instantiate
        unit = config.get('PROPERTY', 'Unit')
        self.http = http_thread("http thread", self.node_id, type, unit, self.payload_queue)
        self.http.daemon = True
        self.http.start()
        self.logger.info('dispenser client http thread initialized')

    def init_logger(self, ):
        ## Retrive parameters from configuration file
        debug_level = config.get('DEBUG', 'LogLevel')
        property_type = config.get('PROPERTY', 'Type')

        ## Logger
        if debug_level == 'Info':
            level = logging.INFO
        else:
            level = logging.DEBUG

        self.logger = logging.getLogger(property_type)
        self.logger.setLevel(level)
        ch = logging.StreamHandler()
        #ch = logging.FileHandler('dispenser_client.log')
        ch.setLevel(level)
        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        ch.setFormatter(formatter)
        self.logger.addHandler(ch)

    def init_camera(self, ):
        ## Retrive parameters from configuration file
        resolution = config.get('CAMERA', 'Resolution')
        node_id = config.get('PROPERTY', 'Id')
        shape = config.get('CAMERA', 'Shape')
        width = config.getint('CAMERA', 'Width')
        height = config.getint('CAMERA', 'Height')
        channel = config.getint('CAMERA', 'Channel')
        rotation = config.getint('CAMERA', 'Rotation')

        ## Camera
        self.camera = picamera.PiCamera()
        self.camera.rotation = rotation
        self.camera.resolution = resolution
        self.node_id = node_id
        self.shape = shape
        size = (width, height, channel)
        self.image = np.empty(size, dtype=np.uint8)
        self.camera.start_preview(fullscreen=False, window = (100,20,0,0))
        self.logger.info('dispenser client camera initialized, start_preview executed')

    def init_ir_sensor(self, ):
        GPIO.setwarnings(False)
        GPIO.setmode(GPIO.BCM)
        GPIO.setup(4, GPIO.IN) #motion sensor
        self.logger.info('dispenser client GPIO check')

    def capture(self):
        self.camera.capture(self.image, 'rgb')
        image_temp = self.image.astype(np.float64)
        image_64 = base64.b64encode(image_temp).decode('ascii')
        payload = {'NodeID': self.node_id, 'Timestamp': time.time() ,'Image': image_64, 'Shape': self.shape}
        headers = {'Content_Type': 'application/json', 'Accept': 'text/plain'}
        self.payload_queue.put((payload, headers, self.route)) # dispenser thread

    def update_route(self, new_route):
        self.route = new_route

    def update_node_id(self, new_node_id):
        self.node_id = new_node_id

    def update_unit(self, new_unit):
        self.unit = new_unit

class http_thread(threading.Thread):
    def __init__(self, name, node_id, type, unit, payload_queue):
        ###########################################################################################
        # Logger
        level = self.log_level(config.get('DEBUG', 'LogLevel'))
        self.logger = logging.getLogger('http thread')
        self.logger.setLevel(level)
        ch = logging.StreamHandler()
        #ch = logging.FileHandler('dispenser_client.log')
        ch.setLevel(level)
        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        ch.setFormatter(formatter)
        self.logger.addHandler(ch)
        ###########################################################################################

        # Initialize thread
        threading.Thread.__init__(self)
        self.name = name
        self.node_id = node_id
        self.type = type
        self.unit = unit
        self.payload_queue = payload_queue
        self.storage_queue = queue.Queue()

        # second_http thread instantiate
        self.second_http = second_http_thread("http thread", node_id, type, unit, self.storage_queue)
        self.second_http.daemon = True
        self.second_http.start()
        self.logger.info('dispenser client second http thread initialized')

    def log_level(self, level):
        if level == 'Info':
            return logging.INFO
        else:
            return logging.DEBUG

    def run(self):
        while 1:
            if self.payload_queue.qsize():
                payload, headers, route = self.payload_queue.get()
                try:
                    result = requests.post(route, json=payload, headers=headers)
                    code = result.status_code
                    self.logger.debug(result.json())
                    if code == 200:
                        self.logger.info('HTTP request received.')
                    else:
                        self.storage_queue.put((payload, headers, route))
                except:
                    self.logger.info("Failed to establish connection with server. Try again in 5s.")
                    self.storage_queue.put((payload, headers, route))

class second_http_thread(threading.Thread):
    def __init__(self, name, node_id, type, unit, storage_queue):
        ###########################################################################################
        # Logger
        level = self.log_level(config.get('DEBUG', 'LogLevel'))
        self.logger = logging.getLogger('second http thread')
        self.logger.setLevel(level)
        ch = logging.FileHandler('dispenser_client.log')
        ch.setLevel(level)
        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        ch.setFormatter(formatter)
        self.logger.addHandler(ch)
        ###########################################################################################

        # Initialize thread
        threading.Thread.__init__(self)
        self.name = name
        self.node_id = node_id
        self.type = type
        self.unit = unit
        self.storage_queue = storage_queue

    def log_level(self, level):
        if level == 'Info':
            return logging.INFO
        else:
            return logging.DEBUG

    def run(self):
        while 1:
            if self.storage_queue.qsize():
                time.sleep(5) # wait for 5 seconds before second attempt
                payload, headers, route = self.storage_queue.get()
                try:
                    result = requests.post(route, json=payload, headers=headers)
                    code = result.status_code
                    self.logger.debug(result.json())
                    if code == 200:
                        self.logger.info("second attempt successful.")
                    else:
                        self.logger.info("second attempt failed with code: " + str(code))
                        self.storage_queue.put((payload, headers, route))
                except:
                    self.storage_queue.put((payload, headers, route))
                    self.logger.info("Failed to establish connection with server again. Try again in 30s.")
                    time.sleep(25)

if __name__ == "__main__":

    # Initialization
    client = DispenserClient()

    #Main loop
    while 1:
        try:
            if GPIO.input(4):
                time.sleep(0.25)
            else:
                cur_time = time.time()
                respond = client.capture()
                time.sleep(2)
        except KeyboardInterrupt:
            sys.exit()


# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>
#         Fabian Pedregosa <fabian.pedregosa@inria.fr>
#
# License: BSD 3 clause

from math import log

import numpy as np
import pytest

from sklearn import datasets
from sklearn.linear_model import ARDRegression, BayesianRidge, Ridge
from sklearn.utils import check_random_state
from sklearn.utils._testing import (
    assert_almost_equal,
    assert_array_almost_equal,
    assert_array_less,
)
from sklearn.utils.extmath import fast_logdet

diabetes = datasets.load_diabetes()

def test_bayesian_ridge_scores():
    """Check scores attribute shape"""
    X, y = diabetes.data, diabetes.target

    clf = BayesianRidge(compute_score=True)
    clf.fit(X, y)

    assert clf.scores_.shape == (clf.n_iter_ + 1,)

def test_bayesian_ridge_score_values():
    """Check value of score on toy example.

    Compute log marginal likelihood with equation (36) in Sparse Bayesian
    Learning and the Relevance Vector Machine (Tipping, 2001):

    - 0.5 * (log |Id/alpha + X.X^T/lambda| +
             y^T.(Id/alpha + X.X^T/lambda).y + n * log(2 * pi))
    + lambda_1 * log(lambda) - lambda_2 * lambda
    + alpha_1 * log(alpha) - alpha_2 * alpha

    and check equality with the score computed during training.
    """

    X, y = diabetes.data, diabetes.target
    n_samples = X.shape[0]
    # check with initial values of alpha and lambda (see code for the values)
    eps = np.finfo(np.float64).eps
    alpha_ = 1.0 / (np.var(y) + eps)
    lambda_ = 1.0

    # value of the parameters of the Gamma hyperpriors
    alpha_1 = 0.1
    alpha_2 = 0.1
    lambda_1 = 0.1
    lambda_2 = 0.1

    # compute score using formula of docstring
    score = lambda_1 * log(lambda_) - lambda_2 * lambda_
    score += alpha_1 * log(alpha_) - alpha_2 * alpha_
    M = 1.0 / alpha_ * np.eye(n_samples) + 1.0 / lambda_ * np.dot(X, X.T)
    M_inv_dot_y = np.linalg.solve(M, y)
    score += -0.5 * (
        fast_logdet(M) + np.dot(y.T, M_inv_dot_y) + n_samples * log(2 * np.pi)
    )

    # compute score with BayesianRidge
    clf = BayesianRidge(
        alpha_1=alpha_1,
        alpha_2=alpha_2,
        lambda_1=lambda_1,
        lambda_2=lambda_2,
        max_iter=1,
        fit_intercept=False,
        compute_score=True,
    )
    clf.fit(X, y)

    assert_almost_equal(clf.scores_[0], score, decimal=9)

def test_bayesian_ridge_parameter():
    # Test correctness of lambda_ and alpha_ parameters (GitHub issue #8224)
    X = np.array([[1, 1], [3, 4], [5, 7], [4, 1], [2, 6], [3, 10], [3, 2]])
    y = np.array([1, 2, 3, 2, 0, 4, 5]).T

    # A Ridge regression model using an alpha value equal to the ratio of
    # lambda_ and alpha_ from the Bayesian Ridge model must be identical
    br_model = BayesianRidge(compute_score=True).fit(X, y)
    rr_model = Ridge(alpha=br_model.lambda_ / br_model.alpha_).fit(X, y)
    assert_array_almost_equal(rr_model.coef_, br_model.coef_)
    assert_almost_equal(rr_model.intercept_, br_model.intercept_)

def test_bayesian_sample_weights():
    # Test correctness of the sample_weights method
    X = np.array([[1, 1], [3, 4], [5, 7], [4, 1], [2, 6], [3, 10], [3, 2]])
    y = np.array([1, 2, 3, 2, 0, 4, 5]).T
    w = np.array([4, 3, 3, 1, 1, 2, 3]).T

    # A Ridge regression model using an alpha value equal to the ratio of
    # lambda_ and alpha_ from the Bayesian Ridge model must be identical
    br_model = BayesianRidge(compute_score=True).fit(X, y, sample_weight=w)
    rr_model = Ridge(alpha=br_model.lambda_ / br_model.alpha_).fit(
        X, y, sample_weight=w
    )
    assert_array_almost_equal(rr_model.coef_, br_model.coef_)
    assert_almost_equal(rr_model.intercept_, br_model.intercept_)

def test_toy_bayesian_ridge_object():
    # Test BayesianRidge on toy
    X = np.array([[1], [2], [6], [8], [10]])
    Y = np.array([1, 2, 6, 8, 10])
    clf = BayesianRidge(compute_score=True)
    clf.fit(X, Y)

    # Check that the model could approximately learn the identity function
    test = [[1], [3], [4]]
    assert_array_almost_equal(clf.predict(test), [1, 3, 4], 2)

def test_bayesian_initial_params():
    # Test BayesianRidge with initial values (alpha_init, lambda_init)
    X = np.vander(np.linspace(0, 4, 5), 4)
    y = np.array([0.0, 1.0, 0.0, -1.0, 0.0])  # y = (x^3 - 6x^2 + 8x) / 3

    # In this case, starting from the default initial values will increase
    # the bias of the fitted curve. So, lambda_init should be small.
    reg = BayesianRidge(alpha_init=1.0, lambda_init=1e-3)
    # Check the R2 score nearly equals to one.
    r2 = reg.fit(X, y).score(X, y)
    assert_almost_equal(r2, 1.0)

def test_prediction_bayesian_ridge_ard_with_constant_input():
    # Test BayesianRidge and ARDRegression predictions for edge case of
    # constant target vectors
    n_samples = 4
    n_features = 5
    random_state = check_random_state(42)
    constant_value = random_state.rand()
    X = random_state.random_sample((n_samples, n_features))
    y = np.full(n_samples, constant_value, dtype=np.array(constant_value).dtype)
    expected = np.full(n_samples, constant_value, dtype=np.array(constant_value).dtype)

    for clf in [BayesianRidge(), ARDRegression()]:
        y_pred = clf.fit(X, y).predict(X)
        assert_array_almost_equal(y_pred, expected)

def test_std_bayesian_ridge_ard_with_constant_input():
    # Test BayesianRidge and ARDRegression standard dev. for edge case of
    # constant target vector
    # The standard dev. should be relatively small (< 0.01 is tested here)
    n_samples = 10
    n_features = 5
    random_state = check_random_state(42)
    constant_value = random_state.rand()
    X = random_state.random_sample((n_samples, n_features))
    y = np.full(n_samples, constant_value, dtype=np.array(constant_value).dtype)
    expected_upper_boundary = 0.01

    for clf in [BayesianRidge(), ARDRegression()]:
        _, y_std = clf.fit(X, y).predict(X, return_std=True)
        assert_array_less(y_std, expected_upper_boundary)

def test_update_of_sigma_in_ard():
    # Checks that `sigma_` is updated correctly after the last iteration
    # of the ARDRegression algorithm. See issue #10128.
    X = np.array([[1, 0], [0, 0]])
    y = np.array([0, 0])
    clf = ARDRegression(max_iter=1)
    clf.fit(X, y)
    # With the inputs above, ARDRegression prunes both of the two coefficients
    # in the first iteration. Hence, the expected shape of `sigma_` is (0, 0).
    assert clf.sigma_.shape == (0, 0)
    # Ensure that no error is thrown at prediction stage
    clf.predict(X, return_std=True)

def test_toy_ard_object():
    # Test BayesianRegression ARD classifier
    X = np.array([[1], [2], [3]])
    Y = np.array([1, 2, 3])
    clf = ARDRegression(compute_score=True)
    clf.fit(X, Y)

    # Check that the model could approximately learn the identity function
    test = [[1], [3], [4]]
    assert_array_almost_equal(clf.predict(test), [1, 3, 4], 2)

@pytest.mark.parametrize("n_samples, n_features", ((10, 100), (100, 10)))
def test_ard_accuracy_on_easy_problem(global_random_seed, n_samples, n_features):
    # Check that ARD converges with reasonable accuracy on an easy problem
    # (Github issue #14055)
    X = np.random.RandomState(global_random_seed).normal(size=(250, 3))
    y = X[:, 1]

    regressor = ARDRegression()
    regressor.fit(X, y)

    abs_coef_error = np.abs(1 - regressor.coef_[1])
    assert abs_coef_error < 1e-10

def test_return_std():
    # Test return_std option for both Bayesian regressors
    def f(X):
        return np.dot(X, w) + b

    def f_noise(X, noise_mult):
        return f(X) + np.random.randn(X.shape[0]) * noise_mult

    d = 5
    n_train = 50
    n_test = 10

    w = np.array([1.0, 0.0, 1.0, -1.0, 0.0])
    b = 1.0

    X = np.random.random((n_train, d))
    X_test = np.random.random((n_test, d))

    for decimal, noise_mult in enumerate([1, 0.1, 0.01]):
        y = f_noise(X, noise_mult)

        m1 = BayesianRidge()
        m1.fit(X, y)
        y_mean1, y_std1 = m1.predict(X_test, return_std=True)
        assert_array_almost_equal(y_std1, noise_mult, decimal=decimal)

        m2 = ARDRegression()
        m2.fit(X, y)
        y_mean2, y_std2 = m2.predict(X_test, return_std=True)
        assert_array_almost_equal(y_std2, noise_mult, decimal=decimal)

def test_update_sigma(global_random_seed):
    # make sure the two update_sigma() helpers are equivalent. The woodbury
    # formula is used when n_samples < n_features, and the other one is used
    # otherwise.

    rng = np.random.RandomState(global_random_seed)

    # set n_samples == n_features to avoid instability issues when inverting
    # the matrices. Using the woodbury formula would be unstable when
    # n_samples > n_features
    n_samples = n_features = 10
    X = rng.randn(n_samples, n_features)
    alpha = 1
    lmbda = np.arange(1, n_features + 1)
    keep_lambda = np.array([True] * n_features)

    reg = ARDRegression()

    sigma = reg._update_sigma(X, alpha, lmbda, keep_lambda)
    sigma_woodbury = reg._update_sigma_woodbury(X, alpha, lmbda, keep_lambda)

    np.testing.assert_allclose(sigma, sigma_woodbury)

@pytest.mark.parametrize("dtype", [np.float32, np.float64])
@pytest.mark.parametrize("Estimator", [BayesianRidge, ARDRegression])
def test_dtype_match(dtype, Estimator):
    # Test that np.float32 input data is not cast to np.float64 when possible
    X = np.array([[1, 1], [3, 4], [5, 7], [4, 1], [2, 6], [3, 10], [3, 2]], dtype=dtype)
    y = np.array([1, 2, 3, 2, 0, 4, 5]).T

    model = Estimator()
    # check type consistency
    model.fit(X, y)
    attributes = ["coef_", "sigma_"]
    for attribute in attributes:
        assert getattr(model, attribute).dtype == X.dtype

    y_mean, y_std = model.predict(X, return_std=True)
    assert y_mean.dtype == X.dtype
    assert y_std.dtype == X.dtype

@pytest.mark.parametrize("Estimator", [BayesianRidge, ARDRegression])
def test_dtype_correctness(Estimator):
    X = np.array([[1, 1], [3, 4], [5, 7], [4, 1], [2, 6], [3, 10], [3, 2]])
    y = np.array([1, 2, 3, 2, 0, 4, 5]).T
    model = Estimator()
    coef_32 = model.fit(X.astype(np.float32), y).coef_
    coef_64 = model.fit(X.astype(np.float64), y).coef_
    np.testing.assert_allclose(coef_32, coef_64, rtol=1e-4)

# TODO(1.5) remove
@pytest.mark.parametrize("Estimator", [BayesianRidge, ARDRegression])
def test_bayesian_ridge_ard_n_iter_deprecated(Estimator):
    """Check the deprecation warning of `n_iter`."""
    depr_msg = (
        "'n_iter' was renamed to 'max_iter' in version 1.3 and will be removed in 1.5"
    )
    X, y = diabetes.data, diabetes.target
    model = Estimator(n_iter=5)

    with pytest.warns(FutureWarning, match=depr_msg):
        model.fit(X, y)

# TODO(1.5) remove
@pytest.mark.parametrize("Estimator", [BayesianRidge, ARDRegression])
def test_bayesian_ridge_ard_max_iter_and_n_iter_both_set(Estimator):
    """Check that a ValueError is raised when both `max_iter` and `n_iter` are set."""
    err_msg = (
        "Both `n_iter` and `max_iter` attributes were set. Attribute"
        " `n_iter` was deprecated in version 1.3 and will be removed in"
        " 1.5. To avoid this error, only set the `max_iter` attribute."
    )
    X, y = diabetes.data, diabetes.target
    model = Estimator(n_iter=5, max_iter=5)

    with pytest.raises(ValueError, match=err_msg):
        model.fit(X, y)


import scrapy
from datetime import datetime
import re

class AthuarSpider(scrapy.Spider):
    name = "spyder1"
    allowed_domains = ['']
    start_urls = [
        ''
    ]

    def parse(self, response):
        house_page_links = response.css('div.div-block-update1 a::attr(href)')
        yield from response.follow_all(house_page_links, self.parse_houses)

        pagination_links = response.css('div.div-block-40 a::attr(href)')
        yield from response.follow_all(pagination_links, self.parse)

    def parse_houses(self, response):
        specs = response.xpath('/html/body/div[3]/div[1]/div[2]/div/div[3]/div[3]//text()').getall()
        specs = ''.join(specs)
        specs = self.parse_house_specs(specs)
        title = response.xpath('//*[@id="detalhes"]/h1/text()').get()
        yield {
            'title': title,
            'description': response.xpath('/html/body/div[3]/div[1]/div[2]/div/div[3]/div[2]/div/text()').get(),
			'price': response.xpath('/html/body/div[3]/div[1]/div[1]/div/div[2]/span/text()').get(),
			'suites': specs['suites'],
			'rooms': specs['rooms'],
			'garage': specs['garage'],
			'bathrooms': specs['bathrooms'],
			'area': specs['area'],
			'date': datetime.now().strftime("%Y-%m-%d"),
			'address': response.xpath('/html/body/div[3]/div[1]/div[2]/div/div[3]/div[1]/div[1]/text()').get(),
            'url': response.url,
            'type': self.get_type(title)
        }

    def parse_house_specs(self, specs):
        specs_dict = dict()
        specs = specs.replace('\n','').replace('\r','').replace(' ', '')
        # result string
        # '2Banheiro(s)2Quarto(s)SimChurrasqueiraSimMuradoSimPortãoSimGradeSimSalãodefesta(s)31,00(m²)ÁreaTotal31,00(m²)ÁreaConstruída31,00(m²)ÁreaÚtil31,00(m²)ÁreaPrivativa'
        # select first character before name
        specs_dict['bathrooms'] = self.find_1_character(specs, 'Banheiro')
        specs_dict['rooms'] = self.find_1_character(specs, 'Quarto')
        specs_dict['garage'] = self.find_1_character(specs, 'Garagem')
        specs_dict['area'] = self.find_only_number(specs, ['(m²)ÁreaÚtil', '(m²)ÁreaPrivativa', '(m²)ÁreaTotal'])
        # specs_dict['address'] = self.find_1_character('Banheiro')
        specs_dict['suites'] = self.find_1_character(specs, 'Suíte')

        return specs_dict

    def find_1_character(self, specs, substr):
        idx = specs.find(substr)
        if idx < 1: return '0'
        return specs[idx-1]

    def find_only_number(self, specs, substrs, window=6):
        for substr in substrs:
            idx = specs.find(substr)
            if idx >= 0:
                break
        if idx < 1: return '0'
        try:
            return self.getNumbers(specs[idx-window:idx])
        except Exception as e:
            self.logger.warning(e)
            return '0'

    def getNumbers(self, str):
        array = re.findall("([0-9]+[,.]+[0-9]+)", str)
        return array[0]

    def get_type(self, title):
        types = ['APARTAMENTO', 'SOBRADO', 'TERRENO', 'COBERTURA', 'CASA']
        for t in types:
            if title.find(t) >= 0:
                return t.lower()

        return ''


import numpy as np, torch, torch.nn as nn
from torch.autograd import Variable
from torch.nn.functional import normalize, softmax
from word2vec import word2vec

class MLPAttentionWithoutQuery(nn.Module):
    def __init__(self, Dv):
        super().__init__()
        self.W = nn.Sequential(
            nn.Linear(Dv, Dv, bias=False),
            nn.Tanh(),
            nn.Linear(Dv, 1, bias=False)
        )

        for p in self.parameters():
            if p.dim() > 1:
                nn.init.xavier_normal_(p)
            else:
                nn.init.constant_(p)

    def masked_softmax(self, logits, mask, dim=1, epsilon=1e-9):
        """ logits, mask has same size """
        masked_logits = logits.masked_fill(mask == 0, -1e9)
        max_logits = torch.max(masked_logits, dim=dim, keepdim=True)[0]
        exps = torch.exp(masked_logits - max_logits)
        masked_exps = exps * mask.float()
        masked_sums = masked_exps.sum(dim, keepdim=True) + epsilon
        return masked_exps / masked_sums

    def forward(self, v, mask):
        """
        :param: v: (N1, N2, ..., NN), L, Dv
        :param: mask: (N1, N2, ..., NN), L
        We perform attention over "L"
        :return: v: (N1, N2, ..., NN), Dv ; weighted averaged vector
        :return: a: (N1, N2, ..., NN), L  ; weights for vectors (0<=ai<=1, sum to 1)
        """
        z = self.W(v).squeeze(dim=-1)  # (N1, N2, ..., NN), L
        a = self.masked_softmax(logits=z, mask=mask, dim=-1)
        # a: (N1, N2, ..., NN), L
        NNs, L, D = v.shape[:-2], v.shape[-2], v.shape[-1]
        return torch.bmm(
            a.view(np.prod(NNs), L).unsqueeze(dim=-2),  # (N1*N2*...*NN), 1, L
            v.view(np.prod(NNs), L, D)  # (N1*N2*...*NN), L, D
        ).squeeze(  # (N1*N2*...*NN), 1, D
            dim=-2  # (N1*N2*...*NN), D
        ).view(*NNs, -1)  # (N1, N2, ..., NN), D

class TextLSTM(nn.Module):
    def __init__(self,
                 input_size, hidden_size,
                 num_layers=1, bidirectional=False, dropout=0, bias=True,
                 batch_first=True,
                 device='cpu'):
        super(TextLSTM, self).__init__()
        self.batch_first = batch_first
        self.bidirectional = bidirectional
        self.num_layers = num_layers
        self.hidden_size = hidden_size
        self.device = device
        if self.num_layers == 1: dropout = 0
        self.rnn = nn.LSTM(
            input_size=input_size,
            hidden_size=hidden_size,
            num_layers=num_layers,
            bias=bias,
            batch_first=batch_first,
            bidirectional=bidirectional,
            dropout=dropout
        )

    def zero_init(self, batch_size):
        nd = 1 if not self.bidirectional else 2
        h0 = Variable(torch.zeros((self.num_layers * nd, batch_size, self.hidden_size))).to(self.device)
        c0 = Variable(torch.zeros((self.num_layers * nd, batch_size, self.hidden_size))).to(self.device)
        return (h0, c0)

    def forward(self, inputs, length, rnn_init=None, is_sorted=False):
        if rnn_init is None:
            rnn_init = self.zero_init(inputs.size(0))
        if not is_sorted:
            sort_idx = torch.sort(length, descending=True)[1]
            inputs = inputs[sort_idx]
            length = length[sort_idx]
            h0, c0 = rnn_init
            rnn_init = (h0[:, sort_idx, :], c0[:, sort_idx, :])
            unsort_idx = torch.sort(sort_idx)[1]
        x_pack = nn.utils.rnn.pack_padded_sequence(inputs, length, batch_first=self.batch_first)
        output, (hn, cn) = self.rnn(x_pack, rnn_init)
        output, _ = nn.utils.rnn.pad_packed_sequence(output, batch_first=self.batch_first)
        if not is_sorted:
            output = output[unsort_idx]
            hn = hn[:, unsort_idx, :]
            cn = cn[:, unsort_idx, :]
        return output, (hn, cn)

class abae_attention(nn.Module):
    def __init__(self, d_embed):
        super(abae_attention, self).__init__()
        self.M = nn.Linear(d_embed, d_embed)
        self.M.weight.data.uniform_(-0.1, 0.1) # -0.1과 0.1 사이의 값으로 초기화

    def forward(self, e_i):
        y_s = torch.mean(e_i, dim=-1)
        d_i = torch.bmm(e_i.transpose(1, 2), self.M(y_s).unsqueeze(2)).tanh()
        a_i = d_i / sum(torch.exp(d_i))
        return a_i.squeeze(1)

class ABAE(nn.Module):
    def __init__(self, E, T):
        super(ABAE, self).__init__()
        n_vocabs, d_embed = E.shape
        n_aspects, d_embed = T.shape
        self.E = nn.Embedding(n_vocabs, d_embed)
        self.T = nn.Embedding(n_aspects, d_embed)
        self.attention = abae_attention(d_embed)
        self.linear = nn.Linear(d_embed, n_aspects)
        self.E.weight = nn.Parameter(torch.from_numpy(E), requires_grad=False)
        self.T.weight = nn.Parameter(torch.from_numpy(T), requires_grad=True)

    def forward(self, pos, negs):
        p_t, z_s = self.predict(pos)
        r_s = normalize(torch.mm(self.T.weight.t(), p_t.t()).t(), dim=-1)
        e_n = self.E(negs).transpose(-2, -1)
        z_n = normalize(torch.mean(e_n, dim=-1), dim=-1)
        return r_s, z_s, z_n

    def predict(self, x):
        e_i = self.E(x).transpose(1, 2)
        a_i = self.attention(e_i)
        z_s = normalize(torch.bmm(e_i, a_i).squeeze(2), dim=-1)
        p_t = softmax(self.linear(z_s), dim=1)
        return p_t, z_s

    def aspects(self):
        E_n = normalize(self.E.weight, dim=1)
        T_n = normalize(self.T.weight, dim=1)
        projection = torch.mm(E_n, T_n.t()).t()
        return projection

class ABSC(nn.Module):
    def __init__(self, args):
        super().__init__()
        self.args = args
        self.w2v = word2vec(self.args.tokenized_path)
        self.w2v.embed(self.args.w2v_path, self.args.emsize)
        self.w2v.aspect(self.args.n_aspects)

        self.ABAE = ABAE(self.w2v.E, self.w2v.T)
        if self.args.device is 'cuda':
            self.ABAE = self.ABAE.cuda()
        self.BiLSTM = TextLSTM(
            input_size=self.args.emsize, hidden_size=self.args.nhid // 2,
            bidirectional=True, bias=True, num_layers=self.args.nlayers,
            device=self.args.device)
        self.AttentionLayer = MLPAttentionWithoutQuery(self.args.nhid)
        self.Classifier = nn.Linear(self.args.nhid, 5)

    def forward(self, text, length, mask):
        """
        :param: text, mask: N, L
        :param: length: N,
        """
        x = self.WordEmb(text)  # N, L, Dw

        x = self.BiLSTM(inputs=x, length=length)[0]  # N, L, Dh

        x = self.AttentionLayer(v=x, mask=mask)  # N, Dh
        return self.Classifier(x)  # N, 5


import numpy as np
import torch

def cart2cyl(input_xyz):
    rho = np.sqrt(input_xyz[..., 0] ** 2 + input_xyz[..., 1] ** 2)
    phi = np.arctan2(input_xyz[..., 1], input_xyz[..., 0])
    return np.concatenate((rho.reshape(-1, 1), phi.reshape(-1, 1), input_xyz[..., 2:]), axis=-1)

def cyl2cart(input_xyz_polar):
    x = input_xyz_polar[..., 0] * np.cos(input_xyz_polar[..., 1])
    y = input_xyz_polar[..., 0] * np.sin(input_xyz_polar[..., 1])
    return np.concatenate((x.reshape(-1, 1), y.reshape(-1, 1), input_xyz_polar[..., 2:]), axis=-1)

def rotate_points_along_z(points, angle):
    """
    Args:
        points: (N, 3 + C or 2 + C)
        angle: float, angle along z-axis, angle increases x ==> y
    Returns:

    """
    cosa = np.cos(angle)
    sina = np.sin(angle)
    rot_matrix = np.array([
        [cosa,  sina, 0],
        [-sina, cosa, 0],
        [0, 0, 1]
    ]).astype(float)
    if points.shape[1]==2:
        points_rot = np.matmul(points, rot_matrix[:2, :2])
    elif points.shape[1]>2:
        points_rot = np.matmul(points[:, 0:3], rot_matrix)
        points_rot = np.concatenate((points_rot, points[:, 3:]), axis=-1)
    else:
        raise IOError('Input points should have the shape: (N, 3 + C or 2 + C).')
    return points_rot

def rotate_points_along_z_torch(points, angle):
    """
    Args:
        points: (N, 2 + C) or (B, 2 + C)
        angle: float or tensor of shape (B), angle along z-axis, angle increases x ==> y
    Returns:

    """
    if len(points.shape) == 2:
        points = points.unsqueeze(0)
    if isinstance(angle, float):
        angle = torch.tensor([angle], device=points.device)
    else:
        assert isinstance(angle, torch.Tensor)
        assert points.shape[0] == 1 or angle.shape[0] == points.shape[0]
    cosa = torch.cos(angle)
    sina = torch.sin(angle)
    rot_matrix = torch.stack([
        torch.stack([cosa,  sina], dim=-1),
        torch.stack([-sina, cosa], dim=-1)
    ], dim=1).float().to(points.device)
    if points.shape[0] == 1 and angle.shape[0] > 1:
        points = torch.tile(points, (len(rot_matrix), 1, 1))
    points_rot = torch.bmm(points[..., 0:2], rot_matrix)
    points_rot = torch.cat((points_rot, points[..., 2:]), dim=-1)
    return points_rot

def mask_values_in_range(values, min,  max):
    return np.logical_and(values>min, values<max)

def mask_points_in_box(points, pc_range):
    n_ranges = len(pc_range) // 2
    list_mask = [mask_values_in_range(points[:, i], pc_range[i],
                                       pc_range[i+n_ranges]) for i in range(n_ranges)]
    return np.array(list_mask).all(axis=0)

def mask_points_in_range(points: np.array, dist: float) -> np.array:
    """

    :rtype: np.array
    """
    return np.linalg.norm(points[:, :2], axis=1) < dist

def get_tf_matrix_torch(vectors, inv=False):
    device = vectors.device
    n, _ = vectors.shape
    xs = vectors[:, 0]
    ys = vectors[:, 1]
    angles = vectors[:, 2]
    cosa = torch.cos(angles)
    sina = torch.sin(angles)
    ones = torch.ones_like(angles)
    zeros = torch.zeros_like(angles)
    rot_matrix = torch.zeros((n, 3, 3), device=device, requires_grad=True)
    rot_matrix[:, 0, 0] = cosa
    rot_matrix[:, 0, 1] = -sina
    rot_matrix[:, 1, 0] = sina
    rot_matrix[:, 1, 1] = cosa
    shift_matrix = torch.zeros_like(rot_matrix, requires_grad=True)
    shift_matrix[:, 0, 1] = xs
    shift_matrix[:, 1, 0] = ys
    shift_matrix[:, [0, 1, 2], [0, 1, 2]] = 1.0
    if inv:
        mat = torch.einsum('...ij, ...jk', rot_matrix, shift_matrix)
    else:
        mat = torch.einsum('...ij, ...jk', shift_matrix, rot_matrix)
    return mat, rot_matrix, shift_matrix

def pose_err_global2relative_torch(poses, errs):
    """
    Calcaulate relative pose transformation based on the errorneous global positioning
    :param poses: Nx2 or Nx3, first row is ego pose, other rows are the coop poses
    :param errs: Nx3, first row is ego pose error and other rows for coop pose errors
    :return: (N-1)x3, relative localization errors between ego and coop vehicles
    """
    if poses.shape[-1]==2:
        poses = torch.cat([poses, torch.zeros_like(poses[:, 0:1])], dim=-1)
    poses_err = poses + errs

    R01, _, _ = get_tf_matrix_torch(-poses[:1], inv=True)
    R10_hat, _, _ = get_tf_matrix_torch(poses_err[:1])
    R20, _, _ = get_tf_matrix_torch(poses[1:])
    R02_hat, _, _ = get_tf_matrix_torch(-poses_err[1:], inv=True)

    delta_R21 = torch.einsum('...ij, ...jk', R01, R20)
    delta_R21 = torch.einsum('...ij, ...jk', delta_R21, R02_hat)
    delta_R21 = torch.einsum('...ij, ...jk', delta_R21, R10_hat)

    x = delta_R21[0, 2]
    y = delta_R21[1, 2]
    theta = torch.atan2(delta_R21[1, 0], delta_R21[0, 0])
    return torch.stack([x, y, theta], dim=-1)


import argparse
import numpy as np
import time
import ntpath
import style
import ArrayUtils
from bokeh.plotting import figure, output_file, show
from bokeh.models import Span, Label, Tabs, Panel
from bokeh.models.widgets import Div
from bokeh.layouts import column, row, widgetbox
from PresentMonResult import PresentMonResult, LoadFromCsv

# Debug?
debug = False

# FUNCTIONS
def CreateHistogram(data, width, histTitle, xLabel, yLabel, indicateMu=True):
    """
    Generates a Bokeh histogram from a numpy array of data.
    :param data: Any numpy array of ints or floats
    :param width: Target rendering width
    :param histTitle: Chart title
    :param xLabel: X-axis label
    :param yLabel: Y-axis label
    :param indicateMu: Whether or not to draw a vertical line indicating the data mean
    :return: A Bokeh quad-graph with range from mu +/- (4*sigma).
    """
    # Get data from result:
    # indicateMu = True, indicateSigma=0
    # TODO: Add automatic fps conversion in PresentMonResult class
    # TODO: Add parallellization for large datasets
    mu = np.mean(data)
    sigma = np.std(data)

    # Create plot (hard coded settings for now)
    p = figure(title=histTitle, x_axis_label=xLabel,
               y_axis_label=yLabel)
    p.x_range.end = mu + 4 * sigma
    p.x_range.start = mu - 4 * sigma
    p.width = width
    p.toolbar.logo = None
    hist, edges = np.histogram(data, density=True, bins=min(2000, data.size))

    p.quad(top=hist, bottom=0, left=edges[:-1], right=edges[1:],
           fill_color=style.histogramStyle['fill_color'], line_color=style.histogramStyle['line_color'])

    # Add markers for mean, stdev
    if indicateMu:
        muLine = Span(location=mu, dimension='height', line_color='red', line_width=1)
        p.renderers.extend([muLine])

    return p

def CreateTabbedHistogram(dataSet, width):
    """
    Creates a panel containing two histograms, one with frametimes and one with framerate
    :param dataSet: A numpy array of frametimes in ms
    :param width: Target rendering width
    :return: A Bokeh panel with one tab for each histogram
    """
    # Create the charts
    frequencyData = 1000. /  np.array(dataSet)

    timeHistogram = CreateHistogram(dataSet,
                                    width,
                                    histTitle="Frametimes",
                                    xLabel="Frame time (ms)",
                                    yLabel="Relative frequency")

    frequencyHistogram = CreateHistogram(frequencyData,
                                         width,
                                         histTitle="Framerate",
                                         xLabel="Framerate (fps)",
                                         yLabel="Relative frequency")

    # Create panels
    timeTab = Panel(child=timeHistogram, title="Frametimes")
    frequencyTab = Panel(child=frequencyHistogram, title="FPS")

    # Create tabs
    return Tabs(tabs=[timeTab, frequencyTab])

def CreateLineDiagram(frameData, timeData, width, chartTitle, xLabel, yLabel):
    """
    Creates a line diagram over time from a numpy array
    :param frameData: A numpy array containing framerate or frametimes
    :param timeData: A numpy array containing timestamps
    :param width: Target rendering width
    :param chartTitle: The title of the chart
    :param xLabel: X-axis label
    :param yLabel: Y-axis label
    :return: A Bokeh line diagram with frameData plotted against timeData
    """

    # Generate figure
    p = figure(title=chartTitle, x_axis_label=xLabel, y_axis_label=yLabel)
    p.width = width
    p.toolbar.logo = None

    # Add line
    p.line(timeData, frameData, line_width=0.5)

    return p

def CreateTabbedLineDiagram(dataSet, width):
    """
    Generates a panel with two tabs, one for framerate and one for frametimes
    :param dataSet: A PresentMonResult object
    :param width: Target rendering width of the panel
    :return: A Bokeh panel containing two tabs
    """
    # Create charts
    frequencyData = 1000. / dataSet.msBetweenPresents

    timeChart = CreateLineDiagram(dataSet.msBetweenPresents, dataSet.timeStamps, width, "Framtimes", "Runtime (s)",
                                  "Frame time (ms)")
    frequencyChart = CreateLineDiagram(frequencyData, dataSet.timeStamps, width, "Framerate", "Runtime (s)",
                                       "Framerate (fps)")

    # Create panels
    timeTab = Panel(child=timeChart, title="Frametimes")
    frequencyTab = Panel(child=frequencyChart, title="Framerate")

    return Tabs(tabs=[timeTab, frequencyTab])

def GenerateTextStatistics(data, width):
    """
    Generates a Bokeh Div containing statistics in text form.
    :param data: A numpy array of frametimes in ms.
    :param width: Target rendering width.
    :return: A Bokeh Div based on the input data.
    """
    # Get components:
    basicStatistics = GenerateBasicStatisticsText(data)
    frameToFrameStatistics = GenerateFrameToFrameStatistics(data)
    thresholdStatistics = GenerateThresholdStatistics(data, [60, 90, 120, 144])

    # Generate graphic
    text = "
" + \
           basicStatistics + frameToFrameStatistics + thresholdStatistics + \
           "
"
    div = Div(text=text, width=width)
    return div

# Text block generation:
def GenerateBasicStatisticsText(data):
    """
    Generates a basic statistics section in HTML format.
    :param data: A PresentMonResult object.
    :return: Basic statistics as an HTML string (header + paragraphs).
    """
    nFrames = data.timeStamps.size

    # Mean, meadian and standard deviation times
    meanFrametime = np.mean(data.msBetweenPresents)
    medianFrameTime = np.median(data.msBetweenPresents)
    stdDevFrameTime = np.std(data.msBetweenPresents)

    # Corresponding framerates:
    meanFramerate = 1000. / meanFrametime
    medianFramerate = 1000. / medianFrameTime
    stdDevFramerate = 1000. / stdDevFrameTime

    # Generate and return text
    return "
Basic statistics
" + \
           "
Number of processed frames: " + str(nFrames) + \
           "
Median: " + "{:10.3f}".format(medianFramerate) + " fps (" + "{:10.3f}".format(medianFrameTime) + " ms)" + \
           "
Mean: " + "{:10.3f}".format(meanFramerate) + " fps (" + "{:10.3f}".format(meanFrametime) + " ms)" + \
           "
Standard deviation: " + "{:10.3f}".format(stdDevFramerate) + " fps (" + "{:10.3f}".format(
        stdDevFrameTime) + " ms)"

def GenerateFrameToFrameStatistics(data):
    """
    Generates frame-to-frame statistics in HTML format.
    :param data: A PresentMonResult object.
    :return: Frame-to-frame statistics as an HTML string (header + paragraphs).
    """
    # Calulate average frame-to-frame difference magnitude
    diffsMs = ArrayUtils.getArrDiffs(data.msBetweenPresents)

    maxDiffAmp = np.max(diffsMs)
    minDiffAmp = np.min(diffsMs)
    avgDiffAmp = np.mean(diffsMs)

    # Generate and return text
    return "
Frame-to-frame statistics
" + \
           "
Average magnitude of difference: " + "{:10.3f}".format(avgDiffAmp) + " ms
" + \
           "
Maximum magnitude of difference: " + "{:10.3f}".format(maxDiffAmp) + " ms
" + \
           "
Minimum magnitude of difference: " + "{:10.3f}".format(minDiffAmp) + " ms
"

def GenerateThresholdStatistics(data, thresholdFramerates):
    """
    Generates threshold statistics in HTML data.
    :param data: A PresentMonResult object.
    :param thresholdFramerates: A list of framerates to use as thresholds.
    :return: Threshold statistics as an HTML string (header + paragraphs).
    """
    totalTimeSeconds = data.timeStamps[-1]

    thresholds = []
    for frameRate in thresholdFramerates:
        thresholds.append((frameRate, [], 0.0))

    # Thresholds (time spent above x fps)
    # (threshold fps, list of frames, fraction of total time spent)
    for frame in data.msBetweenPresents:
        for index in range(0, len(thresholds)):
            # for threshold in thresholds:
            if (1000. / frame) > thresholds[index][0]:
                thresholds[index][1].append(frame)
                thresholds[index] = (
                    thresholds[index][0], thresholds[index][1],
                    thresholds[index][2] + (frame / 1000) / totalTimeSeconds)

    # Piece together the values and return
    text = "
Framerate threshold statistics:
"
    for threshold in thresholds:
        text = text + "
Fraction of time spent above " + str(threshold[0]) + " fps: " + "{:10.3f}".format(
            threshold[2] * 100) + "%
"

    return text

# SETTINGS
plotWidth = 1068

# Parse arguments
parser = argparse.ArgumentParser()
parser.add_argument("-debug", help="Enable debug mode", action="store_true")
parser.add_argument("-file", help="Path to a PresentMon generated CSV-file")

startTime = time.time()

args = parser.parse_args()
fileSource = "TestData/csgo-2.csv"
if args.file:
    fileSource = args.file
elif args.debug:
    debug = True
else:
    parser.error('Either file or debug mode most be selected')

# Load data
data = LoadFromCsv(fileSource)

# Set output file
output_file("Analysis.html")

# Generate header
fileName = ntpath.basename(fileSource)
header = Div(text="

Analysis: {}

".format(fileName), width=plotWidth, style=style.headerStyle)

# Generate plots
histogram = CreateTabbedHistogram(data.msBetweenPresents, plotWidth)  # Relative frequency histograms
frameToFrameHistogram =  CreateHistogram(ArrayUtils.getArrDiffs(data.msBetweenPresents),
                                    plotWidth,
                                    histTitle="Frame-to-frame deviations",
                                    xLabel="Frame time difference (ms)",
                                    yLabel="Relative frequency")
lineDiagram = CreateTabbedLineDiagram(data, plotWidth)  # Framerate/frametimes as line diagram
textStatistics = GenerateTextStatistics(data, plotWidth)  # Statistics in text form

endTime = time.time()  # For debug diagnostics
if debug:
    print("Script runtime: " + str(endTime - startTime))

# Output and show
show(
    column(
        header,
        row(
            column(
                Div(text="

Performance distribution

", width=plotWidth, style=style.subheaderStyle),
                histogram,
                Div(text="

Performance timeline

", width=plotWidth, style=style.subheaderStyle),
                lineDiagram,
                Div(text="

Frame-to-frame deviations

", width=plotWidth, style=style.subheaderStyle),
                frameToFrameHistogram
            ),
            textStatistics
        )
    )
)


# -*- coding: utf-8 -*-
"""
Created on Thu Dec 13 12:53:59 2018

@author: Grani
"""

import requests

phonebook = {
        "Dani": "79060485763"
}

localhost = "http://127.0.0.1:5000"

def show_phonebook():
    response = requests.get(localhost + "/phonebook")
    if response.status_code == 200:
        return response.json()
    else:
        return "I'm calling police"

def add_contact(name, phone):
    response = requests.post(localhost + "/addcontact/" + name + "/" + phone)
    if response.status_code == 200:
        return response.json()
    else:
        return "I'm calling police"

def show_contact(name):
    response = requests.get(localhost + "/showcontact/" + name)
    if response.status_code == 200:
        return response.json()
    else:
        return "I'm calling police"

def del_contact(name):
    response = requests.post(localhost + "/removecontact/" + name)
    if response.status_code == 200:
        return response.json()
    else:
        return "I'm calling police"

def update_contact(name, phone):
    response = requests.post(localhost + "/updatecontact/" + name + "/" + phone)
    if response.status_code == 200:
        return response.json()
    else:
        return "I'm calling police"


"""
AJAX for Registry Viewer plugin
"""

from yapsy.IPlugin import IPlugin
from flask import Response
from Registry import Registry
import binascii
import json
import logging

class FaRegviewAjax(IPlugin):
    def __init__(self):
        self.display_name = 'Regview Ajax'
        self.popularity = 0
        self.cache = True
        IPlugin.__init__(self)

    def activate(self):
        IPlugin.activate(self)
        return

    def deactivate(self):
        IPlugin.deactivate(self)
        return

    def check(self, evidence, path_on_disk):
        """Checks if the file is compatible with this plugin"""
        return True

    def mimetype(self, mimetype):
        """Returns the mimetype of this plugins get command"""
        return "application/json"

    def get(self, evidence, helper, path_on_disk, request):
        """Returns the result of this plugin to be displayed in a browser"""
        method = helper.get_request_value(request, 'method')
        if not method:
            # TODO CHANGE ERROR
            logging.error('Method required for Regview AJAX')
            raise IOError
        elif method == "base":
            return self.base_tree(path_on_disk)
        elif method == "children":
            return self.get_children(request, helper, path_on_disk)
        elif method == "values":
            return self.values(request, helper, path_on_disk)

        # TODO CHANGE ERROR
        logging.error('Unknown method "' + method + '" provided to Regview AJAX')
        raise IOError

    def base_tree(self, path_on_disk):
        data = self.get_sub_keys("", path_on_disk)
        # TODO REPLACE WITH DICTIONARY AND JSONIFY, SEE: http://stackoverflow.com/questions/12435297/how-do-i-jsonify-a-list-in-flask
        return Response(json.dumps(data), mimetype='application/json')

    def get_children(self, request, helper, path_on_disk):
        node_id = helper.get_request_value(request, 'node_id')
        if not node_id:
            return "[]"
        data = self.get_sub_keys(node_id, path_on_disk)
        # TODO REPLACE WITH DICTIONARY AND JSONIFY, SEE: http://stackoverflow.com/questions/12435297/how-do-i-jsonify-a-list-in-flask
        return Response(json.dumps(data), mimetype='application/json')

    def values(self, request, helper, path_on_disk):
        node_id = helper.get_request_value(request, 'node_id')
        #response.content_type = 'application/json'
        if not node_id:
            return "[]"
        data = get_values(node_id, path_on_disk)
        # TODO REPLACE WITH DICTIONARY AND JSONIFY, SEE: http://stackoverflow.com/questions/12435297/how-do-i-jsonify-a-list-in-flask
        return Response(json.dumps(data), mimetype='application/json')

    def get_sub_keys(self, key, path_on_disk):
        registry = get_registry(path_on_disk)
        subkeys = get_subkeys(key, registry)
        registry_keys = []
        for subkey in subkeys:
            if len(key) > 0:
                fqkp = key + "\\" + subkey
            else:
                fqkp = subkey
            sk_ds = {'title': subkey, 'refKey': fqkp}
            if get_subkeys(fqkp, registry):
                sk_ds['folder'] = True
                sk_ds['lazy'] = True
            registry_keys.append(sk_ds)
        return registry_keys

def get_registry(pod):
    return Registry.Registry(pod)

def parse_reg(key, depth=0):
    reg_str = '\t' * depth + key.path() + "\n"
    if depth < 6:
        for subkey in key.subkeys():
            reg_str += parse_reg(subkey, depth + 1)
    return reg_str

def get_values(key_name, pod):
    #for value in [v for v in key.values() if v.value_type() == Registry.RegSZ or v.value_type() == Registry.RegExpandSZ]:
    reg = get_registry(pod)
    results = []
    try:
        key = reg.open(key_name)
        for value in key.values():
            if value.value_type_str() == "RegBin":
                results.append({ 'name': value.name(), 'type': value.value_type_str(), 'value': "0x" + str(binascii.hexlify(value.value())) })
            else:
                results.append({ 'name': value.name(), 'type': value.value_type_str(), 'value': value.value() })
        return results
    except Registry.RegistryKeyNotFoundException:
        logging.warn("Registry plugin could not find the key: " + key_name)
        return None

def get_subkeys(key_name, reg):
    try:
        subkeys = []
        key = reg.open(key_name)
        for subkey in key.subkeys():
            subkeys.append(subkey.name())
        return subkeys
    except Registry.RegistryKeyNotFoundException:
        logging.warn("Registry plugin could not find the key: " + key_name)
        return None


# -*- coding: latin-1 -*-
import serial
import time

channel = 'COM3'

arduino = serial.Serial(channel, 9600, timeout=1)  # Creates arduino object
time.sleep(3)

def turn_left(servoNumber):
    toLeft = str(servoNumber) + ' l'
    arduino.write(bytes(toLeft, 'latin-1'))

def turn_right(servonumber):
    toRight = str(servonumber) + ' l'
    arduino.write(bytes(toRight, 'latin-1'))

while True:
    command = input("What servo and left or right (1/2 l/r: ")
    # print(bytes(command, "latin-1"))
    # sends command to arduino via serial port
    if command[0] == 1:
        if command[2] == 'l':
            turn_left(1)

        if command[2] == 'r':
            turn_right(1)

    if command[0] == 2:
        if command[2] == 'l':
            turn_left(2)

        if command[2] == 'r':
            turn_right(2)

    value = arduino.readlines()  # reads all lines sent from arduino via Serial.print()
    for line in value:
        print(line.decode(encoding='latin-1'))

    if command == 'q':
        break


from ipywidgets import widgets

from pandas_profiling.report.presentation.core.image import Image

class WidgetImage(Image):
    def render(self):
        image = self.content["image"]
        if self.content["image_format"] == "svg":
            image.replace("svg ", 'svg class="img-responsive center-img"')
        else:
            image = ''.format(
                image=image, alt=self.content["alt"]
            )

        widget = widgets.HTML(image)
        if "caption" in self.content and self.content["caption"] is not None:
            caption = widgets.HTML(
                '
{caption}
'.format(
                    caption=self.content["caption"]
                )
            )
            return widgets.VBox([widget, caption])
        else:
            return widget


from __future__ import annotations
from sklearn.neural_network import MLPRegressor
from forecastingbackend.MLModels.ModelInterface.MLModelBuilder import MLModelBuilder

class MLPBuilder(MLModelBuilder):
	def __init__(self) -> None:
		self.reset()

	def reset(self) -> None:
		self._model = MLPRegressor()

	@property
	def release_model(self):
		completed_model = self._model
		self.reset()
		return completed_model

	def build_model(self, hyper_param_dict: dict):
		if hyper_param_dict is None:
			self._model = MLPRegressor()
		else:
			self._model = MLPRegressor(
				hidden_layer_sizes=hyper_param_dict["hidden_layer_sizes"],
				activation=hyper_param_dict["activation"],
				learning_rate=hyper_param_dict["learning_rate"],
				solver=hyper_param_dict["solver"]
			)
		return self.release_model


"""******************************** 开始
    author:惊修
    time:$
   ******************************* 结束"""
from django.urls import path
from .views import *

app_name = 'userprofile'

urlpatterns = [
    path('login/', user_login, name='login'),
    path('logout/', user_logout, name='logout'),
    path('register/',user_register, name='register'),
    path('delete/<int:id>', user_delete, name='delete'),
    path('edit/<int:id>', profile_edit, name='profile_eidt'),

]


from ultralytics import YOLO
import cv2
# Load a model
model = YOLO('./yolov8x.pt')  # load an official detection model

# Track with the model
results = model(f'bus.jpg')[0]
boxes = results.boxes
names = results.names
img = results.orig_img
if boxes is not None:
    for box in reversed(boxes):
        conf = int(box.conf.squeeze() * 100)
        x1,y1,x2,y2 = (int(i) for i in box.xyxy.squeeze())
        cv2.putText(img, f'{names[int(box.cls)]} {conf}%', (x1,y1-10), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (0,0,255), 2)
        cv2.rectangle(img, (x1,y1), (x2,y2), (0,0,255), 4)
        print(f'{names[int(box.cls)]}  {x1}-{y1}-{x2}-{y2}')
        print('=========================================================')
    cv2.imshow('img', img)
    cv2.waitKey(0)


from django.conf.urls import url, include

from authenticating.views import registration, login

urlpatterns = [
    url(r'^register/$', registration, name='register'),
    url(r'^login/$', login, name='mylogin'),
    url(r'', include('django.contrib.auth.urls')),
    url(r'', include('social.apps.django_app.urls', namespace='social'))
]


from flask import current_app as app
from flask import request, jsonify
from flask_login import login_user, logout_user, login_required, current_user

from sqlalchemy.exc import IntegrityError
from http import HTTPStatus

from audio_processing.models import User
from audio_processing.app import db, login_manager

@login_manager.user_loader
def load_user(id):
    return User.query.filter_by(id=id).first()

@app.route('/auth/login',  methods=['POST'])
def login():
  google_auth_id = request.form.get('google_auth_id')
  if not google_auth_id:
    return "need valid google account", HTTPStatus.BAD_REQUEST

  google_email = request.form.get('google_email')
  if not google_email:
    return "needs valid email", HTTPStatus.BAD_REQUEST

  user = User.query.filter_by(google_auth_id=google_auth_id, google_email=google_email).first()
  if user:
    print("logging in user:", user)
    login_user(user)
    return jsonify({'message': f"logging in existing user: {user.user_name}"}), HTTPStatus.OK

  user_name = request.form.get('user_name')
  if not user_name:
    return jsonify({'error': 'need non empty user name'}), HTTPStatus.BAD_REQUEST

  # otherwise, create this user
  user = User(
    user_name=user_name,
    google_email=google_email,
    google_auth_id=google_auth_id,
  )
  db.session.add(user)
  try:
    db.session.commit()
  except IntegrityError:
    db.session.rollback()
    return jsonify({'error': 'malicious attempt to create user - user already exists'}), HTTPStatus.BAD_REQUEST

  login_user(user)
  return jsonify({'message': 'created new user'}), HTTPStatus.CREATED

@app.route('/auth/logout',  methods=['POST'])
@login_required
def logout():
  logout_user()
  return jsonify({'message': 'user logged out'}), HTTPStatus.OK


# -*- coding: utf-8 -*-
from threading import Thread  # 多线程库
from queue import Queue  # 队列库
import requests
import re  # 正则库
from openpyxl import Workbook  # 操作xlsx文件的库
import time
import cchardet  # 用来检测网页编码的库

class UrlCheck(Thread):
    def __init__(self, url_queue, list_result):
        super().__init__()
        self.url_queue = url_queue  # 要探测的url队列
        self.list_result = list_result  # 结果数组
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (compatible; Baiduspider-render/2.0; +http://www.baidu.com/search/spider.html)'  # 百度爬虫的ua，可能绕过一些反爬
        }

    def run(self):
        while True:
            try:
                url = self.url_queue.get()
                if 'http' not in url:
                    url = "http://" + url
                req = self.download(url)
                if req:
                    title = re.search(r'<title>(.*?)</title>', req.text, re.I | re.S)
                    if title:
                        title = title.group(1)
                    else:
                        title = ""
                    list_result2 = [url, req.status_code, title]
                    self.list_result.append(list_result2)
                    print(url, '状态码：', req.status_code, '标题：', title)
            finally:
                self.url_queue.task_done()

    def download(self, url, retries=3):
        requests.packages.urllib3.disable_warnings()
        try:
            req = requests.get(url, headers=self.headers, timeout=3, verify=False)
        except requests.Timeout:
            req = None
            if retries > 0:
                return self.download(url, retries - 1)
        except requests.RequestException as err:
            req = None
            print(f"fetching {url} error: {err}")
        else:
            encoding = cchardet.detect(req.content)['encoding']
            req.encoding = encoding
        return req

if __name__ == '__main__':
    u_queue = Queue()
    checker = [k.strip() for k in open('urls.txt', encoding='utf-8')]
    list_result = []
    for url in checker:
        u_queue.put(url)
    print("开始探测：")
    for i in range(100):
        bdm = UrlCheck(u_queue, list_result)
        bdm.daemon = True
        bdm.start()

    u_queue.join()

    wb = Workbook()
    ws = wb.active
    ws.append(['url', '状态码', '标题'])
    for list1 in list_result:
        ws.append(list1)
    wb.save(time.strftime("%Y%m%d%H%M%S", time.localtime())+'.xlsx')

    print('done!')


from dash import html, dcc, Input, Output, State
from styles.styles import *

def sliders(label, description, min, max, step, marks, value, id_name):
    br = html.Br()
    label = html.Label(label)
    desc = html.P(description, className="description")
    slider = dcc.Slider(id=id_name, min=min, max=max, step=step, marks={i: str(i) for i in marks}, value=value)
    return [br, label, desc, slider]

low_marks = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]
generations = [1, 2, 3, 4, 5]
high_marks = [20, 30, 40, 50, 60]

top_p_slider = sliders("Top P", top_p_description, 0.1, 1.0, 0.1, low_marks, 0.8, "top-p")
temperature_slider = sliders("Temperature", temperature_description, 0.1, 1.0, 0.1, low_marks, 1.0, "temp")
output_length = sliders("Poem Length", output_len_description, 20, 60, 10, high_marks, 30, "poem-length")
options_number = sliders("Number of poems", options_num_description, 1, 5, 1, generations, 1, "n-poems")

sliders_elements = top_p_slider + temperature_slider + output_length + options_number

header = html.Div(style=header_style, children=[
    html.H1(children='AiLan-P03m', style=header_text_style)
    ])

subheader = html.Div(children=[
    html.H2('Generador de poemas basado en Transformers')], style=div_header_child_style)

body = html.Div([
    html.Div(children=sliders_elements, style=inputs_div_style, id="inputs-left"),
    html.Div(children=[
        html.Label('Fragmento de input'),
        html.P(className="description", children="Insertar el fragmento de poesía a partir del cual al modelo hará la generación"),
        dcc.Textarea(style=text_area, id="input-text",
                     value="Esto es un ejemplo\ndel input que debes dar\nsustituye estas líneas\npor tu poema")
    ], style=inputs_div_style, id="inputs-right")
], style=body_style)

button = html.Div(style=under_div_style, children=[
    html.Button("Generar", id="submit-button", n_clicks=0),
])

footer = dcc.Loading(children=html.Div(id="my-poems", style=footer_style), type="graph", fullscreen=False)


# coding=utf-8
import numpy as np
from feature import Counter, Feature
import sqlite3

db = sqlite3.connect('ptt.db')
cur = db.execute('select * from articles')
f = Feature()
a = 0
for i in cur:
    a += 1
    try:
        f.features(i[5])
    except:
        print(i[5])
    print(f.size())

f.store()


import torch
import opt_einsum as oe

device = "cuda"
big_core = torch.randn(
    2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, dtype=torch.float64, device=device
)
batches_of_small_cores = [
    torch.randn(512, 25, 25, 2, dtype=torch.float64, device=device) for _ in range(16)
]

equation = "αβγi,αβγj,αβγk,αβγl,αβγm,αβγn,αβγo,αβγp,αβγq,αβγr,αβγs,αβγt,αβγu,αβγv,αβγw,αβγx,ijklmnopqrstuvwxω->αβγω"
print(oe.contract_path(equation, *batches_of_small_cores, big_core, optimize="auto"))
result = oe.contract(equation, *batches_of_small_cores, big_core, optimize="auto")


# https://programmers.co.kr/learn/courses/30/lessons/42586?language=python3
def solution(progresses, speeds):
    from collections import deque
    prs = deque(progresses)
    sps = deque(speeds)
    deploy_cnt = []
    while prs:
        for i in range(len(prs)):
            prs[i] += sps[i]

        cnt = 0
        while prs and prs[0] >= 100:
            prs.popleft()
            sps.popleft()
            cnt += 1

        if cnt > 0:
            deploy_cnt.append(cnt)
    return deploy_cnt

def solution(progresses, speeds):
    from collections import deque
    required_day = deque([(100-p) / s for p, s in zip(progresses, speeds)])
    deploy_cnt = []

    while required_day:
        day = required_day.popleft()
        cnt = 1

        while required_day and required_day[0] < day:
            required_day.popleft()
            cnt += 1
        deploy_cnt.append(cnt)
    return deploy_cnt

if __name__ == "__main__":
    print(solution([93, 30, 55], [1, 30, 5]))
    print(solution([93, 30, 55], [1, 30, 5]) == [2, 1])
    print(solution([95, 90, 99, 99, 80, 99], [1, 1, 1, 1, 1, 1]))
    print(solution([95, 90, 99, 99, 80, 99], [1, 1, 1, 1, 1, 1]) == [1, 3, 2])

[스택큐] 기능개발.py


import os
import sys
import numpy as np
import pandas as pd
from scipy import interp
from sklearn import metrics
import math
from sklearn.inspection import permutation_importance

# Model 폴더 찾기 위한 방법
pathA = os.path.join(os.path.dirname(__file__), os.path.pardir, "../../../")
pathB = os.path.join(os.path.join(pathA))
basePath = os.path.abspath(pathB)
# Model Path 등록

sys.path.append(basePath)

from Common.Logger.Logger import logger

log = logger("log")

class graph:
    def __init__(self, param, classes=None):
        self.classes = classes
        self.nClasses = 0
        self.param = param
        if classes is not None:
            self.nClasses = len(self.classes)

    # input : yTest, yPred
    # output : output
    # Check : yPred는 decision_function으로 계산해야 함. smpark
    def roc(self, yTest, yPred):
        fpr = dict()
        tpr = dict()
        roc_auc = dict()

        # for i in range(self.nClasses):
        #     try:
        #         fpr[i], tpr[i], _ = metrics.roc_curve(yTest[:, i], yPred[:, i])
        #         roc_auc[i] = metrics.auc(fpr[i], tpr[i])
        #     except:
        #         continue

        fpr["micro"], tpr["micro"], _ = metrics.roc_curve(yTest.ravel(), yPred.ravel())
        roc_auc["micro"] = metrics.auc(fpr["micro"], tpr["micro"])

        # all_fpr = np.unique(np.concatenate([fpr[i] for i in range(self.nClasses)]))

        # Then interpolate all ROC curves at this points
        # mean_tpr = np.zeros_like(all_fpr)
        # for i in range(self.nClasses):
            # mean_tpr += interp(all_fpr, fpr[i], tpr[i])

        # Finally average it and compute AUC
        # mean_tpr /= self.nClasses

        # fpr["macro"] = all_fpr
        # tpr["macro"] = mean_tpr
        # roc_auc["macro"] = metrics.auc(fpr["macro"], tpr["macro"])

        microROCPosition = []
        for i in range(len(fpr['micro'])):
            # if (not math.isnan(float(fpr['micro'][i]))) and (not math.isnan(float(tpr['micro'][i]))):
            microROCPosition.append(
                {
                    "X": '{:.4f}'.format(fpr['micro'][i]),
                    "Y": '{:.4f}'.format(tpr['micro'][i])
                }
            )

        # macroROCPosition = []
        # for i in range(len(fpr['macro'])):
        #     if (not math.isnan(float(fpr['macro'][i]))) and (not math.isnan(float(tpr['macro'][i]))):
        #         macroROCPosition.append(
        #             {
        #                 "X": '{:.4f}'.format(fpr['macro'][i]),
        #                 "Y": '{:.4f}'.format(tpr['macro'][i])
        #             }
        #         )

        output = {
            "AI_CD": self.param["SERVER_PARAM"]["AI_CD"] if "SERVER_PARAM" in self.param else None,
            "MDL_IDX": self.param["MDL_IDX"] if "MDL_IDX" in self.param else None,
            "MODEL_NAME": self.param["MODEL_NAME"] if "MODEL_NAME" in self.param else None,

            "GRAPH_TYPE": "ROC_Curve",
            "LEGEND_X": "False Positive Rate",
            "LEGEND_Y": "True Positive Rate",
            "GRAPH_DATA": [
                {
                    "GRAPH_NAME": "ROC Curve",
                    "AREA": '{:.4f}'.format(roc_auc["micro"]),
                    "GRAPH_POSITION": microROCPosition,
                }
                # {
                #     "GRAPH_NAME": "Macro Average ROC Curve",
                #     "AREA": '{:.4f}'.format(roc_auc["micro"]),
                #     "GRAPH_POSITION": macroROCPosition,
                # }
            ]
        }

        # for i in range(self.nClasses):
        #     ROCPositionClsses = []
        #     for j in range(len(fpr[i])):
        #         if (not math.isnan(float(fpr[i][j]))) and (not math.isnan(float(tpr[i][j]))):
        #             ROCPositionClsses.append(
        #                 {
        #                     "X": '{:.4f}'.format(fpr[i][j]),
        #                     "Y": '{:.4f}'.format(tpr[i][j])
        #                 }
        #             )

        #     output["GRAPH_DATA"].append(
        #         {
        #             "GRAPH_NAME": "ROC Curve of class {}".format(self.classes[i]),
        #             "AREA": '{:.4f}'.format(roc_auc[i]),
        #             "GRAPH_POSITION": ROCPositionClsses,
        #         }
        #     )

        return output

    # input : yTest, yPred
    # output : output
    # Check : yPred는 predict_proba로 계산해야 함. smpark
    def precisionRecall(self, yTest, yPred):
        precision = dict()
        recall = dict()
        average_precision = dict()
        graphData = []

        precision["micro"], recall["micro"], _ = metrics.precision_recall_curve(yTest.ravel(), yPred.ravel())
        average_precision["micro"] = metrics.average_precision_score(yTest, yPred, average="micro")

        microPRPosition = []
        for i in range(len(precision['micro'])):
            # if (not math.isnan(float(precision['micro'][i]))) and (not math.isnan(float(recall['micro'][i]))):
            microPRPosition.append(
                {
                    "X": '{:.4f}'.format(recall['micro'][i]),
                    "Y": '{:.4f}'.format(precision['micro'][i])
                }
            )

        # for i in range(self.nClasses):
        #     preRecallPosition = []
        #     precision[i], recall[i], _ = metrics.precision_recall_curve(
        #         yTest[:, i],
        #         yPred[:, i]
        #     )

        #     for j in range(len(precision[i])):
        #         if (not math.isnan(float(precision[i][j]))) and (not(math.isnan(float(recall[i][j])))):
        #             preRecallPosition.append(
        #                 {
        #                     "X": '{:.4f}'.format(recall[i][j]),
        #                     "Y": '{:.4f}'.format(precision[i][j])
        #                 }
        #             )

        #     graphData.append({
        #         "GRAPH_NAME": "Precision Recall Curve of class {}".format(self.classes[i]),
        #         "GRAPH_POSITION": preRecallPosition,
        #     })

        # output = {
        #     "AI_CD": self.param["SERVER_PARAM"]["AI_CD"],
        #     "MDL_IDX": self.param["MDL_IDX"],
        #     "MODEL_NAME": self.param["MODEL_NAME"],
        #     "GRAPH_TYPE": "Precision_Recall_Curve",
        #     "LEGEND_X": "Recall",
        #     "LEGEND_Y": "Precision",
        #     "GRAPH_DATA": graphData
        # }

        output = {
            "AI_CD": self.param["SERVER_PARAM"]["AI_CD"] if "SERVER_PARAM" in self.param else None,
            "MDL_IDX": self.param["MDL_IDX"] if "MDL_IDX" in self.param else None,
            "MODEL_NAME": self.param["MODEL_NAME"] if "MODEL_NAME" in self.param else None,

            "GRAPH_TYPE": "Precision_Recall_Curve",
            "LEGEND_X": "recall",
            "LEGEND_Y": "precision",
            "GRAPH_DATA": [
                {
                    "GRAPH_NAME": "PR Curve",
                    "AREA": '{:.4f}'.format(average_precision["micro"]),
                    "GRAPH_POSITION": microPRPosition,
                }
            ]
        }

        return output

    # input : yTest, yPred
    # output : output
    # Check : yPred는 predict로 계산해야 함. smpark
    # Check2 : yTest, yPred 둘 다 binarize -> decode 필요함(1차원으로 만들어 줘야함)
    def confusionMatrix(self, yTest, yPred, classes):

        if type(yTest[0]) != str:
            labels = range(0, len(classes))
        else:
            labels = classes

        cm = metrics.confusion_matrix(yTest, yPred, normalize="all", labels=labels)
        cm = cm.tolist()
        newCm = []

        for data in cm:
            tmpCm = []
            for tmp in data:
                tmpCm.append(float('{:.4f}'.format(tmp)))
            newCm.append(tmpCm)

        output = {
            "AI_CD": self.param["SERVER_PARAM"]["AI_CD"] if "SERVER_PARAM" in self.param else None,
            "MDL_IDX": self.param["MDL_IDX"] if "MDL_IDX" in self.param else None,
            "MODEL_NAME": self.param["MODEL_NAME"] if "MODEL_NAME" in self.param else None,
            "GRAPH_TYPE": "Confusion_Matrix",
            "LEGEND_X": self.classes,
            "LEGEND_Y": self.classes,
            "GRAPH_DATA": [{
                "GRAPH_NAME": "Confusion Matrix",
                "GRAPH_POSITION": newCm,
            }]
        }
        return output

    # input : yTest, yPred
    # output : output
    # Check : yPred 는 Predict로 계산해야 함. smpark
    def regPlot(self, yTest, yPred):

        graphData = []

        m, b = np.polyfit(yTest, yPred, 1)

        pt = m * yTest + b

        xMin = min(yTest)
        xMax = max(yTest)

        yMin = min(pt)
        yMax = max(pt)

        for i in range(len(yTest)):
            if (not math.isnan(float(yTest[i]))) and (not math.isnan(float(yPred[i]))):
                graphData.append(
                    {
                        "X": '{:.4f}'.format(yTest[i]),
                        "Y": '{:.4f}'.format(yPred[i])
                    }
                )
        output = {
            "AI_CD": self.param["SERVER_PARAM"]["AI_CD"] if "SERVER_PARAM" in self.param else None,
            "MDL_IDX": self.param["MDL_IDX"] if "MDL_IDX" in self.param else None,
            "MODEL_NAME": self.param["MODEL_NAME"] if "MODEL_NAME" in self.param else None,
            "GRAPH_TYPE": "Reg_Plot",
            "LEGEND_X": "Y Label",
            "LEGEND_Y": "Y Pred",
            "BASE_GRAPH_DATA": [
                {
                    "X": '{:.4f}'.format(xMin),
                    "Y": '{:.4f}'.format(yMin)
                },
                {
                    "X": '{:.4f}'.format(xMax),
                    "Y": '{:.4f}'.format(yMax)
                }
            ],
            "GRAPH_DATA": [{
                "GRAPH_NAME": "Reg Plot",
                "GRAPH_POSITION": graphData,
            }]
        }
        return output

    def featureImportance(self, colNames, modelFi):

        fImp = list(modelFi)
        fImpAll = 0
        featureImp = []

        for i in range(len(colNames)):
            fImpAll += fImp[i]

        for j in range(len(fImp)):
            fImp[j] = float(fImp[j]/fImpAll)

        dictFi = dict(zip(colNames, fImp))
        dictFi = sorted(dictFi.items(), reverse=True, key=lambda item: item[1])

        for k in range(len(dictFi)):
            featureImp.append({
                    "name": '{}'.format(dictFi[k][0]),
                    "value": '{:.4f}'.format(dictFi[k][1])
            })

        output = {
            "AI_CD": self.param["SERVER_PARAM"]["AI_CD"] if "SERVER_PARAM" in self.param else None,
            "MDL_IDX": self.param["MDL_IDX"] if "MDL_IDX" in self.param else None,
            "MODEL_NAME": self.param["MODEL_NAME"] if "MODEL_NAME" in self.param else None,
            "GRAPH_TYPE": "Feature_Importance",
            "GRAPH_DATA": [{
                "GRAPH_NAME": "Feature Importance",
                "GRAPH_POSITION": featureImp,
            }]
        }
        return output

    def permutation_fi(self, model, xTest, yTest, colNames, n_repeats=None):

        pImp = permutation_importance(model, xTest, yTest, n_repeats=n_repeats, random_state=42)
        pImp_mean = pImp.importances_mean
        pImpAll = 0

        permutationImp = []
        for i in range(len(colNames)):
            if pImp_mean[i] < 0:
                pImp_mean[i] = 0.0
            pImpAll += pImp_mean[i]

        for j in range(len(pImp_mean)):
            pImp_mean[j] = float(pImp_mean[j]/pImpAll)

        dictPi = dict(zip(colNames, pImp_mean))
        dictPi = sorted(dictPi.items(), reverse=True, key=lambda item: item[1])

        for k in range(len(dictPi)):
            permutationImp.append({
                    "name": '{}'.format(dictPi[k][0]),
                    "value": '{:.4f}'.format(dictPi[k][1])
            })

        output = {
            "AI_CD": self.param["SERVER_PARAM"]["AI_CD"] if "SERVER_PARAM" in self.param else None,
            "MDL_IDX": self.param["MDL_IDX"] if "MDL_IDX" in self.param else None,
            "MODEL_NAME": self.param["MODEL_NAME"] if "MODEL_NAME" in self.param else None,
            "GRAPH_TYPE": "Feature_Importance",
            "GRAPH_DATA": [{
                "GRAPH_NAME": "Feature Importance",
                "GRAPH_POSITION": permutationImp,
            }]
        }

        return output

    # input : yTest, yPred
    # output : output
    # Check : yPred 는 Predict로 계산해야 함. smpark
    # Check2 : yTest는 numpy array로 와야함.
    # 고려사항 : 샘플링 필요할 수 있음.
    def distributionPlot(self, yTest, yPred):
        yTestGraphData = []
        yPredGraphData = []
        graphData = []

        for i in range(len(yTest)):
            if not math.isnan(float(yTest[i])):
                yTestGraphData.append(
                    {
                        "X": '{:.4f}'.format(i),
                        "Y": '{:.4f}'.format(yTest[i]),
                    }
                )

        graphData.append(
            {
                "GRAPH_NAME": "Distribution Plot of Y Test Data",
                "GRAPH_POSITION": yTestGraphData,
            }
        )

        for i in range(len(yPred)):
            if not math.isnan(float(yPred[i])):
                yPredGraphData.append(
                    {
                        "X": '{:.4f}'.format(i),
                        "Y": '{:.4f}'.format(yPred[i]),
                    }
                )

        graphData.append(
            {
                "GRAPH_NAME": "Distribution Plot of Y Pred Data",
                "GRAPH_POSITION": yPredGraphData,
            }
        )

        output = {
            "AI_CD": self.param["SERVER_PARAM"]["AI_CD"] if "SERVER_PARAM" in self.param else None,
            "MDL_IDX": self.param["MDL_IDX"] if "MDL_IDX" in self.param else None,
            "MODEL_NAME": self.param["MODEL_NAME"] if "MODEL_NAME" in self.param else None,
            "GRAPH_TYPE": "Distribution_Plot",
            "LEGEND_X": "Amount of Test Dataset",
            "LEGEND_Y": "Y Test and Y Pred",
            "GRAPH_DATA": graphData
        }
        return output


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from CinePred.data.importing import import_data
from CinePred.data.preprocessing import *
from CinePred.data.featuring import *
from CinePred.data.transformers import GenreOHE
from google.cloud import storage

from sklearn.model_selection import cross_val_score, GridSearchCV
from xgboost import XGBRegressor, plot_importance
import numpy as np
from joblib import dump, load

def preproc(df, path="gs://wagon-data-722-cinepred/data/cat_acteur.csv"):
    '''
        Clean the dataframe

        Input: dataframe
        Output: dataframe cleaned and sorted by budget
    '''
    # NA & columns:
    df = add_success_movies_per_actors(df, path=path)
    df = add_number_of_movies_actor1_in_Timeline(df, path=path)
    df = add_number_of_movies_actor2_in_Timeline(df, path=path)
    df = add_number_of_movies_actor3_in_Timeline(df, path=path)
    df = add_total_income_of_last_movie_of_actors_in_Timeline(df, path=path)

    df = keep_columns(df,
                      column_names=[
                          'worlwide_gross_income', 'year', 'date_published',
                          'genre', 'duration', 'budget', 'production_company',
                          'director', 'writer', 'shifted', 'nb_movies_actor1',
                          'nb_movies_actor2', 'nb_movies_actor3',
                          'last income', 'imdb_title_id','actors',
                          'description','avg_vote','country','title'
                      ])
    df = remove_na_rows(df)

    # date_published
    df['date_published'] = convert_to_date(df[['date_published']])

    # day of the year
    df['date_sin'] = add_sin_features(df[['date_published']])
    df['date_cos'] = add_cos_features(df[['date_published']])
    df.drop(columns='date_published', inplace=True)

    # year, duration
    df['year'] = convert_to_int(df[['year']])
    df['duration'] = convert_to_int(df[['duration']])

    # genre
    ohe = GenreOHE()
    ohe.fit(df) # la colonne 'genre' est spécifié dans la classe
    df = ohe.transform(df)

    # budget
    df['budget'] = convert_budget_column(df[['budget']])
    df = df[df['budget'] > 100]
    df = add_inflation(df, 'budget')
    df['budget'] = log_transformation(df[['budget']])

    # income
    df['worlwide_gross_income'] = convert_income(df[['worlwide_gross_income']])
    df = add_inflation(df, 'worlwide_gross_income')
    df['worlwide_gross_income'] = log_transformation(df[['worlwide_gross_income']])

    # Cumsum
    df = Add_Ones(df)
    df = Add_number_of_movies_per_prod_company_in_Timeline(df)
    df = Add_number_of_movies_per_directors_in_Timeline(df)
    df = Add_number_of_movies_per_writer_in_Timeline(df)
    df = Remove_Ones(df)

    # sort & index:
    df.sort_values('budget', inplace=True)
    df.reset_index(inplace=True)
    df.drop(columns=['index'], inplace=True)

    return df

def feature_importance(df):
    X = df.drop(columns=['worlwide_gross_income'])
    y = df['worlwide_gross_income']
    model = XGBRegressor(learning_rate=0.1, max_depth=2)
    model.fit(X, y)
    print(model.feature_importances_)
    plot_importance(model)

def preproc_demoday(df, path = "gs://wagon-data-722-cinepred/data/cat_acteur.csv"):
    # df = add_success_movies_per_actors(df, path=path)
    # df = add_number_of_movies_actor1_in_Timeline(df, path=path)
    # df = add_number_of_movies_actor2_in_Timeline(df, path=path)
    # df = add_number_of_movies_actor3_in_Timeline(df, path=path)
    # df = add_total_income_of_last_movie_of_actors_in_Timeline(df, path=path)

    # df = remove_na_rows(df)

    # df['date_published'] = convert_to_date(df[['date_published']])

    # # day of the year
    # df['date_sin'] = add_sin_features(df[['date_published']])
    # df['date_cos'] = add_cos_features(df[['date_published']])
    # df.drop(columns='date_published', inplace=True)

    # year, duration
    # df['year'] = convert_to_int(df[['year']])
    # df['duration'] = convert_to_int(df[['duration']])

    # genre
    # ohe = GenreOHE()
    # ohe.fit(df) # la colonne 'genre' est spécifié dans la classe
    # df = ohe.transform(df)

    # budget
    # df['budget'] = convert_budget_column(df[['budget']])
    # df = df[df['budget'] > 100]
    # df = add_inflation(df, 'budget')
    df['budget'] = log_transformation(df[['budget']])

    # # income
    # df['worlwide_gross_income'] = convert_income(df[['worlwide_gross_income']])
    # df = add_inflation(df, 'worlwide_gross_income')
    # df['worlwide_gross_income'] = log_transformation(df[['worlwide_gross_income']])

    # Cumsum
    # df = Add_Ones(df)
    # df = Add_number_of_movies_per_prod_company_in_Timeline(df)
    # df = Add_number_of_movies_per_directors_in_Timeline(df)
    # df = Add_number_of_movies_per_writer_in_Timeline(df)
    # df = Remove_Ones(df)

    # sort & index:
    # df.sort_values('budget', inplace=True)
    # df.reset_index(inplace=True)
    # df.drop(columns=['index'], inplace=True)

    return df

def get_best_params(model, X, y):
    # Inspect all pipe components parameters to find the one you want to gridsearch
    # Instanciate grid search
    grid_search = GridSearchCV(
        model,
        param_grid={
            # Access any component of the pipeline, as far back as you want
            'learning_rate': [0.01, 0.1],
            'n_estimators': [100, 200, 300],
            'max_depth': [2]
        },
        cv=5,
        scoring="neg_mean_absolute_error",
        verbose=2)

    grid_search.fit(X, y)
    return grid_search.best_params_

def get_mae(df):
    X = df.drop(columns=['worlwide_gross_income'])
    y = df['worlwide_gross_income']
    model = XGBRegressor(learning_rate=0.1, max_depth=2)

    return cross_val_score(model, X, y, cv=5, scoring='neg_mean_absolute_error')

def predict(df):
    '''
        Input: a preprocessed df
        Output: 2 scores MAE scores '''

    mid = int(len(df) / 2)
    df1 = df.iloc[:mid].copy()
    df2 = df.iloc[mid:].copy()

    score1 = np.mean(np.abs(get_mae(df1))) # {'learning_rate': 0.1, 'max_depth': 2, 'n_estimators': 200}
    score2 = np.mean(np.abs(get_mae(df2))) # {'learning_rate': 0.1, 'max_depth': 2, 'n_estimators': 300}
    return [round(score1, 3), round(score2, 3)]

def predict2(df):
    ''' Get the mae on the full dataset (crossvalidated) '''

    score = np.abs(get_mae(df))  # {'learning_rate': 0.1, 'max_depth': 2, 'n_estimators': 300}
    score = round(np.mean(score),2)
    return(score)

def predict_fromX(model, df):
    prediction = model.predict(df)
    return 10 ** prediction[0]

def save_model(fitted_model, file_name="model.joblib"):
    dirname = os.path.dirname(__file__)
    filepath = os.path.join(dirname, 'models/'+file_name)
    dump(fitted_model, filepath)

def load_model(file_name="model.joblib"):
    #dirname = os.path.dirname(__file__)
    #filepath = os.path.join(dirname, 'models/'+file_name)

    # filepath = 'gs://wagon-data-722-cinepred/model/' + file_name

    BUCKET_NAME = "wagon-data-722-cinepred"
    storage_filename = 'model/model.joblib'
    local_filename = "model.joblib"

    client = storage.Client()
    bucket = client.bucket(BUCKET_NAME)
    blob = bucket.blob(storage_filename)
    blob.download_to_filename(local_filename)

    return load(local_filename)

def get_fitted_model(df):
    # Select only film with most income
    mid = int(len(df) / 2)
    df1 = df.iloc[:mid].copy()
    df2 = df.iloc[mid:].copy()

    # Create X and y
    X = df2.drop(columns=['worlwide_gross_income'])
    y = df2['worlwide_gross_income']
    model = XGBRegressor(learning_rate=0.1, max_depth=2)
    model.fit(X,y)

    score = cross_val_score(model, X, y, cv=5, scoring='neg_mean_absolute_error')
    print(f'mae = {round(np.mean(np.abs(score)),2)}')
    return model

if __name__ == '__main__':
    # Import
    print("----- IMPORT DATA ------")
    #df = import_data(link = 'gs://wagon-data-722-cinepred/data/IMDb_movies.csv')
    #df_preproc = preproc(df)
    df_preproc = import_data(link='gs://wagon-data-722-cinepred/data/preprocessed.csv')

    # Prepare
    print("----- CLEAN DATA ------")
    df_preproc = df_preproc.drop(columns=['production_company', 'director', 'writer'])
    df_preproc = df_preproc.drop(columns=['imdb_title_id','actors','description','avg_vote','country','title'])

    # Predict
    print("----- PREDICT DATA ------")
    #print(predict(df_preproc))

    print("----- GET FITTED MODEL ------")
    model = get_fitted_model(df_preproc)

    print("----- SAVE MODEL ------")
    save_model(model, "model.joblib")

    print("----- LOAD MODEL ------")
    model = load_model("model.joblib")

    print("----- PREDICT MODEL ------")
    prediction = predict_fromX(
        model,df_preproc.head(1).drop(columns=['worlwide_gross_income']))
    print(prediction)
    #print(predict(df))
    #print(predict2(df))
    #get_mae(df)


from django.db import models

class Category(models.Model):
    parent_category = models.ForeignKey(
        "Category", verbose_name='Родительская категория',
        related_name='children', null=True, blank=True,
        on_delete=models.CASCADE
    )
    name = models.CharField(verbose_name='Название', max_length=200)

    class Meta:
        verbose_name = 'Категория'
        verbose_name_plural = 'Категории'

    def __str__(self):
        return '{}'.format(self.name)

class Product(models.Model):
    article = models.CharField(verbose_name='Код', max_length=15)
    name = models.CharField(verbose_name='Наименование', max_length=200)
    price = models.DecimalField(verbose_name='Цена', max_digits=100, decimal_places=2)
    special_price = models.DecimalField(verbose_name='Цена специального предложения', max_digits=100, decimal_places=2)
    count = models.IntegerField(verbose_name='Количество')
    measurement = models.CharField(verbose_name='Ед. измерения', max_length=5)
    united_purchases = models.BooleanField(verbose_name='Совместные покупки', default=False)
    image = models.ImageField(verbose_name='Изображение', upload_to='images', null=True, blank=True)
    show_main = models.BooleanField(verbose_name='Отображать на главной', default=False)
    description = models.TextField(verbose_name='Описание')
    category = models.ForeignKey(
        Category, verbose_name='Категория',
        related_name='products', on_delete=models.CASCADE
    )

    class Meta:
        verbose_name = 'Продукт'
        verbose_name_plural = 'Продукты'

    def __str__(self):
        return '{}'.format(self.name)

class Properties(models.Model):
    key = models.CharField(verbose_name='Название свойства', max_length=100)
    value = models.CharField(verbose_name='Значение свойства', max_length=100)
    product = models.ForeignKey(
        Product, verbose_name='Продукт', related_name='options',
        on_delete=models.CASCADE
    )

    class Meta:
        verbose_name = 'Опиция'
        verbose_name_plural = 'Опции'

    def __str__(self):
        return '{} - {}:{}'.format(self.product, self.key, self.value)


import requests

stran = requests.get(
    "https://www.imdb.com/search/title/?title_type=feature&num_votes=1000000,&sort=user_rating,desc&count=250"
)

# print(stran)
# print(stran.text)
with open("stran1.html", "w") as dat:
    dat.write(stran.text)

for i in range(10):
    stran = requests.get(
        f"https://www.imdb.com/search/title/?title_type=feature&num_votes=10000,&sort=user_rating,desc&count=250&start={1+i*250}&ref_=adv_nxt"
    )
    with open(f"stran{i}.html", "w") as dat:
        dat.write(stran.text)


from sqlalchemy import select
from sqlalchemy.dialects.postgresql import insert

import exceptions
import models
from database.models import AccountCookies
from repositories.base import Repository

__all__ = ('AccountCookiesRepository',)

class AccountCookiesRepository(Repository):

    def get_by_account_name(self, account_name: str) -> models.AuthCookies:
        statement = select(AccountCookies).where(AccountCookies.account_name == account_name)
        with self._session_factory() as session:
            account_cookies = session.scalar(statement)
        if account_cookies is None:
            raise exceptions.DoesNotExistInDatabase('Cookies not found')
        return models.AuthCookies(account_name=account_name, cookies=account_cookies.cookies)

    def update(self, *, account_name: str, cookies: dict[str, str]):
        statement = (
            insert(AccountCookies)
            .values(account_name=account_name, cookies=cookies)
            .on_conflict_do_update(
                index_elements=('account_name',),
                set_={'cookies': cookies},
                where=(AccountCookies.account_name == account_name),
            )
        )
        with self._session_factory() as session:
            with session.begin():
                session.execute(statement)


import unittest
import sys
from math import floor
from datetime import datetime
#Used to import StockAnalyzer from parent folder
sys.path.append("..")
from Stock_Analyzer import StockAnalyzer

class TestStockAnalyzer(unittest.TestCase):
    def setUp(self):
        self.sa = StockAnalyzer()

    def test_get_date_to_string(self):
        r=self.sa.get_date_to_string(datetime(1994, 5, 20))
        e="May 20 1994"
        self.assertEqual(r,e)

    def test_get_string_to_date_object(self):
        #class datetime.datetime(year, month, day, hour=0, minute=0, second=0, microsecond=0, tzinfo=None)¶
        r=self.sa.get_string_to_date_object("May 20 1994")
        e=datetime(1994, 5, 20)
        self.assertEqual(r,e)

    def test_get_sorted_dates_array(self):
        reverse=False
        e=[]
        e.append(self.sa.get_string_to_date_object('Dec  11 2014'))
        e.append(self.sa.get_string_to_date_object('May 20 2014'))
        e.append(self.sa.get_string_to_date_object('May 21 2013'))
        e.append(self.sa.get_string_to_date_object('Apr 22 2013'))
        r = self.sa.get_sorted_dates_array(['May 21 2013', 'Dec 11 2014', 'Apr 22 2013', 'May 20 2014'], reverse)

        #Testing that they are in sorted order
        for i in range(1,len(r)):
            cur_date=r[i]
            prev_date=r[i-1]
            if reverse==True:
                self.assertTrue( cur_date<=prev_date )
            else:
                self.assertTrue( cur_date>=prev_date )

        #Testing that they consist from input dates
        for date in e:
            r.remove(date)
        self.assertTrue(len(r)==0)

    def test_calculate_cashflow_growth(self):
        #CF_growth= (CF_current - CF_previous)/CF_current

        ci={'May 20 2014': 10, 'Dec 11 2014': 50, 'May 11 2015' :60}
        r=self.sa._calculate_cashflow_growth(ci)
        print(r)
        self.assertEqual(r['May 11 2015'], .1667)
        self.assertEqual(r['Dec 11 2014'], .8)

        #Check that nothing else is calculated as we only have 2 periods
        #to report
        self.assertTrue(len(r)==2)

    def tearDown(self):
        pass


import requests
import bs4
from datetime import datetime
import re

def get_wiki(language):
    date = datetime.now().strftime("%Y%m%d000000")

    link = f'https://{language}.wikipedia.org/wiki/Special:FeedItem/featured/{date}/{language}'
    res = requests.get(link)

    wiki = bs4.BeautifulSoup(res.text, "lxml")

    if language == "de":
        elems = wiki.select('.hauptseite-box-content')
        text = elems[0].getText()
    elif language == "fr":
        header = wiki.find_all('h2')[1]
        text = ""
        for tag in wiki.find("h1").next_siblings:
            if tag.name == "ul":
                break
            elif tag.name == "p":
                text += tag.getText()
        elems = wiki.select('p')
        text = elems[4]
    else:
        elems = wiki.select('p')
        text = elems[0]

    imgs = wiki.select('img')
    img = imgs[0]
    img['src'] = 'https:'+img['src']

    for a in text.findAll('a'):
        a['href'] = 'https://en.wikipedia.org'+a['href']

    return str(img), str(text)

def get_wiki_for_twitter():
    date = datetime.now().strftime("%Y%m%d000000")

    link = f'https://en.wikipedia.org/wiki/Special:FeedItem/featured/{date}/en'
    res = requests.get(link)

    wiki = bs4.BeautifulSoup(res.text, "lxml")

    elems = wiki.select('p')

    link = 'https://en.wikipedia.org' + \
        elems[0].select('a', href=True)[0]['href']

    imgs = wiki.select('img')
    img_url = 'https:'+imgs[0]['src']
    img_url = re.sub('.jpg/.*?px', '.jpg/1200px', img_url, flags=re.DOTALL)

    text = elems[0].getText()

    return str(text), link, img_url


from tabulate import tabulate
import os
import numpy as np
import matplotlib.pyplot as plt
from statistics import stdev

class StatisticsCalculator:

	def __init__(self,ToutLearner,ToutProposer,drty):
		self.TimeOutLearner = ToutLearner
		self.TimeOutProposer = ToutProposer
		self.Table = []
		self.fileDirectory = drty
		self.avgCpuTimes = []
		self.avgElapsedTimes = []
		self.stdCpuTimes = []
		self.stdElapsedTimes = []
		self.varyingParameterValues = []
		self.varyingParameter = None

	def createDirectory(self):
		if not os.path.exists(self.fileDirectory):
			os.makedirs(self.fileDirectory)

	def addTableHeader(self,varyingParameterName , param2Name , param3Name):
		tableHeader = ["Timeout","Preemtion","TimeOutLearner","TimeOutProposer",varyingParameterName,param2Name,param3Name,"Safety","Liveness","AvgElpTime","SDElpTime","ElpTimeRange","AvgCpuTime","SDCpuTime","CpuTimeRange"]
		self.Table.append(tableHeader)
		self.varyingParameter = varyingParameterName

	def plot(self,title,parameter1Name,parameter1Value,parameter2Name,parameter2Value):
		plt.figure()
		plt.title(title, size = 14)
		plt.xlabel(parameter1Name, size = 14)
		plt.ylabel(parameter2Name, size = 14)
		plt.plot(parameter1Value,parameter2Value,'-r',label='Label')
		plt.legend(loc='upper left')
		self.createDirectory()
		filename = './' + self.fileDirectory +'/' + title +'.png'
		plt.savefig(filename)
		plt.close()

	def addEntry(self,varyingParameter,cpuTimeList,elapsedTimeList,isTimeOut,isPreemtionEnabled,param2Value,param3Value,isSafe,isLive):
		tableRow = []
		tableRow.append(isTimeOut)
		tableRow.append(isPreemtionEnabled)
		tableRow.append(self.TimeOutLearner)
		tableRow.append(self.TimeOutProposer)
		tableRow.append(round(varyingParameter,5))
		tableRow.append(round(param2Value,5))
		tableRow.append(round(param3Value,5))
		tableRow.append(isSafe)
		tableRow.append(isLive)
		tableRow.append(round(sum(elapsedTimeList) / len(elapsedTimeList),5))
		tableRow.append(round(stdev(elapsedTimeList),5))
		tableRow.append(round(max(elapsedTimeList) - min(elapsedTimeList),5))
		tableRow.append(round(sum(cpuTimeList) / len(cpuTimeList),5))
		tableRow.append(round(stdev(cpuTimeList),5))
		tableRow.append(round(max(cpuTimeList) - min(cpuTimeList),5))
		self.Table.append(tableRow)

		self.avgElapsedTimes.append(sum(elapsedTimeList) / len(elapsedTimeList))
		self.avgCpuTimes.append(sum(cpuTimeList) / len(cpuTimeList))
		self.stdElapsedTimes.append(stdev(elapsedTimeList))
		self.stdCpuTimes.append(stdev(cpuTimeList))
		self.varyingParameterValues.append(varyingParameter)

	def report(self):
		print(tabulate(self.Table))
		self.plot("Average Cpu Time vs " + self.varyingParameter ,self.varyingParameter,self.varyingParameterValues,"Average Cpu Time" , self.avgCpuTimes  )
		self.plot("Average Elapsed Time vs " + self.varyingParameter ,self.varyingParameter,self.varyingParameterValues,"Average Elapsed Time" , self.avgElapsedTimes  )
		self.plot("Standard Deviation - Elapsed Time vs " + self.varyingParameter ,self.varyingParameter,self.varyingParameterValues,"Standard Deviation - Elapsed Time" , self.stdElapsedTimes  )
		self.plot("Standard Deviation - Cpu Time vs " + self.varyingParameter ,self.varyingParameter,self.varyingParameterValues,"Standard Deviation - CPU Time" , self.stdCpuTimes  )
		filename = './' + self.fileDirectory +'/' + self.varyingParameter +'Analysis.txt'
		f = open(filename,'w')
		f.write(tabulate(self.Table))
		f.flush()
		f.close()