import numpy as np

from .newton_root import bisection, init_newton, newton, newton_s
from .utils import tensor_H

def _iss_single(s, X, W, r_inv):
    """
    Update of the demixing matrix with the iterative source steering of the s-th source
    """
    _, __, n_frames = X.shape

    # demix
    Y = W @ X

    # OP: n_frames * n_src
    v_num = (Y * r_inv[None, :, :]) @ np.conj(Y[:, s, :, None])  # (n_freq, n_src, 1)
    # OP: n_frames * n_src
    v_denom = r_inv[None, :, :] @ np.abs(Y[:, s, :, None]) ** 2
    # (n_freq, n_src, 1)

    # OP: n_src
    v = v_num[:, :, 0] / v_denom[:, :, 0]
    # OP: 1
    v[:, s] = 1.0 - np.sqrt(n_frames) / np.sqrt(v_denom[:, s, 0])

    # update demixed signals
    # OP: n_frames * n_src
    W[:] -= v[:, :, None] * W[:, None, s, :]

def _ip_single(s, X, W, r_inv):
    """
    Performs update of the s-th demixing vector using
    the iterative projection rules
    """

    n_freq, n_chan, n_frames = X.shape

    # Compute Auxiliary Variable
    # shape: (n_freq, n_chan, n_chan)
    V = np.matmul((X * r_inv[None, None, :]), tensor_H(X)) / n_frames

    WV = np.matmul(W, V)
    rhs = np.eye(n_chan)[None, :, s]  # s-th canonical basis vector
    W[:, s, :] = np.conj(np.linalg.solve(WV, rhs))

    # normalize
    denom = np.matmul(
        np.matmul(W[:, None, s, :], V[:, :, :]), np.conj(W[:, s, :, None])
    )
    W[:, s, :] /= np.sqrt(denom[:, :, 0])

def _ip_double(s1, s2, X, W, r_inv):
    """
    Performs a joint update of the s1-th and s2-th demixing vectors
    usint the iterative projection 2 rules
    """
    n_frames = X.shape[-1]

    # Compute Auxiliary Variable
    # shape: (n_freq, n_chan, n_chan)
    V = [
        (X * r_inv[None, i, None, :]) @ tensor_H(X) / n_frames
        for i, s in enumerate([s1, s2])
    ]

    _ip_double_sub(s1, s2, V, W)

def _ip_double_sub(s1, s2, V, W):

    n_freq, n_chan, _ = W.shape

    # right-hand side for computation of null space basis
    # no_update = [i for i in range(n_src) if i != s]
    rhs = np.eye(n_chan)[None, :, [s1, s2]]

    # Basis for demixing vector
    H = []
    HVH = []
    for i, s in enumerate([s1, s2]):
        H.append(np.linalg.solve(W @ V[i], rhs))
        HVH.append(tensor_H(H[i]) @ V[i] @ H[i])

    # Now solve the generalized eigenvalue problem
    lmbda_, R = np.linalg.eig(np.linalg.solve(HVH[0], HVH[1]))

    # Order by decreasing order of eigenvalues
    I_inv = lmbda_[:, 0] > lmbda_[:, 1]
    lmbda_[I_inv, :] = lmbda_[I_inv, ::-1]
    R[I_inv, :, :] = R[I_inv, :, ::-1]

    for i, s in enumerate([s1, s2]):
        denom = np.sqrt(np.conj(R[:, None, :, i]) @ HVH[i] @ R[:, :, i, None])
        W[:, s, None, :] = tensor_H(H[i] @ (R[:, :, i, None] / denom))

def _ip_double_two_channels(X, W, r_inv):
    """
    Specialized update rule for the 2 sources/2 channels case.
    In this, a globally optimal update of the surrogate function exists.
    """

    n_freq, n_chan, n_frames = X.shape
    s1, s2 = 0, 1

    assert n_chan == 2, "This update rule is only valid for two channels"

    # Compute Auxiliary Variable
    # shape: (n_freq, n_chan, n_chan)
    V = [
        (X * r_inv[None, i, None, :]) @ tensor_H(X) / n_frames
        for i, s in enumerate([s1, s2])
    ]

    # Now solve the generalized eigenvalue problem
    lmbda_, R = np.linalg.eig(np.linalg.solve(V[0], V[1]))

    # Order by decreasing order of eigenvalues
    I_inv = lmbda_[:, 0] < lmbda_[:, 1]
    lmbda_[I_inv, :] = lmbda_[I_inv, ::-1]
    R[I_inv, :, :] = R[I_inv, :, ::-1]

    for i, s in enumerate([s1, s2]):
        denom = np.sqrt(np.conj(R[:, None, :, i]) @ V[i] @ R[:, :, i, None])
        W[:, s, None, :] = tensor_H(R[:, :, i, None] / denom)

def _parametric_background_update(n_src, W, Cx):
    """
    Update the backgroud part of a parametrized demixing matrix
    """

    W_target = W[:, :n_src, :]  # target demixing matrix
    J = W[:, n_src:, :n_src]  # background demixing matrix

    tmp = np.matmul(W_target, Cx)
    J[:, :, :] = tensor_H(np.linalg.solve(tmp[:, :, :n_src], tmp[:, :, n_src:]))

def _joint_demix_background(s, n_src, X, W, r_inv, Cx):
    """
    Joint update of one demixing vector and one block
    """

    n_freq, n_chan, n_frames = X.shape

    # right-hand side for computation of null space basis
    # no_update = [i for i in range(n_src) if i != s]
    update = [s] + list(range(n_src, n_chan))
    rhs = np.eye(n_chan)[None, :, update]

    # Compute Auxiliary Variable
    # shape: (n_freq, n_chan, n_chan)
    V = (np.matmul((X * r_inv[None, None, :]), tensor_H(X))) / n_frames

    # Basis for demixing vector
    Hw = np.linalg.solve(W @ V, rhs)
    HVH = np.matmul(tensor_H(Hw), V @ Hw)

    # Basis for the background
    Hb = np.linalg.solve(W @ Cx, rhs)
    HCH = np.matmul(tensor_H(Hb), Cx @ Hb)

    # Now solve the generalized eigenvalue problem
    B_H = np.linalg.cholesky(HCH)
    B_inv = np.linalg.inv(tensor_H(B_H))
    V_tilde = np.linalg.solve(B_H, HVH) @ B_inv
    lmbda_, R = np.linalg.eigh(V_tilde)
    R = np.matmul(B_inv, R)

    # The target demixing vector requires normalization
    R[:, :, -1] /= np.sqrt(lmbda_[:, -1, None])

    # Assign to target and background
    W[:, s, None, :] = tensor_H(Hw @ R[:, :, -1, None])
    W[:, n_src:, :] = tensor_H(Hb @ R[:, :, :-1])

def _block_ip(sources, X, W, V):
    """
    Block iterative projection update
    This is a joint update of the demixing vectors of sources
    that are not independent
    """

    n_freq, n_chan, n_frames = X.shape

    rhs = np.eye(n_chan)[None, :, sources]
    W[:, sources, :] = tensor_H(np.linalg.solve(W @ V, rhs))

    for s in sources:
        # normalize
        denom = np.matmul(
            np.matmul(W[:, None, s, :], V[:, :, :]), np.conj(W[:, s, :, None])
        )
        W[:, s, :] /= np.sqrt(denom[:, :, 0])

def _ipa_make_T(u, q, k):

    n_freq, n_chan = u.shape

    o = list(range(n_chan))
    o.remove(k)

    T = np.zeros((n_freq, n_chan, n_chan), dtype=u.dtype)
    T[:] = np.eye(n_chan)[None, :, :]
    T[:, o, k] = np.conj(q)
    T[:, k, :] = np.conj(u)

    return T

def _ipa(V, W, k):
    """
    Implementation of Iterative Projection with Adjustment (IPS) updates for IVA using the global solution based
    on Newton-Raphson root finding
    """
    n_freq, n_chan, _ = W.shape

    o = list(range(n_chan))
    o.remove(k)

    Vk = W @ V[k] @ tensor_H(W)
    Vk_inv = np.linalg.inv(Vk)

    # shape (n_freq, n_chan - 1, 1)
    a = np.real(
        [(W[:, None, k, :] @ V[m] @ np.conj(W[:, k, :, None]))[..., 0] for m in o]
    ).transpose([1, 0, 2])
    a_inv_sq = 1.0 / np.sqrt(a)

    # shape (n_freq, n_chan - 1, 1)
    b = np.array(
        [(W[:, None, k, :] @ V[m] @ np.conj(W[:, m, :, None]))[..., 0] for m in o]
    ).transpose([1, 0, 2])

    u, q = _ipa_sub(k, Vk, Vk_inv, a, a_inv_sq, b)

    T = _ipa_make_T(u[:, :, 0], q[:, :, 0], k)
    W = T @ W

    return W

def _ipa2(k, Y, W, r_inv):

    n_freq, n_chan, n_frames = Y.shape

    o = list(range(n_chan))
    o.remove(k)

    Vk = (Y * r_inv[k, None, None, :]) @ tensor_H(Y) / n_frames
    Vk = 0.5 * (Vk + np.conj(Vk))
    Vk_inv = np.linalg.inv(Vk)

    # shape (n_freq, n_chan - 1, 1)
    a = (r_inv[None, o, :]) @ np.abs(Y[:, k, :, None]) ** 2
    a /= n_frames
    a_inv_sq = 1.0 / np.sqrt(a)

    # shape (n_freq, n_chan - 1, 1)
    b = (Y[:, o, :] * r_inv[None, o, :]) @ np.conj(Y[:, k, :, None])
    b = np.conj(b)
    b /= n_frames

    u, q = _ipa_sub(k, Vk, Vk_inv, a, a_inv_sq, b)

    # compute the new separation matrix
    T = _ipa_make_T(u[:, :, 0], q[:, :, 0], k)
    W[:] = T @ W

    # update the output signal
    Yk = tensor_H(u) @ Y
    for i, ell in enumerate(o):
        Y[:, ell, :] += np.conj(q[:, i, :]) * Y[:, k, :]
    Y[:, [k], :] = Yk

def _ipa_sub(k, Vk, Vk_inv, a, a_inv_sq, b):

    n_freq, n_chan, _ = Vk.shape
    o = list(range(n_chan))
    o.remove(k)

    # shape (n_freq, n_chan, n_chan)
    C = np.conj(Vk_inv[:, :, o][:, o, :])

    # shape (n_freq, n_chan, n_chan)
    U = a_inv_sq * C * tensor_H(a_inv_sq)
    # sometimes, the hermitian symmetry is lost due to small numerical innacuracies
    # and numpy.linalg.eigh returns negative eigenvalues, throwing off the algorithm,
    # we thus enforce hermtian symmetry of U here
    U = 0.5 * (U + tensor_H(U))

    # shapes (n_freq, n_chan), (n_freq, n_chan - 1, n_chan - 1)
    phi, Sigma = np.linalg.eigh(U)

    # shape (n_freq, n_chan - 1, 1)
    ag = a_inv_sq * np.conj(Vk_inv[:, o, k, None])

    # shape (n_freq)
    g_C_inv_g = np.sum(np.abs(tensor_H(Sigma) @ ag) ** 2 / phi[:, :, None], axis=(1, 2))
    z = np.real(Vk_inv[:, k, k] - g_C_inv_g)

    # shape (n_freq, n_chan, 1)
    v = tensor_H(Sigma) @ (-U @ (a_inv_sq * b) - ag)

    # normalize so that the largest eigenvalue is one
    phi_max = phi[:, -1]
    phi_hat = phi / phi_max[:, None]
    v_hat = v[:, :, 0] / phi_max[:, None]
    z_hat = z / phi_max

    # make an array to receive the solution
    q = np.zeros((n_freq, n_chan - 1, 1), dtype=Vk.dtype)
    lambda_ = np.zeros(n_freq, dtype=np.float)

    # when v is very small, the solution is given by the dominant eigenvector
    # of U
    I_small = np.linalg.norm(v[:, :, 0], axis=1) < 1e-10
    if np.sum(I_small) > 0:

        # there is one more case we need to account for
        # when z is larger than the largest eigenvalue,
        # the optimal q vector is zero, and lambda = z
        I_z = np.logical_and(I_small, z >= phi_max)
        lambda_[I_z] = z[I_z]

        # when z is smaller than the largest eigenvalue
        # then lambda = largest eigenvalue and q is the
        # corresponding eigenvector
        I_ow = np.logical_and(I_small, z < phi_max)
        uv = Sigma[I_ow, :, -1, None]  # leading eigenvector
        lambda_[I_ow] = phi[I_ow, -1]

        scale = np.sqrt(phi[I_ow, -1] - z[I_ow]) / np.sqrt(
            np.real(tensor_H(uv) @ U[I_ow] @ uv)[:, 0, 0]
        )
        q[I_ow, :, 0] = (
            a_inv_sq[I_ow, :, 0] * (uv[:, :, 0] * scale[:, None]) + b[I_ow, :, 0]
        )

    # initial value for root finding
    # anything larger than the largest eigenvalue should be fine
    I_big = np.logical_not(I_small)

    # use the smart initialization
    lambda_0 = init_newton(phi_hat[I_big, :], v_hat[I_big, :], z_hat[I_big])

    # newton
    lambda_[I_big] = newton(
        np.maximum(lambda_0, z_hat[I_big]),
        phi_hat[I_big, :],
        v_hat[I_big, :],
        z_hat[I_big],
        atol=1e-5,
        max_iter=1000,
        verbose=False,
        plot=False,
    )

    # rescale
    lambda_[I_big] *= phi_max[I_big]

    # finally piece together the optimum vector
    t = v[I_big, :] / (lambda_[I_big, None, None] - phi[I_big, :, None])
    q[I_big, :] = a_inv_sq[I_big] * (Sigma[I_big] @ t) - b[I_big] / a[I_big]

    # build the other demixing vector
    x = np.ones((n_freq, n_chan, 1), dtype=Vk.dtype)
    x[:, o, :] = -np.conj(q)
    u = Vk_inv[:, :, k, None] - Vk_inv[:, :, o] @ np.conj(q)

    I = np.where(lambda_ <= 0.0)[0]
    if len(I) > 0:
        print("Detected bad cases, investigate:")
        v_not_hat = -U @ (a_inv_sq * b) - ag
        for i in I:
            print("*****")
            print("freq:", i)
            print("U=")
            print(U[i])
            print("v=")
            print(v_not_hat[i])
            print("z=", z[i])
            print("phi=")
            print(phi[i])
            print("v_hat=")
            print(v[i])
        print("*****")
        raise ValueError(
            "Numerical problem. Please report to "
            " https://github.com/fakufaku/auxiva-ipa"
        )

    u *= 1.0 / np.sqrt(lambda_[:, None, None])

    return u, q


import logging

from google.appengine.api import memcache
from dataviz.restclient import RestClient

URL = "api.geonames.org"
USERNAME = ""

geonames = RestClient(URL, "")

def locate(name):
    city = memcache.get(name)
    logging.debug("Memcache has %r for key %r" % (city, name))
    if city is None:
        logging.warn("Cache miss for geonames, key %r" % name)
        response = geonames.get("searchJSON", q=name, username=USERNAME,
                                maxRows=1)
        city = {}
        if "totalResultsCount" in response and response['totalResultsCount']:
            city = {
                'name': response['geonames'][0]['toponymName'],
                'coord': [
                    response['geonames'][0]['lng'],
                    response['geonames'][0]['lat']
                ],
            }
        result = memcache.add(name, city)
        logging.debug("Memcached %r with key %r: %r" % (city, name, result))
    return city


from django.shortcuts import render

# Create your views here.
from django.http import HttpResponse
from django.contrib.auth.decorators import login_required
from django.shortcuts import get_object_or_404, render

from dcodex.util import get_request_dict
import logging

from .models import *

def index(request):
    return HttpResponse("Hello, world. You're at the DCodex Carlson index.")

def location_siglum( request, location_id, siglum_text ):
    location = get_object_or_404(Location, id=location_id)
    sublocations = location.sublocation_set.all()
    siglum = get_object_or_404(Siglum, name=siglum_text)
    collation = location.collation_set.first()

    siglum = get_object_or_404(Siglum, name=siglum_text)

    verse_labels = location.closest_verse_labels().all()

    transcription = None
    manuscript = None
    bible_verse = None
    if siglum.witness.manuscript and len(verse_labels) > 0:
        manuscript = siglum.witness.manuscript
        bible_verse = verse_labels[0].bible_verse()
        transcription = manuscript.transcription( bible_verse )

    attestation = siglum.witness.get_attestation( sublocation=sublocation, parallel=parallel, corrector=corrector )

    return render(request, 'dcodex_carlson/location.html', {'location': location, 'manuscript':manuscript, 'bible_verse':bible_verse, 'transcription':transcription, 'sublocations': sublocations, 'siglum':siglum, 'witness':siglum.witness, 'parallel':None, 'verse_labels':verse_labels,} )

def location_siglum_parallel( request, location_id, siglum_text, parallel_code ):
    location = get_object_or_404(Location, id=location_id)
    sublocations = location.sublocation_set.all()
    siglum = get_object_or_404(Siglum, name=siglum_text)
    parallel = get_object_or_404(Parallel, code=parallel_code)
    verse_labels = [verse_label for verse_label in location.closest_verse_labels().all() if verse_label.parallel == parallel]

    transcription = None
    manuscript = None
    bible_verse = None
    if siglum.witness.manuscript and len(verse_labels) > 0:
        manuscript = siglum.witness.manuscript
        bible_verse = verse_labels[0].bible_verse()
        transcription = manuscript.transcription( bible_verse )

    return render(request, 'dcodex_carlson/location.html', {'location': location, 'manuscript':manuscript, 'bible_verse':bible_verse,'transcription':transcription, 'sublocations': sublocations, 'siglum':siglum, 'witness':siglum.witness, 'parallel':parallel, 'verse_labels':verse_labels,} )

def location( request, location_id ):
    location = get_object_or_404(Location, id=location_id)
    sublocations = location.sublocation_set.all()

    return render(request, 'dcodex_carlson/location.html', {'location': location, 'sublocations': sublocations} )

def attestations( request ):
    request_dict = get_request_dict(request)

    reading_code = request_dict.get('code')
    sublocation_id = request_dict.get('sublocation_id')
    sublocation = get_object_or_404(SubLocation, id=sublocation_id)

    parallels = sublocation.get_parallels()
    html = ""
    for parallel in parallels:
        parallel_switch = ""
        if parallel:
            parallel_switch = "/%s " % parallel.code
        html += "<div>" + str(parallel_switch) + sublocation.code_attestations_string(reading_code, parallel) + "</div>"

    return HttpResponse(html)

def set_attestation( request ):
    request_dict = get_request_dict(request)

    reading_code = request_dict.get('code')
    corrector = request_dict.get('corrector')

    sublocation_id = request_dict.get('sublocation_id')
    sublocation = get_object_or_404(SubLocation, id=sublocation_id)

    witness_id = request_dict.get('witness_id')
    witness = get_object_or_404(Witness, id=witness_id)

    parallel = None
    parallel_id = request_dict.get('parallel_id')
    parallel_code = request_dict.get('parallel_code')
    if parallel_id:
        parallel = get_object_or_404(Parallel, id=parallel_id)
    elif parallel_code:
        parallel = get_object_or_404(Parallel, code=parallel_code)

    logger = logging.getLogger(__name__)

    logger.error("Parallel in view")
    logger.error(parallel)

    response = witness.set_attestation( sublocation=sublocation, code=reading_code, parallel=parallel, corrector=corrector )
    return HttpResponse("OK" if response else "FAIL")

def set_text( request ):
    request_dict = get_request_dict(request)

    corrector = request_dict.get('corrector')

    sublocation_id = request_dict.get('sublocation_id')
    sublocation = get_object_or_404(SubLocation, id=sublocation_id)

    witness_id = request_dict.get('witness_id')
    witness = get_object_or_404(Witness, id=witness_id)

    parallel = None
    parallel_id = request_dict.get('parallel_id')
    parallel_code = request_dict.get('parallel_code')
    if parallel_id:
        parallel = get_object_or_404(Parallel, id=parallel_id)
    elif parallel_code:
        parallel = get_object_or_404(Parallel, code=parallel_code)

    logger = logging.getLogger(__name__)

    logger.error("Parallel in view")
    logger.error(parallel)

    attestation = witness.get_attestation( sublocation=sublocation, parallel=parallel, corrector=corrector )
    if attestation:
        attestation.text = request_dict.get('text')
        attestation.save()
        return HttpResponse("OK")

    return HttpResponse("FAIL")


from Application.Api.Domain.PlexUser import PlexUser
from Application.Api.Domain.Vote import Vote
from typing import List

class Movie(object):
    def __init__(self,
                 movie_id: int,
                 imdb_id: str,
                 movie_name: str,
                 creator: PlexUser,
                 approved: bool,
                 declined: bool,
                 deleted: bool,
                 private: bool,
                 channel_id: int,
                 trakt_id: int,
                 votes: List[Vote]):
        self.movie_id: int = movie_id
        self.imdb_id: str = imdb_id
        self.movie_name: str = movie_name
        self.creator: PlexUser = creator
        self.approved: bool = approved
        self.declined: bool = declined
        self.deleted: bool = deleted
        self.private: bool = private
        self.channel_id: int = channel_id
        self.trakt_id: int = trakt_id
        self.votes: List[Vote] = votes


import pygame
from pygame_constants import *

class GameButton:

    # Initiate button: location, text, size, locked or not
    def __init__(self, location, title, lock=False):
        self.location = location
        self.size = (location.X_END - location.X, location.Y_END - location.Y)
        self.title = title
        self.title_center = ((location.X_END + location.X) // 2, (location.Y_END + location.Y) // 2)
        self.backgroundTile = pygame.Surface(self.size)
        if lock:
            self.locked = False
            self.lockButton()
        else:
            self.locked = True
            self.unlockButton()

    def lockButton(self):
        if not self.locked:
            self.title_view = FONT_COURIER_20.render(self.title, True, WHITE, GREY)
            self.title_rect = self.title_view.get_rect()
            self.title_rect.center = self.title_center
            self.backgroundTile.fill(GREY)
            self.locked = True

    def unlockButton(self):
        if self.locked:
            self.title_view = FONT_COURIER_20.render(self.title, True, BLACK, ORANGE)
            self.title_rect = self.title_view.get_rect()
            self.title_rect.center = self.title_center
            self.backgroundTile.fill(ORANGE)
            self.locked = False

    # Draw the button
    def blitButton(self, screen):
        screen.blit(self.backgroundTile, (self.location.X, self.location.Y))
        screen.blit(self.title_view, self.title_rect)

    def isLocked(self):
        return self.locked


import sqlite3
import os
from pathlib import Path
from flask import Blueprint
from pypika import Query, Table, Field
from flask import jsonify, request

company = Blueprint('company', __name__,
                    template_folder='templates')

def get_db_connection():
    THIS_FOLDER = Path(__file__).parent.resolve()
    DATABASE = os.path.join(THIS_FOLDER, "database.db")
    conn = sqlite3.connect(DATABASE)
    conn.row_factory = sqlite3.Row
    return conn

@company.route('/company/add', methods=['POST'])
def add_inventory():
    connection = get_db_connection()
    data = request.get_json()
    inventory_table = Table('company')
    query = Query.into(inventory_table).insert(None,
                                               data['name'],
                                               data['email'],
                                               data['password']).get_sql()

    cursor = connection.cursor()
    cursor.execute(query)
    id = cursor.lastrowid
    connection.commit()
    connection.close()
    return jsonify({'message': 'Company created yay', 'id': id})


# -*- coding: utf-8 -*-
"""
Created on Tue Jun  1 19:12:47 2021

@author: gama0
"""

import requests
from bs4 import BeautifulSoup
import time
from tqdm import tqdm
from fake_useragent import UserAgent
import copy
import os
# import numpy as np

# a list of textures to NOT include
filters = ['AcousticFoam',
  'Candy',
  'ChristmasTreeOrnament',
  'Facade',
  'Fence',
  'Fingerprint',
  'Foam',
  'Metal Walkway',
  'OfficeCeiling',
  'Pathway',
  'Paint00',
  'Painting',
  'PineNeedles',
  'Porcelain',
  'Road',
  'RockBrush',
  'Rust001',
  'Sign',
  'Scratches',
  'Smear',
  'Sticker',
  'Tape',
  'TreeEnd',
  'SurfaceImperfections',
  'Footsteps',
  'Substance']

# ambientCG website:
output_path = '../dataset/zip/'
home = 'https://ambientcg.com/'
url = "https://ambientcg.com/list?category=&type=Material&sort=Alphabet"

if not os.path.exists(output_path):
    os.makedirs(output_path)

#%%
def main():
    # agent to request
    user_agent = UserAgent()
    download_url = []

    method_url = [url + '&method=PBRMultiAngle']

    # for methods with multiple pages of content (1 page has 180 links)
    offsets = [0, 180, 360, 540, 720, 900]
    for offset in offsets:
        method_url.append(url + '&method=PBRApproximated' + '&offset={}'.format(offset))
        method_url.append(url + '&method=PBRPhotogrammetry' + '&offset={}'.format(offset))
        method_url.append(url + '&method=PBRProcedural' + '&offset={}'.format(offset))

    #%%
    # =========================== request the list to "download_url"===================
    for i in method_url:

        re = requests.get(i, headers = { 'user-agent': user_agent.random })
        soup = BeautifulSoup(re.text,'html.parser')

        elems = soup.find_all('a')
        for elem in elems:
            # texture_url + elem = each download url
            if elem.get('href').startswith("/view?id="):
                n = elem.get('href').split('/')[-1]
                download_url.append(home + n)

    copy_download_url = copy.deepcopy(download_url)

    # =========================== filters ===================
    # ( I don't know the reason why there's 99 elements need to be removed, but it can't do it completely at once.)
    n = 0
    for _ in range(5):
        for i in copy_download_url:
            for word in filters:
                if i.split('=')[-1].startswith(word):
                    # print('remove this!', i)
                    copy_download_url.remove(i)
                    n += 1

    copy_download_url.sort()
    # np.savetxt('test.txt',copy_download_url,delimiter="\n", fmt="%s")

    #%%
    # =========================== request download url ===================
    counts = 0
    for i in tqdm(range(len(copy_download_url))):
    # ==========================
        # 如果爬的過程斷掉，從多少開始...
        if i<counts:
            # counts += 1
            continue
    # ==========================
        # print(copy_download_url[i])
        re = requests.get(copy_download_url[i], # https://ambientcg.com/view?id=Tiles098
                        headers = { 'user-agent': user_agent.random })
        soup = BeautifulSoup(re.text,'html.parser')

        # if you want to choose png or JPG
        zip_url = []
        elems = soup.find_all('a', class_= "TextLink")
        for elem in elems:
            zip_url.append(elem.get('href'))

        r = requests.get(zip_url[0], # https://ambientcg.com/get?file=Tiles098_1K-JPG.zip
                        headers = { 'user-agent': user_agent.random })

        with open(output_path + zip_url[0].split('=')[-1], "wb") as zipfile:
            zipfile.write(r.content)

        time.sleep(0.1)
        counts += 1

if __name__ == "__main__":
    main()


import codecs
import csv
import string
import time
import re

import nltk
from nltk.corpus import stopwords

class TrumpDataReader:

    def __init__(self, training_file, case="-t"):
        self.tweets_training = []
        self.tweets_generating = []
        self.case = case

        self.negating = False

        if training_file:
            self.read_and_process_data(training_file, self.case)

    def read_and_process_data(self, training_file, case):
        """
        :param training_file: file to be read and processed.
        :return: void
        """
        with codecs.open(training_file, 'r', 'utf-8') as f:
            f.readline()  # formatting line
            i = 0
            k = 1000
            start_time = time.time()
            for row in f:
                self.negating = False
                tweet = row.split("§")[0]
                clean_tweet_training = self.process_tweet(tweet, "-t")
                clean_tweet_generating = self.process_tweet(tweet, "-lm")
                if clean_tweet_training != "":
                    self.tweets_training.append(clean_tweet_training)
                    self.tweets_generating.append(clean_tweet_generating)
                    #print(clean_tweet_training)
                    #print(clean_tweet_generating)

                if i % k == 0:
                    end_time = time.time()
                    duration = end_time - start_time
                    print(duration)
                i += 1
            print('Done!')

    def process_tweet(self, tweet, case):
        """
        :param tweet: one tweet read from the training document, only contains the body of the tweet.
        :return: tweet, tweet body with dimensionality reduced.
        """
        # tokens = nltk.word_tokenize(tweet) # Gör att kontrollen för länkar inte fungerar
        tokens = tweet.split()

        if tokens[0] == "RT":
            return ""

        for index in range(len(tokens)):
            tokens[index] = self.process_token(tokens[index], case)

        while "" in tokens:
            tokens.remove("")

        if case == "-t":
            tokens = set(tokens)
        tweet = self.list_to_string(tokens)

        return tweet

    def process_token(self, token, case):
        """
        Method to process a single token in a tweet
        :param token: one word from tweet
        :return: processed token
        """
        if token[0] == "@":
            token = "@user"

        elif token[:4] == "http" or token[:4] == "www." or token[len(token)-4:] == ".com":
            token = "adress.com"

        elif token == "&":
            token = ""

        else:
            if case == "-t":

                token = token.lower().strip()

                stop_words = set(stopwords.words("english"))  # Går det snabbare om man har self.stopwords?

                # if token in string.punctuation:
                #    self.negating = False
                #    return ""

                if any(char in token for char in ".,:;()!?"):
                    self.negating = False

                if token in stop_words:
                    return ""

                if token[len(token)-3:] == "n't":
                    self.negating = True

                # if token == "n't":
                #    self.negating = True
                #    return ""
                token = re.sub(r'\s[^\s\w]+\s', ' ', token)
                token = re.sub(r'\d+\s?|\n|[^\s\w]', '', token)

                if self.negating:
                    token = "NOT_" + token

            elif case == "-lm":
                pass

        return token

    def list_to_string(self, split_tweet):
        """
        :param split_tweet: tweet split into list
        :return: split_tweet joined to a string
        """
        return " ".join(split_tweet)

def main():
    TrumpDataReader("Data/tweet_data.txt")

if __name__ == "__main__":
    main()


import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.utils import resample

import tensorflow as tf

def load_image(filename, label):
    file = tf.io.read_file(filename)
    img = tf.image.decode_png(file, channels=3)

    IMG_SIZE = 224
    img = tf.image.resize_with_pad(img, IMG_SIZE, IMG_SIZE)

    return img, label

def get_data(selected_fold:int=2, batch_size=28, seed:int=0):
    '''

    The data outlines a 5-fold cross-validation strategy.

    Parameters:
    ----------

    selected_fold: int
        Fold to be selected (1-5).

        The 2nd fold has as the highest proportion of training images.

    seed: int
        Radom state for reproducability.

    Returns:
    -------

    Tuple
        train_tensor, validation_tensor, test_tensor
    '''
    classes = dict(benign=0, malignant=1)
    fold_info = pd.read_csv('./data/BreaKHis_v1/Folds.csv')
    fold_info['label'] = fold_info['filename'].str.extract('(malignant|benign)')
    selected_fold_data = fold_info.query(f'fold == @selected_fold').copy()

    train = selected_fold_data.query("mag >= 200 and grp == 'train'")
    test = selected_fold_data.query("mag >= 200 and grp == 'test'")
    train.shape, test.shape

    X_train, X_valid, y_train, y_valid = train_test_split(
        train['filename'],
        train['label'].map(classes),
        random_state=seed
    )

    train_df = pd.concat([pd.Series(X_train, name='filename'), pd.Series(y_train, name='label')], axis=1)

    class_1 = train_df[train_df['label'] == 1]
    class_0 = train_df[train_df['label'] == 0]

    class_1_downsampled = resample(class_1, replace=False, n_samples=len(class_0), random_state=seed)

    train_downsampled = pd.concat([class_0, class_1_downsampled], axis=0)

    train_downsampled = train_downsampled.sample(frac=1, random_state=seed).reset_index(drop=True)

    X_train_downsampled = train_downsampled['filename']
    y_train_downsampled = train_downsampled['label']

    X_train, X_valid, y_train, y_valid = train_test_split(X_train_downsampled, y_train_downsampled, random_state=seed)

    train_tensor = tf.data.Dataset.from_tensor_slices((X_train, y_train)) \
                    .map(load_image).batch(batch_size)

    validation_tensor = tf.data.Dataset.from_tensor_slices((X_valid, y_valid)) \
                        .map(load_image).batch(batch_size)

    test = test.sample(frac=1, random_state=seed)
    test_tensor = tf.data.Dataset.from_tensor_slices((test['filename'], test['label'] \
                .map(classes))) \
                .map(load_image).batch(batch_size)

    return train_tensor, validation_tensor, test_tensor


import sys
from dataclasses import dataclass, field
from itertools import repeat
from pathlib import Path
from typing import Optional, Union

from ..common import deep
from ..common.binary_reader import BinaryReader
from ..common.deep import Json
from ..common.gltf import parse_glb
from ..common.logging import get_logger
from .license_validation import validate_license

logger = get_logger(__name__)

@dataclass(frozen=True)
class Vrm0MaterialProperty:
    name: str
    shader: str
    render_queue: Optional[int]
    keyword_map: dict[str, bool]
    tag_map: dict[str, str]
    float_properties: dict[str, float]
    vector_properties: dict[str, list[float]]
    texture_properties: dict[str, int]

    @staticmethod
    def create(json_dict: Json) -> "Vrm0MaterialProperty":
        fallback = Vrm0MaterialProperty(
            name="Undefined",
            shader="VRM_USE_GLTFSHADER",
            render_queue=None,
            keyword_map={},
            tag_map={},
            float_properties={},
            vector_properties={},
            texture_properties={},
        )
        if not isinstance(json_dict, dict):
            return fallback

        name = json_dict.get("name")
        if not isinstance(name, str):
            name = fallback.name

        shader = json_dict.get("shader")
        if not isinstance(shader, str):
            shader = fallback.shader

        render_queue = json_dict.get("renderQueue")
        if not isinstance(render_queue, int):
            render_queue = fallback.render_queue

        raw_keyword_map = json_dict.get("keywordMap")
        if isinstance(raw_keyword_map, dict):
            keyword_map = {
                k: v for k, v in raw_keyword_map.items() if isinstance(v, bool)
            }
        else:
            keyword_map = fallback.keyword_map

        raw_tag_map = json_dict.get("tagMap")
        if isinstance(raw_tag_map, dict):
            tag_map = {k: v for k, v in raw_tag_map.items() if isinstance(v, str)}
        else:
            tag_map = fallback.tag_map

        raw_float_properties = json_dict.get("floatProperties")
        if isinstance(raw_float_properties, dict):
            float_properties = {
                k: v
                for k, v in raw_float_properties.items()
                if isinstance(v, (float, int))
            }
        else:
            float_properties = fallback.float_properties

        raw_vector_properties = json_dict.get("vectorProperties")
        if isinstance(raw_vector_properties, dict):
            vector_properties: dict[str, list[float]] = {}
            for k, v in raw_vector_properties.items():
                if not isinstance(v, list):
                    continue
                float_v: list[float] = []
                ok = True
                for e in v:
                    if not isinstance(e, (float, int)):
                        ok = False
                        break
                    float_v.append(float(e))
                if ok:
                    vector_properties[k] = float_v
        else:
            vector_properties = fallback.vector_properties

        raw_texture_properties = json_dict.get("textureProperties")
        if isinstance(raw_texture_properties, dict):
            texture_properties = {
                k: v for k, v in raw_texture_properties.items() if isinstance(v, int)
            }
        else:
            texture_properties = fallback.texture_properties

        return Vrm0MaterialProperty(
            name=name,
            shader=shader,
            render_queue=render_queue,
            keyword_map=keyword_map,
            tag_map=tag_map,
            float_properties=float_properties,
            vector_properties=vector_properties,
            texture_properties=texture_properties,
        )

@dataclass
class ImageProperties:
    name: str
    filepath: Path
    filetype: str

@dataclass
class ParseResult:
    filepath: Path
    json_dict: dict[str, Json] = field(default_factory=dict)
    spec_version_number: tuple[int, int] = (0, 0)
    spec_version_str: str = "0.0"
    spec_version_is_stable: bool = True
    vrm0_extension: dict[str, Json] = field(init=False, default_factory=dict)
    vrm1_extension: dict[str, Json] = field(init=False, default_factory=dict)
    hips_node_index: Optional[int] = None
    image_properties: list[ImageProperties] = field(init=False, default_factory=list)
    vrm0_material_properties: list[Vrm0MaterialProperty] = field(
        init=False, default_factory=list
    )
    skins_joints_list: list[list[int]] = field(init=False, default_factory=list)
    skins_root_node_list: list[int] = field(init=False, default_factory=list)

def remove_unsafe_path_chars(filename: str) -> str:
    unsafe_chars = {
        0: "\x00",
        1: "\x01",
        2: "\x02",
        3: "\x03",
        4: "\x04",
        5: "\x05",
        6: "\x06",
        7: "\x07",
        8: "\x08",
        9: "\t",
        10: "\n",
        11: "\x0b",
        12: "\x0c",
        13: "\r",
        14: "\x0e",
        15: "\x0f",
        16: "\x10",
        17: "\x11",
        18: "\x12",
        19: "\x13",
        20: "\x14",
        21: "\x15",
        22: "\x16",
        23: "\x17",
        24: "\x18",
        25: "\x19",
        26: "\x1a",
        27: "\x1b",
        28: "\x1c",
        29: "\x1d",
        30: "\x1e",
        31: "\x1f",
        34: '"',
        42: "*",
        47: "/",
        58: ":",
        60: "<",
        62: ">",
        63: "?",
        92: "\\",
        124: "|",
    }  # 32:space #33:!
    remove_table = str.maketrans(
        "".join([chr(unsafe_char) for unsafe_char in unsafe_chars]),
        "".join(repeat("_", len(unsafe_chars))),
    )
    safe_filename = filename.translate(remove_table)
    return safe_filename

#  "accessorの順に" データを読み込んでリストにしたものを返す
def decode_bin(
    json_dict: dict[str, Json], binary: bytes
) -> list[list[Union[int, float, list[int], list[float]]]]:
    br = BinaryReader(binary)
    # This list indexed by accessor index
    decoded_binary: list[list[Union[int, float, list[int], list[float]]]] = []
    buffer_view_dicts = json_dict.get("bufferViews")
    if not isinstance(buffer_view_dicts, list):
        buffer_view_dicts = []
    accessor_dicts = json_dict.get("accessors")
    if not isinstance(accessor_dicts, list):
        return []
    type_num_dict = {"SCALAR": 1, "VEC2": 2, "VEC3": 3, "VEC4": 4, "MAT4": 16}
    for accessor_index, accessor_dict in enumerate(accessor_dicts):
        if not isinstance(accessor_dict, dict):
            continue
        accessor_type = accessor_dict.get("type")
        if not isinstance(accessor_type, str):
            continue
        type_num = type_num_dict.get(accessor_type)
        if not isinstance(type_num, int):
            logger.warning(f"Unrecognized accessor type: {accessor_type}")
            continue
        buffer_view_index = accessor_dict.get("bufferView")
        if not isinstance(buffer_view_index, int):
            logger.warning(
                f"accessors[{accessor_index}] doesn't have bufferView that is not implemented yet"
            )
            decoded_binary.append([])
            continue
        buffer_view_dict = buffer_view_dicts[buffer_view_index]
        if not isinstance(buffer_view_dict, dict):
            continue
        buffer_view_byte_offset = buffer_view_dict.get("byteOffset")
        if not isinstance(buffer_view_byte_offset, int):
            buffer_view_byte_offset = 0
        br.set_pos(buffer_view_byte_offset)
        data_list: list[Union[int, float, list[int], list[float]]] = []
        accessor_count = accessor_dict.get("count")
        if not isinstance(accessor_count, int):
            accessor_count = 0
        component_type = accessor_dict.get("componentType")
        if not isinstance(component_type, int):
            raise ValueError(f"Unsupported component type: {component_type}")
        for _ in range(accessor_count):
            if type_num == 1:
                single_data = br.read_as_data_type(component_type)
                data_list.append(single_data)
            else:
                multiple_data = []
                for _ in range(type_num):
                    multiple_data.append(br.read_as_data_type(component_type))
                data_list.append(multiple_data)
        decoded_binary.append(data_list)

    return decoded_binary

@dataclass
class VrmParser:
    filepath: Path
    extract_textures_into_folder: bool
    make_new_texture_folder: bool
    license_validation: bool
    decoded_binary: list[list[Union[int, float, list[int], list[float]]]] = field(
        init=False, default_factory=list
    )
    json_dict: dict[str, Json] = field(init=False, default_factory=dict)

    def parse(self) -> ParseResult:
        # bin chunkは一つだけであることを期待
        json_dict, _ = parse_glb(self.filepath.read_bytes())
        self.json_dict = json_dict

        if self.license_validation:
            validate_license(self.json_dict)

        parse_result = ParseResult(filepath=self.filepath, json_dict=self.json_dict)
        self.vrm_extension_read(parse_result)
        self.material_read(parse_result)

        return parse_result

    def vrm_extension_read(self, parse_result: ParseResult) -> None:
        vrm1_dict = deep.get(self.json_dict, ["extensions", "VRMC_vrm"])
        if isinstance(vrm1_dict, dict):
            self.vrm1_extension_read(parse_result, vrm1_dict)
            return

        vrm0_dict = deep.get(self.json_dict, ["extensions", "VRM"])
        if isinstance(vrm0_dict, dict):
            self.vrm0_extension_read(parse_result, vrm0_dict)

    def vrm0_extension_read(
        self, parse_result: ParseResult, vrm0_dict: dict[str, Json]
    ) -> None:
        spec_version = vrm0_dict.get("specVersion")
        if isinstance(spec_version, str):
            parse_result.spec_version_str = spec_version
        parse_result.vrm0_extension = vrm0_dict

        human_bones = deep.get(vrm0_dict, ["humanoid", "humanBones"], [])
        if not isinstance(human_bones, list):
            raise ValueError("No human bones")

        hips_node_index: Optional[int] = None
        for human_bone in human_bones:
            if isinstance(human_bone, dict) and human_bone.get("bone") == "hips":
                index = human_bone.get("node")
                if isinstance(index, int):
                    hips_node_index = index

        if not isinstance(hips_node_index, int):
            logger.warning("No hips bone index found")
            return

        parse_result.hips_node_index = hips_node_index

    def vrm1_extension_read(
        self, parse_result: ParseResult, vrm1_dict: dict[str, Json]
    ) -> None:
        parse_result.vrm1_extension = vrm1_dict
        parse_result.spec_version_number = (1, 0)
        parse_result.spec_version_is_stable = False

        hips_node_index = deep.get(
            vrm1_dict, ["humanoid", "humanBones", "hips", "node"]
        )
        if not isinstance(hips_node_index, int):
            logger.warning("No hips bone index found")
            return
        parse_result.hips_node_index = hips_node_index

    # ここからマテリアル
    def material_read(self, parse_result: ParseResult) -> None:
        material_dicts = self.json_dict.get("materials")
        if not isinstance(material_dicts, list):
            return
        for index in range(len(material_dicts)):
            parse_result.vrm0_material_properties.append(
                Vrm0MaterialProperty.create(
                    deep.get(
                        self.json_dict,
                        ["extensions", "VRM", "materialProperties", index],
                    )
                )
            )

if __name__ == "__main__":
    VrmParser(
        Path(sys.argv[1]),
        extract_textures_into_folder=True,
        make_new_texture_folder=True,
        license_validation=True,
    ).parse()


import numpy as np
import cupy as cp
from .layer import Layer
from regularisers import l2

class DenseLayer(Layer):

    def __init__(self, layer_name, incoming_chans=None, output_dim=None, with_bias=True,
                 weight_regulariser=None, weight_initialiser="normal"):
        super().__init__(layer_name)
        self.incoming_chans = incoming_chans
        self.output_dim = output_dim
        self.with_bias = with_bias
        self.weight_regulariser = weight_regulariser
        self.downstream_X = None
        self.weight_initialiser = weight_initialiser

        if incoming_chans is not None and output_dim is not None:
            if self.weight_initialiser == "glorot_uniform":
                limit = np.sqrt(6.0 / (self.incoming_chans + self.output_dim))
                weights = np.random.uniform(low=-limit,
                                            high=limit,
                                            size=(self.incoming_chans,
                                                self.output_dim)).astype(np.float32)
            elif self.weight_initialiser == "normal":
                weights = 0.01*np.random.randn(self.incoming_chans, self.output_dim).astype(np.float32)
            self.learned_params = {"weights": weights}
            self.grads = {"weights": np.zeros_like(weights)}

            if with_bias:
                bias = np.zeros(output_dim).astype(np.float32)
                self.learned_params.update({"bias": bias})
                self.grads.update({"bias": np.zeros_like(bias, dtype=np.float32)})
        else:
            self.learned_params = {}
            self.grads = {}

    def __repr__(self):
        return "DenseLayer({}, incoming_chans={}, output_dim={}, weight_regulariser={})".format(
                                       self.layer_name,
                                       self.incoming_chans,
                                       self.output_dim,
                                       repr(self.weight_regulariser)
                                       )

    def forward(self, X, test_mode=False):
        xp = cp.get_array_module(X)
        if not test_mode:
            self.downstream_X = X

        out = xp.dot(X, self.learned_params["weights"])
        if self.with_bias:
            out += self.learned_params["bias"][xp.newaxis, :]

        return out

    def backward(self, upstream_dx):
        xp = cp.get_array_module(upstream_dx)
        if self.with_bias:
            self.grads["bias"] = xp.sum(upstream_dx, axis=0)
        self.grads["weights"] = xp.dot(self.downstream_X.T, upstream_dx)
        if self.weight_regulariser:
            self.grads["weights"] += (
                self.weight_regulariser.backward(self.learned_params["weights"])
            )

        return xp.dot(upstream_dx, self.learned_params["weights"].T)

    def save_to_h5(self, open_f, save_grads=True):
        base_dset = open_f.create_dataset(self.layer_name + "/layer_info", dtype=np.float32)
        base_dset.attrs["type"] = self.__class__.__name__
        base_dset.attrs["incoming_chans"] = self.incoming_chans
        base_dset.attrs["output_dim"] = self.output_dim
        base_dset.attrs["with_bias"] = self.with_bias

        dset = open_f.create_dataset(self.layer_name + "/weights",
                                     self.learned_params['weights'].shape,
                                     dtype=self.learned_params['weights'].dtype)
        dset[:] = cp.asnumpy(self.learned_params['weights'])
        if self.weight_regulariser is not None:
            dset.attrs["weight_regulariser_type"] = np.string_(self.weight_regulariser.type)
            dset.attrs["weight_regulariser_strength"] = np.string_(self.weight_regulariser.strength)
        if self.with_bias:
            dset_bias = open_f.create_dataset(self.layer_name + "/bias",
                                              self.learned_params['bias'].shape,
                                              dtype=self.learned_params['bias'].dtype)
            dset_bias[:] = cp.asnumpy(self.learned_params['bias'])

        if save_grads:
            dset_grads = open_f.create_dataset(self.layer_name + "/grads/weights",
                                               self.grads['weights'].shape,
                                               dtype=self.learned_params['weights'].dtype)
            dset_grads[:] = cp.asnumpy(self.grads["weights"])
            if self.with_bias:
                dset_grads_bias = open_f.create_dataset(self.layer_name + "/grads/bias",
                                                        self.grads['bias'].shape,
                                                        dtype=self.learned_params['bias'].dtype)
                dset_grads_bias[:] = cp.asnumpy(self.grads["bias"])

    def load_from_h5(self, open_f, load_grads=True):
        self.incoming_chans = open_f[self.layer_name + '/layer_info'].attrs['incoming_chans']
        self.output_dim = open_f[self.layer_name + '/layer_info'].attrs['output_dim']
        self.with_bias = open_f[self.layer_name + '/layer_info'].attrs['with_bias']

        weight_regulariser_type = open_f[self.layer_name + '/weights'].attrs.get("weight_regulariser_type", None)
        if weight_regulariser_type:
            weight_regulariser_strength = open_f[self.layer_name + '/weights'].attrs["weight_regulariser_strength"]
            if weight_regulariser_type == b"l2":
                self.weight_regulariser = l2.l2(strength=float(weight_regulariser_strength))

        self.learned_params['weights'] = open_f[self.layer_name + '/weights'][:]
        if self.with_bias:
            self.learned_params['bias'] = open_f[self.layer_name + '/bias'][:]
        if load_grads:
            self.grads['weights'] = open_f[self.layer_name + '/grads/weights'][:]
            if self.with_bias:
                self.grads['bias'] = open_f[self.layer_name + '/grads/bias'][:]


import datetime

class CardValidationError(Exception):
    pass

class CardValidator:

    def __init__(self, repo):
        self.__repository = repo

    def validate(self, card):
        errors = []
        if type(card.entityId) != int:
            errors.append("The id should be integer!")
        if type(card.CNP) != int or len(str(card.CNP)) != 13:
            errors.append("The CNP should have 13 digits!")
        for c in self.__repository.read():
            if c.CNP == card.CNP:
                errors.append("Duplicate CNP!")
        x = card.birthDate.split(".")
        if len(x) != 3:
            errors.append("Wrong birth date")
        else:
            try:
                int(x[0])
                int(x[1])
                int(x[2])
            except ValueError:
                errors.append("Wrong birth date")
            if not 0 < int(x[0]) < 32:
                errors.append("The birth day should be between 1 and 31!")
            if not 0 < int(x[1]) < 13:
                errors.append("The birth month must be between 1 and 12!")
            if not int(x[2]) < datetime.datetime.now().year:
                errors.append("The birth year must smaller than the current year!")
        y = card.registrationDate.split(".")
        if len(y) != 3:
            errors.append("Wrong registration date")
        else:
            try:
                int(y[0])
                int(y[1])
                int(y[2])
            except ValueError:
                errors.append("Wrong registration date")
            if not 0 < int(y[0]) < 32:
                errors.append("The registration day should be between 1 and 31!")
            if not 0 < int(y[1]) < 13:
                errors.append("The registration month must be between 1 and 12!")
            if not int(y[2]) < datetime.datetime.now().year:
                errors.append("The registration year must smaller than the current year!")
        if errors:
            raise CardValidationError(errors)


import torch
import torch.nn as nn
from torch.nn import functional as F

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.encoder = nn.Sequential(
            nn.Conv2d(1, 32, kernel_size=5, stride=1, bias=False), nn.BatchNorm2d(32), nn.ReLU(), nn.MaxPool2d(2, 2),
            nn.Conv2d(32, 64, kernel_size=5, stride=1, bias=False), nn.BatchNorm2d(64), nn.ReLU(), nn.MaxPool2d(2, 2),
        )
        self.classifier = nn.Sequential(nn.Linear(64 * 4 * 4, 128),
                                        nn.Dropout(0.5),
                                        nn.ReLU(),
                                        nn.Linear(128, 10))

    def forward(self, x):
        h = self.encoder(x)
        h = torch.flatten(h, 1)
        h = self.classifier(h)
        output = F.log_softmax(h, dim=1)
        return output


from typing import List, Tuple, Any, Dict, Optional
from ..utils import *
from ..event_series import *

POS_ROUND_DIGITS = 3
SOFT_COLLIDE_SHIFT = 0.2

BAG_RADIUS = 0.5
BAG_MASS = 20
BAG_DENSITY = 40

BELT_WIDTH = 1.5
BELT_UNIT_MASS = 15

SPEED_STEP = 0.1
SPEED_ROUND_DIGITS = 3

MODEL_BELT_LEN = 313.25

_G = 9.8
_FRICTON_COEFF = 0.024
_THETA_1 = 1/(6.48*BELT_WIDTH*BELT_WIDTH*BAG_DENSITY)
_THETA_2_L = _G*_FRICTON_COEFF*BELT_UNIT_MASS
_k_3 = 4000
_THETA_3 = 0.0031
_THETA_4_L = _G*_FRICTON_COEFF
_k_2 = 30

_ETA = 0.8

# THETAS_ORIG = [2.3733e-4, 8566.3, 0.0031, 51.6804]

def _P_Zhang(length, V, T):
    return _THETA_1*V*T*T + (_THETA_2_L*length + _k_3)*V + \
        _THETA_3*T*T/V + (_THETA_4_L*length + _k_2)*T + V*V*T/3.6

def consumption_Zhang(length, speed, n_bags):
    if speed == 0:
        return 0
    Q_g = n_bags * BAG_MASS / length
    T = Q_g * speed * 3.6
    return _P_Zhang(length, speed, T) / _ETA

def consumption_linear(length, speed, n_bags):
    return _G * _FRICTION_COEFF * BAG_MASS * n_bags * speed / _ETA

##
# ^ weird stuff up here ...
#

class EnergySpender(ChangingValue):
    """
    Class which records energy consumption
    """
    def __init__(self, data_series: EventSeries, length, consumption='zhang'):
        super().__init__(data_series)
        if type(consumption) == 'float':
            self._consumption = lambda v, n: consumption
        elif consumption == 'zhang':
            self._consumption = lambda v, n: consumption_Zhang(length, v, n)
        elif consumption == 'linear':
            self._consumption = lambda v, n: consumption_linear(length, v, n)
        else:
            raise Exception('consumption: ' + consumption)

    def update(self, time, speed, n_bags):
        super().update(time, self._consumption(speed, n_bags))

class CollisionException(Exception):
    """
    Thrown when objects on conveyor collide
    """
    def __init__(self, args):
        super().__init__('[{}] Objects {} and {} collided in position {}'
                        .format(args[3], args[0], args[1], args[2]))
        self.obj1 = args[0]
        self.obj2 = args[1]
        self.pos = args[2]
        self.handler = args[3]

class AutomataException(Exception):
    pass

def search_pos(ls: List[Tuple[Any, float]], pos: float,
               return_index: bool = False,
               preference: str = 'nearest') -> Tuple[Any, float]:
    assert len(ls) > 0, "what are you gonna find pal?!"
    return binary_search(ls, differs_from(pos, using=lambda p: p[1]),
                         return_index=return_index, preference=preference)

def shift(objs, d):
    return [(o, round(p + d, POS_ROUND_DIGITS)) for (o, p) in objs]

# Automata for simplifying the modelling of objects movement
_model_automata = {
    'pristine': {'resume': 'moving', 'change': 'dirty'},
    'moving': {'pause': 'pristine'},
    'dirty': {'change': 'dirty', 'start_resolving': 'resolving'},
    'resolving': {'change': 'resolving', 'end_resolving': 'resolved'},
    'resolved': {'resume': 'moving'}
}

class ConveyorModel:
    """
    Datatype which allows for modeling the conveyor with
    objects moving on top of it.

    A conveyor is modeled as a line with some checkpoints
    placed along it. Only one checkpoint can occupy a given position.

    Objects can be placed on a conveyor. All the objects on a conveyor
    move with the same speed - the current speed of a conveyor. Current speed
    of a conveyor can be changed.

    We want to have the following operations:
    - put an object to a conveyor (checking for collisions)
    - remove an object from a conveyor
    - change the conveyor speed
    - update the positions of objects as if time T has passed with a current
      conveyor speed (T might be negative, moving objects backwards)
    - calculate time T until earliest event of object reaching a checkpoint
      (or the end of the conveyor), and also return those checkpoint and object
    """
    def __init__(self, env: DynamicEnv, length: float, max_speed: float,
                 checkpoints: List[Tuple[Any, float]], energy_series: EventSeries = None,
                 model_id: AgentId = ('world', 0)):
        assert length > 0, "Conveyor length <= 0!"
        assert max_speed > 0, "Conveyor max speed <= 0!"

        checkpoints = sorted(checkpoints, key=lambda p: p[1])
        if len(checkpoints) > 0:
            assert checkpoints[0][1] >= 0, "Checkpoints with position < 0!"
            assert checkpoints[-1][1] < length, "Checkpoint with position >= conveyor length!"
            for i in range(len(checkpoints) - 1):
                assert checkpoints[i][1] < checkpoints[i+1][1], \
                    "Checkpoints with equal positions!"

        # constants
        self.env = env
        self.model_id = model_id
        self.checkpoints = checkpoints
        self.checkpoint_positions = {cp: pos for (cp, pos) in checkpoints}
        self.length = length
        self.max_speed = max_speed

        # variables
        self.speed = 0
        self.objects = {}
        self.object_positions = []

        self._spender = None
        if energy_series is not None:
            self._spender = EnergySpender(energy_series, self.length)

        self._state = 'pristine'
        self._resume_time = 0
        self._resolved_events = set()
        self._speed_changes = 0
        self._speed_sum = 0

    def _stateTransfer(self, action):
        try:
            self._state = _model_automata[self._state][action]
            if action == 'change' and self._spender is not None:
                self._spender.update(self.env.time(), self.speed, len(self.objects))
        except KeyError:
            print('RAISED')
            raise AutomataException(
                '{}: Invalid action `{}` in state `{}`;\n  speed - {}m/s\n  cps - {};\n  objs - {}'
                .format(self.model_id, action, self._state, self.speed,
                        self.checkpoints, self.object_positions))

    def totalEnergySpent(self):
        return self._spender.total(self.env.time())

    def checkpointPos(self, cp: Any) -> Optional[float]:
        if cp[0] == 'conv_end':
            return self.length
        return self.checkpoint_positions.get(cp, None)

    def nextCheckpoint(self, pos: float) -> Tuple[Any, float]:
        return search_pos(self.checkpoints, pos, preference='next')

    def nearestObject(self, pos: float, after=None, speed=None, not_exact=False,
                      preference='nearest') -> Optional[Tuple[Any, float]]:
        if len(self.object_positions) == 0:
            return None

        if after is not None:
            if speed is None:
                speed = self.speed
            objs = shift(self.object_positions, after * speed)
        else:
            objs = self.object_positions

        res = search_pos(objs, pos, preference=preference, return_index=True)
        if res is not None:
            (oid, o_pos), idx = res
            if not_exact and o_pos == pos:
                if preference == 'prev':
                    idx -= 1
                elif preference == 'next':
                    idx += 1
                else:
                    raise Exception('please dont use nearest with not exact')
                if idx < 0 or idx > len(objs):
                    return None
                oid, o_pos = objs[idx]
            return self.objects[oid], o_pos
        return None

    def working(self) -> bool:
        return self.speed > 0

    def setSpeed(self, speed: float):
        assert speed >= 0 and speed <= self.max_speed, \
            "Speed is not in [0, max_speed]!"
        if self.speed != speed:
            self._stateTransfer('change')
            self.speed = speed
            self._speed_changes += 1
            self._speed_sum += speed

    def putObject(self, obj_id: int, obj: Any, pos: float, soft_collide=True, return_nearest=False):
        assert obj_id not in self.objects, "Clashing object ID!"
        pos = round(pos, POS_ROUND_DIGITS)

        nearest = None
        if len(self.objects) > 0:
            (n_obj_id, n_pos), n_idx = search_pos(self.object_positions, pos, return_index=True)
            if n_pos == pos:
                if soft_collide:
                    print('{}: TRUE COLLISION: #{} and #{} on {}'.format(self.model_id, obj_id, n_obj_id, pos))
                    i = n_idx
                    p_pos = pos
                    while i < len(self.object_positions) and self.object_positions[i][1] >= p_pos:
                        p_pos = round(p_pos + SOFT_COLLIDE_SHIFT, POS_ROUND_DIGITS)
                        self.object_positions[i] = (self.object_positions[i][0], p_pos)
                        i += 1
                else:
                    raise CollisionException((obj, self.objects[n_obj_id], pos, self.model_id))
            elif n_pos < pos:
                n_idx += 1
            nearest = (n_obj_id, n_pos)
        else:
            n_idx = 0

        self.objects[obj_id] = obj
        self.object_positions.insert(n_idx, (obj_id, pos))
        self._stateTransfer('change')

        if return_nearest:
            return nearest

    def objPos(self, obj_id: int):
        for (oid, pos) in self.object_positions:
            if oid == obj_id:
                return pos
        return None

    def removeObject(self, obj_id: int):
        pos_idx = None
        for (i, (oid, pos)) in enumerate(self.object_positions):
            if oid == obj_id:
                pos_idx = i
                break

        self.object_positions.pop(pos_idx)
        obj = self.objects.pop(obj_id)
        self._stateTransfer('change')
        return obj

    def shift(self, d):
        self.object_positions = shift(self.object_positions, d)

    def skipTime(self, time: float, clean_ends=True):
        if time == 0:
            return 0

        self._stateTransfer('change')
        d = time * self.speed
        if len(self.objects) == 0:
            return d

        last_pos_orig = self.object_positions[-1][1]
        self.shift(d)

        if clean_ends:
            while len(self.object_positions) > 0 and self.object_positions[0][1] < 0:
                obj_id, _ = self.object_positions.pop(0)
                self.objects.pop(obj_id)
            while len(self.object_positions) > 0 and self.object_positions[-1][1] > self.length:
                obj_id, pos = self.object_positions.pop()
                # print('HAHA NE ZHDAL? A VOT, NAHUI IDET: {}, {} -> {}; {}s, {}m/s'.format(obj_id, last_pos_orig, pos, time, self.speed))
                self.objects.pop(obj_id)

        return d

    def nextEvents(self, obj=None, cps=None, skip_immediate=True,
                   skip_resolved=True) -> List[Tuple[Any, Any, float]]:
        if self.speed == 0:
            return []

        obj = def_list(obj, self.objects.keys())
        obj_positions = [(oid, pos) for (oid, pos) in self.object_positions
                         if oid in obj]

        cps = def_list(cps, self.checkpoint_positions.keys())
        c_points = [(cp, pos) for (cp, pos) in self.checkpoints if cp in cps]
        c_points.append((('conv_end', self.model_id[1]), self.length))

        def _skip_cond(obj_id, cp_idx, pos):
            cp, cp_pos = c_points[cp_idx]
            if skip_resolved and (obj_id, cp) in self._resolved_events:
                return True
            return cp_pos <= pos if skip_immediate else cp_pos < pos

        cp_idx = 0
        events = []
        for (obj_id, pos) in obj_positions:
            assert pos >= 0 and pos <= self.length, \
                "`nextEvents` on conveyor with undefined object positions!"

            while cp_idx < len(c_points) and _skip_cond(obj_id, cp_idx, pos):
                cp_idx += 1

            if cp_idx < len(c_points):
                cp = c_points[cp_idx]
                obj = self.objects[obj_id]
                diff = (cp[1] - pos) / self.speed
                events.append((obj, cp[0], diff))

        events.sort(key=lambda p: p[2])
        return events

    def immediateEvents(self, obj=None, cps=None) -> List[Tuple[Any, Any, float]]:
        return [ev for ev in self.nextEvents(obj=obj, cps=cps, skip_immediate=False) if ev[2] == 0]

    def pickUnresolvedEvent(self) -> Union[Tuple[Any, Any, float], None]:
        assert self.resolving(), "picking event for resolution while not resolving"
        evs = self.nextEvents(skip_immediate=False)
        if len(evs) == 0:
            return None

        obj, cp, diff = evs[0]
        if diff > 0:
            return None

        self._resolved_events.add((obj.id, cp))
        # print('HEEEEEEEY {} {} {}'.format(self.model_id, obj, cp))
        return obj, cp, diff

    def resume(self):
        self._stateTransfer('resume')
        self._resume_time = self.env.time()

    def pause(self):
        self._stateTransfer('pause')
        time_diff = self.env.time() - self._resume_time
        assert time_diff >= 0, "Pause before resume??"
        self.skipTime(time_diff)

    def startResolving(self):
        self._stateTransfer('start_resolving')

    def endResolving(self):
        self._resolved_events = set()
        self._stateTransfer('end_resolving')

    def pristine(self):
        return self._state == 'pristine'

    def dirty(self):
        return self._state == 'dirty'

    def resolving(self):
        return self._state == 'resolving'

    def resolved(self):
        return self._state == 'resolved'

    def moving(self):
        return self._state == 'moving'

def all_next_events(models: Dict[int, ConveyorModel], **kwargs):
    res = []
    for conv_idx, model in models.items():
        evs = model.nextEvents(**kwargs)
        res = merge_sorted(res, [(conv_idx, ev) for ev in evs],
                           using=lambda p: p[1][2])
    return res

def all_unresolved_events(models: Dict[int, ConveyorModel]):
    while True:
        had_some = False
        for conv_idx, model in models.items():
            if model.dirty():
                model.startResolving()
            if model.resolving():
                ev = model.pickUnresolvedEvent()
                if ev is not None:
                    yield (conv_idx, ev)
                    had_some = True
        if not had_some:
            break

def find_max_speed(up_model, up_pos, up_speed, dist_to_end, max_speed, bag_radius=BAG_RADIUS):
    speed = max_speed
    while speed > 0:
        end_time = dist_to_end / speed
        up_nearest = up_model.nearestObject(up_pos, after=end_time, speed=up_speed)
        if up_nearest is None:
            break

        _, obj_pos = up_nearest
        if abs(up_pos - obj_pos) >= 2*bag_radius:
            break
        elif dist_to_end == 0:
            return 0
        speed = round(speed - SPEED_STEP, SPEED_ROUND_DIGITS)

    return max(0, speed)


__author__ = "Sebastian Heinlein <devel@glatzor.de>"
__all__ = ("get_tests_dir", "Chroot", "AptDaemonTestCase")

import inspect
import os
import shutil
import subprocess
import sys
import time
import tempfile
import unittest

if sys.version_info.major > 2:
    from http.server import HTTPServer
    from http.server import SimpleHTTPRequestHandler as HTTPRequestHandler
else:
    from BaseHTTPServer import HTTPServer
    from SimpleHTTPServer import SimpleHTTPRequestHandler as HTTPRequestHandler

import apt_pkg
import apt.auth

PY3K = sys.version_info.major > 2

class MockQueue(object):

    """A fake TransactionQueue which only provides a limbo attribute."""

    def __init__(self):
        self.limbo = {}

class Chroot(object):

    """Provides a chroot which can be used by APT."""

    def __init__(self, prefix="tmp"):
        self.path = tempfile.mkdtemp(prefix)

    def setup(self):
        """Setup the chroot and modify the apt configuration."""
        for subdir in ["alternatives", "info", "parts", "updates", "triggers"]:
            path = os.path.join(self.path, "var", "lib", "dpkg", subdir)
            os.makedirs(path)
        for fname in ["status", "available"]:
            with open(os.path.join(self.path, "var", "lib", "dpkg", fname),
                      "w"):
                pass
        os.makedirs(os.path.join(self.path, "var/cache/apt/archives/partial"))
        os.makedirs(os.path.join(self.path, "var/lib/apt/lists"))
        os.makedirs(os.path.join(self.path, "var/lib/apt/lists/partial"))
        os.makedirs(os.path.join(self.path, "etc/apt/apt.conf.d"))
        os.makedirs(os.path.join(self.path, "etc/apt/sources.list.d"))
        os.makedirs(os.path.join(self.path, "etc/apt/preferences.d"))
        os.makedirs(os.path.join(self.path, "etc/apt/trusted.gpg.d"))
        os.makedirs(os.path.join(self.path, "var/log"))
        os.makedirs(os.path.join(self.path, "media"))

        # Make apt use the new chroot
        dpkg_wrapper = os.path.join(get_tests_dir(), "dpkg-wrapper.sh")
        apt_key_wrapper = os.path.join(get_tests_dir(), "fakeroot-apt-key")
        config_path = os.path.join(self.path, "etc/apt/apt.conf.d/10chroot")
        with open(config_path, "w") as cnf:
            cnf.write("Dir::Bin::DPkg %s;" % dpkg_wrapper)
            cnf.write("Dir::Bin::Apt-Key %s;" % apt_key_wrapper)
        apt_pkg.read_config_file(apt_pkg.config, config_path)
        apt_pkg.init_system()
        apt_pkg.config["Dir"] = self.path

    def remove(self):
        """Remove the files of the chroot."""
        apt_pkg.config.clear("Dir")
        apt_pkg.config.clear("Dir::State::Status")
        apt_pkg.init()
        shutil.rmtree(self.path)

    def add_trusted_key(self):
        """Add glatzor's key to the trusted ones."""
        apt.auth.add_key_from_file(os.path.join(get_tests_dir(),
                                                "repo/glatzor.gpg"))

    def install_debfile(self, path, force_depends=False):
        """Install a package file into the chroot."""
        cmd_list = ["fakeroot", "dpkg", "--root", self.path,
                    "--log=%s/var/log/dpkg.log" % self.path]
        if force_depends:
            cmd_list.append("--force-depends")
        cmd_list.extend(["--install", path])
        cmd = subprocess.Popen(cmd_list,
                               env={"PATH": "/sbin:/bin:/usr/bin:/usr/sbin"})
        cmd.communicate()

    def add_test_repository(self, copy_list=True, copy_sig=True):
        """Add the test repository to the to the chroot."""
        return self.add_repository(os.path.join(get_tests_dir(), "repo"),
                                   copy_list, copy_sig)

    def add_cdrom_repository(self):
        """Emulate a repository on removable device."""
        # Create the content of a fake cdrom
        media_path = os.path.join(self.path, "tmp/fake-cdrom")
        # The cdom gets identified by the info file
        os.makedirs(os.path.join(media_path, ".disk"))
        with open(os.path.join(media_path, ".disk/info"), "w") as info:
            info.write("This is a fake CDROM")
        # Copy the test repository "on" the cdrom
        shutil.copytree(os.path.join(get_tests_dir(), "repo"),
                        os.path.join(media_path, "repo"))

        # Call apt-cdrom add
        mount_point = self.mount_cdrom()
        os.system("apt-cdrom add -m -d %s "
                  "-o 'Debug::Acquire::cdrom'=True "
                  "-o 'Acquire::cdrom::AutoDetect'=False "
                  "-o 'Dir'=%s" % (mount_point, self.path))
        self.unmount_cdrom()

        config_path = os.path.join(self.path, "etc/apt/apt.conf.d/11cdrom")
        with open(config_path, "w") as cnf:
            cnf.write('Debug::Acquire::cdrom True;\n'
                      'Acquire::cdrom::AutoDetect False;\n'
                      'Acquire::cdrom::NoMount True;\n'
                      'Acquire::cdrom::mount "%s";' % mount_point)

    def mount_cdrom(self):
        """Copy the repo information to the CDROM mount point."""
        mount_point = os.path.join(self.path, "media/cdrom")
        os.symlink(os.path.join(self.path, "tmp/fake-cdrom"), mount_point)
        return mount_point

    def unmount_cdrom(self):
        """Remove all files from the mount point."""
        os.unlink(os.path.join(self.path, "media/cdrom"))

    def add_repository(self, path, copy_list=True, copy_sig=True):
        """Add a sources.list entry to the chroot."""
        # Add a sources list
        lst_path = os.path.join(self.path, "etc/apt/sources.list")
        with open(lst_path, "w") as lst_file:
            lst_file.write("deb file://%s ./ # Test repository" % path)
        if copy_list:
            filename = apt_pkg.uri_to_filename("file://%s/." % path)
            shutil.copy(os.path.join(path, "Packages"),
                        "%s/var/lib/apt/lists/%s_Packages" % (self.path,
                                                              filename))
            if os.path.exists(os.path.join(path, "Release")):
                shutil.copy(os.path.join(path, "Release"),
                            "%s/var/lib/apt/lists/%s_Release" % (self.path,
                                                                 filename))
            if copy_sig and os.path.exists(os.path.join(path, "Release.gpg")):
                shutil.copy(os.path.join(path, "Release.gpg"),
                            "%s/var/lib/apt/lists/%s_Release.gpg" % (self.path,
                                                                     filename))

class AptDaemonTestCase(unittest.TestCase):

    @classmethod
    def setupClass(cls):
        # Start with a clean APT configuration to remove changes
        # of previous tests
        for key in set([key.split("::")[0] for key in apt_pkg.config.keys()]):
            apt_pkg.config.clear(key)
        apt_pkg.init_config()

    def start_fake_polkitd(self, options="all"):
        """Start a fake PolicyKit daemon.

        :param allowed_actions: comma separated list of allowed actions.
                                Defaults to all
        """
        try:
            env = os.environ.copy()
            env["DBUS_SYSTEM_BUS_ADDRESS"] = self.dbus_address
        except AttributeError:
            env = None
        dir = get_tests_dir()
        proc = subprocess.Popen([os.path.join(dir, "fake-polkitd.py"),
                                "--allowed-actions", options],
                                env=env)
        self.addCleanup(self._kill_process, proc)
        return proc

    def start_session_aptd(self, chroot="/", debug=True):
        """Start an aptdaemon which listens on the session D-Bus.

        :param chroot: path to the chroot
        """
        env = os.environ.copy()
        try:
            env["DBUS_SYSTEM_BUS_ADDRESS"] = self.dbus_address
        except AttributeError:
            pass
        try:
            env.pop("http_proxy")  # Unset a local proxy, see LP #1050799
        except KeyError:
            pass
        dir = get_tests_dir()
        if dir == "/usr/share/aptdaemon/tests":
            path = "/usr/sbin/aptd"
        else:
            path = os.path.join(dir, "../aptd")
        cmd = ["python3", path, "--disable-plugins",
               "--chroot", chroot]
        if debug:
            cmd.append("--debug")
        proc = subprocess.Popen(cmd, env=env)
        self.addCleanup(self._kill_process, proc)
        return proc

    def start_dbus_daemon(self):
        """Start a private D-Bus daemon and return its process and address."""
        proc, dbus_address = start_dbus_daemon()
        self.addCleanup(self._kill_process, proc)
        self.dbus_address = dbus_address

    def start_keyserver(self, filename=None):
        """Start a fake keyserver on hkp://localhost:19191

        Keyword arguments:
        filename -- Optional path to a GnuPG pulic key file which should
            be returned by lookups. By default the key of the test repo
            is used.
        """
        dir = tempfile.mkdtemp(prefix="keyserver-test-")
        self.addCleanup(shutil.rmtree, dir)
        os.mkdir(os.path.join(dir, "pks"))
        if filename is None:
            filename = os.path.join(get_tests_dir(), "repo/glatzor.gpg")
        shutil.copy(filename, os.path.join(dir, "pks", "lookup"))

        pid = os.fork()
        if pid == 0:
            # quiesce server log
            os.dup2(os.open('/dev/null', os.O_WRONLY), sys.stderr.fileno())
            os.chdir(dir)
            httpd = HTTPServer(('localhost', 19191), HTTPRequestHandler)
            httpd.serve_forever()
            os._exit(0)
        else:
            self.addCleanup(self._kill_pid, pid)

        # wait a bit until server is ready
        time.sleep(0.5)

    def _kill_pid(self, pid):
        os.kill(pid, 15)
        os.wait()

    def _kill_process(self, proc):
        proc.kill()
        proc.wait()

def get_tests_dir():
    """Return the absolute path to the tests directory."""
    # Try to detect a relative tests dir if we are running from the source
    # directory
    try:
        path = inspect.getsourcefile(sys.modules["aptdaemon.test"])
    except KeyError:
        path = inspect.getsourcefile(inspect.currentframe())
    path = os.path.realpath(path)
    relative_path = os.path.join(os.path.dirname(path), "../tests")
    if os.path.exists(os.path.join(relative_path, "repo/Packages")):
        return os.path.normpath(relative_path)
    # Fallback to an absolute path
    elif os.path.exists("/usr/share/aptdaemon/tests/repo/Packages"):
        return "/usr/share/aptdaemon/tests"
    else:
        raise Exception("Could not find tests direcotry")

def start_dbus_daemon():
    """Start a private D-Bus daemon and return its process and address."""
    config_path = os.path.join(get_tests_dir(), "dbus.conf")
    proc = subprocess.Popen(["dbus-daemon", "--nofork",
                             "--print-address", "--config-file",
                             config_path],
                            stdout=subprocess.PIPE)
    output = proc.stdout.readline().strip()
    if PY3K:
        dbus_address = output.decode()
    else:
        dbus_address = output
    return proc, dbus_address

# vim: ts=4 et sts=4


import urllib.request
import re
import xlsxwriter

def download_page(pageUrl):
    try:
        page = urllib.request.urlopen(pageUrl)
        text = codecs.decode(page.read(), encoding = 'utf-8')
        return text
    except:
        return('Error')

commonUrl = 'http://aranea.juls.savba.sk/guest/run.cgi/first?corpname=AranFinn_x&reload=&iquery=&queryselector=cqlrow&lemma=&lpos=&phrase=&word=&wpos=&char=&cql='
wordlist_sharp = ['ter%C3%A4v%C3%A4', 'jyrkk%C3%A4', 'vihlova', 'tarkka', 'korotettu', 'kirpe%C3%A4', 'kipakka', 'ter%C3%A4v%C3%A4kulmainen', 'yl%C3%A4vireinen', '%C3%A4t%C3%A4kk%C3%A4', 'pist%C3%A4v%C3%A4', 'k%C3%A4rjek%C3%A4s', 'fiksu', 't%C3%A4sm%C3%A4llinen', 'ankara', 'tuima', 'tyylik%C3%A4s']
wordlist_size = ['iso', 'aikuinen', 'suuri', 'mittava', 'pieni', 'nuori', 'snadi', 'pikku']
wordlist_smooth = ['sile%C3%A4', 'sujuva', 'kitkaton', 'juoheva', 'sutjakka', 'sulava', 'pehme%C3%A4', 'luonteva', 'luonnikas']
regTag = re.compile('<.*?>', flags = re.DOTALL)
regNumber = re.compile('\"(.*?) hits')
regFrequency = re.compile('\((.*?) per million\)')
regLeftContext = re.compile('<td class="lc "(.*?)	', flags = re.DOTALL)
regRightContext = re.compile('<td class="rc "(.*?)	', flags = re.DOTALL)
questionnaire_sharp = xlsxwriter.Workbook('questionnaire_sharp.xlsx')
sharp_worksheet1 = questionnaire_sharp.add_worksheet('Финский')
sharp_worksheet2 = questionnaire_sharp.add_worksheet('Финский стандартная форма')
questionnaire_size = xlsxwriter.Workbook('questionnaire_size.xlsx')
size_worksheet1 = questionnaire_size.add_worksheet('Финский')
size_worksheet2 = questionnaire_size.add_worksheet('Финский стандартная форма')
questionnaire_smooth = xlsxwriter.Workbook('questionnaire_smooth.xlsx')
smooth_worksheet1 = questionnaire_smooth.add_worksheet('Финский')
smooth_worksheet2 = questionnaire_smooth.add_worksheet('Финский стандартная форма')
sharp_worksheet1.write(0, 0, 'лексема')
sharp_worksheet1.write(0, 1, 'язык')
sharp_worksheet1.write(0, 2, 'микрофрейм')
sharp_worksheet1.write(0, 3, 'контекст на языке')
sharp_worksheet1.write(0, 4, 'фрейм')
sharp_worksheet1.write(0, 5, 'такс. класс')
sharp_worksheet1.write(0, 6, 'поле')
sharp_worksheet1.write(0, 7, 'тип значения')
sharp_worksheet1.write(0, 8, 'пример')
sharp_worksheet1.write(0, 9, 'комментарий')
size_worksheet1.write(0, 0, 'лексема')
size_worksheet1.write(0, 1, 'язык')
size_worksheet1.write(0, 2, 'микрофрейм')
size_worksheet1.write(0, 3, 'контекст на языке')
size_worksheet1.write(0, 4, 'фрейм')
size_worksheet1.write(0, 5, 'такс. класс')
size_worksheet1.write(0, 6, 'поле')
size_worksheet1.write(0, 7, 'тип значения')
size_worksheet1.write(0, 8, 'пример')
size_worksheet1.write(0, 9, 'комментарий')
smooth_worksheet1.write(0, 0, 'лексема')
smooth_worksheet1.write(0, 1, 'язык')
smooth_worksheet1.write(0, 2, 'микрофрейм')
smooth_worksheet1.write(0, 3, 'контекст на языке')
smooth_worksheet1.write(0, 4, 'фрейм')
smooth_worksheet1.write(0, 5, 'такс. класс')
smooth_worksheet1.write(0, 6, 'поле')
smooth_worksheet1.write(0, 7, 'тип значения')
smooth_worksheet1.write(0, 8, 'пример')
smooth_worksheet1.write(0, 9, 'комментарий')
for i in range(len(wordlist_sharp)):
    sharp_worksheet2.write(0, i + 2, wordlist_sharp[i])
for i in range(len(wordlist_size)):
    size_worksheet2.write(0, i + 2, wordlist_size[i])
for i in range(len(wordlist_smooth)):
    smooth_worksheet2.write(0, i + 2, wordlist_smooth[i])

with open('nouns sharp.txt', encoding = 'utf-8') as wordlist_sharp_n:
    wordlist_sharp_n = list(wordlist_sharp_n)
    i_sharp = 1
    for i in range(len(wordlist_sharp_n)):
        translation, word = wordlist_sharp_n[i].split(' ')
        word = word[:-1]
        sharp_worksheet2.write(i + 1, 0, translation)
        sharp_worksheet2.write(i + 1, 1, word)
        for j in range(len(wordlist_sharp)):
            sharp = wordlist_sharp[j]
            url = commonUrl + '%5Blemma+%3D+%22' + sharp + '%22%5D+%5Blemma+%3D+%22' + word + '%22%5D&default_attr=word'
            page = download_page(url)
            if regNumber.search(page) != None:
                number = (regNumber.search(page)).group()
                number = (number.lstrip('"')).rstrip(' hits')
                number = int(number)
                freq = (regFrequency.search(page)).group()
                freq = (freq.lstrip('(')).rstrip(' per million)')
                freq = float(freq)
                LeftContext = (regLeftContext.search(page)).group()
                LeftContext = re.sub('\n', '', LeftContext)
                LeftContext = re.sub('  ', '', LeftContext)
                RightContext = (regRightContext.search(page)).group()
                RightContext = re.sub('\n', '', RightContext)
                RightContext = re.sub('  ', '', RightContext)
                example = re.sub(regTag, '', LeftContext + ' ' + sharp + ' ' + RightContext)
                if number >= 10:
                    sharp_worksheet1.write(i_sharp, 0, word)
                    sharp_worksheet1.write(i_sharp, 1, 'финский')
                    sharp_worksheet1.write(i_sharp, 2, 'острый' + translation)
                    sharp_worksheet1.write(i_sharp, 3, word)
                    sharp_worksheet1.write(i_sharp, 6, 'острый')
                    sharp_worksheet1.write(i_sharp, 8, example)
                    sharp_worksheet2.write(i + 1, j + 2, freq)
                    i_sharp += 1

with open('nouns size.txt', encoding = 'utf-8') as wordlist_size_n:
    i_size = 1
    wordlist_size_n = list(wordlist_size_n)
    for i in range(len(wordlist_size_n)):
        translation, word = wordlist_size_n[i].split(' ')
        word = word[:-1]
        size_worksheet2.write(i + 1, 0, translation)
        size_worksheet2.write(i + 1, 1, word)
        for j in range(len(wordlist_size)):
            size = wordlist_size[j]
            url = commonUrl + '%5Blemma+%3D+%22' + size + '%22%5D+%5Blemma+%3D+%22' + word + '%22%5D&default_attr=word'
            page = download_page(url)
            if regNumber.search(page) != None:
                number = (regNumber.search(page)).group()
                number = (number.lstrip('"')).rstrip(' hits')
                number = int(number)
                freq = (regFrequency.search(page)).group()
                freq = (freq.lstrip('(')).rstrip(' per million)')
                freq = float(freq)
                LeftContext = (regLeftContext.search(page)).group()
                LeftContext = re.sub('\n', '', LeftContext)
                LeftContext = re.sub('  ', '', LeftContext)
                RightContext = (regRightContext.search(page)).group()
                RightContext = re.sub('\n', '', RightContext)
                RightContext = re.sub('  ', '', RightContext)
                example = re.sub(regTag, '', LeftContext + ' ' + size + ' ' + RightContext)
                if number >= 10:
                    size_worksheet1.write(i_size, 0, word)
                    size_worksheet1.write(i_size, 1, 'финский')
                    if j < 4:
                        size_worksheet1.write(i_size, 2, 'большой' + translation)
                    else:
                        size_worksheet1.write(i_size, 2, 'маленький' + translation)
                    size_worksheet1.write(i_size, 3, word)
                    size_worksheet1.write(i_size, 6, 'размер')
                    size_worksheet1.write(i_size, 8, example)
                    size_worksheet2.write(i + 1, j + 2, freq)
                    i_size += 1

with open('nouns smooth.txt', encoding = 'utf-8') as wordlist_smooth_n:
    i_smooth = 1
    wordlist_smooth_n = list(wordlist_smooth_n)
    for i in range(len(wordlist_smooth_n)):
        translation, word = wordlist_smooth_n[i].split(' ')
        word = word[:-1]
        smooth_worksheet2.write(i + 1, 0, translation)
        smooth_worksheet2.write(i + 1, 1, word)
        for j in range(len(wordlist_smooth)):
            smooth = wordlist_smooth[j]
            url = commonUrl + '%5Blemma+%3D+%22' + smooth + '%22%5D+%5Blemma+%3D+%22' + word + '%22%5D&default_attr=word'
            page = download_page(url)
            if regNumber.search(page) != None:
                number = (regNumber.search(page)).group()
                number = (number.lstrip('"')).rstrip(' hits')
                number = int(number)
                freq = (regFrequency.search(page)).group()
                freq = (freq.lstrip('(')).rstrip(' per million)')
                freq = float(freq)
                LeftContext = (regLeftContext.search(page)).group()
                LeftContext = re.sub('\n', '', LeftContext)
                LeftContext = re.sub('  ', '', LeftContext)
                RightContext = (regRightContext.search(page)).group()
                RightContext = re.sub('\n', '', RightContext)
                RightContext = re.sub('  ', '', RightContext)
                example = re.sub(regTag, '', LeftContext + ' ' + smooth + ' ' + RightContext)
                if number >= 10:
                    smooth_worksheet1.write(i_smooth, 0, word)
                    smooth_worksheet1.write(i_smooth, 1, 'финский')
                    smooth_worksheet1.write(i_smooth, 2, 'гладкий' + translation)
                    smooth_worksheet1.write(i_smooth, 3, word)
                    smooth_worksheet1.write(i_smooth, 6, 'гладкий')
                    smooth_worksheet1.write(i_smooth, 8, example)
                    smooth_worksheet2.write(i + 1, j + 2, freq)
                    i_smooth += 1

questionnaire_sharp.close()
questionnaire_size.close()
questionnaire_smooth.close()


"""
Author: hzh huzihe@whu.edu.cn
Date: 2023-08-19 20:38:37
LastEditTime: 2023-08-19 20:39:12
FilePath: /pyplot/MachineLearning/gnss_fcm.py
Descripttion:
"""

import numpy as np
from fcmeans import FCM
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import Normalizer
import matplotlib.pyplot as plt
import matplotlib.cm as cm

import pandas as pd
from sklearn import metrics

from joblib import dump, load

def gnss_fcm_train_model(traindata, model):
    gnssdata = pd.read_csv(traindata)
    originX = gnssdata[["postResp", "priorR", "elevation", "SNR"]]
    y = gnssdata["los"]

    # 数据标准化
    X = Normalizer().fit_transform(originX)

    # 聚类
    fcm = FCM()
    fcm.n_clusters = 2
    fcm.fit(X)

    dump(fcm, model)

def gnss_fcm_predict(model, testdata):
    gnssdata = pd.read_csv(testdata)
    # x = gnssdata.drop(["week","second","sat","los","priorResp","postR","P","L","azimuth",],axis=1,)
    originX = gnssdata[["postResp", "priorR", "elevation", "SNR"]]
    satInfo = gnssdata[["week", "second", "sat"]]
    y = gnssdata["los"]

    X = Normalizer().fit_transform(originX)

    fcm = load(model)
    labels = fcm.predict(X)
    labels = 1 - labels  # 将预测结果和 nlos实际意义做对应调整

    sil_samples = metrics.silhouette_samples(X, labels)

    from sklearn.metrics import accuracy_score

    score = accuracy_score(y, labels)
    print(
        "fcm silhouette score = {:.3}, accuracy score = {:.3}\n".format(
            sil_samples.mean(), score
        )
    )

    satInfo.insert(loc=len(satInfo.columns), column="los", value=labels)
    return satInfo

if __name__ == "__main__":
    trimble_path = "./data/ml-data/20230511/trimble.res1"
    X6833B_path = "./data/ml-data/20230511/X6833B.res1"
    ublox_path = "./data/ml-data/20230511/ublox.res1"
    CK6n_path = "./data/ml-data/20230511/CK6n.res1"

    trimble_modelpath = "./data/ml-data/model/gnss_fcm_trimble.model"
    X6833B_modelpath = "./data/ml-data/model/gnss_fcm_X6833B.model"
    ublox_modelpath = "./data/ml-data/model/gnss_fcm_ublox.model"
    CK6n_modelpath = "./data/ml-data/model/gnss_fcm_CK6n.model"

    # gnss_fcm_train_model(trimble_path, trimble_modelpath)
    # gnss_fcm_train_model(X6833B_path, X6833B_modelpath)
    # gnss_fcm_train_model(ublox_path, ublox_modelpath)
    # gnss_fcm_train_model(CK6n_path, CK6n_modelpath)

    satinfo = gnss_fcm_predict(trimble_modelpath, trimble_path)
    satinfo = gnss_fcm_predict(X6833B_modelpath, X6833B_path)
    satinfo = gnss_fcm_predict(ublox_modelpath, ublox_path)
    satinfo = gnss_fcm_predict(CK6n_modelpath, CK6n_path)


#!/usr/bin/python
# -*- coding: utf-8 -*-

from flask import Flask
from flask import request
from flask import current_app as app
from bokeh.layouts import layout, column
from bokeh.models.widgets import Paragraph, Div
from bokeh.models.widgets.inputs import Select

from dominate.tags import *
from dominate.util import raw

from pywrapbokeh import WrapBokeh

app = Flask(__name__)

fruits = [('0', 'Bananna'), ('1', 'Blueberry'), ('2', 'Orange'), ('3', 'Peach'), ('4', 'Apple')]

@app.route("/", methods=['GET', 'POST'])
def hello():
    global layout_done

    # create pywrapbokeh object
    widgets = WrapBokeh("/", app.logger)

    # see https://github.com/Knio/dominate
    widgets.dominate_document()  # create dominate document
    with widgets.dom_doc.body:  # add css elements here...
        style(raw("""body {background-color:powderblue;}"""))

    # check/get page metrics and get args for the page
    args, _redirect_page_metrics = widgets.process_req(request)
    if not args: return _redirect_page_metrics
    app.logger.info("{} : args {}".format("/", args))

    selected_fruit = args.get("sel_fruit", "Unknown")
    if selected_fruit is not "Unknown":
        selected_fruit = fruits[int(selected_fruit)][1]  # get name of fruit

    # check args here for actions that would redirect to another page
    # if args.get( name ) == something: return redirect( new_url )

    # add bokeh widgets to the pywrapbokeh object
    # - order does not matter
    # - API pattern,
    #   - WrapBokeh.add( <name_of_widget>, <bokeh API>(..., [css_classes=[<name_of_widget>]]))
    #   - if you want to affect widget properties (color, font size, etc) you need to set css_classes
    widgets.add("sel_fruit", Select(options=fruits, value=None, title="Select Fruit", css_classes=['sel_fruit']))
    widgets.add_css("sel_fruit", {'select': {'background-color': "#6495ED"}})

    widgets.init()

    # start a bokeh layout
    doc_layout = widgets.layout()

    # append items to the document
    doc_layout.children.append(column(Div(text="""
Hello World!
"""),
                                      Paragraph(text="""Your favorite fruit is {}""".format(selected_fruit))))

    # append a pywrapbokeh widget
    doc_layout.children.append(widgets.get("sel_fruit"))

    # render page
    return widgets.render(doc_layout)

app.run(host="0.0.0.0", port=6800, debug=False)


# -*- coding: UTF-8 -*-
from django.contrib import admin
from . import models as m
from django import forms
from tinymce.widgets import TinyMCE
from modeltranslation.admin import TranslationAdmin

class PageAdminForm(forms.ModelForm):
    class Meta:
        model = m.Page
        widgets = {
            'text_uk': TinyMCE(attrs={'cols': 90, 'rows': 30},),
            'text_ru': TinyMCE(attrs={'cols': 90, 'rows': 30},),
            'text_en': TinyMCE(attrs={'cols': 90, 'rows': 30},),
        }

class PageAdmin(TranslationAdmin):
    radio_fields = {"category": admin.VERTICAL}
    prepopulated_fields = {"slug": ("title",)}
    list_filter = ('category', 'is_active')
    list_display = ('title', 'category', 'is_active')
    search_fields = ('title', 'short_text', 'text')
    form = PageAdminForm

    fieldsets = [
        (u'Content', {'fields': ('title', 'short_text', 'text', 'is_active')}),
        (None, {'fields': ('slug', 'category', 'image', 'order')}),
    ]

    def get_prepopulated_fields(self, request, obj=None):
        if obj is not None:
            return {}
        return self.prepopulated_fields

    def get_readonly_fields(self, request, obj=None):
        if obj:
            return ['slug']
        return []

    class Media:
        js = (
            'modeltranslation/js/force_jquery.js',
            'http://ajax.googleapis.com/ajax/libs/jqueryui/1.8.24/jquery-ui.min.js',
            'modeltranslation/js/tabbed_translation_fields.js',
        )
        css = {
            'screen': ('modeltranslation/css/tabbed_translation_fields.css',),
        }

class ClientAdmin(admin.ModelAdmin):
    pass

class ClientQuoteAdmin(TranslationAdmin):
    class Media:
        js = (
            'modeltranslation/js/force_jquery.js',
            'http://ajax.googleapis.com/ajax/libs/jqueryui/1.8.24/jquery-ui.min.js',
            'modeltranslation/js/tabbed_translation_fields.js',
        )
        css = {
            'screen': ('modeltranslation/css/tabbed_translation_fields.css',),
        }

class TopSliderAdmin(admin.ModelAdmin):
    list_display = ('title', 'language', 'is_active', 'url')
    list_filter = ('language', 'is_active')

admin.site.register(m.TopSlider, TopSliderAdmin)
admin.site.register(m.Page, PageAdmin)
admin.site.register(m.Client, ClientAdmin)
admin.site.register(m.ClientQuote, ClientQuoteAdmin)


##############################################################################
#    商品イメージと仮登録セッションIDを紐づけて本登録を行うAPI
##############################################################################
import os
import cv2
import numpy as np
import json
import datetime
from datetime import datetime as dt
from typing import Any, Dict, List
from configparser import ConfigParser
from sqlalchemy.orm.exc import NoResultFound

# 独自モジュール読み込み
import app.log as log
import app.common as common
from model.temporary_products import TemporaryProduct
from model.products import Product
logger = log.get_logger("register")

# 設定ファイル読み込み
config = ConfigParser()
config.read("settings.conf", encoding="utf-8")

##### 設定読み込み ####################
# 画像保管先を表すディレクトリーパス
DESTINATION_DIRECTORY_PATH = config.get("register", "destination_directory_path")

def execute(request) -> Dict[str, Any]:
    """JPEG形式の画像を整数の配列で表したデータとセッションIDを紐づけて本登録を行います。

    Arguments:
        request -- POST リクエスト
            {
                // 仮登録テーブルに紐づけられたセッションID
                "session_id": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",

                // JPEG圧縮した画像を uint8 配列で並べたデータ
                "image": [uint8, uint8, ...]
            }

    Returns:
        Dict[str, Any] -- 処理結果
            {
                // 本登録に成功したかどうか
                "success": False or True,

                // 本登録に失敗した原因を表すメッセージ
                "message": "..."
            }
    """
    logger.info(f"API Called.")

    # リクエストパラメーター取り出し
    session_id = request.json["session_id"]
    image = common.convert_request_image_to_ndarray(request.json["image"])
    logger.info(f"仮登録セッションID: [{session_id}]")

    # セッションIDのファイル名でサーバーに商品画像を保管する
    if not os.path.exists(DESTINATION_DIRECTORY_PATH):
        os.makedirs(DESTINATION_DIRECTORY_PATH, exist_ok=True)
    image_path = os.path.join(DESTINATION_DIRECTORY_PATH, f"{session_id}.jpg")
    cv2.imwrite(image_path, image)

    with common.create_session() as session:
        # 仮登録テーブルの該当レコードを取得
        try:
            target_temporary_product = session \
                .query(TemporaryProduct) \
                .filter(TemporaryProduct.session_id == session_id) \
                .one()
        except NoResultFound:
            response = {
                "success": False,
                "message": f"指定されたセッションIDから仮登録テーブル上の該当するレコードを特定できませんでした: {session_id}",
            }
            logger.info(f"API Exit: {response}")
            return response

        # 本登録テーブルに追加
        session.add(Product(
            image_path=image_path,
            expiration_date=target_temporary_product.expiration_date,
            consumed=False,
            added_shopping_list=False,
            created_time=dt.now()
        ))

        # 仮登録テーブルから該当レコードを削除
        session.delete(target_temporary_product)

        session.commit()

    response = {
        "success": True,
        "message": None,
    }
    logger.info(f"API Exit: {response}")
    return response


import unittest
import excalibur
import datetime

class TestTimeConversion(unittest.TestCase):
    def test_convert_timestamp_between_timezones(self):
        dt_obj = datetime.datetime(2019, 5, 1, 18, 20, 28)
        dt_obj_converted = excalibur.convert_timestamp_between_timezones(
            dt_obj,
            excalibur.get_utc_timezone_name(),
            excalibur.get_ny_timezone_name()
        )
        expected_dt_obj = datetime.datetime(2019, 5, 1, 14, 20, 28)
        # expected_dt_obj = excalibur.set_datetime_obj_timezone(expected_dt_obj, excalibur.get_ny_timezone_name())
        self.assertEqual(dt_obj_converted.replace(tzinfo=None), expected_dt_obj)


import pytest

from some_lang import ast
from some_lang.lang_frontend import Context

def test_function_defs():
    ctx = Context().module("def ident(?) -> ?:\n    ident(x) = x\nprint (ident 42)")
    assert ctx.env.apply("ident")("ARG") == "ARG"

def test_polymorphic_function_not_yet_supported():
    src = """
def ident(?) -> ?:
    ident(x) = x

print (inc (ident 1))
print (not (ident true))
"""
    ctx = Context()
    ctx.define("not", lambda x: not x, "Bool -> Bool")
    ctx.define("inc", lambda x: x + 1, "Int -> Int")

    with pytest.raises(TypeError):
        ctx.module(src)