import os
import random
import torch
from scipy.sparse import csr_matrix
import numpy as np
import time
from collections import defaultdict

class DataLoader:
    def __init__(self, task_dir):
        self.task_dir = task_dir

        if  task_dir == 'data/Dis_5fold_user/' or task_dir == 'data/Dis_5fold_item/' or  task_dir == 'data/new_last-fm/'or  task_dir == 'data/new_amazon-book/'or  task_dir == 'data/new_alibaba-fashion/':
            self.all_cf = self.read_cf(self.task_dir + 'train_1.txt')  # all_cf  (np.array)
            self.test_cf = self.read_cf(self.task_dir + 'test_1.txt')
        else:
            self.all_cf = self.read_cf(self.task_dir + 'train.txt')
            self.test_cf = self.read_cf(self.task_dir + 'test.txt')

        self.n_users = max(max(self.all_cf[:,0]),max(self.test_cf[:,0])) + 1
        self.n_items = max(max(self.all_cf[:,1]),max(self.test_cf[:,1])) + 1
        self.known_user_set = self.cf_to_set(self.all_cf)
        self.test_user_set = self.cf_to_set(self.test_cf)

        n_all = self.all_cf.shape[0]
        rand_idx = np.random.permutation(n_all)
        self.all_cf = self.all_cf[rand_idx]

        self.triple = self.read_triples('kg.txt')

        self.arraytriple = np.asarray(self.triple)
        self.n_ent = max(max(self.arraytriple[:, 0]), max(self.arraytriple[:, 2])) + 1
        self.n_nodes = self.n_ent + self.n_users    # user + entity
        self.n_rel = max(self.arraytriple[:,1]) + 1

        if task_dir == 'data/Dis_5fold_item/'or  task_dir == 'data/new_last-fm/'or  task_dir == 'data/new_amazon-book/'or  task_dir == 'data/new_alibaba-fashion/':
            self.item_set = self.cf_to_item_set(self.all_cf)
            self.facts_cf, self.train_cf = self.generate_inductive_train(self.all_cf)
        else:
            self.facts_cf = self.all_cf[0:n_all*6//7]
            self.train_cf = self.all_cf[n_all*6//7:]

        self.fact_triple = self.cf_to_triple(self.facts_cf)
        self.train_triple = self.cf_to_triple(self.train_cf)
        self.test_triple = self.cf_to_triple(self.test_cf)

        # add inverse
        self.d_triple  = self.double_triple(self.triple)

        # add interact and user into triplets
        self.fact_data, self.known_data = self.interact_triple(self.d_triple)

        # use user-kg
        if  task_dir == 'data/Dis_5fold_user/' or task_dir == 'data/Dis_5fold_item/' :
            self.readukg = 1
            self.ukg = self.read_user_kg()
            self.n_rel += 1
            self.fact_data += self.ukg
            self.known_data += self.ukg
            print('load user kg')
        else:
            self.readukg = 0

        self.load_graph(self.fact_data)
        self.load_test_graph(self.known_data)

        self.train_q, self.train_a, self.train_w = self.load_train_query(self.train_triple)
        self.test_q,  self.test_a  = self.load_query(self.test_triple)

        self.n_train = len(self.train_q)
        self.n_test = len(self.test_q)

        print('n_facts:',len(self.facts_cf), 'n_test_cf:', len(self.test_cf), 'n_train:', self.n_train,  'n_test:', self.n_test)
        print('users:', self.n_users,'items:', self.n_items, 'other entities:', self.n_ent - self.n_items )

    def read_cf(self, file_name):
        inter_mat = list()
        lines = open(file_name, "r").readlines()
        for al in lines:
            tmps = al.strip()
            inters = [int(i) for i in tmps.split(" ")]

            u_id, pos_ids = inters[0], inters[1:]
            pos_ids = list(set(pos_ids))
            for i_id in pos_ids:
                inter_mat.append([u_id, i_id])

        return np.array(inter_mat)   #[[u1,i1],[u1,i2],……,[un,in]]

    def read_triples(self, filename):
        triples = []
        with open(os.path.join(self.task_dir, filename)) as f:
            for line in f:
                h, r, t = line.strip().split()
                h, r, t = int(h), int(r), int(t)

                triples.append([h,r,t])

        return triples

    def double_triple(self, triples):
        new_triples = []
        for triple in triples:
            h, r, t = triple
            new_triples.append([t, r+self.n_rel, h])

        return triples + new_triples

    def interact_triple(self, triples):
        # consider  additional relations  'interact' and 'in-interact'.
        copy_tri = triples.copy()
        for id, [h, r, t] in enumerate(copy_tri):
            copy_tri[id] = [h + self.n_users, r + 2, t + self.n_users]

        fact_user_triple = []

        for uitriple in self.fact_triple:
            u , i = uitriple[0], uitriple[2]

            fact_user_triple.append([u, 0, i])
            fact_user_triple.append([i, 1, u])

        train_user_triple = []
        for uitriple in self.train_triple:
            u , i = uitriple[0], uitriple[2]

            train_user_triple.append([u, 0, i])
            train_user_triple.append([i, 1, u])
        return copy_tri + fact_user_triple,   copy_tri + fact_user_triple + train_user_triple

    def cf_to_triple(self, cf):
        cf = list(cf)
        newtriple = []
        for [u,i] in cf :
            if u >= self.n_users:
                continue
            i = i + self.n_users
            newtriple.append([u, 0, i ])
        return newtriple

    def cf_to_set(self, cf):
        cf = list(cf)
        user_set = defaultdict(list)
        for [u, i] in cf :
            if u >= self.n_users:
                print("unexpected users")
                continue
            user_set[u].append(i + self.n_users)
        return user_set

    def read_user_kg(self):
        ukg = []
        with open(os.path.join(self.task_dir, 'ukg.txt')) as f:
            for line in f:
                h, r, t = line.strip().split()
                h, r, t = int(h), int(r), int(t)
                if h >= self.n_users or t >= self.n_users:
                    continue
                ukg.append([h, 2*self.n_rel + 2, t])
                ukg.append([t, 2*self.n_rel + 3, h])
        return ukg

    def cf_to_item_set(self, cf):  # item_set[i] = [u1, u3, u4……]
        cf = list(cf)
        item_set = defaultdict(list)
        for [u, i] in cf :
            if u >= self.n_users:
                print("unexpected users")
                continue
            item_set[i].append(u)
        return item_set

    def check_item_inkg(self, triple):    # return d(item) in kg
        it_inkg = np.zeros((self.n_items,1))
        for h,r,t in triple:
            if h < self.n_items:
                it_inkg[h] += 1
            if t < self.n_items:
                it_inkg[t] += 1
        return it_inkg

    def generate_inductive_train(self, cf):
        fcf = cf.tolist()
        n_train = 0
        train_cf = []
        ind_item = []
        while(n_train < len(cf) / 8):
            item = random.randint(0, self.n_items-1)
            if item in ind_item:
                continue
            for u in self.item_set[item]:
                train_cf.append([u,item])
                fcf.remove([u,item])      # as fact_cf
            ind_item.append(item)
            n_train += len(self.item_set[item])

        return np.array(fcf), np.array(train_cf)

    def load_graph(self, triples):
        idd = np.concatenate([np.expand_dims(np.arange(self.n_nodes),1), (2*self.n_rel+2)*np.ones((self.n_nodes, 1)), np.expand_dims(np.arange(self.n_nodes),1)], 1)

        self.KG = np.concatenate([np.array(triples), idd], 0)

        self.n_fact = len(self.KG)
        self.M_sub = csr_matrix((np.ones((self.n_fact,)), (np.arange(self.n_fact), self.KG[:,0])), shape=(self.n_fact, self.n_nodes))

    def load_test_graph(self, triples):
        idd = np.concatenate([np.expand_dims(np.arange(self.n_nodes),1), (2*self.n_rel+2)*np.ones((self.n_nodes, 1)), np.expand_dims(np.arange(self.n_nodes),1)], 1)

        self.tKG = np.concatenate([np.array(triples), idd], 0)

        self.tn_fact = len(self.tKG)
        self.tM_sub = csr_matrix((np.ones((self.tn_fact,)), (np.arange(self.tn_fact), self.tKG[:,0])), shape=(self.tn_fact, self.n_nodes))

    def load_train_query(self, triples):
        triples.sort(key=lambda x:(x[0], x[1]))
        pos_items = defaultdict(lambda:list())
        neg_items = defaultdict(list)

        for trip in triples:
            h, r, t = trip
            pos_items[(h,r)].append(t)
            while True:
                neg_item = np.random.randint(low=self.n_users, high=self.n_users + self.n_items, size=1)[0]  #重要修改！low=n_users
                if neg_item not in self.known_user_set[h]:
                    break
            neg_items[(h,r)].append(neg_item)

        queries = []
        answers = []
        wrongs = []
        for key in pos_items:
            queries.append(key)
            answers.append(np.array(pos_items[key]))
            wrongs.append(np.array(neg_items[key]))

        return queries, answers, wrongs

    def load_query(self, triples):
        triples.sort(key=lambda x:(x[0], x[1]))
        trip_hr = defaultdict(lambda:list())

        for trip in triples:
            h, r, t = trip
            trip_hr[(h,r)].append(t)

        queries = []
        answers = []
        for key in trip_hr:
            queries.append(key)
            answers.append(np.array(trip_hr[key]))
        return queries, answers

    def get_neighbors(self, nodes, mode='train'):

        if mode=='train':
            KG = self.KG
            M_sub = self.M_sub
        else:
            KG = self.tKG
            M_sub = self.tM_sub

        # nodes: n(node) x 2 with (batch_idx, node_idx)
        node_1hot = csr_matrix((np.ones(len(nodes)), (nodes[:,1], nodes[:,0])), shape=(self.n_nodes, nodes.shape[0]))
        edge_1hot = M_sub.dot(node_1hot)
        edges = np.nonzero(edge_1hot)
        sampled_edges = np.concatenate([np.expand_dims(edges[1],1), KG[edges[0]]], axis=1)     # (batch_idx, head, rela, tail)
        sampled_edges = torch.LongTensor(sampled_edges).cuda()

        # index to nodes
        head_nodes, head_index = torch.unique(sampled_edges[:,[0,1]], dim=0, sorted=True, return_inverse=True)
        tail_nodes, tail_index = torch.unique(sampled_edges[:,[0,3]], dim=0, sorted=True, return_inverse=True)

        sampled_edges = torch.cat([sampled_edges, head_index.unsqueeze(1), tail_index.unsqueeze(1)], 1)

        mask = sampled_edges[:,2] == (self.n_rel*2 + 2)
        _, old_idx = head_index[mask].sort()
        old_nodes_new_idx = tail_index[mask][old_idx]

        return tail_nodes, sampled_edges, old_nodes_new_idx

    def get_batch(self, batch_idx, steps=2, data='train'):
        if data=='train':
            query, answer, wrongs = np.array(self.train_q), self.train_a, self.train_w
        if data=='test':
            query, answer = np.array(self.test_q), np.array(self.test_a)

        subs = []
        rels = []
        objs = []
        pos = []
        neg =[]
        subs = query[batch_idx, 0]
        rels = query[batch_idx, 1]

        if data =='train':
            pos = answer[batch_idx[0]:batch_idx[-1]+1]
            neg = wrongs[batch_idx[0]:batch_idx[-1]+1]
            return subs, rels, pos, neg
        else :
            objs = np.zeros((len(batch_idx), self.n_nodes))
            for i in range(len(batch_idx)):
                objs[i][answer[batch_idx[i]]] = 1
            return subs, rels, objs

    def shuffle_train(self,):

        if self.task_dir == 'data/Dis_5fold_item/' :
            self.facts_cf, self.train_cf = self.generate_inductive_train(self.all_cf)
            self.fact_triple = self.cf_to_triple(self.facts_cf)
            self.train_triple = self.cf_to_triple(self.train_cf)

            self.fact_data,_ = self.interact_triple(self.d_triple)
            if self.readukg == 1:
                self.fact_data += self.ukg
            self.load_graph(self.fact_data)
            self.train_q, self.train_a, self.train_w = self.load_train_query(self.train_triple)
            self.n_train = len(self.train_q)
        elif self.task_dir == 'data/new_last-fm/'or  self.task_dir == 'data/new_amazon-book/'or  self.task_dir == 'data/new_alibaba-fashion/':
            self.train_triple = np.array(self.train_triple)
            n_all = len(self.train_triple)
            rand_idx = np.random.permutation(n_all)
            self.train_triple = self.train_triple[rand_idx].tolist()
            self.train_q, self.train_a, self.train_w = self.load_train_query(self.train_triple)
            self.n_train = len(self.train_q)
        else:
            fact_triple = np.array(self.fact_triple)
            train_triple = np.array(self.train_triple)
            all_ui_triple = np.concatenate([fact_triple, train_triple], axis=0)
            n_all = len(all_ui_triple)
            rand_idx = np.random.permutation(n_all)
            all_ui_triple = all_ui_triple[rand_idx]

            self.fact_triple = all_ui_triple[:n_all*6//7].tolist()
            self.train_triple = all_ui_triple[n_all*6//7:].tolist()

            self.fact_data,_ = self.interact_triple(self.d_triple)
            if self.readukg == 1:
                self.fact_data += self.ukg
            self.load_graph(self.fact_data)
            self.train_q, self.train_a, self.train_w = self.load_train_query(self.train_triple)
            self.n_train = len(self.train_q)


import os
import requests

def download_file_from_url(url: str, save_path: str = "download") -> None:
    """Downloads/chunks file from URL

    Parameters
    ----------
        url : str
        save_path (str, optional): str, default "download"
            local directory to save file in
    """
    with requests.get(url, allow_redirects=True, stream=True) as r:
        r.raise_for_status()
        with open(save_path, "wb") as f:
            for chunk in r.iter_content(chunk_size=8192):
                f.write(chunk)

def download_zipped_file(url: str, headers: dict = None):
    return None


"""add pitch table

Revision ID: ca98fc844cc0
Revises: 7cdd9feac9d4
Create Date: 2020-09-20 23:39:39.191540

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'ca98fc844cc0'
down_revision = '7cdd9feac9d4'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('pitches',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('pitch', sa.Text(), nullable=True),
    sa.Column('category_id', sa.Integer(), nullable=True),
    sa.Column('date', sa.DateTime(), nullable=True),
    sa.Column('user_id', sa.Integer(), nullable=True),
    sa.ForeignKeyConstraint(['user_id'], ['users.id'], ),
    sa.PrimaryKeyConstraint('id')
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('pitches')
    # ### end Alembic commands ###


from collections import Counter

t = int(input())

def minops(a, b, c):
    if min(a, b, c) < 0:
        return float('inf')
    if b == 0 and c == 0:
        return 0
    if b > 0 and c > 0:
        x = min(b, c)
        return x + minops(a + x, b - x, c - x)
    ar = 1 + minops(a - 1, b + 1, c)
    br = 1 + minops(a - 1, b, c + 1)
    cr = 1 + minops(a, b - 2, c + 2)
    dr = 1 + minops(a, b + 2, c - 2)
    return min(ar, br, cr, dr)

for _ in range(t):
    n = int(input())
    arr = list(map(int, input().split()))
    ctr = Counter()
    for el in arr:
        ctr[el % 3] += 1
    print(minops(ctr[0], ctr[1], ctr[2]))


import inspect
import logging
from uuid import uuid1

from django.conf import settings
from rest_framework import serializers
from wrapt import decorator

HIDE_ANNOTATION = 'hide'

class DynamicSerializer(serializers.Serializer):
    def get_fields(self):
        fields = super().get_fields()

        requested_fields = self.context['request'].query_params.get('fields')
        if not requested_fields:
            return fields

        filtered_fields = set(requested_fields.split(',')).intersection(fields)
        return {k: v for k, v in fields.items() if k in filtered_fields}

def get_logger(logger_name=settings.LOGGER_NAME):
    logger = logging.getLogger(logger_name)
    logger.propagate = False
    return logger

def log(logger_inst=get_logger(), lvl: int = logging.INFO, hide_output=False):
    @decorator
    def _log(wrapped, instance, args, kwargs):
        func_name = f'{wrapped.__module__}.{wrapped.__qualname__}'
        extra = {'call_id': uuid1().hex, 'function': func_name}

        try:
            params = inspect.getfullargspec(wrapped)
            extra['input_data'] = get_logged_args(params, [instance] + list(args) if instance else args, kwargs)
            logger_inst.log(level=lvl, msg=f'call {func_name}', extra=extra)

            result = wrapped(*args, **kwargs)

            extra['result'] = normalize_for_log(result) if not hide_output else 'hidden'
            logger_inst.log(level=lvl, msg=f'return {func_name}', extra=extra)
            return result
        except Exception as e:
            if hasattr(e, 'return_value'):
                return e.return_value

            raise

    return _log

def get_logged_args(params, args, kwargs):
    result = {}
    annotations = params.annotations

    for i, v in enumerate(args[:len(params.args)]):
        arg_name = params.args[i]
        result[arg_name] = 'hidden' if _is_hidden(arg_name, annotations) else normalize_for_log(v)

    varargs = params.varargs
    if varargs:
        if _is_hidden(varargs, annotations):
            result['*args'] = f'hidden {len(args) - len(params.args)} args'
        else:
            result['*args'] = [normalize_for_log(i) for i in args[len(params.args):]]

    for k, v in kwargs.items():
        is_kwarg_hidden = _is_hidden(k, annotations) or (
            params.varkw and k not in params.kwonlyargs and k not in params.args
        )
        result[k] = 'hidden' if is_kwarg_hidden else normalize_for_log(v)

    return result

def normalize_for_log(value):
    if isinstance(value, bool) or value is None:
        return str(value)
    elif isinstance(value, dict):
        return {k: normalize_for_log(v) for k, v in value.items()}
    elif isinstance(value, (list, set, frozenset, tuple)):
        return type(value)(normalize_for_log(i) for i in value)
    return str(value)

def _is_hidden(name, annotations):
    return annotations.get(name) == HIDE_ANNOTATION


import numpy as np
import pytest
from numpy import testing as npt

from ifg.calculator import IfgCalculator
from ifg.units_converter import convert_theta_to_temperature

COMPLICATED_MESH = (
    [3.0, 5.0],
    [5.0, 10.0, 15.0],
    np.array(
        [
            [4.909741, 8.182902],
            [3.092943, 5.154905],
            [2.36035732, 3.9339289],
        ]
    ),
)

@pytest.fixture(
    params=[
        ([1.0], 2.0, np.array([[3.014607]])),
        ([3.0], 5.0, np.array([[4.909741]])),
        ([3.0, 5.0], 5.0, np.array([[4.909741, 8.182902]])),
        COMPLICATED_MESH,
    ],
)
def mesh_example(request):
    return request.param

class TestConverterFromThetaToTemperature:
    def test_correct_conversion(self, mesh_example):
        theta, volume, temperature = mesh_example
        # GIVEN: theta and specific volume (atomic)
        # WHEN: conversion to temperature occurs
        # THEN: the result equals to the expected one
        npt.assert_allclose(convert_theta_to_temperature(theta, volume), temperature)

@pytest.mark.skip("broken after input methods refactoring")
def test_correct_mesh_creation():
    from ifg.calculator import _make_mesh

    # GIVEN: theta array, volume array and corresponding temperatures
    theta, volume, temperature = COMPLICATED_MESH
    # WHEN: mesh is created upon these volumes and temperatures
    vv, tt = _make_mesh(volume, temperature)
    # THEN: volume grid is created as if w/o theta
    npt.assert_allclose(
        vv,
        [
            [5.0, 10.0, 15.0],
            [5.0, 10.0, 15.0],
        ],
    )
    # THEN: temperature grid is created with respect to both volumes and thetas
    npt.assert_allclose(
        tt,
        np.array(
            [
                [4.909741, 3.092943, 2.36035732],
                [8.182902, 5.154905, 3.9339289],
            ]
        ),
    )

@pytest.mark.skip("broken after input methods refactoring")
def test_correct_simple_mesh_creation():
    from ifg.calculator import _make_mesh

    # GIVEN: simple (non-theta) volumes and temperatures
    volumes = [1.0, 2.0]
    temperatures = [10.0, 20.0, 30.0]
    # WHEN: mesh grid is created
    vv, tt = _make_mesh(volumes, temperatures)
    # THEN: correct simple mesh is created
    npt.assert_allclose(vv, [[1.0, 2.0], [1.0, 2.0], [1.0, 2.0]])
    npt.assert_allclose(tt, [[10.0, 10.0], [20.0, 20.0], [30.0, 30.0]])

@pytest.mark.skip("broken after with_ removal")
def test_cannot_input_theta_before_volume():
    # GIVEN: freshly created IfgCalculator()
    # WHEN: `with_theta` is called before volume input
    # THEN: exception is raised
    with pytest.raises(ValueError) as e:
        IfgCalculator().with_theta([1.0, 2.0])
        assert e.match(
            "specific volume should be defined before using theta for temperature input"
        )


import serial
from time import sleep
from serial import Serial

ser = serial.Serial("/dev/ttyS0",38400)

while True:
    ser.write(b'1')

    rd=ser.read().decode('utf-8')
    sleep(1)
    dl=ser.inWaiting()
    rd+=ser.read(dl)
    print(rd)


# -*- encoding:utf-8 -*-

# 网抑云 lrc歌曲m3u生成器
# 版本: 11.0
import platform
import sys
import codecs
import hashlib
import json
import os
import urllib.request, urllib.parse, urllib.error
from http import cookiejar
import gzip
import signal
from sys import argv
from io import StringIO

if len(argv) < 2:
    print("请传入歌单id")
    sys.exit()

# 歌单id设置 设置为 argv[1] 将使用 " python make_lrc_music_m3u.py 歌单id " 这种方式传入
playlistId = argv[1]

# 播放列表存放位置
m3udir = "./播放列表/"

# 相对于播放列表存放位置的 音乐存放位置
mp3dir_in_m3udir = "../音乐/"

# 是否按m3u分类
if len(argv) > 2:
    sortBym3u = True
else:
    sortBym3u = False

# 是否下载歌词
downLrc = True

# 是否下载音乐
down128Music = True

# 账号cookie
# 由于不登录只会返回前10首歌 更多的需要登录 (歌单创建者 失效了可以打开一次输出的url再复制) 别人得到了这段cookie相当于能登录你的账号 请务必不要泄露
# Chrome打开网抑云 -> 登录 -> 按F12打开 开发者工具 -> 切换到 Console -> 输入 document.cookie 按回车 -> 复制输出内容替换下面的双引号
cookie = ""

translationTable = str.maketrans("àèéùâêîôûçë", "aeuaeeiouce")

# Ctrl + C 退出
def signal_handler(signal, frame):
   print('Ctrl + C, exit now...')
   sys.exit(1)

signal.signal(signal.SIGINT, signal_handler)

# 加载头部 防ban
cookieJar = cookiejar.CookieJar()
opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cookieJar))
opener.addheaders = [
    ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9'),
    ('Accept-Encoding', 'gzip, deflate'),
    ('Accept-Language', 'zh-CN,zh-TW;q=0.9,zh;q=0.8,en-US;q=0.7,en;q=0.6'),
    ('Connection', 'keep-alive'),
    ('Cookie', cookie),
    ('DNT', '1'),
    ('Host', 'music.163.com'),
    ('Upgrade-Insecure-Requests', '1'),
    ('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36')
]
urllib.request.install_opener(opener)

# 判断文件是否存在
def hasFile(fileName):
    # 文件名不区分大小写
    for i in listdir:
        if fileName.upper() == i.upper():
            return i

    return ''

# 寻找存在的音频文件
fType = ''
def findHasMusicFileFullFileName(name):
    global fType
    fType = ''
    fileTyps = ['.flac', '.ape', '.wav', '.mp3']
    for t in fileTyps:
        fullFileName = hasFile(name + t)
        if fullFileName != '':
            fType = t
            return fullFileName

    return ''

# 半角转全角
def half2full(ustring):
    rstring = ""
    for uchar in ustring:
        inside_code = ord(uchar)
        if inside_code == 32:
            inside_code = 12288
        elif inside_code >= 32 and inside_code <= 126:
            inside_code += 65248

        rstring += chr(inside_code)
    return rstring

# 发送请求
def urlGetJsonLoad(url):
    gzdata = ''
    try:
        gzdata = urllib.request.urlopen(url, timeout=10)
    except Exception as e:
        print('connect error: ', url)
        return {'code': ''}
    try:
        if gzdata.info().get('Content-Encoding') == 'gzip':
            gziper = gzip.GzipFile(fileobj=gzdata)
            return json.loads(gziper.read().decode('utf-8'))
        else:
            return json.loads(gzdata.read().decode('utf-8'))
    except Exception as e:
        print('decode error: ', url)
        return {'code': ''}

# 替换文件名不允许字符
def replaceName(name):
    name = name.replace('?', half2full('?'))
    name = name.replace('*', half2full('*'))
    name = name.replace('/', half2full('/'))
    name = name.replace('\\', half2full('\\'))
    name = name.replace('<', half2full('<'))
    name = name.replace('>', half2full('>'))
    name = name.replace(':', half2full(':'))
    name = name.replace('\"', half2full('\"'))
    name = name.replace('|', half2full('|'))
    name = name.replace('[', half2full('['))
    name = name.replace(']', half2full(']'))
    name = name.strip()
    return name

# 获取歌词
def getLrc(tracksId):
    url = 'http://music.163.com/api/song/lyric?lv=-1&tv=-1&id=' + tracksId
    dataS = urlGetJsonLoad(url)
    if dataS['code'] != 200:
        ecode = str(dataS['code'])
        print('errorCode: ' + ecode)
        return ''
    else:
        if 'lrc' not in dataS or 'lyric' not in dataS['lrc'] or dataS['lrc']['lyric'] == None:
            return ''

        if 'tlyric' not in dataS or 'lyric' not in dataS['tlyric'] or dataS['tlyric']['lyric'] == None:
            return dataS['lrc']['lyric']

        # 按换行分割
        lrcL = dataS['lrc']['lyric'].splitlines()
        tlyricL = dataS['tlyric']['lyric'].splitlines()
        tlyricD = {}
        lrcStr = ''

        # 分割翻译
        for tlyric in tlyricL:
            tl = tlyric.split(']', 1)
            # 防止有时间但翻译为空
            if len(tl) > 1:
                tlyricD[tl[0]] = tl[1]

        # 合并歌词
        for lrc in lrcL:
            l = lrc.split(']', 1)
            if l[0] in tlyricD:
                lrcStr += l[0] + ']' + l[1] + '\t' + tlyricD[l[0]] + '\n'
            else:
                lrcStr += lrc + '\n'

        return lrcStr

# 下载歌曲
def downMusic(tracksId, fileName):
    url = 'http://music.163.com/song/media/outer/url?id=' + tracksId + '.mp3'
    try:
        print('Download music: ' + fileName.replace(m3udir, '').replace(mp3dir, ''))
    except:
        print('Download music: ' + tracksId)
    try:
        urllib.request.urlretrieve(url, fileName)
        if os.path.getsize(fileName) < 20000:
            # 小于10k这音频肯定有问题 给它扬了
            print('Need VIP')
            os.remove(fileName)
            return False
    except:
        print('error')
        return False
    return True

# 写出文件
def writeToFile(name, text):
    try:
        print('Write to file: ' + name.replace(m3udir, '').replace(mp3dir, ''))
    except:
        print('Write to file: ')
    try:
        file = codecs.open(name, "w", "utf-8")
        file.write(text)
        file.close()
    except:
        print('error')

# 生成播放列表
m3uText = "#EXTM3U"
def addPlaylist(mp3Title, mp3Name):
    global m3uText
    m3uText += "\n#EXTINF:" + mp3Title + "\n" + mp3dir.replace("/", "\\") + mp3Name

# 确保需要的文件夹
if not os.path.isdir(m3udir):
    os.mkdir(m3udir)

if not os.path.isdir(m3udir + mp3dir_in_m3udir):
    os.mkdir(m3udir + mp3dir_in_m3udir)

# 弄个文件来存对应关系
db = {}
if os.path.isfile(m3udir + mp3dir_in_m3udir + 'db.json'):
    with codecs.open(m3udir + mp3dir_in_m3udir + 'db.json', "r", "utf-8") as dbf:
        try:
            db = json.load(dbf)
        except:
            print('Error DB')
else:
    print('NO DB')

# 获取歌单
url = 'http://music.163.com/api/v6/playlist/detail?n=2147483647&id=' + playlistId
print(url)
dataL = urlGetJsonLoad(url)
if dataL['code'] != 200:
    ecode = str(dataL['code'])
    print('errorCode: ' + ecode)
else:
    m3uName = replaceName(dataL['playlist']['name'])
    allNum = len(dataL['playlist']['tracks'])
    nowNum = 0

    # 如按歌单分文件夹,不存在文件夹就创建
    if sortBym3u:
        mp3dir = mp3dir_in_m3udir + m3uName + "/"
        if not os.path.isdir(m3udir + mp3dir):
            os.mkdir(m3udir + mp3dir)

    else:
        mp3dir = mp3dir_in_m3udir

    # 查找所有文件
    listdir = os.listdir(m3udir + mp3dir)
    noFileTxt = ''

    # 循环歌单
    for tracks in dataL['playlist']['tracks']:
        nowNum += 1
        print("\r" + str(nowNum) + '/ ' + str(allNum), end=' ')
        fileName = ''
        fileNameAndroid = ''
        fileNameReverse = ''
        fileNameReverseAndroid = ''
        # 循环歌手
        i = len(tracks['ar']) - 1
        for artist in tracks['ar']:
            if i > 0:
                fileName += artist['name'] + ","
                fileNameAndroid += artist['name'] + " "
                fileNameReverse = "," + artist['name'] + fileNameReverse
                fileNameReverseAndroid = " " + artist['name'] + fileNameReverse
                i -= 1
            else:
                fileName += artist['name']
                fileNameAndroid += artist['name']
                fileNameReverse = artist['name'] + fileNameReverse
                fileNameReverseAndroid = artist['name'] + fileNameReverse

        # PC的命名规则 转全角
        fileName += " - " + tracks['name'].strip()
        fileName = replaceName(fileName)
        # 按照空格分隔歌手,例如Android端
        fileNameAndroid += " - " + tracks['name'].strip()
        fileNameAndroid = replaceName(fileNameAndroid.replace('/', ' ').replace('*', ' ').replace('+', half2full('+')).replace('\"', '”'))
        fileNameAndroidOld = fileNameAndroid.translate(translationTable)
        # 歌手翻转的
        fileNameReverse += " - " + tracks['name'].strip()
        fileNameReverseAndroid += " - " + tracks['name'].strip()
        fileNameReverse = replaceName(fileNameReverse)
        fileNameReverseAndroid = replaceName(fileNameReverseAndroid)
        fileNameReverseAndroidOld = fileNameReverseAndroid.translate(translationTable)

        tid = str(tracks['id'])
        # 检查存在的文件
        fullFileNameAndroid = findHasMusicFileFullFileName(fileNameAndroid)

        if fullFileNameAndroid == '':
            fullFileName = ""
            lastFindName = ""
            # 从db找，找到了就重命名
            if tid in db:
                lastFindName = db[tid]
                fullFileName = findHasMusicFileFullFileName(lastFindName)
            # PC 命名方式重命名为Android命名方式
            if fileNameAndroid != fileName:
                lastFindName = fileName
                fullFileName = findHasMusicFileFullFileName(lastFindName)
            # 最近网易整理了一下歌手,处理一下已有文件翻转的情况
            if fullFileName == '' and fileNameAndroid != fileNameReverse:
                lastFindName = fileNameReverse
                fullFileName = findHasMusicFileFullFileName(lastFindName)
            if fullFileName == '' and fileNameAndroid != fileNameReverseAndroid:
                lastFindName = fileNameReverseAndroid
                fullFileName = findHasMusicFileFullFileName(lastFindName)
            # 最近整理发型客户端突然支持法语子字符了 大量文件改名
            if fullFileName == '' and fileNameAndroid != fileNameAndroidOld:
                lastFindName = fileNameAndroidOld
                fullFileName = findHasMusicFileFullFileName(lastFindName)
            if fullFileName == '' and fileNameAndroid != fileNameReverseAndroidOld:
                lastFindName = fileNameReverseAndroidOld
                fullFileName = findHasMusicFileFullFileName(lastFindName)

            if fullFileName != '':
                try:
                    print('Rename file: ' + fullFileName)
                except:
                    print('Rename file: ')
                try:
                    os.rename(m3udir + mp3dir + fullFileName, m3udir + mp3dir + fileNameAndroid + fType)
                    fullFileNameAndroid = fileNameAndroid + fType
                    try:
                        os.remove(lastFindName + '.lrc')
                    except:
                        pass
                except:
                    print('Rename error!')
                fullFileNameAndroid = fileNameAndroid + fType

        if fullFileNameAndroid == '':
            fullFileNameAndroid = fileNameAndroid + '.mp3'
            if down128Music:
                # 这里将下载一个无封面的128kbps的版本 听个响
                if downMusic(tid, m3udir + mp3dir + fullFileNameAndroid):
                    db[tid] = fileNameAndroid
                else:
                    noFileTxt += fileNameAndroid + '\r\n'
            else:
                print('NO File: ' + fileNameAndroid)
                noFileTxt += fileNameAndroid + '\r\n'
        else:
            db[tid] = fileNameAndroid

        # 如果需要下载歌词,不存在歌词就下载
        if downLrc:
            if not hasFile(fileNameAndroid + ".lrc"):
                lrcS = getLrc(tid)
                if len(lrcS) > 0:
                    writeToFile(m3udir + mp3dir + fileNameAndroid + ".lrc", lrcS)

        # 添加到播放列表 理论上这里fullFileName是不会为空的
        if fullFileNameAndroid != '':
            addPlaylist(tracks['name'], fullFileNameAndroid)

    # 写db文件
    writeToFile(m3udir + mp3dir_in_m3udir + 'db.json', json.dumps(db, ensure_ascii=False))

    # 没有文件的让人类处理
    if noFileTxt != '':
        writeToFile(m3udir + mp3dir + "noFile.txt", noFileTxt)

    # 写播放列表文件
    writeToFile(m3udir + m3uName + ".m3u", m3uText)


import argparse
import csv
import multiprocessing
import os
import pathlib
import subprocess
from contextlib import nullcontext

import torch
from diffusers import StableDiffusionOnnxPipeline, StableDiffusionPipeline
from torch import autocast

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

prompt = "a photo of an astronaut riding a horse on mars"
from pathlib import Path

path = Path("~/models/stable-diffusion-v1-4").expanduser()

# path = "CompVis/stable-diffusion-v1-4"

def get_inference_pipeline(precision, backend):
    """
    returns HuggingFace diffuser pipeline
    cf https://github.com/huggingface/diffusers#text-to-image-generation-with-stable-diffusion
    note: could not download from CompVis/stable-diffusion-v1-4 (access restricted)
    """

    assert precision in ("half", "single"), "precision in ['half', 'single']"
    assert backend in ("pytorch", "onnx"), "backend in ['pytorch', 'onnx']"
    if backend == "pytorch":
        pipe = StableDiffusionPipeline.from_pretrained(
            # "CompVis/stable-diffusion-v1-4",
            path,
            revision="main" if precision == "single" else "fp16",
            use_auth_token=True,
            torch_dtype=torch.float32 if precision == "single" else torch.float16,
        )
    else:
        pipe = StableDiffusionOnnxPipeline.from_pretrained(
            "CompVis/stable-diffusion-v1-4",
            use_auth_token=True,
            revision="onnx",
            provider="CPUExecutionProvider"
            if device.type == "cpu"
            else "CUDAExecutionProvider",
            torch_dtype=torch.float32 if precision == "single" else torch.float16,
        )

    # Disable safety
    disable_safety = True
    if disable_safety:

        def null_safety(images, **kwargs):
            return images, False

        pipe.safety_checker = null_safety

    def null_feature_extractor(images, return_tensors="pt", **kwargs):
        class Dummy:
            def __init__(self):
                self.pixel_values = torch.zeros((1,))

            def to(self, *args, **kwargs):
                return self

        return Dummy()

    pipe.feature_extractor = null_feature_extractor

    del pipe.vae.encoder

    if backend == "pytorch":
        pipe = pipe.to(device)
    return pipe

def do_inference(pipe, n_samples, precision, num_inference_steps):
    torch.cuda.empty_cache()
    context = (
        autocast if (device.type == "cuda" and precision == "half") else nullcontext
    )
    context = nullcontext  # disable autocast
    with context("cuda"):
        images = pipe(
            prompt=[prompt] * n_samples, num_inference_steps=num_inference_steps
        ).images

    return images

def get_inference_time(pipe, n_samples, n_repeats, precision, num_inference_steps):
    from torch.utils.benchmark import Timer

    timer = Timer(
        stmt="do_inference(pipe, n_samples, precision, num_inference_steps)",
        setup="from __main__ import do_inference",
        globals={
            "pipe": pipe,
            "n_samples": n_samples,
            "precision": precision,
            "num_inference_steps": num_inference_steps,
        },
        num_threads=multiprocessing.cpu_count(),
    )
    profile_result = timer.timeit(
        n_repeats
    )  # benchmark.Timer performs 2 iterations for warmup
    return round(profile_result.mean, 2)

def get_inference_memory(pipe, n_samples, precision, num_inference_steps):
    if not torch.cuda.is_available():
        return 0

    torch.cuda.empty_cache()
    context = (
        autocast if (device.type == "cuda" and precision == "half") else nullcontext
    )
    context = nullcontext  # disable autocast
    with context("cuda"):
        images = pipe(
            prompt=[prompt] * n_samples, num_inference_steps=num_inference_steps
        ).images

    mem = torch.cuda.memory_reserved()
    return round(mem / 1e9, 2)

def run_benchmark(n_repeats, n_samples, precision, backend, num_inference_steps):
    """
    * n_repeats: nb datapoints for inference latency benchmark
    * n_samples: number of samples to generate (~ batch size)
    * precision: 'half' or 'single' (use fp16 or fp32 tensors)

    returns:
    dict like {'memory usage': 17.70, 'latency': 86.71'}
    """

    pipe = get_inference_pipeline(precision, backend)

    logs = {
        "memory": 0.00
        if device.type == "cpu"
        else get_inference_memory(pipe, n_samples, precision, num_inference_steps),
        "latency": get_inference_time(
            pipe, n_samples, n_repeats, precision, num_inference_steps
        ),
    }
    print(f"n_samples: {n_samples}\tprecision: {precision}\tbackend: {backend}")
    print(logs, "\n")
    return logs

def get_device_description():
    """
    returns descriptor of cuda device such as
    'NVIDIA RTX A6000'
    """
    if device.type == "cpu":
        name = subprocess.check_output(
            "grep -m 1 'model name' /proc/cpuinfo", shell=True
        ).decode("utf-8")
        name = " ".join(name.split(" ")[2:]).strip()
        return name
    else:
        return torch.cuda.get_device_name()

def run_benchmark_grid(grid, n_repeats, num_inference_steps):
    """
    * grid : dict like
        {
            "n_samples": (1, 2),
            "precision": ("single", "half"),
        }
    * n_repeats: nb datapoints for inference latency benchmark
    """

    csv_fpath = pathlib.Path(__file__).parent.parent / "benchmark_tmp.csv"
    # create benchmark.csv if not exists
    if not os.path.isfile(csv_fpath):
        header = ["device", "precision", "runtime", "n_samples", "latency", "memory"]
        with open(csv_fpath, "w") as f:
            writer = csv.writer(f)
            writer.writerow(header)

    # append new benchmark results to it if benchmark_tmp.csv already exists
    with open(csv_fpath, "a") as f:
        writer = csv.writer(f)
        device_desc = get_device_description()
        for n_samples in grid["n_samples"]:
            for precision in grid["precision"]:
                for backend in grid["backend"]:
                    try:
                        new_log = run_benchmark(
                            n_repeats=n_repeats,
                            n_samples=n_samples,
                            precision=precision,
                            backend=backend,
                            num_inference_steps=num_inference_steps,
                        )
                    except Exception as e:
                        if "CUDA out of memory" in str(
                            e
                        ) or "Failed to allocate memory" in str(e):
                            print(str(e))
                            torch.cuda.empty_cache()
                            new_log = {"latency": -1.00, "memory": -1.00}
                        else:
                            raise e

                    latency = new_log["latency"]
                    memory = new_log["memory"]
                    new_row = [
                        device_desc,
                        precision,
                        backend,
                        n_samples,
                        latency,
                        memory,
                    ]
                    writer.writerow(new_row)

if __name__ == "__main__":

    parser = argparse.ArgumentParser()

    parser.add_argument(
        "--samples",
        default="1",
        type=str,
        help="Comma sepearated list of batch sizes (number of samples)",
    )

    parser.add_argument(
        "--steps", default=50, type=int, help="Number of diffusion steps."
    )

    parser.add_argument(
        "--repeats",
        default=3,
        type=int,
        help="Number of repeats.",
    )

    args = parser.parse_args()

    grid = {
        "n_samples": tuple(map(int, args.samples.split(","))),
        # Only use single-precision for cpu because "LayerNormKernelImpl" not implemented for 'Half' on cpu,
        # Remove autocast won't help. Ref:
        # https://github.com/CompVis/stable-diffusion/issues/307
        # "precision": ("single",) if device.type == "cpu" else ("single", "half"),
        "precision": ("half",),  # if device.type == "cpu" else ("single", "half"),
        # Only use onnx for cpu, until issues are fixed by upstreams. Ref:
        # https://github.com/huggingface/diffusers/issues/489#issuecomment-1261577250
        # https://github.com/huggingface/diffusers/pull/440
        # "backend": ("pytorch", "onnx") if device.type == "cpu" else ("pytorch",),
        "backend": ("pytorch",),
    }
    run_benchmark_grid(grid, n_repeats=args.repeats, num_inference_steps=args.steps)


"""This package contains classes implementing custom Bayesian Optimization method from the paper.
"""

import numpy as np

from bayes_opt import Events
from sklearn.gaussian_process.kernels import Matern

class IterationCounter(object):
    """Iteration counter implementation that works as an event listener for :py:class:`BayesianOptimization`."""

    def __init__(self):
        self.counter = 0

    def inc(self):
        self.counter += 1

    def update(self, event, *args, **kwargs):
        if event == Events.OPTIMIZATION_STEP:
            self.inc()

class CustomKernel(Matern):
    """Custom kernel based on a Matern one.
    Modifies covariance of an unseen variable with itself in a such way that dispersion of values predicted by
        a Gaussian Process is increased in the needed area.
    The detailed description is in the paper.
    """

    def __init__(self,
                 length_scale=1.0,
                 length_scale_bounds=(1e-5, 1e5),
                 nu=1.5,
                 iteration_counter=None,
                 sigmoid_scale=1e-3,
                 sigmoid_x_scale=10,
                 sigmoid_x_shift=0,
                 sigmoid_decay_iterations=3
                 ):
        """Constructor.

        :param length_scale: `length_scale` for Matern kernel
        :param length_scale_bounds: `length_scale_bounds` for Matern kernel
        :param nu: `nu` for Matern kernel
        :param iteration_counter: An object that contains `counter` property to get current optimization iteration.
            If None, iteration number is always zero
        :param sigmoid_scale: Vertical scale of a sigmoid
        :param sigmoid_x_scale: Horizontal scale of a sigmoid
        :param sigmoid_x_shift: Horizontal shift of a sigmoid
        :param sigmoid_decay_iterations: Maximal iteration number at which covariance modification is applied
        """
        super().__init__(length_scale, length_scale_bounds, nu)
        self.iteration_counter = iteration_counter
        self.sigmoid_scale = sigmoid_scale
        self.sigmoid_x_scale = sigmoid_x_scale
        self.sigmoid_x_shift = sigmoid_x_shift
        self.sigmoid_decay_iterations = sigmoid_decay_iterations

    def diag(self, X):
        res = super().diag(X)

        iteration_n = 0
        if self.iteration_counter is not None:
            iteration_n = self.iteration_counter.counter

        sigmoid = 1 / (1 + np.exp(-(X[:, 0] + self.sigmoid_x_shift) * self.sigmoid_x_scale))
        sigmoid_multiplier = self.sigmoid_scale * (
                max(self.sigmoid_decay_iterations - iteration_n, 0) / self.sigmoid_decay_iterations)
        mulitplier = 1 + (1 - sigmoid) * sigmoid_multiplier
        res = np.multiply(res, mulitplier)
        return res


"""Add subscribe column to user table

Revision ID: 4e435ff8ba74
Revises: bbba2255e00
Create Date: 2015-02-09 10:36:45.935116

"""

# revision identifiers, used by Alembic.
revision = '4e435ff8ba74'
down_revision = 'bbba2255e00'

from alembic import op
import sqlalchemy as sa

def upgrade():
    op.add_column('user', sa.Column('subscribed', sa.Boolean, default=True))
    query = 'UPDATE "user" SET subscribed=true;'
    op.execute(query)

def downgrade():
    op.drop_column('user', 'subscribed')


import random
from typing import Literal
from icecream import ic

CUADRILLA = {
    'Marion': 'Olivier',
    'Leire': 'Sergio',
    'Nora': 'Jon',
    'Naiara': 'Jorge',
    'Marta': 'Andrés',
    'Estela': 'Odei',
    'Idoia': 'Eneko',
}

LISTA_COLORES = [
    '#0da68c',
    '#24b568',
    '#6fc45c',
    '#ced42f',
    '#6c9600',
    '#26A69A',
    '#4CAF50',
    '#7CB342',
    '#C0CA33',
    '#FDD835',
    '#FFB300',
    '#FF9800',
    '#FB8C00',
    '#F4511E',
]
LISTA_COLORES_AVATAR = [
    '#263238',
    '#37474F',
    '#455A64',
    '#546E7A',
    '#607D8B',
    '#5C6BC0',
    '#3949AB',
    '#283593',
    '#1A237E',
    '#880E4F',
    '#AD1457',
    '#C2185B',
    '#D81B60',
    '#E91E63',
]

async def lista_cuadrilla() -> list[str]:
    lista_nombres = []
    for item in CUADRILLA.items():
        lista_nombres.append(item[0])
        lista_nombres.append(item[1])
    return sorted(lista_nombres)

async def crear_dict_persona_color(tipo:Literal["avatar", "grafico"]='grafico') -> dict:
    lista = LISTA_COLORES if tipo == 'grafico' else LISTA_COLORES_AVATAR
    persona_color_dict = {}
    lista_nombres = await lista_cuadrilla()
    for nombre, color in zip(lista_nombres, lista):
        persona_color_dict[nombre] = color

    return persona_color_dict

#! No se usa
def generar_color_aleatorio():
    return '#{:02x}{:02x}{:02x}'.format(random.randint(100, 255),
                                       random.randint(100, 255),
                                       random.randint(100, 255))

async def get_color_async(nombre:str, tipo:Literal["avatar", "grafico"]='grafico') -> str:
    """Devuelve el color asociado a la persona

    Parameters
    ----------
    nombre : str
        _description_

    Returns
    -------
    str
        _description_
    """
    persona_color_dict = await crear_dict_persona_color(tipo)
    return persona_color_dict[nombre]

class Switch:
    def __init__(self, initial_state:bool=True) -> None:
        self.state = initial_state

    def accionar(self) -> None:
        self.state = not self.state


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Utilities to handle and manipulate AirQualityStationCollection objects
"""

import json
import logging
import warnings

from collections import defaultdict
from itertools import groupby

from itaqa.core.AirQualityStation import AirQualityStation
from itaqa.core.AirQualityStationCollection import AirQualityStationCollection
from itaqa.utils.AQS_utils import merge_AQS_data

logger = logging.getLogger(__name__)

def group_by_name(AQSC):
    """Return a dict with as key the name of the station and as value a list of AQS objects"""
    AQS_by_name = defaultdict(list)
    for k, g in groupby(AQSC.AQS_list, lambda x: x.name):
        for station in g:
            AQS_by_name[k].append(station)
    return AQS_by_name

def merge_by_group(AQSC, AQS_group):
    """Merge multiple groups of AQS and return a list of merged AQS objects"""
    for k in AQS_group:
        # Setup new resulting AQS
        new_AQS = AirQualityStation(k)
        new_AQS.set_address(region=AQS_group[k][0].region,
                            province=AQS_group[k][0].province,
                            comune=AQS_group[k][0].comune)
        # TODO: Compute geolocation and check if they are not so nearby
        new_AQS.metadata['premerge_history'] = {}
        frames = []

        for station in AQS_group[k]:
            frames.append(station.data.set_index('Timestamp'))
            # TODO: Refactor to handle stations with more than one pollutant before merge
            cols = station.data.columns.to_list()
            cols.remove('Timestamp')
            pollutant = cols[0]
            new_AQS.metadata['premerge_history'][pollutant] = {}
            new_AQS.metadata['premerge_history'][pollutant]['name'] = station.name
            new_AQS.metadata['premerge_history'][pollutant]['geolocation'] = station.geolocation

        # Take first frame and merge all the others
        merged_df = frames.pop()
        for frame in frames:
            merged_df = merged_df.merge(frame, how='outer', left_index=True, right_index=True)
        merged_df.reset_index(inplace=True)
        new_AQS.data = merged_df

        # Remove from AQSC the merged stations and add the new one
        AQSC.remove([AQS.uuid for AQS in AQS_group[k]])
        AQSC.add(new_AQS)

def remove_empty_stations(AQSC):
    """Remove AQS without any data inside"""
    empty_stations = [AQS.uuid for AQS in AQSC.AQS_list if AQS.data.size < 1]
    if empty_stations:
        AQSC.remove(empty_stations)

def merge_AQSC(AQSC1, AQSC2):
    """
    Merge the data of two AQSCs and return a new AQSC

    Given two AQSCs, check the amount of AQSs and merge them
    This function is to be used to merge AQSs containing data relative to different timestamps
    For example to merge two collections relative to different years
    """
    if len(AQSC1.AQS_list) != len(AQSC2.AQS_list):
        warning.warn("The AQSCs contain a different number of AQSs")

    new_AQSC = AirQualityStationCollection()

    AQSC1_names = [AQS.name for AQS in AQSC1.AQS_list]
    AQSC2_names = [AQS.name for AQS in AQSC2.AQS_list]

    AQS_names = list(set(AQSC1_names) & set(AQSC2_names))
    AQS_names.sort()

    for AQS_name in AQS_names:
        AQS1 = AQSC1.search(AQS_name)
        AQS2 = AQSC2.search(AQS_name)
        merged_AQS = merge_AQS_data([AQS1, AQS2])
        new_AQSC.add(merged_AQS)

    return new_AQSC

def merge_AQSC_multiple(AQSC_list):
    """
    Merge the data of multiple AQSCs and return a new AQSC

    Given a list of AQSC, check the amount of AQS contained in each and merge them
    This function is to be used to merge AQSs containing data relative to different timestamps
    For example to merge two collections relative to different years
    """
    new_AQSC = AirQualityStationCollection()

    all_names = list()
    for AQSC in AQSC_list:
        all_names.append([AQS.name for AQS in AQSC.AQS_list])

    AQS_names = list(set().union(*all_names))
    AQS_names.sort()

    for AQS_name in AQS_names:
        AQSs_to_merge = list()
        for AQSC in AQSC_list:
            AQS = AQSC.search(AQS_name)
            AQSs_to_merge.append(AQS)
        merged_AQS = merge_AQS_data(AQSs_to_merge)
        new_AQSC.add(merged_AQS)

    return new_AQSC


"""
Sets up and runs advanced benchmarking tests for developers.

It is designed to submit many jobs simultaneously on an HPC system.
Please see the README.md file for more information.
"""

import os
import pandas as pd
import shutil
import json
import subprocess
import numpy as np

def set_up_calcs(basedir, env, testfile="test.py"):
    """
    Set up calculations by creating directories and preparing files.

    Parameters
    ----------
    basedir : str
        The base directory where calculations will be set up.
    env : str
        The environment variable to be used in the calculations.
    testfile : str, optional
        The name of the test file to use for calculations.

    Returns
    -------
    None
    """
    if os.path.exists(basedir):
        user_input = (
            input(
                f"The directory {basedir} already exists."
                + "Are you sure you want to continue? (yes/no): "
            )
            .strip()
            .lower()
        )
        if user_input != "yes":
            print("Operation cancelled.")
            return

    # Read the CSV file
    df = pd.read_csv("pressure_benchmarks.csv")

    # Loop through the rows of the dataframe
    for index, row in df.iterrows():
        species = row["species"]
        rho = row["rho"]
        rho_round = round(rho, 3)
        temp = row["temp"]
        temp_round = round(temp, 3)

        # Create the sub-folder
        subdir = os.path.join(basedir, f"{species}/rho_{rho_round}/T_{temp_round}")
        os.makedirs(subdir, exist_ok=True)

        # Read the original submit.slurm file
        with open("submit.slurm", "r") as file:
            filedata = file.read()

        # Replace the target string
        filedata = filedata.replace("env={ENV}", f"env={env}")

        # Write the modified submit.slurm file to the new location
        with open(subdir + "/submit.slurm", "w") as file:
            file.write(filedata)

        # same for python script
        shutil.copy(testfile, subdir + "/test.py")

        # Make a JSON file containing all the information from the row
        row_dict = row.to_dict()
        with open(os.path.join(subdir, "input.json"), "w") as f:
            json.dump(row_dict, f, indent=4)

    print("Setup complete.")

def run_calcs(basedir):
    """
    Run calculations by submitting jobs to an HPC system.

    Parameters
    ----------
    basedir : str
        The base directory where calculations are set up.

    Returns
    -------
    None
    """
    # Read the CSV file
    df = pd.read_csv("pressure_benchmarks.csv")

    # Loop through the rows of the dataframe
    for index, row in df.iterrows():
        species = row["species"]
        rho = row["rho"]
        rho_round = round(rho, 3)
        temp = row["temp"]
        temp_round = round(temp, 3)

        # Define the sub-folder
        subdir = os.path.join(basedir, f"{species}/rho_{rho_round}/T_{temp_round}")

        # Check if the directory exists
        if not os.path.exists(subdir):
            print(f"Directory does not exist: {subdir}")
            continue

        # Check if submit.slurm exists in the directory
        if not os.path.exists(os.path.join(subdir, "submit.slurm")):
            print(f"submit.slurm does not exist in: {subdir}")
            continue

        # Execute sbatch submit.slurm
        try:
            subprocess.run(["sbatch", "submit.slurm"], check=True, cwd=subdir)
            print(f"Job submitted for {species}, rho_{rho}, T_{temp}")
        except subprocess.CalledProcessError as e:
            print(f"Failed to submit job for {species}, rho_{rho}, T_{temp}: {e}")

# Function to check if the values are close
def check_close(input_val, output_val, atol=1e-9, rtol=1e-2):
    """
    Check if two values are close to each other within a tolerance.

    By default it checks for a relative tolerance of 1%.
    Absolute tolerance is not considered.

    Parameters
    ----------
    input_val : float
        The first value to compare.
    output_val : float
        The second value to compare.
    atol : float, optional
        The absolute tolerance parameter.
    rtol : float, optional
        The relative tolerance parameter.

    Returns
    -------
    bool
        True if the values are close within the specified tolerances, False otherwise.
    """
    return np.isclose(input_val, output_val, atol=atol, rtol=rtol)

# Function to calculate the percentage error
def calculate_percentage_error(value1, value2):
    """
    Calculate the percentage error between two values.

    Parameters
    ----------
    value1 : float
        The first value (reference value).
    value2 : float
        The second value (value to compare).

    Returns
    -------
    float
        The percentage error between the two values.
    """
    if value1 == 0 and value2 == 0:
        return 0  # Avoid division by zero if both values are zero
    try:
        return round(abs((value1 - value2) / value1) * 100, 3)
    except ZeroDivisionError:
        return np.inf  # Return infinity if value1 is zero

# Function to gather benchmark results and save to a new file
def gather_benchmark_results(basedir, new_filename):
    """
    Gather benchmark results from JSON files and save them to a new CSV file.

    Parameters
    ----------
    basedir : str
        The base directory where the benchmark results are stored.
    new_filename : str
        The name of the new CSV file to save the results.

    Returns
    -------
    None
    """
    # Read the existing dataframe
    df = pd.read_csv("pressure_benchmarks.csv")

    # Prepare a list to hold the new data
    new_data = []

    # Loop through the rows of the dataframe
    for index, row in df.iterrows():
        species = row["species"]
        rho = row["rho"]
        temp = row["temp"]

        # Define the sub-folder
        subdir = os.path.join(
            basedir, f"{species}/rho_{round(rho, 3)}/T_{round(temp, 3)}"
        )

        # Initialize the outcome and percentage errors
        outcome = "fail"
        pc_err_st = pc_err_vir = pc_err_id = time_s = None

        # Read input.json and output.json files
        try:
            with open(os.path.join(subdir, "input.json"), "r") as f:
                input_data = json.load(f)

            with open(os.path.join(subdir, "output.json"), "r") as f:
                output_data = json.load(f)

            # Extract the relevant values
            P_st_rr_input = input_data["P_st_rr"]
            P_vir_nocorr_input = input_data["P_vir_nocorr"]
            P_id_input = input_data["P_id"]

            P_st_rr_output = output_data["P_st_rr"]
            P_vir_nocorr_output = output_data["P_vir_nocorr"]
            P_id_output = output_data["P_id"]
            time_s = round(output_data["time"], 2)

            # Calculate percentage errors
            pc_err_st = calculate_percentage_error(P_st_rr_input, P_st_rr_output)
            pc_err_vir = calculate_percentage_error(
                P_vir_nocorr_input, P_vir_nocorr_output
            )
            pc_err_id = calculate_percentage_error(P_id_input, P_id_output)

            # Check if the values are close within the specified relative tolerance
            # The original calculations were converged to 1%
            rtol = 1e-2  # 1%
            if (
                np.isclose(P_st_rr_input, P_st_rr_output, rtol=rtol)
                and np.isclose(P_vir_nocorr_input, P_vir_nocorr_output, rtol=rtol)
                and np.isclose(P_id_input, P_id_output, rtol=rtol)
            ):
                outcome = "pass"

        except Exception as e:
            print(f"Error reading files for {species}, rho_{rho}, T_{temp}: {e}")
            outcome = "fail"

        # Append the new row to the data list
        new_data.append(
            [species, rho, temp, outcome, pc_err_st, pc_err_vir, pc_err_id, time_s]
        )

    # Create the new dataframe with additional columns for percentage errors
    new_df_columns = [
        "species",
        "rho",
        "temp",
        "outcome",
        "pc_err_st",
        "pc_err_vir",
        "pc_err_id",
        "time_s",
    ]
    new_df = pd.DataFrame(new_data, columns=new_df_columns)

    # Save the new dataframe to the specified CSV file
    new_df.to_csv(new_filename, index=False)
    print(f"Results saved to {new_filename}")

def analyze_benchmark_results(csv_file):
    """
    Read benchmark results from a CSV file and analyzes the data.

    Parameters
    ----------
    csv_file : str
        The path to the CSV file containing benchmark results

    Returns
    -------
    None
    """
    # Read the CSV file into a DataFrame
    df = pd.read_csv(csv_file)

    # Calculate the number of tests passed
    total_tests = len(df)
    passed_tests = len(df[df["outcome"] == "pass"])
    pass_fraction = passed_tests / total_tests

    # Calculate median and max for error columns
    err_st_stats = {"median": df["pc_err_st"].median(), "max": df["pc_err_st"].max()}
    err_vir_stats = {
        "median": df["pc_err_vir"].median(),
        "max": df["pc_err_vir"].max(),
    }
    err_id_stats = {"median": df["pc_err_id"].median(), "max": df["pc_err_id"].max()}

    # Calculate mean, median, quartiles, and max for time column
    time_stats = {
        "mean": df["time_s"].mean(),
        "median": df["time_s"].median(),
        "q1": df["time_s"].quantile(0.25),
        "q3": df["time_s"].quantile(0.75),
        "max": df["time_s"].max(),
    }

    # Format the table
    table = f"""
    Benchmarking Results Analysis
    -----------------------------
    Tests passed: {passed_tests} / {total_tests} ({pass_fraction:.2%})

    Error Statistics
    -----------------------------------------
    | Error Type  | Median     | Max        |
    |-------------|------------|------------|
    | pc_err_st   | {err_st_stats['median']:10.2f} | {err_st_stats['max']:10.2f} |
    | pc_err_vir  | {err_vir_stats['median']:10.2f} | {err_vir_stats['max']:10.2f} |
    | pc_err_id   | {err_id_stats['median']:10.2f} | {err_id_stats['max']:10.2f} |

    Time Statistics (s)
    ------------------------
    | Statistic |   Value  |
    |-----------|----------|
    | Mean      | {time_stats['mean']:8.2f} |
    | Median    | {time_stats['median']:8.2f} |
    | Q1        | {time_stats['q1']:8.2f} |
    | Q3        | {time_stats['q3']:8.2f} |
    | Max       | {time_stats['max']:8.2f} |
    """
    print(table)

def calc_time_diff(csv_ref, csv_new):
    """
    Calculate the average percentage difference in timings between two atoMEC versions.

    Parameters
    ----------
    csv_ref : str
        Filepath to the reference CSV file.
    csv_new : str
        Filepath to the new CSV file to compare against the reference.

    Returns
    -------
    float
        The average absolute percentage difference of the 'time_s' column
        between the reference and new datasets.

    Notes
    -----
    The percentage difference is calculated using the formula:
    100 * abs(time_new - time_ref) / time_ref
    This formula gives the average of the absolute relative differences
    from the reference to the new values.
    """
    # Read the time_s column from each CSV into a DataFrame
    df_ref = pd.read_csv(csv_ref)["time_s"]
    df_new = pd.read_csv(csv_new)["time_s"]

    # Calculate the absolute percentage difference
    time_diff_pc = 100 * (df_ref - df_new) / df_ref

    # Return the mean of these percentage differences, rounded to 2 decimal places
    return round(time_diff_pc.mean(), 2)

def comp_benchmark_results(csv_ref, csv_new):
    """
    Compare benchmark results between a reference CSV file and a new CSV file.

    Prints individual test and timings results, and finally does a row-by-row
    percentage difference of timings.

    Parameters
    ----------
    csv_ref : str
        Filepath to the reference CSV file containing benchmark results.
    csv_new : str
        Filepath to the new CSV file containing benchmark results to be compared.

    Returns
    -------
    None
    """
    # Print results from the reference CSV
    print("\nResults from reference csv")
    print("--------------------------")
    analyze_benchmark_results(csv_ref)

    # Print results from the new CSV
    print("\nResults from new csv")
    print("--------------------------")
    analyze_benchmark_results(csv_new)

    # Calculate and print the average time percentage difference
    avg_time_diff = calc_time_diff(csv_ref, csv_new)
    print("\n-------------------------------------")
    print(f" Average time % difference = {avg_time_diff}% ")
    print("-------------------------------------")


#!/usr/bin/env python
#

"""General logging interface for the application."""

import logging
import logging.handlers
import settings

class Logger():
    """Provides a general logging interface for the application."""
    __instance = None

    class FileLogger():
        """Provides an interface to log in files."""
        log = None

        def __init__(self):
            """ Logger's initialisation based on settings parameters. """
            self.log = logging.getLogger(settings.APPLICATION_NAME)
            logging.NOLOG = 60
            logging.addLevelName(logging.NOLOG, 'NOLOG')
            #SYSLOG LEVEL
            if settings.LOG_LEVEL == 'CRITICAL':
                self.log.setLevel(logging.CRITICAL)
            elif settings.LOG_LEVEL == 'ERROR':
                self.log.setLevel(logging.ERROR)
            elif settings.LOG_LEVEL == 'WARNING':
                self.log.setLevel(logging.WARNING)
            elif settings.LOG_LEVEL == 'INFO':
                self.log.setLevel(logging.INFO)
            elif settings.LOG_LEVEL == 'DEBUG':
                self.log.setLevel(logging.DEBUG)
            else:
                #Superior than CRITICAL
                self.log.setLevel(logging.NOLOG)

            #File handler
            filelog = logging.handlers.RotatingFileHandler(
                        settings.LOG_FILENAME,
                        maxBytes=settings.LOG_FILESIZE,
                        backupCount=settings.LOG_NBLOGFILES
                      )

            #Formatter
            if len(settings.LOG_FORMATTER) == 0:
                formatter = logging._defaultFormatter
            else:
                formatter = logging.Formatter(settings.LOG_FORMATTER)

            filelog.setFormatter(formatter)
            self.log.addHandler(filelog)

    def __init__(self):
        """Create instance of Syslog logger if doesn't exists."""
        if not Logger.__instance :
            Logger.__instance = Logger.FileLogger()

    def warn(self,message):
        """Produce log message with WARNING level."""
        if not Logger.__instance :
            Logger.__instance = Logger.FileLogger()
        Logger.__instance.log.warn(message)

    def debug(self,message):
        """Produce log message with DEBUG level."""
        if not Logger.__instance :
            Logger.__instance = Logger.FileLogger()
        Logger.__instance.log.debug(message)

    def info(self,message):
        """Produce log message with INFO level."""
        if not Logger.__instance :
            Logger.__instance = Logger.FileLogger()
        Logger.__instance.log.info(message)

    def error(self,message):
        """Produce log message with ERROR level."""
        if not Logger.__instance :
            Logger.__instance = Logger.FileLogger()
        Logger.__instance.log.error(message)

    def critical(self,message):
        """Produce log message with CRITICAL level."""
        if not Logger.__instance :
            Logger.__instance = Logger.FileLogger()
        Logger.__instance.log.critical(message)

    def __new__(Logger):
        """Create and return instance of Syslog logger if doesn't exists."""
        if not Logger.__instance :
            Logger.__instance = Logger.FileLogger()
        return Logger.__instance

logger = Logger()

def warn(message):
    """Produce log message with WARNING level."""
    logger.warn(message)

def debug(message):
    """Produce log message with DEBUG level."""
    logger.debug(message)

def info(message):
    """Produce log message with INFO level."""
    logger.info(message)

def error(message):
    """Produce log message with ERROR level."""
    logger.error(message)

def critical(message):
    """Produce log message with CRITICAL level."""
    logger.critical(message)


from bokeh.io.output import output_file
from bokeh.plotting import figure, output_file, show

if __name__ =="__main__":
    output_file('gaficado_simple.html')
    fig = figure()

    total_vals = int(input('Cuantos valores quieres graficar? '))
    x_vals = list(range(total_vals))
    y_vals = []

    for i in x_vals:
        val = int(input(f'Valor Y para {i}'))
        y_vals.append(val)

    fig.line(x_vals,y_vals, line_width = 2)
    show(fig)


"""Implemenation of pixel map.

It is used for storing colors of image pixels and calculating their entropy.
"""
import numpy as np
from dataclasses import dataclass

@dataclass
class Pixel:
    """Data class representing single pixel of the image.

    Pixel consist of three RGB colors.
    """
    red: np.uint8
    green: np.uint8
    blue: np.uint8

    @property
    def color(self):
        """Whole color value of the pixel."""
        return (self.red << 16) + (self.green << 8) + self.blue

    def __sub__(self, other):
        return PixelDifference(
            clamp(np.int16(self.red) - np.int16(other.red), -32768, 32767),
            clamp(np.int16(self.green) - np.int16(other.green), -32768, 32767),
            clamp(np.int16(self.blue) - np.int16(other.blue), -32768, 32767)
        )

    def __add__(self, other):
        return Pixel(
            clamp(self.red + other.red, 0, 255),
            clamp(self.green + other.green, 0, 255),
            clamp(self.blue + other.blue, 0, 255)
        )

def get_pixel(pixel_difference):
    return Pixel(
        clamp(pixel_difference.red, 0, 255),
        clamp(pixel_difference.green, 0, 255),
        clamp(pixel_difference.blue, 0, 255),
    )

def clamp(number, minimum, maximum):
    if number > maximum:
        return maximum
    if number < minimum:
        return minimum
    return number

@dataclass
class PixelDifference:
    """Data class representing difference between two pixels.

    PixelDifference consist of three RGB color values. However they can be bigger than 255 and negative.
    """
    red: np.int16
    green: np.int16
    blue: np.int16

    def __add__(self, other):
        return PixelDifference(
            self.red + other.red,
            self.green + other.green,
            self.blue + other.blue
        )

    def __sub__(self, other):
        return PixelDifference(
            self.red - other.red,
            self.green - other.green,
            self.blue - other.blue
        )

class PixelMap:
    """Map wrapper of the image pixels.

    Pixel Map stores pixels of the image.
    """

    def __init__(self, height, width):
        """
        Initialize new instance of Pixel Map with a given height and width of an image.

        Args:
            height (int): height of the pixel table to create
            width (int): width of the pixel table to create
        """
        self.width = width
        self.height = height
        self.pixels = np.empty((height, width), dtype=Pixel)

    def __getitem__(self, idx):
        """Return pixel on the given idx coordinates.

        -- idx - tuple of (row, col)
        """
        row, col = idx
        return self.pixels[row, col]

    @classmethod
    def from_vector(cls, vector, height, width):
        assert vector.size == height * width
        new_pixel_map = cls(height, width)
        for row in range(height):
            for col in range(width):
                pixel = vector[row*width + col]
                new_pixel_map.new_pixel(row, col, pixel.red, pixel.green, pixel.blue)

        return new_pixel_map

    def new_pixel(self, row, col, red, green, blue):
        """Assign new pixel to (row, col) with given RGB color ingredients.

        -- row - pixel row, y-coordinate
        -- col - pixel col, x-coordinates
        -- red - red ingredient
        -- green - green ingredient
        -- blue - blue ingredient
        """
        assert row < self.height and col < self.width, f"Out of range: [{row}, {col}]"
        pixel = Pixel(red % 256, green % 256, blue % 256)
        self.pixels[row, col] = pixel

    @property
    def size(self):
        """Size of the pixel map, which is height * width."""
        return self.height * self.width

    def __str__(self):
        return str(self.pixels)

    def get_blocks(self, block_side):
        """Divide pixel table into blocks and return them as vectors.

        Blocks are squared size with side length of block_side. They are flattened into vectors.
        -- block_side - the length of blocks sides"""
        blocks = []
        for row_idx in range(0, self.height, block_side):
            for col_idx in range(0, self.width, block_side):
                blocks.append([(pixel.red, pixel.green, pixel.blue) for pixel in np.ndarray.flatten(self.pixels[row_idx:row_idx+block_side, col_idx:col_idx+block_side])])
        return np.array(blocks, dtype=np.uint8)


# import libraries
from dataframe_maker.dataframe_maker import DFGuns
from plot_maker.plot_maker import PlotMaker
from dash import Dash
import dash_table
import dash_html_components as html
import dash_core_components as dcc
import dash_bootstrap_components as dbc
from dash.dependencies import Input, Output

# initialise Dash app
app = Dash(__name__,
           external_stylesheets=[dbc.themes.COSMO])

# generate dataframe
df = DFGuns().return_dataframe()

# generate charts
charts = PlotMaker(df)

app.layout = dbc.Container([
    dbc.Row(children=[  # header
        dbc.Row(className='jumbotron',

                children=[
                    html.H1('Firearms & Gun Homicide statistics Dashboard.'),
                    html.P('Data has been provided where it exists. Median values provided where numeric data absent.')
                ])]),
    dbc.Row([
        dbc.Col(children=[
            dbc.Label('Chunk size',
                      style={'color': 'pink'}),
            dbc.FormGroup(dbc.RadioItems(id='main_chart_dd_viewby',
                                         options=[{'label': 'Country', 'value': 'country'},
                                                  {'label': 'Region', 'value': 'region'},
                                                  {'label': 'Sub-Region', 'value': 'sub_region'}],
                                         value='country')),
            dbc.Label('X-Axis',
                      style={'color': 'pink'}),
            dbc.FormGroup(dbc.RadioItems(id='main_chart_x',
                                         options=[{'label': 'Population', 'value': 'pop_2017'},
                                                  {'label': 'Civilian Firearms',
                                                   'value': 'firearms_owned_civilians'},
                                                  {'label': 'Registered Firearms',
                                                   'value': 'registered_firearms'},
                                                  {'label': 'Unregistered Firearms',
                                                   'value': 'Unregistered firearms'},
                                                  {'label': 'Firearms per 100 people',
                                                   'value': 'firearms_per_100_persons'},
                                                  {'label': 'Murder Rate per 100,000', 'value': 'Rate'}],
                                         value='pop_2017')),
            dbc.Label('Y-Axis',
                      style={'color': 'pink'}),
            dbc.FormGroup(dbc.RadioItems(id='main_chart_y',
                                         options=[{'label': 'Population', 'value': 'pop_2017'},
                                                  {'label': 'Civilian Firearms',
                                                   'value': 'firearms_owned_civilians'},
                                                  {'label': 'Registered Firearms',
                                                   'value': 'registered_firearms'},
                                                  {'label': 'Unregistered Firearms',
                                                   'value': 'Unregistered firearms'},
                                                  {'label': 'Firearms per 100 people',
                                                   'value': 'firearms_per_100_persons'},
                                                  {'label': 'Murder Rate per 100,000', 'value': 'Rate'}],
                                         value='firearms_owned_civilians')),
            dbc.Label('Marker Size',
                      style={'color': 'pink'}),
            dbc.FormGroup(dbc.RadioItems(id='main_chart_z',
                                         options=[{'label': 'Population', 'value': 'pop_2017'},
                                                  {'label': 'Civilian Firearms',
                                                   'value': 'firearms_owned_civilians'},
                                                  {'label': 'Registered Firearms',
                                                   'value': 'registered_firearms'},
                                                  {'label': 'Unregistered Firearms',
                                                   'value': 'Unregistered firearms'},
                                                  {'label': 'Firearms per 100 people',
                                                   'value': 'firearms_per_100_persons'},
                                                  {'label': 'Murder Rate per 100,000', 'value': 'Rate'}],
                                         value='Rate'))
        ]),
        dcc.Graph(id='main_chart')
    ]),
    dash_table.DataTable(
        id='table-dark',
        columns=[{'name': 'Country', 'id': 'country'},
                 {'name': 'Region', 'id': 'region'},
                 {'name': 'Sub Region', 'id': 'sub_region'},
                 {'name': 'Population', 'id': 'pop_2017'},
                 {'name': 'Civilian Firearms', 'id': 'firearms_owned_civilians'},
                 {'name': 'Registered Firearms', 'id': 'registered_firearms'},
                 {'name': 'Unregistered Firearms', 'id': 'Unregistered firearms'},
                 {'name': 'Firearms per 100 people', 'id': 'firearms_per_100_persons'},
                 {'name': 'Murder Rate per 100,000', 'id': 'Rate'}],
        data=charts.df.to_dict('records'),
        sort_action='native',
        style_table={'width': '98%',
                     'align': 'center'}
    )
])

# callback for main_chart
@app.callback(output=Output(component_id='main_chart',
                            component_property='figure'),
              inputs=[Input(component_id='main_chart_dd_viewby',
                            component_property='value'),
                      Input(component_id='main_chart_x',
                            component_property='value'),
                      Input(component_id='main_chart_y',
                            component_property='value'),
                      Input(component_id='main_chart_z',
                            component_property='value')
                      ])
def callback(selector_region, x_axis, y_axis, z_axis):
    return charts.main_scatter(value_selection=selector_region,
                               x_axis=x_axis,
                               y_axis=y_axis,
                               z_axis=z_axis)

# run app
if __name__ == '__main__':
    app.run_server(debug=False,
                   port=5000)


from utils import config
from utils.lstmnet import LstmNet
import numpy as np
import tensorflow as tf
import pickle
from time import time
import argparse

import os
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID";
os.environ["CUDA_VISIBLE_DEVICES"] = '1'
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

import keras.backend.tensorflow_backend as K
from keras.optimizers import Adam

N_TIMESTEPS = 64
N_FEATURES = 64

def load_data_train():
    if os.path.isfile(os.path.join(config.BASE_PATH, 'data_heart.npz')) == True:
        data = np.load(os.path.join(config.BASE_PATH, 'data_heart.npz'))
        X_train = data['X_train']
        Y_train = data['Y_train']
    return X_train, Y_train

def load_data_val():
    if os.path.isfile(os.path.join(config.BASE_PATH, 'data_heart.npz')) == True:
        data = np.load(os.path.join(config.BASE_PATH, 'data_heart.npz'))
        X_val = data['X_val']
        Y_val = data['Y_val']
    return X_val, Y_val

def load_data_test():
    if os.path.isfile(os.path.join(config.BASE_PATH, 'data_heart.npz')) == True:
        data = np.load(os.path.join(config.BASE_PATH, 'data_heart.npz'))
        X_test = data['X_test']
        Y_test = data['Y_test']
    return X_test, Y_test

def main(X_train, Y_train, X_val, Y_val, X_test, Y_test, n_layers, h_units):
    '''
    Train and predict.

    Parameters
    ----------
    n_layers : int, number of hidden LSTM layers
    c_units : int, number of hidden units for each layer
    '''

    batch_size = 64
    epochs = 100

    print('\nModel with {} layer(s) and {} hidden units each'.format(n_layers, h_units))
    start = time()
    model = LstmNet.build(input_shape=(N_TIMESTEPS, N_FEATURES),
                          n_layers = n_layers,
                          c_units = [h_units]*n_layers,
                          n_classes = 2)
    model.compile(loss='binary_crossentropy', optimizer=Adam(), metrics=['accuracy'])
    model.fit(X_train, Y_train,
              validation_data=(X_val, Y_val),
              batch_size=batch_size,
              epochs=epochs,
              verbose=0)
    end = time()

    tr_loss = model.history.history['loss'][-1]
    tr_val_loss = model.history.history['val_loss'][-1]
    tr_acc = model.history.history['acc'][-1]
    tr_val_acc = model.history.history['val_acc'][-1]
    print('Training completed in {}s : loss {}, val_loss {}, acc {}, val_acc {}'
          .format(end-start, tr_loss, tr_val_loss, tr_acc, tr_val_acc))

    n_param = model.count_params()
    acc = model.evaluate(X_test, Y_test, verbose=0)[-1]
    print('Model parameters : {}, Prediction accuracy : {}'.format(n_param, acc))
#     model.save('model/heart_lstm_layers{}_units{}_bs{}_epochs{}.h5'
#                .format(n_layers, h_units, batch_size, epochs))

if __name__=='__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--layers', type=int, help='number of LSTM layers')
    parser.add_argument('--units', type=int, help='number of hidden units per layer')
    args = parser.parse_args()

    np.random.seed(1)
    tf.set_random_seed(1)
    cfg = tf.ConfigProto()
    cfg.gpu_options.allow_growth = True
    sess = tf.Session(config=cfg)
    K.set_session(sess)

    X_train, Y_train = load_data_train()
    X_val, Y_val = load_data_val()
    X_test, Y_test = load_data_test()

    main(X_train, Y_train, X_val, Y_val, X_test, Y_test, args.layers, args.units)


import logging
import os
import socket
from socket import AF_INET, SOCK_RAW, IPPROTO_IP, IPPROTO_ICMP
import struct
import select
import time
from collections import namedtuple

import icmputils.constants as const
from icmputils.utils.ip import IPv4Header

LOG = logging.getLogger(__name__)

class TimeoutError(RuntimeError):
    def __init__(self, timeout):
        msg = f'Timeout within {timeout} sec'
        super().__init__(msg)

# ICMP header is type(8), code(8), checksum(16), id(16), sequence(16)
_IcmpHeader = namedtuple('_IcmpHeader', [
    'type',
    'code',
    'checksum',
    'id',
    'seq_num',
])

class IcmpHeader(_IcmpHeader):
    _format = 'BBHHH'

    def pack(self):
        return struct.pack(self._format, *self)

    @classmethod
    def unpack(cls, byte_obj):
        return IcmpHeader(*struct.unpack(cls._format, byte_obj))

    def __len__(self):
        return struct.calcsize(self._format)

    @property
    def function(self):
        return self.type, self.code

def get_icmp_checksum(binary_data):
    def carry_around_add(a, b):
        c = a + b
        return (c & 0xffff) + (c >> 16)

    s = 0
    for i in range(0, len(binary_data), 2):
        w = binary_data[i] + (binary_data[i+1] << 8)
        s = carry_around_add(s, w)
    return ~s & 0xffff

def parse_ping_packet(packet):
    if not isinstance(packet, (bytes, bytearray)):
        raise TypeError('Packet parameter must be bytes or bytearray!')
    ip_header = IPv4Header.unpack(packet[:20])
    icmp_header = IcmpHeader.unpack(packet[20:28])
    return ip_header, icmp_header, packet[28:]

def generate_ping_ip_payload(id, seq_id, payload_size=56):
    if payload_size < const.MIN_PAYLOAD_SIZE:
        raise ValueError(
            f'Payload size must be greater than {const.MIN_PAYLOAD_SIZE}')

    dummy_header = IcmpHeader(*const.ICMP_ECHO_REQUEST, 0, id, seq_id)
    payload = struct.pack('d', time.clock()).ljust(payload_size, b'\x00')

    checksum = get_icmp_checksum(dummy_header.pack() + payload)
    header = IcmpHeader(*const.ICMP_ECHO_REQUEST, checksum, id, seq_id)

    return header.pack() + payload

def send_one_ping(sock, dest_addr, id, seq_id, payload_size=56):
    """Send one ping to the given `dest_addr`."""
    packet_data = generate_ping_ip_payload(id, seq_id, payload_size)
    sock.sendto(packet_data, (dest_addr, 0))

def receive_one_ping(sock, id, timeout):
    """Receive ping from the socket."""
    time_left = timeout
    while time_left > 0:
        started_select = time.clock()
        readable, *_ = select.select([sock], [], [], time_left)
        ended_select = time.clock()
        time_left -= (ended_select - started_select)

        if len(readable) == 0:
            break

        packet, addr = sock.recvfrom(1024)

        ip_header, icmp_header, icmp_payload = parse_ping_packet(packet)
        if icmp_header.function == const.ICMP_ECHO_REPLY and icmp_header.id == id:
            time_sent = struct.unpack('d', icmp_payload[:struct.calcsize('d')])[0]
            ip_payload_size = len(packet) - len(ip_header)
            time_spent = (ended_select - time_sent) * 100000
            return (ip_payload_size, socket.getfqdn(addr[0]), addr[0],
                    icmp_header.seq_num, ip_header.ttl, time_spent)
    raise TimeoutError(timeout)

def _ping(dest_ip, timeout, seq_id, payload_size):
    """Returns either the delay (in seconds) or none on timeout."""
    id = os.getpid() & 0xFFFF
    with socket.socket(AF_INET, SOCK_RAW, IPPROTO_ICMP) as sock:
        send_one_ping(sock, dest_ip, id, seq_id, payload_size)
        ping_result = receive_one_ping(sock, id, timeout)
    return ping_result

def ping(hostname, timeout=2, count=4, payload_size=56, log_to_file=False):
    if log_to_file:
        fh = logging.FileHandler(
            '%d-ping-%s.log' % (time.time(), hostname), 'w')
        LOG.addHandler(fh)
    try:
        dest_addr = socket.gethostbyname(socket.gethostbyname(hostname))
        frame_size = payload_size + 28
        LOG.info('PING %s (%s) %d(%d) bytes of data.',
                 hostname, dest_addr, payload_size, frame_size)

        for seq_id in range(1, count+1):
            try:
                ping_result = _ping(dest_addr, timeout, seq_id, payload_size)
                LOG.info(
                    '%d bytes from %s (%s): icmp_seq=%d ttl=%d time=%0.1f ms',
                    *ping_result)
            except TimeoutError as e:
                LOG.info(e)
    except socket.gaierror as e:
        LOG.error(e)
    except PermissionError:
        LOG.error('Must be superuser')


from django.conf.urls import patterns, url

from workout import views

urlpatterns = patterns('',
    # ex: /workout/
    url(r'^$', views.index, name='index'),
    # ex: /workout/5/
    url(r'^(?P<poll_id>\d+)/$', views.detail, name='detail'),
    # ex: /workout/5/results/
    url(r'^(?P<poll_id>\d+)/results/$', views.results, name='results'),
    # ex: /workout/5/vote/
    url(r'^(?P<poll_id>\d+)/vote/$', views.vote, name='vote'),
    url(r'^applejack/$', views.makeEvent, name='event'),
)


from tkinter import *
from tkinter import filedialog
import string
from errors import errorCodes, error
import math
import numpy as np
import matplotlib.pyplot as plt

class seventhLab():
    def __init__(self, window):
        super().__init__()
        self.window = window    # родительское окно в методе
        self.method = "1"
        self.coefficients = []
        self.inputs = []
        self.initUI()
        self.Y = []
        self.X = []
        self.matrix_length = 3

    def initUI(self):
        self.window.title("Сплайны #7")

        mainMenu = Menu(self.window, tearoff=0)  # создаём вкладку в меню
        self.window.config(menu=mainMenu)
        fileMenu = Menu(mainMenu, tearoff=0)  # вкладываем туда кнопки
        fileMenu.add_command(label="Открыть", command=self.on_open)
        mainMenu.add_cascade(label="Файл", menu=fileMenu)

        buttons_frame = Frame(master=self.window, padx=10, pady=5)
        self.clear_button = Button(text="Очистить", master=buttons_frame, width=10, command=self.clear)
        self.clear_button.pack(side=LEFT)
        self.start_button = Button(text="Рассчитать", master=buttons_frame, width=10, command=self.calculate)
        self.start_button.pack(side=LEFT)
        buttons_frame.pack(side=TOP)

        main_frame_x = Frame(master=self.window)
        Label(master=main_frame_x, text="x: ").pack(side=LEFT)
        input1 = Entry(master=main_frame_x, width=10)
        input2 = Entry(master=main_frame_x, width=10)
        input3 = Entry(master=main_frame_x, width=10)
        input4 = Entry(master=main_frame_x, width=10)
        input5 = Entry(master=main_frame_x, width=10)
        input1.pack(side=LEFT)
        input2.pack(side=LEFT)
        input3.pack(side=LEFT)
        input4.pack(side=LEFT)
        input5.pack(side=LEFT)

        main_frame_x.pack(side=TOP, padx=10)

        main_frame_y = Frame(master=self.window)
        Label(master=main_frame_y, text="y: ").pack(side=LEFT)
        input6 = Entry(master=main_frame_y, width=10)
        input7 = Entry(master=main_frame_y, width=10)
        input8 = Entry(master=main_frame_y, width=10)
        input9 = Entry(master=main_frame_y, width=10)
        input10 = Entry(master=main_frame_y, width=10)
        input5.pack(side=LEFT)
        input6.pack(side=LEFT)
        input7.pack(side=LEFT)
        input8.pack(side=LEFT)
        input9.pack(side=LEFT)
        input10.pack(side=LEFT)
        main_frame_y.pack(side=TOP, padx=10)

        sub_frame = Frame(master=self.window)
        Label(master=sub_frame, text="Рассчитать от x:").pack(side=LEFT)
        input11 = Entry(master=sub_frame, width=10)
        input11.pack(side=LEFT)
        sub_frame.pack(side=TOP)

        self.inputs = [input1, input2, input3, input4, input5, input6, input7, input8, input9, input10, input11]

        result_frame = Frame(master=self.window)
        result_frame.pack(side=TOP)
        self.result_label = Label(master=result_frame, font="Courier 16", pady=10)
        self.result_label.pack(side=TOP)

    def get_coef(self):
        coefs = []
        for input in self.inputs:
            coefs.append(float(input.get()))
        return coefs

    def print_result(self, result):
        self.result_label.config(text="y = " + result)

        window = plt.subplots()
        x = np.arange(self.X[0], self.X[-1], 1)
        y = []
        for i in range(len(x)):
            y.append(self.calculate_digit(x[i]))
        plt.plot(x, y)
        plt.show()

    def sortMatrix(self, matrix):
        # сортировка строк по убыванию (пузырёк)
        rowMaxElem = 0
        for i in range(self.matrix_length):
            for j in range(i + 1, self.matrix_length):
                # сравнение строк матрицы
                column = 0
                while matrix[i][column] == matrix[j][column]:
                    column += 1
                    if column == self.matrix_length + 1:
                        error(errorCodes.IDENTIAL_LINES)
                        return
                if abs(matrix[i][column]) < abs(matrix[j][column]):  # по убыванию
                    matrix[i], matrix[j] = matrix[j], matrix[i]

    def calculate_gaus(self, matrix):

        self.sortMatrix(matrix)

        # зануляем всё что ниже диагонали
        for step in range(self.matrix_length - 1):
            for row in range(step + 1, self.matrix_length):
                if(matrix[step][step] == 0):
                    continue
                coefficient = matrix[row][step] / -matrix[step][step]
                for column in range(self.matrix_length + 1):
                    matrix[row][column] += round(coefficient * matrix[step][column], 10)
            self.sortMatrix(matrix)

        # находим неизвестные
        step = 0
        results = []
        for row in range(self.matrix_length - 1, -1, -1):
            count = 0
            for column in range(self.matrix_length - 1, self.matrix_length - step - 1, -1):
                matrix[row][self.matrix_length] -= matrix[row][column] * results[count]
                count += 1
            results.append(matrix[row][self.matrix_length] / matrix[row][self.matrix_length - step - 1])
            step += 1

        results.reverse()

        return np.array([[results[0]], [results[1]], [results[2]]])

    def get_step_x(self, i):
        return self.X[i + 1] - self.X[i]

    def get_step_x2(self, i):
        return self.X[i] - self.X[i - 1]

    def get_step_y(self, i):
        return self.Y[i + 1] - self.Y[i]

    def calculate(self):
        self.coefficients = self.get_coef()
        result = self.calculate_digit(self.coefficients[10])
        self.print_result(str(result))

    def calculate_digit(self, x):
        self.X = np.array(self.coefficients[0:5])
        self.Y = np.array(self.coefficients[5:10])

        C = np.empty((3, 3))
        for i in range(3):
            for j in range(3):
                if i == j:
                    C[i][j] = (self.get_step_x(i) + self.get_step_x(i + 1)) / 3
                elif i == j - 1:
                    C[i][j] = self.get_step_x(i + 1) / 6
                elif i == j + 1:
                    C[i][j] = self.get_step_x(i) / 6
                else:
                    C[i][j] = 0

        D = np.empty((3, 1))
        for i in range(3):
            D[i][0] = self.get_step_y(i + 1) / self.get_step_x(i + 1) - self.get_step_y(i) / self.get_step_x(i)

        CD = np.hstack((C, D))

        M = np.round(np.array(self.calculate_gaus(CD)), 2)
        M = np.vstack((0, M, 0))

        i = 0
        while x > self.X[i]:
            i += 1

        result = 0
        result += M[i - 1][0] * (self.X[i] - x) ** 3 / 6 / self.get_step_x2(i)
        result += M[i][0] * (x - self.X[i - 1]) ** 3 / 6 / self.get_step_x2(i)
        result += (self.Y[i - 1] - M[i - 1][0] * self.get_step_x2(i) ** 2 / 6) * (self.X[i] - x) / self.get_step_x2(i)
        result += (self.Y[i] - M[i][0] * self.get_step_x2(i) ** 2 / 6) * (x - self.X[i - 1]) / self.get_step_x2(i)

        return result

    def clear(self):
        for input in self.inputs:
            input.delete(0, END)
        self.result_label.config(text="")

    def on_open(self):
        ftypes = [('Текстовые файлы', '*.txt'), ('Все файлы', '*')]
        fold = filedialog.askopenfilename(filetypes=ftypes)
        try:
            text = self.read_file(fold)
        except FileNotFoundError:
            error(errorCodes.FILE_NOT_FOUND)
            return

        self.coefficients = text.split(" ")
        for index, input in enumerate(self.inputs):
            input.delete(0, END)
            input.insert(0, self.coefficients[index])

    def read_file(self, filename):
        with open(filename, "r") as f:
            text = f.read()
        return text

def startLab7():
    window = Tk()
    app = seventhLab(window)
    window.mainloop()