'''
Main data structures of blockchain.

Rather than fully decode the data into python object,
we access the required fields from cbor data.
It's more efficient this way at most scenario.

We also try to cache raw data, to prevent re-serialization.
'''
import cbor
import base64
import binascii
import itertools
from collections import defaultdict
from .utils import hash_serialized, hash_data, verify
from .address import Address, AddressContent
from .random import Random
from . import config

class DecodedBase(object):
    def __init__(self, data, raw=None, hash=None):
        self.data = data
        self._raw = raw
        self._hash = hash

    @classmethod
    def from_raw(cls, raw, hash=None):
        return cls(cbor.loads(raw), raw, hash)

    def raw(self):
        if not self._raw:
            self._raw = cbor.dumps(self.data)
        return self._raw

    def hash(self):
        if not self._hash:
            self._hash = hash_serialized(self.raw())
        return self._hash

class DecodedBlockHeader(DecodedBase):
    def prev_header(self):
        return self.data[1][1]

    def slot(self):
        '''
        (epoch, slotid)

        slotid: None means genesis block.
        '''
        if self.is_genesis():
            epoch = self.data[1][3][0]
            slotid = None
        else:
            epoch, slotid = self.data[1][3][0]
        return epoch, slotid

    def is_genesis(self):
        return self.data[0] == 0

    def difficulty(self):
        if self.is_genesis():
            n, = self.data[1][3][1]
        else:
            n, = self.data[1][3][2]
        return n

    def tx_count(self):
        if not self.is_genesis():
            return self.data[1][2][0][0]

    def protocol_magic(self):
        return self.data[1][0]

    def leader_key(self):
        assert not self.is_genesis()
        return self.data[1][3][1]

    def unknowns(self):
        return self.data[1][4][2]

class DecodedTransaction(DecodedBase):
    @classmethod
    def build(cls, inputs, outputs, attrs=None):
        '''
        inputs: [(txid, ix)]
        outputs: [(address, n)]
        '''
        data = (
            cbor.VarList([  # inputs
                (0, cbor.Tag(24, cbor.dumps((txid, ix))))
                for txid, ix in inputs
            ]),
            cbor.VarList([  # outputs
                (cbor.loads(Address.decode_base58(addr).encode()), c)
                for addr, c in outputs
            ]),
            attrs or {}  # attrs
        )
        return cls(data)

    def tx(self):
        from .wallet import Tx
        return Tx(self.hash(), self.inputs(), self.outputs())

    def inputs(self):
        from .wallet import TxIn
        return set(TxIn(*cbor.loads(item.value))
                   for tag, item in self.data[0]
                   if tag == 0)

    def outputs(self):
        from .wallet import TxOut
        return [TxOut(cbor.dumps(addr), c)
                for addr, c in self.data[1]]

class DecodedTxAux(DecodedBase):
    '(Tx, TxWitness)'
    @classmethod
    def build(cls, tx, witnesses):
        '''
        witnesses: [(pubkey, signature)]
        '''
        assert len(tx.inputs()) == len(witnesses)
        return cls((
            tx.data,
            [(0, cbor.Tag(24, cbor.dumps((pk, sig))))
             for pk, sig in witnesses]
        ))

    def transaction(self):
        return DecodedTransaction(self.data[0])

    def verify(self, undo):
        '''
        undo: [TxOut]
        '''
        for txin, wit, out in itertools.zip_longest(
                self.transaction().inputs(),
                self.data[1],
                undo):
            if wit[0] != 0:
                print('only support pubkey witness')
                return False
            pk, sig = cbor.loads(wit[1].value)
            if not out:
                return False
            addr = Address.decode(out.addr)
            if not addr.verify_pubkey(pk):
                return False
            if not verify('tx', pk, sig, hash_data(self.data[0])):
                print('sig verify fail')
                return False
        return True

class DecodedBlock(DecodedBase):
    def header(self):
        return DecodedBlockHeader([self.data[0], self.data[1][0]])

    def is_genesis(self):
        return self.data[0] == 0

    def transactions(self):
        assert not self.is_genesis()
        # GenericBlock -> MainBody -> [(Tx, TxWitness)]
        return [DecodedTransaction(tx) for tx, _ in self.data[1][1][0]]

    def txaux(self):
        assert not self.is_genesis()
        # GenericBlock -> MainBody -> [(Tx, TxWitness)]
        return [DecodedTxAux(data) for data in self.data[1][1][0]]

    def txs(self):
        'Transaction list in wallet format.'
        return map(lambda t: t.tx(), self.transactions())

    def utxos(self):
        'Set of inputs spent, and UTxO created.'
        from .wallet import TxIn
        txins = set()
        utxo = {}
        # Process in reversed order, so we can remove inputs spent by current block.
        for t in reversed(self.transactions()):
            tx = t.tx()
            for idx, txout in enumerate(tx.outputs):
                # new utxo
                txin = TxIn(tx.txid, idx)
                if txin not in txins:
                    utxo[txin] = txout
            for txin in tx.inputs:
                txins.add(txin)
        return txins, utxo

    def unknowns(self):
        return self.data[1][2][0]

    def leaders(self):
        assert self.is_genesis()
        return self.data[1][1]

def genesis_block(prev_header, epoch, leaders):
    'create genesis block'
    return DecodedBlock([
        0,
        [[
            config.PROTOCOL_MAGIC,
            prev_header.hash(),
            hash_data(leaders),
            [epoch, prev_header.difficulty()],
            {}
        ], leaders],
        {}
    ])

def avvm_pk(s):
    s = s.replace('_', '/').replace('-', '+')
    s = base64.b64decode(s)
    assert len(s) == 32
    return s

def genesis_balances():
    return [(AddressContent.redeem(avvm_pk(k)).address(), int(v))
            for k, v in config.GENESIS['avvmDistr'].items()] + \
        [(Address.decode_base58(k), int(v))
         for k, v in config.GENESIS['nonAvvmBalances'].items()]

def bootstrap_distr(c):
    # bootstrap
    stakeholders = config.GENESIS['bootStakeholders']
    sum_weight = sum(stakeholders.values())
    result = []
    if c < sum_weight:
        for holder, weight in stakeholders.items():
            result.append((binascii.unhexlify(holder), min(c, weight)))
            c -= weight
            if c < 0:
                break
    else:
        d, m = divmod(c, sum_weight)
        stakes = [weight * d for _, weight in stakeholders.items()]
        if m > 0:
            ix = Random(hash_data(m)).number(len(stakeholders))
            stakes[ix] += m
        result = zip(map(binascii.unhexlify, stakeholders.keys()), stakes)

    return result

def addr_stakes(addr, c):
    dist = addr.attrs.get(0)
    if dist is None:
        return bootstrap_distr(c)
    else:
        # TODO
        assert False

def genesis_stakes():
    stakes = defaultdict(int)
    for addr, c in genesis_balances():
        for holder, n in addr_stakes(addr, c):
            stakes[holder] += n
    return stakes

def fts(slotcount, seed, coinsum, stakelist):
    'if stakelist is consistant with coinsum, no exception'
    assert coinsum > 0

    # generate coin indexes.
    rnd = Random(seed)
    indices = [(slot, rnd.number(coinsum)) for slot in range(slotcount)]
    indices.sort(key=lambda t: t[1])  # sort by coin index

    leaders = []
    it = iter(stakelist)
    current_holder, upper_range = next(it)
    for slot, idx in indices:
        while True:
            if idx <= upper_range:
                leaders.append((slot, current_holder))
                break

            current_holder, c = next(it)
            upper_range += c

    assert len(leaders) == slotcount
    leaders.sort(key=lambda t: t[0])  # sort by slot index
    return [leader for _, leader in leaders]

def genesis_block0():
    'create the first genesis block from config.'
    stakes = genesis_stakes().items()
    # TODO cardano-sl used HM.toList to sort genesis stakes, so...
    leaders = fts(
        config.GENESIS['protocolConsts']['k'] * 10,
        config.GENESIS['ftsSeed'].encode(),
        sum(n for _, n in stakes),
        stakes,
    )
    return DecodedBlock([
        0,
        [[
            config.PROTOCOL_MAGIC,
            config.GENESIS_HASH,
            hash_data(leaders),
            [0, 0],
            {}
        ], leaders],
        {}
    ])

if __name__ == '__main__':
    config.use('mainnet')
    blk = genesis_block0()
    leaders = blk.leaders()

    from cardano.storage import Storage
    store = Storage('test_db')
    db_leaders = store.genesis_block(0).leaders()

    # compare with official result.
    for i in range(10):
        print(leaders[i], db_leaders[i])

{'Tx': 'wallet.Tx', 'TxIn': 'wallet.TxIn', 'TxOut': 'wallet.TxOut'}


"""Clip likes

Revision ID: f659e63d8f95
Revises: 8da406f568b8
Create Date: 2018-06-20 13:07:58.907447

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'f659e63d8f95'
down_revision = '8da406f568b8'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('comments', sa.Column('content', sa.String(), nullable=True))
    op.drop_column('comments', 'media_id')
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('comments', sa.Column('media_id', sa.VARCHAR(), autoincrement=False, nullable=True))
    op.drop_column('comments', 'content')
    # ### end Alembic commands ###


import numpy as np
import pandas as pd
from collections import Counter

DATASET_PATH = "./data/UCI HAR Dataset/"

INPUT_SIGNAL_TYPES = [
    "body_acc_x_",
    "body_acc_y_",
    "body_acc_z_",
    "body_gyro_x_",
    "body_gyro_y_",
    "body_gyro_z_",
    "total_acc_x_",
    "total_acc_y_",
    "total_acc_z_"
]

def load_x(X_signals_paths):
    X_signals = []
    for signal_type_path in X_signals_paths:
        with open(signal_type_path, "r") as f:
            X_signals.append([np.array(serie, dtype=np.float32)
                             for serie in [row.replace('  ', ' ').strip().split(' ') for row in f]])
    return np.transpose(X_signals, (1, 2, 0))   # (?, 128, 9)

def load_y(y_path):
    with open(y_path, "r") as f:
        y = np.array([elem for elem in [row.replace('  ', ' ').strip().split(' ') for row in f]],
                     dtype=np.int32)
    y = y.reshape(-1, )
    return y - 1    # (?, 1)

train_x_signals_paths = [
    DATASET_PATH + "train/Inertial Signals/" + signal + "train.txt" for signal in INPUT_SIGNAL_TYPES
]
test_x_signals_paths = [
    DATASET_PATH + "test/Inertial Signals/" + signal + "test.txt" for signal in INPUT_SIGNAL_TYPES
]

train_y_path = DATASET_PATH + "train/y_train.txt"
test_y_path = DATASET_PATH + "test/y_test.txt"

train_x = load_x(train_x_signals_paths)
test_x = load_x(test_x_signals_paths)

train_y = load_y(train_y_path)
test_y = load_y(test_y_path)

train_y_one_hot = np.asarray(pd.get_dummies(train_y), dtype=np.int8)
test_y_one_hot = np.asarray(pd.get_dummies(test_y), dtype=np.int8)

print(train_x.shape, test_x.shape)
print(train_y.shape, Counter(train_y))
print(test_y.shape, Counter(test_y))
print(train_y_one_hot.shape)
print(test_y_one_hot.shape)

np.save("./data/processed/np_train_x.npy", train_x)
np.save("./data/processed/np_train_y.npy", train_y)
np.save("./data/processed/np_train_y_one_hot.npy", train_y_one_hot)
np.save("./data/processed/np_test_x.npy", test_x)
np.save("./data/processed/np_test_y.npy", test_y)
np.save("./data/processed/np_test_y_one_hot.npy", test_y_one_hot)

# (7352, 128, 9) (2947, 128, 9)
# (7352,) Counter({5: 1407, 4: 1374, 3: 1286, 0: 1226, 1: 1073, 2: 986})
# (2947,) Counter({5: 537, 4: 532, 0: 496, 3: 491, 1: 471, 2: 420})
# (7352, 6)
# (2947, 6)


from cnnClassifier.constants import *
from cnnClassifier.components.prepare_base_model import PrepareBaseModel
from cnnClassifier import logger
from cnnClassifier.config.configuration import ConfigurationManager
import torch

STAGE_NAME = "Prepare base model"

class PrepareBaseModelTrainingPipeline:
    def __init__(self):
        pass

    def main(self):
        logger.info(f"Stage: {STAGE_NAME}")
        config_manager = ConfigurationManager()
        config = config_manager.get_prepare_base_model_config()
        prepare_base_model = PrepareBaseModel(config)
        torch.save(prepare_base_model, config.updated_base_model_path)

        logger.info(f"Updated base model saved at {config.updated_base_model_path}")

if __name__ == "__main__":
    try:
        logger.info(f">>>>> Stage {STAGE_NAME} started <<<<<")
        prepare_base_model = PrepareBaseModelTrainingPipeline()
        prepare_base_model.main()
        logger.info(f">>>>> Stage {STAGE_NAME} completed. <<<<< \n")

    except Exception as e:
        logger.exception(e)
        raise e


"""base URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/2.2/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path
from django.urls.conf import include
from core import views
from accounts import views as e

urlpatterns = [

    path('', e.authenticationUser, name='login'),
    path('home/', views.home, name='home'),
    path('example/', views.example, name='example'),
    path('auditoria/', views.auditoria, name='auditoria'),
    path('', include('fiscalizacao.urls')),
    # modificado
    path('', include('monitoramento.urls')),
    #
    path('denuncias/', views.denuncias, name='denuncias'),
    path('fraudes/', views.fraudes, name='fraudes'),
    path('lista_negativa_agr/', views.lista_negativa_agr, name='lista_negativa_agr'),
    path('dossies/', views.dossies, name='dossies'),
    path('admin/', admin.site.urls),
    path('', include('cadastro.urls')),
    path('', include('accounts.urls'))
]


import pyaudio
import numpy as np
import wave
import math
import random

from time import sleep

import pylab as pl

random.seed()

CHUNK = 256
FORMAT = pyaudio.paInt16
CHANNELS = 1
RATE = 48000
MAX_AMP = 2**15 - 1

class Audio(pyaudio.PyAudio):

    def __init__(self, channels=1,
                 rate=RATE,
                 chunk=CHUNK,
                 format=FORMAT):
        pyaudio.PyAudio.__init__(self)

        self.chan = channels
        self.rate = rate
        self.chunk = chunk
        self.format = format
        self.nptype = self.get_np_type(format)

        self.data = np.array([], dtype=self.nptype)

        self.istream = None
        self.ostream = None

        self.chunk_index = 0

    def __del__(self):
        self.terminate()

    def get_chunk(self):
        tmpstr = self.istream.read(self.chunk)
        array = np.fromstring(tmpstr, dtype=self.nptype)
        self.data = np.append(self.data, array)

    def put_chunk(self):
        slice_from = self.chunk_index*self.chunk
        slice_to = min(self.data.shape[0], slice_from + self.chunk)

        if slice_from > self.data.shape[0]:
            raise IndexError
        array = self.data[slice_from:slice_to]
        self.ostream.write(array.tostring())
        self.chunk_index += 1

    def open_input_stream(self):
        self.istream = self.open(format=self.format,
                                 channels=self.chan,
                                 rate=self.rate,
                                 input=True,
                                 frames_per_buffer=self.chunk)

    def close_input_stream(self):
        self.istream.close()
        self.istream = None

    def open_output_stream(self):
        self.ostream = self.open(format=self.format,
                                 channels=self.chan,
                                 rate=self.rate,
                                 output=True)
        self.chunk_index = 0

    def close_output_stream(self):
        self.ostream.close()
        self.ostream = None

    def record(self, time=5.0):
        self.data = np.array([], dtype=self.nptype)
        self.open_input_stream()
        print("Recording...")
        for i in range(0, int(time * self.rate/self.chunk)):
            self.get_chunk()
        print("Done Recording")
        self.close_input_stream()

    def play(self):
        self.open_output_stream()
        self.chunk_index = 0
        print("Playing...")
        while True:
            try:
                self.put_chunk()
            except IndexError:
                break

        sleep(0.4)
        self.close_output_stream()

    def save(self, path):
        raw = self.data.tostring()
        wf = wave.open(path, 'wb')
        wf.setnchannels(self.chan)
        wf.setsampwidth(self.get_sample_size(self.format))
        wf.setframerate(self.rate)
        wf.writeframes(raw)
        wf.close()

    def load(self, path):
        wf = wave.open(path, "rb")
        self.format = self.get_format_from_width(wf.getsampwidth())
        self.nptype = self.get_np_type(self.format)
        self.chan = wf.getnchannels()
        self.rate = wf.getframerate()
        self.data = np.array([], dtype=self.nptype)
        raw = wf.readframes(self.chunk)
        while raw:
            array = np.fromstring(raw, dtype=self.nptype)
            self.data = np.append(self.data, array)
            raw = wf.readframes(self.chunk)
        wf.close()

    def get_np_type(self, type):
        if type == pyaudio.paInt16:
            return np.int16

    def get_pa_type(self, type):
        if type == np.int16:
            return pyaudio.paInt16

    def add_echo(self, repeat, delay):
        length = self.data.shape[0]
        array = np.zeros(length + repeat*delay, dtype=np.float)

        for i in range(0, repeat+1):
            start = i*delay
            end = length + i*delay
            scale = 2**(i+1)
            array[start:end] += self.data / scale
        self.data = np.rint(array).astype(self.nptype)

    def rescale(self, val):
        if not 0 <= val <= 1:
            raise ValueError("Expected scaling factor between 0 and 1")

        peak = np.max(np.abs(self.data))

        rescale_factor = val*MAX_AMP/peak

        self.data = (self.data * rescale_factor).astype(self.nptype)

    def create_tone(self, frequency, length, amplitude):
        if not 0 <= amplitude <= 1:
            raise ValueError("Expected amplitude between 0 and 1")

        s = np.zeros(length, self.nptype)

        for i in range(0, length):
            s[i] = amplitude * MAX_AMP \
                   * math.sin(frequency * i * 2 * math.pi/self.rate)

        self.data = s

    def create_noise(self, length, amplitude):

        if not 0 <= amplitude <= 1:
            raise ValueError("Expected amplitude between 0 and 1")

        s = np.zeros(length, self.nptype)

        for i in range(0, length):
            s[i] = amplitude * MAX_AMP * random.random()

        self.data = s

    def add(self, other):
        length = max(self.data.shape[0], other.data.shape[0])
        array = np.zeros(length, dtype=self.nptype)
        array += self.data / 2.0
        array += other.data / 2.0
        self.data = array

    def __len__(self):
        return self.data.shape[0]

    def get_samplerange(self):
        if self.nptype == np.int16:
            return math.pow(2, 16)

    def compute_fft(self, start, end):
        dur = end - start
        fft = pl.fft(self.data[start:end])
        real_range = np.ceil((dur+1)/2.0)
        fft = fft[0:real_range]
        fft = abs(fft)

        return fft * np.hanning(len(fft))

    def change_speed(self, factor):
        indxs = np.round(np.arange(0, len(self.data), factor))
        indxs = indxs[indxs < len(self.data)].astype(int)
        self.data = self.data[indxs]

    def time_stretch_fft(self, factor, windowsize=1024, overlap=512, apply_hanning=True):
        phase = np.zeros(windowsize)
        if apply_hanning:
            amp_window = np.hanning(windowsize)
        else:
            amp_window = np.ones(windowsize, dtype=np.float)
        result = np.zeros(int(len(self.data) / factor + windowsize))

        for i in np.arange(0, len(self.data)-(windowsize+overlap), overlap*factor, dtype=np.int):
            a1 = self.data[i: i + windowsize]
            a2 = self.data[i + overlap: i + windowsize + overlap]

            s1 = np.fft.fft(amp_window * a1)
            s2 = np.fft.fft(amp_window * a2)
            phase = (phase + np.angle(s2/s1)) % 2*np.pi
            a2_rephased = np.fft.ifft(np.abs(s2)*np.exp(1j*phase))

            i2 = int(i/factor)
            result[i2:i2 + windowsize] += amp_window*np.real(a2_rephased)
        result = ((2**(16-4)) * result/result.max())

        self.data = result.astype(self.nptype)

    def plot_waveform(self, start=0, end=-1, x_unit="samples"):
        array = self.data[start:end]
        num_samples = len(array)
        if x_unit == "samples":
            pl.plot(range(num_samples), array)
            pl.xlabel('Time (Samples)')
        elif x_unit == "time":
            end_time = self.samples_to_time(num_samples)
            y_steps = np.arange(0, end_time, float(end_time) / num_samples)
            pl.plot(y_steps, array)
            pl.xlabel('Time (s)')
        pl.ylabel('Amplitude')
        samplerange = self.get_samplerange()
        pl.ylim([-samplerange/2, samplerange/2])
        pl.show()

    def plot_spectrum(self, array, start=0, end=-1, plot_log=False):
        array = array[start:end]
        len_arr = len(array)
        freq_axis = np.arange(0, len_arr, 1.0)  # * (self.rate / len_arr)
        if plot_log:
            pl.plot(freq_axis/1000, 10*np.log10(array), color='k')
            pl.ylabel('Power (dB)')
        else:
            pl.plot(freq_axis/1000, array, color='k')
        pl.xlabel('Frequency (kHz)')
        pl.show()

def sum_audio(audio_objects):
    length = len(max(audio_objects, key=len))
    scale = 1.0/len(audio_objects)

    array = np.zeros(length, dtype=np.int16)

    for obj in audio_objects:
        array += np.rint(obj.data * scale).astype(np.int16)

    new_object = Audio()
    new_object.data = array

    return new_object

def test_add():
    c = Audio()
    e = Audio()
    g = Audio()

    c.create_tone(261.63, 240000, 0.8)
    e.create_tone(329.63, 240000, 0.8)
    g.create_tone(392.00, 240000, 0.8)

    chord = sum_audio((c, e, g))

    chord.play()

if __name__ == "__main__":
    d = Audio()
    d.load('kdt48.wav')
    d.time_stretch_fft(0.9, 4096, 256)
    d.play()


from datetime import datetime

from solr import SolrConnection

from datahub.core import app
from datahub.util import datetime_add_tz

def connection():
    return SolrConnection(app.config['SOLR_URL'],
                          http_user=app.config.get('SOLR_USER'),
                          http_pass=app.config.get('SOLR_PASSWORD'))

def site_id():
    return app.config.get('SITE_ID', 'datahub.local')

def to_key(entity):
    return 'datahub/%s//%s' % (entity.__tablename__, entity.id)

def flatten_dict(d, prefix='meta', sep='_'):
    """ Flatten a dict to a list of values with each key
    joined to `prefix` by `sep`. """
    flat = {}
    for k, v in d.items():
        key = "".join([prefix, sep, k])
        if isinstance(v, dict):
            flat.update(flatten_dict(v, prefix=key, sep=sep))
        else:
            flat[key] = v
    return flat

def index_add(entity):
    """ Add an SQLAlchemy-controlled entity to the elastic search index
    """
    if not hasattr(entity, '__tablename__'):
        raise TypeError('Can only index entities with a __tablename__')
    if not hasattr(entity, 'to_dict'):
        raise TypeError('Can only index entities with a to_dict')
    conn = connection()
    data = entity.to_dict()
    data['_key'] = to_key(entity)
    data['site_id'] = site_id()
    data['doc_type'] = entity.__tablename__
    flat_data = {}
    for k, v in data.items():
        if isinstance(v, dict):
            flat_data.update(flatten_dict(v, prefix=k))
        else:
            flat_data[k] = v
    data = flat_data
    for k, v in data.items():
        if isinstance(v, datetime):
            data[k] = datetime_add_tz(v)
    #from pprint import pprint
    #pprint(data)
    try:
        conn.add_many([data])
        conn.commit()
    finally:
        conn.close()

def index_delete(entity):
    """ Deleta an SQLAlchemy-controlled entity from the elastic search
    index, catching any NotFoundExceptions. """
    if not hasattr(entity, '__tablename__'):
        raise TypeError('Can only index entities with a __tablename__')
    conn = connection()
    try:
        conn.delete_query('+_key:"%s" AND +site_id:"%s"' % (to_key(entity),
            site_id()))
        conn.commit()
    finally:
        conn.close()

def reset_index():
    """ Delete all entries from the index. """
    conn = connection()
    try:
        conn.delete_query('+site_id:"%s"' % site_id())
        conn.commit()
    finally:
        conn.close()


import requests
from bs4 import BeautifulSoup
import csv

URL = 'https://www.detmir.ru/catalog/index/name/lego/'

HEADERS = {'user-agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:89.0) Gecko/20100101 Firefox/89.0',
           'accept': '*/*'}
HOST = 'https://www.detmir.ru/'
FILE = 'legos.csv'

def get_html(url, params=None):
    r = requests.get(url, headers=HEADERS, params=params)
    return r

def get_pages_count(html):
    soup = BeautifulSoup(html, 'html.parser')
    pagination = soup.find_all('span', class_= 'mhide')
    if pagination:
        return int(pagination[-1].get_text())
    else:
        return 1

def get_content(html):
    soup = BeautifulSoup(html, 'html.parser')
    items = soup.find_all('div', class_='be80pr-1') # Здесь обращаемся к объекту суп и ищем элементы.
    cards = []

    for item in items:
        cards.append(
            {
                'title': item.find('a', class_='cpshbz-0').get_text(strip=True), # Берем заголовок карточки
                'price': item.find('span', class_='price').get_text(strip=True),
                'city': item.find('span', class_='item region').get_text(
                    strip=True),
                'promo': item.find('div', class_='promo').get_text(),
                'link': HOST + item.find('div', class_='link').find('a').get('href'), # Получаем ссылку
            }
        )
    return cards

def save_file(items, path):
    with open(path, 'w', newline='') as file:
        writer = csv.writer(file, delimiter=';')
        writer.writerow(['Наименование', 'Цена', 'Город', 'Промо цена',
                          'Ссылка на страницу с товаром'])
        for item in items:
            writer.writerow([item['title'], item['price'], item['city'],
                             item['promo'], item['link']])

def parser():
    html = get_html(URL)
    if html.status_code == 200:
        legos = []
        print(get_content(html.text))
        pages = get_pages_count(html.text)
        for page in range(1, pages + 1):
            html = get_html(URL, params={'pages': page})
            legos.extend(get_content(html.text))
            if len(legos) > 5:
                save_file(legos, FILE)
            else:
                print('Количество элементов меньше 500!')
        print(legos)
    else:
        print('Error')

parser()


from collections import deque

n, m = map(int, input().split())

k = int(input())

visit = [False] * (k + 1)
horizon_bus = []
vertical_bus = []
bus = [[0]]
isHorizon = [False] * (k + 1)
for _ in range(k):
    b, x1, y1, x2, y2 = list(map(int, input().split()))
    if x1 > x2:
        x2, x1 = x1, x2
    if y1 > y2:
        y2, y1 = y1, y2

    if x1 - x2:
        vertical_bus.append([b, y1, x1, x2])
    else:
        horizon_bus.append([b, x1, y1, y2])
        isHorizon[b] = True
    bus.append([b, x1, y1, x2, y2])
bus.sort(key=lambda x: x[0])
dest = list(map(int, input().split()))
start = dest[:2]
end = dest[2:]

queue = deque()
answer = 0
for index, y, x1, x2 in vertical_bus:
    if start[1] == y and (x1 <= start[0] <= x2):
        queue.append([1, index, start[0], start[1]])
        visit[index] = True
for index, x, y1, y2 in horizon_bus:
    if start[0] == x and (y1 <= start[1] <= y2):
        queue.append([1, index, start[0], start[1]])
        visit[index] = True

def solve():
    while queue:
        count, thisBus, thisX, thisY = queue.popleft()
        thisBusData = bus[thisBus]
        if isHorizon[thisBus]:
            if thisX == end[0] and thisBusData[2] <= end[1] <= thisBusData[4]:
                return count
        else:
            if thisY == end[1] and thisBusData[1] <= end[0] <= thisBusData[3]:
                return count

        for nextBus, nextY, nextX1, nextX2 in vertical_bus:
            if visit[nextBus]:
                continue
            if isHorizon[thisBus]:
                if (nextX1 <= thisX <= nextX2) and (thisBusData[2] <= nextY <= thisBusData[4]):
                    queue.append([count + 1, nextBus, thisX, nextY])
                    visit[nextBus] = True
            else:
                if thisY == nextY and ((thisBusData[3] <= nextX1) != (thisBusData[1] <= nextX2)):
                    queue.append([count + 1, nextBus, nextX1, thisY])
                    visit[nextBus] = True

        for nextBus, nextX, nextY1, nextY2 in horizon_bus:
            if visit[nextBus]:
                continue
            if not isHorizon[thisBus]:
                if (nextY1 <= thisY <= nextY2) and (thisBusData[1] <= nextX <= thisBusData[3]):
                    queue.append([count + 1, nextBus, nextX, thisY])
                    visit[nextBus] = True

            else:
                if thisX == nextX and ((thisBusData[4] <= nextY1) != (thisBusData[2] <= nextY2)):
                    queue.append([count + 1, nextBus, thisX, nextY1])
                    visit[nextBus] = True

print(solve())

백준/Gold/2536. 버스 갈아타기/버스 갈아타기.py


#!/usr/bin/python

from __future__ import print_function
import argparse

parser = argparse.ArgumentParser(description="Using information found in the AHL Data spreadsheet, this creates several PED files that assign phenotypes for the various pathogens or conditions that were tested for. Healthy animals are controls, animals that test positive are cases, and diseased animals that were not tested or test negative are not considered.", epilog = "RS Fraser, April 25, 2017")

parser.add_argument("ped_file", help="The original PED file that is used as a template")
parser.add_argument("ahl_file", help="The AHL file containing the pathogen/condition data. Pigs as columns, conditions as rows. Test positive = 1; anything else is set as a phenotype to ignore")

args = parser.parse_args()

ped = open(args.ped_file, 'r')
ahl = open(args.ahl_file, 'r')

x_dict = {}

for line in ahl:

	line = line.rstrip()
	if "animal_id" in line:
		pig_id_keys = line.split("\t")

	else:
		line_fields = line.split("\t")
		rsid_dict = line_fields[0] + "_dict"
		rsid_dict = {}

		for i in range(len(line_fields)):
			if i == 0:
				rsid = line_fields[i]
			else:
				rsid_dict[pig_id_keys[i]] = line_fields[i]

		x_dict[rsid] = rsid_dict

for k,v in x_dict.items():

	filename_str = k + ".ped"
	filename = open(filename_str, 'w')

	for ped_line in ped:

		ped_line = ped_line.rstrip()
		ped_line_fields = ped_line.split("\t")

		if "-" not in ped_line:
			print(ped_line, file = filename)

		else:

			for k2, v2 in x_dict[k].items():

				if ped_line_fields[0] == k2:

					for i in range(5):
						print(ped_line_fields[i], sep="\t", end="\t", file = filename)

					if "1" in x_dict[k][k2]:
						print("2", sep="", end="\t", file = filename)

					else:
						print("-9", end="\t", file = filename)

					for i in range(6, len(ped_line_fields)):
						print(ped_line_fields[i], sep="\t", end="\t", file = filename)

					print("", file = filename)

	#holy shit did it take awhile to debug this bastard. The issue with this nested for loop is that once the ped file has been iterated over, python doesn't reset it back to the start... so the first key value from teh dictionary would iterate over the entire ped file, and the ped file would be at it's "end". The seek function resets the file's position to the defined value - thus, for every key iteration, it starts the ped file back at the start. Whoosh....
	ped.seek(0)


import json
import random
import uuid
import time
import argparse
import sys
from concurrent.futures import ThreadPoolExecutor

# import the logging library
import logging

# import the boto3 library
import boto3
from botocore.exceptions import ClientError

# create a function that parses json strings and converts them to a dictionary
def parse_json(json_string):
    return json.loads(json_string)

# create a function to create a list of account ids
def create_account_ids():
    account_ids = []
    for i in range(0, 10):
        account_ids.append(str(uuid.uuid4()))
    logging.debug(account_ids)
    return account_ids

# create a function to create a list of site ids for each account
def create_site_ids(account_ids):
    site_ids = {}
    for account_id in account_ids:
        site_ids[account_id] = []
        for i in range(0, 20):
            site_ids[account_id].append(str(uuid.uuid4()))
    logging.debug(site_ids)
    return site_ids

# create a function to create a list of queue ids from a list of site ids
def create_queue_ids(site_ids):
    queue_ids = {}
    for site_id in site_ids:
        queue_ids[site_id] = []
        for i in range(0, 30):
            queue_ids[site_id].append(str(uuid.uuid4()))
    logging.debug(queue_ids)
    return queue_ids

# create a function that generates a dictionary of random data with the schema in ./schema/acceptance.json
def generate_data(schema_file, account_ids, site_ids, queue_ids):
    # create a dictionary to hold the data
    data = {}

    # open the schema file and read the contents
    with open(schema_file, 'r') as s:
        schema = parse_json(s.read())
        logging.debug(schema)

    # iterate over the schema and generate random data
    for key, value in schema.items():
        if value == 'integer':
            data[key] = random.randint(1, 20)
        elif value == 'timestamp':
            data[key] = time.strftime('%Y-%m-%d %H:%M:%S')
        elif value == 'boolean':
            data[key] = bool(random.getrandbits(1))
        elif value == 'special':
            if key == 'account_id':
                # random selection from a list of account ids
                data[key] = random.choice(account_ids)
            elif key == 'site_id':
                # random selection from a list of site ids
                data[key] = random.choice(site_ids[data['account_id']])
            elif key == 'queue_id':
                # random selection from a list of queue ids
                data[key] = random.choice(queue_ids[data['site_id']])
            elif key == 'media':
                data[key] = random.choice(['phone', 'chat', 'other'])
            elif key == 'outcome':
                data[key] = random.choice(['finished', 'escalated', 'transferred'])
            elif key == 'environment_name':
                data[key] = random.choice(['acceptance'])
        # uid
        elif value == 'uuid':
            data[key] = str(uuid.uuid4())
        else:
            logging.warning("Unknown key or value - %s:%s.", key, value)

    # return the data
    logging.debug(data)
    return data

# create a function that connects to AWS MSK and sends the data
# def send_data_msk(data, stream_name, partition_key):
#         # create a client for the kinesis service
#         client = boto3.client('kafka')
#         logging.debug(client)

#         # create a json string from the data
#         json_string = json.dumps(data)

#         response = None
#         try:
#             response = client.send_command(
#                 ClusterArn=stream_name,
#                 Command='SEND-RAW-MESSAGE',
#                 Data=json_string,
#                 PartitionKey=partition_key)
#         except ClientError as e:
#             print(e)
#             logging.exception("Couldn't put record in stream %s.", stream_name)
#             raise
#         logging.debug(response)
#         return response

# create a function that connects to AWS Kinesis and sends the data
def send_data_kinesis(data, stream_name, partition_key, region_name):

    # Create a kinesis client with the cluster arn
    client = boto3.client(
        'kinesis',
        region_name=region_name
    )

    logging.debug(client)

    # create a json string from the data
    json_string = json.dumps(data)

    response = None
    try:
        response = client.put_record(
            StreamName=stream_name,
            Data=json_string,
            PartitionKey=partition_key)
    except ClientError as e:
        print(e)
        logging.exception("Couldn't put record in stream %s.", stream_name)
        raise
    logging.debug(response)
    return response

# create a usage function
def usage():
    print("Usage: python3 datagenerator.py --stream-type <kinesis> \
                                           --stream-name <stream name> \
                                           --partition-key <partition key> \
                                           --region-name <region name> \
                                           --schema-file <schema file> \
                                           --records <number of records> \
                                           --threads <number of threads>")
    exit(1)

# function for the thread pool
def execute(args, account_ids, site_ids, queue_ids):
    # create a counter for the number of records sent
    records_sent = 0

    # create a loop that generates data and sends it to AWS Kinesis
    while records_sent < args.records:
        # generate some data
        data = generate_data(
            args.schema_file,
            account_ids,
            site_ids,
            queue_ids
        )

        # Kinesis or MSK?
        if args.stream_type == 'kinesis':
            # send the data
            response = send_data_kinesis(data, args.stream_name, args.partition_key, args.region_name)
        elif args.stream_type == 'msk':
            # send the data
            #response = send_data_msk(data, args.stream_name, args.partition_key)
            pass

        # log the response
        logging.info(response)

        # increment the counter
        records_sent += 1

# main function
def main():

    # create an argument parser
    parser = argparse.ArgumentParser()

    # add the argument for the number of records to generate
    parser.add_argument('--records', type=int, default=1000)

    # add the argument for the pause between records
    parser.add_argument('--pause', type=int, default=0)

    # add the argument for the log level
    parser.add_argument('--log-level', type=str, default='INFO')

    # add the argument for the log file
    parser.add_argument('--log-file', type=str, default='datagenerator.log')

    # add the argument for the log format
    parser.add_argument('--log-format', type=str, default='%(asctime)s %(thread)d %(levelname)s %(message)s')

    # add the argument for the log date format
    parser.add_argument('--log-date-format', type=str, default='%Y-%m-%d %H:%M:%S')

    # add an argument for stream name
    parser.add_argument('--stream-name', type=str, default='csa-stream')

    # add an argument for partition key
    parser.add_argument('--partition-key', type=str, default='account_id')

    # add an argument for the schema file
    parser.add_argument('--schema-file', type=str, default='./schema/acceptance.json')

    # add an argument for the number of threads
    parser.add_argument('--threads', type=int, default=100)

    # add an argument for the stream type
    parser.add_argument('--stream-type', type=str, default='kinesis')

    # add an argument for the region name
    parser.add_argument('--region-name', type=str, default='us-east-1')

    # print the usage if help is requested
    if '--help' in sys.argv or '-h' in sys.argv:
        usage()

    # parse the arguments
    args = parser.parse_args()

    # update the code below to use the arguments
    # set the log level
    logging.basicConfig(filename=args.log_file, level=args.log_level, format=args.log_format, datefmt=args.log_date_format)

    # generate a fixed list of account ids
    account_ids = create_account_ids()

    # generate a fixed list of site ids for each account
    site_ids = create_site_ids(account_ids)

    # extract the site ids from the site_ids dictionary
    site_ids_list = [site_id for account_id in site_ids for site_id in site_ids[account_id]]
    logging.debug(site_ids_list)

    # generate a fixed list of queue ids for each site
    queue_ids = create_queue_ids(site_ids_list)

     # Make things concurrent now. Implement thread pool
    futures = []
    with ThreadPoolExecutor(max_workers=args.threads) as executor:
        futures.append(executor.submit(execute, args, account_ids, site_ids, queue_ids))

    # wait for the futures to complete
    for future in futures:
        future.result()

# call the main function
if __name__ == '__main__':
    main()


import tensorflow as tf
import numpy as np
from tensorflow import keras
from keras import backend as K

import os
from keras.models import Model, model_from_json, load_model
from keras.utils.vis_utils import plot_model

from keras.losses import Loss

import utils

xs = np.array([-1.0, 0.0, 1.0, 2.0, 3.0, 4.0], dtype=float)
ys = np.array([-3.0, -1.0, 1.0, 3.0, 5.0, 7.0], dtype=float)

model = tf.keras.Sequential([keras.layers.Dense(units=1, input_shape=[1])])
model.compile(optimizer='sgd', loss='mean_squared_error')
model.fit(xs, ys, epochs=500, verbose=0)
print("optimizer='sgd', loss='mean_squared_error': ", model.predict([10.0]))

def my_huber_loss_with_threshold(threshold=1):
    def my_huber_loss(y_true, y_pred):
        error = y_true - y_pred
        is_small_error = tf.abs(error) <= threshold
        small_error_loss = tf.square(error) / 2
        big_error_loss = threshold * (tf.abs(error) - (0.5 * threshold))
        return tf.where(is_small_error, small_error_loss, big_error_loss)

    return my_huber_loss

def my_huber_loss(y_true, y_pred):
    threshold = 1
    error = y_true - y_pred
    is_small_error = tf.abs(error) <= threshold
    small_error_loss = tf.square(error) / 2
    big_error_loss = threshold * (tf.abs(error) - (0.5 * threshold))
    return tf.where(is_small_error, small_error_loss, big_error_loss)

model = tf.keras.Sequential([keras.layers.Dense(units=1, input_shape=[1])])
model.compile(optimizer='sgd', loss=my_huber_loss)
model.fit(xs, ys, epochs=500, verbose=0)
print("optimizer='sgd', loss=my_huber_loss: ", model.predict([10.0]))

class MyHuberLoss(Loss):
    threshold = 1

    def __init__(self, threshold):
        super().__init__()
        self.threshold = threshold

    def call(self, y_true, y_pred):
        error = y_true - y_pred
        is_small_error = tf.abs(error) <= self.threshold
        small_error_loss = tf.square(error) / 2
        big_error_loss = self.threshold * (tf.abs(error) - (0.5 * self.threshold))
        return tf.where(is_small_error, small_error_loss, big_error_loss)

    def get_config(self):
        return {'threshold': self.threshold}
    #
    # @classmethod
    # def from_config(cls, config):
    #     return cls(**config)

# Set options for model save/load.
MUST_SAVE_THE_MODEL = True
MUST_LOAD_MODEL = True

# SAVE_LOAD_MODEL_METHOD = 'model.to_json'
SAVE_LOAD_MODEL_METHOD = 'model.save'

MODEL_DIRECTORY = 'Data\Huber_loss_function\Models'
MODEL_FILE = os.path.join(MODEL_DIRECTORY, 'saved_model.pb')
WEIGHTS_FILE = os.path.join(MODEL_DIRECTORY, 'model_weights.h5')

OPTIMIZER = 'sgd'
LOSS = MyHuberLoss(threshold=1)
# METRICS = ['accuracy']

EPOCHS = 500
# BATCH_SIZE = 32

# Load or create model.
if MUST_LOAD_MODEL and os.path.exists(MODEL_FILE):
    # Load saved model if it exists.

    if SAVE_LOAD_MODEL_METHOD == 'model.to_json':
        json_file = open(MODEL_FILE, 'r')
        loaded_model_json = json_file.read()
        json_file.close()
        model = model_from_json(loaded_model_json)
        print("Model was load from disk!")

        model.load_weights(WEIGHTS_FILE)
        print("Weights loaded!")
        # model.compile(optimizer=OPTIMIZER, loss=LOSS, metrics=METRICS)
    else:
        model = load_model(MODEL_FILE,
                           custom_objects={'MyHuberLoss': MyHuberLoss})
        print("Model was load from disk!")

    plot_model(model, show_shapes=True, show_layer_names=True,
               to_file=os.path.join(MODEL_DIRECTORY, 'loaded-model.png'))
    print(model.summary())
    print("optimizer='sgd', loss=MyHuberLoss(threshold=1): ", model.predict([10.0]))

else:
    # Creating model

    model = tf.keras.Sequential([keras.layers.Dense(units=1, input_shape=[1])])
    model.compile(optimizer=OPTIMIZER, loss=LOSS)
    print("Model was created!")

    try:
        os.makedirs(MODEL_DIRECTORY)
    except:
        pass
    plot_model(model, show_shapes=True, show_layer_names=True,
               to_file=os.path.join(MODEL_DIRECTORY, 'model.png'))
    print(model.summary())

    # TRAIN model
    history = model.fit(xs, ys, epochs=EPOCHS, verbose=0)
    print("optimizer='sgd', loss=MyHuberLoss(threshold=1): ", model.predict([10.0]))
    print('history: ', history)

    # SAVING model and weights (if necessary)
    if MUST_SAVE_THE_MODEL:
        if SAVE_LOAD_MODEL_METHOD == 'model.to_json':
            model_json = model.to_json()
            with open(MODEL_FILE, 'w') as json_file:
                json_file.write(model_json)
                print("Model saved!")
                json_file.close()
            model.save_weights(WEIGHTS_FILE)
            print("Weights saved!")
        else:
            model.save(MODEL_FILE)
            print("Model saved!")

def my_rmse(y_true, y_pred):
    error = y_true - y_pred
    sqr_error = K.square(error)
    mean_sqr_error = K.mean(sqr_error)
    sqrt_mean_sqr_error = K.sqrt(mean_sqr_error)
    return sqrt_mean_sqr_error


# ==========================================================
# Author: Ankush Gupta, Tomas Jakab
# ==========================================================
from __future__ import print_function
from __future__ import absolute_import

from tensorflow.contrib.framework.python.ops import variables
import tensorflow as tf
import os.path as osp

# network definition:
from imm.models.imm_model import IMMModel
from imm.utils.box import Box

import imm.train.cnn_train_multi as tru
from imm.utils.colorize import colorize

import metayaml
from imm.utils.dataset_import import import_dataset
"""
So the main steps are:
  1. create the dataset object
  2. get a model factory
  3. build the training/summary ops
  4. run the training loop.
"""

class model_factory():
  """
  Factory which can be used to
  instantiate models.
  """
  def __init__(self, network, **kwargs):
    self.network = network
    self.net_args = kwargs

  def create(self):
    return self.network(**self.net_args)

def load_configs(file_names):
  """
  Loads the yaml config files.
  """
  # with open(file_name, 'r') as f:
  #   config_str = f.read()
  # config = Box.from_yaml(config_str)
  config = Box(metayaml.read(file_names))
  return config

def main(args):
  config = load_configs(args.configs)
  train_config = config.training
  gpus = range(args.ngpus)

  # get the data and logging (checkpointing) directories:
  data_dir = train_config.datadir
  log_dir = train_config.logdir

  SUBSET = 'train'
  NUM_STEPS = 30000000
  # value at which the gradients are clipped
  GRAD_CLIP = train_config.gradclip

  if args.checkpoint is not None:
    checkpoint_fname = args.checkpoint
  else:
    print(colorize('No checkpoint file specified. Initializing randomly.','red',bold=True))
    checkpoint_fname = osp.join(log_dir,'INVALID')

  opts = {}
  opts['gpu_ids'] = gpus
  opts['log_dir'] = log_dir
  opts['n_summary'] = 10 # number of iterations after which to run the summary-op
  if hasattr(train_config,'n_test'):
    opts['n_test'] = train_config.n_test
  else:
    opts['n_test'] = 500
  opts['n_checkpoint'] = train_config.ncheckpoint # number of iteration after which to save the model

  batch_size = train_config.batch
  graph = tf.Graph()
  with graph.as_default():
    global_step = variables.model_variable('global_step',shape=[],
                                            initializer=tf.constant_initializer(args.reset_global_step),
                                            trainable=False)

    # common model / optimizer parameters:
    lr = args.lr_multiple * tf.train.exponential_decay(train_config.lr.start_val,
                                global_step,
                                train_config.lr.step,
                                train_config.lr.decay,
                                staircase=True)
    if train_config.optim.lower() == 'adam':
      optim = tf.train.AdamOptimizer(lr, name='Adam')
    elif train_config.optim.lower() == 'adadelta':
      optim = tf.train.AdadeltaOptimizer(lr, rho=0.95,epsilon=1e-06,use_locking=False,name='Adadelta')
    elif train_config.optim.lower() == 'adagrad':
      optim = tf.train.AdagradOptimizer(lr, use_locking=False,name='AdaGrad')
    else:
      raise ValueError('Optimizer = %s not suppoerted'%train_config.optim)

    factory = model_factory(IMMModel,
                            config=config.model,
                            global_step=global_step)

    opts['batch_size'] = batch_size
    tf.summary.scalar('lr', lr) # add a summary
    print(colorize('log_dir: ' + log_dir,'green',bold=True))
    print(colorize('BATCH-SIZE: %d'%batch_size,'red',bold=True))

    # dynamic import of a dataset class
    dset_class = import_dataset(train_config.dset)

    # default datasets parameters
    train_dset_params = {}
    test_dset_params = {}

    train_subset = 'train'
    test_subset = 'test'
    if hasattr(train_config, 'train_dset_params'):
      train_dset_params.update(train_config.train_dset_params)
      if 'subset' in train_dset_params:
        train_subset = train_dset_params['subset']
        # delete because not positional kwarg
        del train_dset_params['subset']
    if hasattr(train_config, 'test_dset_params'):
      test_dset_params.update(train_config.test_dset_params)
      if 'subset' in test_dset_params:
        test_subset = test_dset_params['subset']
        # delete because not positional kwarg
        del test_dset_params['subset']

    train_dset = dset_class(train_config.datadir, subset=train_subset,
                            **train_dset_params)
    train_dset = train_dset.get_dataset(batch_size, repeat=True, shuffle=False,
                                        num_preprocess_threads=12)

    if hasattr(train_config, 'max_test_samples'):
      raise ValueError('max_test_samples attribute deprecated')
    test_dset = dset_class(train_config.datadir, subset=test_subset,
                           **test_dset_params)
    test_dset = test_dset.get_dataset(batch_size, repeat=False, shuffle=False,
                                      num_preprocess_threads=12)

    # set up inputs
    training_pl = tf.placeholder(tf.bool)
    handle_pl = tf.placeholder(tf.string, shape=[])
    base_iterator = tf.data.Iterator.from_string_handle(
      handle_pl, train_dset.output_types, train_dset.output_shapes)
    inputs = base_iterator.get_next()

    split_gpus = False
    if hasattr(config.model, 'split_gpus'):
      split_gpus = config.model.split_gpus

    # create the network distributed over multi-GPUs:
    loss, train_op, train_summary_op, test_summary_op, _ = tru.setup_training(
      opts, graph, optim, inputs, training_pl, factory, global_step,
      clip_value=GRAD_CLIP, split_gpus=split_gpus)

    # run the training loop:
    if args.restore_optim:
      restore_vars = 'all'
    else:
      restore_vars = 'model'

    tru.train_loop(opts, graph, loss, train_dset, training_pl, handle_pl,
                   train_op, train_summary_op, test_summary_op, NUM_STEPS,
                   global_step, checkpoint_fname,
                   test_dataset=test_dset,
                   ignore_missing_vars=args.ignore_missing_vars,
                   reset_global_step=args.reset_global_step,
                   vars_to_restore=restore_vars,
                   exclude_vars=[],
                   allow_growth=train_config.allow_growth)

if  __name__=='__main__':
  import argparse
  parser = argparse.ArgumentParser(description='Train Unsupervised Sequence Model')
  parser.add_argument('--configs', nargs='+', default=[], help='Paths to the config files.')
  parser.add_argument('--ngpus',type=int,default=1,required=False,help='Number of GPUs to use for training.')
  parser.add_argument('--lr-multiple',type=float,default=1,help='multiplier on the learning rate.')
  parser.add_argument('--checkpoint',type=str,default=None,
                      help='checkpoint file-name of the *FULL* model to restore.')
  parser.add_argument('--restore-optim',action='store_true',help='Restore the optimizer variables.')
  parser.add_argument('--reset-global-step',type=int,default=-1,help='Force the value of global step.')
  parser.add_argument('--ignore-missing-vars',action='store_true',help='Skip re-storing vars not in the checkpoint file.')
  args = parser.parse_args()
  main(args)


import requests
from bs4 import BeautifulSoup

def getOriginalAspectRatio(title, imdb_number=None):
    BASE_URL = "https://www.imdb.com/"
    HEADERS = {
        'User-Agent': 'Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148'}

    if imdb_number:
        URL = "{}/title/{}/".format(BASE_URL, imdb_number)
    else:
        URL = BASE_URL + "find/?q={}".format(title)
        search_page = requests.get(URL, headers=HEADERS)

        # lxml parser would have been better but not currently supported in Kodi
        soup = BeautifulSoup(search_page.text, 'html.parser')

        title_url_tag = soup.select_one(
            '.ipc-metadata-list-summary-item__t')
        if title_url_tag:
            # we have matches, pick the first one
            title_url = title_url_tag['href']
            imdb_number = title_url.rsplit(
                '/title/', 1)[-1].split("/")[0]
            # this below could have worked instead but for some reason SoupSieve not working inside Kodi
            """title_url = soup.css.select(
                '.ipc-metadata-list-summary-item__t')[0].get('href')
                """

            URL = BASE_URL + title_url

    title_page = requests.get(URL, headers=HEADERS)
    soup = BeautifulSoup(title_page.text, 'html.parser')

    # this below could have worked instead but for some reason SoupSieve not working inside Kodi
    aspect_ratio_tags = soup.find(
        attrs={"data-testid": "title-techspec_aspectratio"})
    if aspect_ratio_tags:
        aspect_ratio_full = aspect_ratio_tags.select_one(
            ".ipc-metadata-list-item__list-content-item").decode_contents()

        """aspect_ratio_full = soup.find(
            attrs={"data-testid": "title-techspec_aspectratio"}).css.select(".ipc-metadata-list-item__list-content-item")[0].decode_contents()
            """

        if aspect_ratio_full:
            aspect_ratio = aspect_ratio_full.split(':')[0].replace('.', '')
    else:
        # check if video has multiple aspect ratios
        URL = "{}/title/{}/technical/".format(BASE_URL, imdb_number)
        tech_specs_page = requests.get(URL, headers=HEADERS)
        soup = BeautifulSoup(tech_specs_page.text, 'html.parser')
        aspect_ratio_li = soup.select_one("#aspectratio").find_all("li")
        if len(aspect_ratio_li) > 1:
            aspect_ratios = []

            for li in aspect_ratio_li:
                aspect_ratio_full = li.select_one(
                    ".ipc-metadata-list-item__list-content-item").decode_contents()
                aspect_ratio = aspect_ratio_full.split(':')[0].replace('.', '')
                aspect_ratios.append(aspect_ratio)

            return aspect_ratios

    return aspect_ratio


from datetime import date, datetime, timedelta, time
import inflect
import sys
p = inflect.engine()

def main():
    fecha_birth = input("Date of birth: ")

    try:
        fecha_birth2 = check_date(fecha_birth)
        if fecha_birth2 == None:
            raise sys.exit('Invalid date')
    except:
        raise sys.exit('Invalid date')

    date_hoy = datetime.combine(date.today(), time(hour=0, minute=0, second=0, microsecond=0))
    time_lapse = date_hoy - fecha_birth2
    minutes = time_lapse / timedelta(minutes=1)
    minutes = int(minutes)
    text = p.number_to_words(minutes, andword="")
    print (text.capitalize(), "minutes")
    #output = p.number_to_words(minutes)
    #print(output.capitalize(), "minutes")

def check_date(birth):
    try:
        birth2 = datetime.strptime(birth, '%Y-%m-%d')
        #print (type(birth2))
        return birth2
    except:
        return None

if __name__ == "__main__":
    main()


"""Emotion model module."""
import random

import pytorch_lightning as pl
import torch
import torch.nn as nn
from torchaudio.transforms import FrequencyMasking, MelSpectrogram, TimeMasking
from torchmetrics.functional import mean_absolute_error, mean_absolute_percentage_error

from emonet.modules import (
    ChunkSpectrogram,
    GRUOutput,
    ReshapeOutput,
    TimeDistributed,
    VadChunk,
    WhiteNoise,
    get_vad,
)
from emonet.utils import get_sample
from emonet.wav_splitter import split_sample

class ConvTimeBlock(nn.Module):
    """Convolutional time block."""

    def __init__(
        self,
        in_channels: int,
        out_channels: int,
        kernel_size: int = 3,
        stride: int = 1,
        padding: int = 1,
        pool_kernel_size: int = 4,
        pool_stride: int = 4,
        dropout: float = 0.3,
    ):
        """
        Create a time-distributed convolutional time block.

        Parameters
        ----------
        in_channels: int
            Number of input channels for convolutional layer.
        out_channels: int
            Number of output channels for convolutional layer.
        kernel_size: int
            Kernel size for convolutional filter.
        stride: int
            Stride size for convolutional filter.
        padding: int
            Padding size for convolutional filter.
        pool_kernel_size: int
            Kernel size for pooling layer.
        pool_stride: int
            Stride size for pooling layer.
        dropout: float
            Dropout probability.
        """
        super().__init__()

        self.block = nn.Sequential(
            TimeDistributed(
                nn.Conv2d(
                    in_channels=in_channels,
                    out_channels=out_channels,
                    kernel_size=kernel_size,
                    stride=stride,
                    padding=padding,
                )
            ),
            TimeDistributed(
                nn.BatchNorm2d(num_features=out_channels),
            ),
            TimeDistributed(
                nn.ReLU(),
            ),
            TimeDistributed(
                nn.MaxPool2d(kernel_size=pool_kernel_size, stride=pool_stride)
            ),
            TimeDistributed(nn.Dropout(p=dropout)),
        )

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        """Pass input through time-distributed convolutional block layer."""
        return self.block(x)

class EmotionRegressor(pl.LightningModule):
    """Emotion regressor model."""

    def __init__(
        self,
        emotion: str,
        lr: float = 0.001,
        weight_decay: float = 0.01,
        snr_low: int = 15,
        snr_high: int = 30,
        freq_mask: int = 20,
        time_mask: int = 20,
        drop: float = 0.3,
        n_mels: int = 128,
        n_fft: int = 128 * 6,
        n_chunks: int = 10,
    ):
        """

        Parameters
        ----------
        emotion: str
            Emotion to train/score.
        lr: float
            Learning rate.
        weight_decay: float
            Weight decay for optimizer.
        snr_low: int
            Signal-to-noise ratio floor for additive white noise augmentation.
        snr_high: int
            Signal-to-noise ratio ceiling for additive white noise augmentation.
        freq_mask: int
            Size of frequency mask for data augmentation.
        time_mask: int
            Size of time mask for data augmentation.
        drop: float
            Dropout rate.
        n_mels: int
            Number of mel filterbanks.
        n_fft: int
            Size of FFT, creates n_fft // 2 + 1 bins.
        n_chunks: int
            Number of time-chunks to create from each sample.
        """
        super().__init__()
        self.emotion = emotion
        self.lr = lr
        self.weight_decay = weight_decay
        self.snr_low = snr_low
        self.snr_high = snr_high
        self.freq_mask = freq_mask
        self.time_mask = time_mask
        self.drop = drop
        self.n_mels = n_mels
        self.n_fft = n_fft
        self.n_chunks = n_chunks
        self.save_hyperparameters()
        self.loss_func = nn.MSELoss()

        self.model = nn.Sequential(
            self.conv_block(), self.lstm_block(), self.linear_block()
        )

    def conv_block(self):
        """Time-distributed CNN block."""
        return nn.Sequential(
            ChunkSpectrogram(self.n_chunks),
            ConvTimeBlock(
                in_channels=1, out_channels=16, pool_kernel_size=2, pool_stride=2
            ),
            ConvTimeBlock(
                in_channels=16, out_channels=32, pool_kernel_size=2, pool_stride=2
            ),
            ConvTimeBlock(
                in_channels=32, out_channels=64, pool_kernel_size=2, pool_stride=2
            ),
            ConvTimeBlock(
                in_channels=64, out_channels=128, pool_kernel_size=2, pool_stride=2
            ),
        )

    @staticmethod
    def lstm_block():
        """LSTM block."""
        return nn.Sequential(
            nn.Flatten(start_dim=2),  # keep batch, timestep dims
            GRUOutput(
                input_size=2048, hidden_size=64, bidirectional=True, batch_first=True
            ),
        )

    def linear_block(self):
        """Fully-connected block."""
        return nn.Sequential(
            nn.Dropout(self.drop),
            ReshapeOutput(),
            nn.Linear(128, 64),
            nn.ReLU(),
            nn.Dropout(self.drop),
            nn.Linear(64, 1),
        )

    def augment_pipe(self, choice, bs):
        """
        Data augmentation pipeline.

        Parameters
        ----------
        choice: int
            Random choice for augmentation.
        bs: int
            Batch size.

        Returns
        -------
        nn.Sequential
            Augmentation pipeline.
        """
        pipes = {
            0: nn.Sequential(
                nn.Identity(), MelSpectrogram(n_mels=self.n_mels, n_fft=self.n_fft)
            ),
            1: nn.Sequential(
                WhiteNoise(bs, self.snr_low, self.snr_high),
                MelSpectrogram(n_mels=self.n_mels, n_fft=self.n_fft),
            ),
            2: nn.Sequential(
                MelSpectrogram(n_mels=self.n_mels, n_fft=self.n_fft),
                FrequencyMasking(self.freq_mask),
            ),
            3: nn.Sequential(
                MelSpectrogram(n_mels=self.n_mels, n_fft=self.n_fft),
                TimeMasking(self.time_mask),
            ),
        }
        return pipes[choice]

    def get_results(self, spec, actual):
        """
        Get model results.

        Parameters
        ----------
        spec: torch.tensor
            Spectrogram.
        actual: float
            Actual emotion score.

        Returns
        -------
        Dict
            Dictionary of model results.
        """
        pred = self.model(spec)

        results = {
            "outputs": pred,
            "loss": self.loss_func(pred, actual),
            "MAE": mean_absolute_error(pred, actual),
            "MAPE": mean_absolute_percentage_error(pred, actual),
        }
        return results

    def training_step(self, batch, idx):
        """Training step."""
        x = batch[0].to(self.device)
        labels = batch[1]
        bs = x.shape[0]
        choice = random.randint(0, 3)
        make_spec = self.augment_pipe(choice, bs)
        spec = make_spec(x).unsqueeze(1)  # add channel dim to spec

        results = self.get_results(spec, labels)
        total_loss = results["loss"]

        self.log_dict(
            {
                "total_train_loss": total_loss,
                "train_loss": results["loss"],
                "train_mae": results["MAE"],
                "train_mape": results["MAPE"],
            },
            prog_bar=True,
            on_step=True,
            on_epoch=True,
            logger=True,
        )
        return total_loss

    def validation_step(self, batch, idx):
        """Validation step."""
        x = batch[0].to(self.device)
        labels = batch[1]
        make_spec = nn.Sequential(MelSpectrogram(n_mels=self.n_mels, n_fft=self.n_fft))
        spec = make_spec(x).unsqueeze(1)  # add channel dim to spec

        results = self.get_results(spec, labels)
        total_loss = results["loss"]

        self.log_dict(
            {
                "total_valid_loss": total_loss,
                "valid_loss": results["loss"],
                "valid_mae": results["MAE"],
                "valid_mape": results["MAPE"],
            },
            prog_bar=True,
            on_step=True,
            on_epoch=True,
            logger=True,
        )
        return total_loss

    def test_step(self, batch, idx):
        """Test step."""
        x = batch[0].to(self.device)
        labels = batch[1]
        raise NotImplementedError

    def predict_step(self, batch, batch_idx, dataloader_idx=0):
        """Predict step."""
        x = batch[0].to(self.device)
        return self.forward(x)

    @staticmethod
    def preprocess(x):
        """Preprocess an audio tensor be applying VAD model."""
        if x.shape[-1] < 16000 * 8:
            raise ValueError(
                "Cannot preprocess tensor less than 8 seconds in duration."
            )
        vad = VadChunk(*get_vad("both"))
        return vad(x)

    def score_file(self, file, sample_rate=16000, vad=True):
        """
        Score a WAV file.

        Splits audio signal into separate 8-second chunks, individually scores each and returns
        mean across splits as final score.

        Parameters
        ----------
        file: Union[str, pathlib.Path]
            Filepath to audio file.
        sample_rate: int
            Sample rate of audio file; default 16,000hz
        vad: bool
            Whether to implement voice activity detection as preprocessing step; default True.

        Returns
        -------
        float
            Emotion score.
        """
        signal, _ = get_sample(file, sample_rate)
        return self.score_signal(signal, vad)

    def score_signal(self, x, vad=True):
        """
        Score a tensor.

        Splits audio signal into separate 8-second chunks, individually scores each and returns
        mean across splits as final score.

        Parameters
        ----------
        x: torch.Tensor
            Audio signal to score.
        vad: bool
            Whether to implement voice activity detection as preprocessing step; default True.

        Returns
        -------
        float
            Emotion score.

        """
        if vad:
            x = self.preprocess(x)
        if x.ndim == 1:
            x = x.unsqueeze(0)  # need dim to match
        return self.forward(x)

    def forward(self, x):
        """
        Score a preprocessed tensor.

        Splits audio signal into separate 8-second chunks, individually scores each and returns
        mean across splits as final score.

        Parameters
        ----------
        x: torch.Tensor

        Returns
        -------
        float
            Emotion score.

        """
        splits = split_sample(x, n_seconds=8)
        spec = MelSpectrogram(n_mels=self.n_mels, n_fft=self.n_fft)
        samples = [spec(s).unsqueeze(1) for s in splits]
        self.model.eval()
        with torch.no_grad():
            preds = [self.model(s) for s in samples]
        return torch.tensor(preds).mean()

    def configure_optimizers(self):
        """Configure optimizer for training."""
        opt = torch.optim.AdamW(
            self.parameters(), lr=self.lr, weight_decay=self.weight_decay
        )
        return {
            "optimizer": opt,
            "lr_scheduler": {
                "scheduler": torch.optim.lr_scheduler.ReduceLROnPlateau(opt),
                "monitor": "train_loss",
                "frequency": 1,
            },
        }


import functools
import time

def status(status, desc):
    assert status in ('OK', 'ERROR')
    return dict(status=status, desc=desc)

def cached_property(ttl):
    """A memoize decorator for class properties."""
    def wrap(fun):
        @functools.wraps(fun)
        def get(self):
            now = time.time()
            try:
                ret, last_update = self._cache[fun]
                if (now - last_update) < ttl:
                    return ret
            except AttributeError:
                self._cache = {}
            except KeyError:
                pass
            ret = fun(self)
            self._cache[fun] = ret, now
            return ret
        return property(get)
    return wrap

def convert_to_international(number, smsc):
    if number.startswith('+'):
        return number
    else:
        if smsc.startswith('+55'): # brazil
            return '+55' + number[1:]
        if smsc.startswith('+39'): # italy
            return '+39' + number
        raise ValueError('convert_to_int. number=%s smsc=%s' % (number, smsc))


from nn.dataset import FoodDataset
import math
import random
from torch.utils.data import DataLoader
import torch.nn as nn
import torch
import datetime
import torch.optim as optim
import argparse
import numpy as np
from itertools import islice
from collections import defaultdict
from torch.utils.tensorboard import SummaryWriter
from nn.models.pretrained_model import PretrainedModel
from pathlib import Path
import json
import sys

# https://github.com/microsoft/ptvsd/issues/943
# import multiprocessing

# multiprocessing.set_start_method("spawn", True)

def count_parameters(model):
    trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
    total = sum(p.numel() for p in model.parameters())
    return trainable, total

def put_text(imgs, texts):
    result = np.empty_like(imgs)
    for i, (img, text) in enumerate(zip(imgs, texts)):
        from PIL import Image
        from PIL import ImageFont
        from PIL import ImageDraw

        img = Image.fromarray((img.transpose((1, 2, 0)) * 255).astype("uint8"), "RGB")

        draw = ImageDraw.Draw(img)
        font = ImageFont.truetype("/usr/share/fonts/TTF/LiberationSans-Regular.ttf", 12)
        # draw.text((x, y),"Sample Text",(r,g,b))
        draw.text((0, 0), text, (255, 255, 255), font=font)
        result[i] = (np.asarray(img).astype("float32") / 255).transpose((2, 0, 1))
        result[i] = (result[i] - result[i].min()) / (result[i].max() - result[i].min())
    return result

def write_imgs_to_dir(batch, dir, imgs, texts, meta):
    for i, (img, text, datapoint) in enumerate(zip(imgs, texts, meta)):
        from PIL import Image

        img = Image.fromarray((img.transpose((1, 2, 0)) * 255).astype("uint8"), "RGB")
        batchdir = dir / f"{batch:05}"
        batchdir.mkdir(parents=True, exist_ok=True)
        img.save(batchdir / f"{i:02}.png")
        (batchdir / f"{i:02}.txt").write_text(text)
        (batchdir / f"{i:02}.json").write_text(json.dumps(datapoint, indent=3))

def write_losses(
    *, writer, running_losses, epoch, batch_idx, epoch_batch_idx, prefix=""
):
    for loss_name, running_loss in running_losses.items():
        avg_loss = np.mean(running_loss)
        loss_name_prefixed = f"{prefix}{loss_name}"
        writer.add_scalar(loss_name_prefixed, avg_loss, batch_idx)
        print(
            "[%d, %5d] %s: %.3f"
            % (epoch, epoch_batch_idx, loss_name_prefixed, avg_loss)
        )

def draw_val_images(
    *,
    output,
    data,
    ings_start_idx,
    image,
    device,
    prediction_keys,
    writer=None,
    ingredient_names,
    logdir,
    epoch,
    batch_idx,
    epoch_batch_idx,
):
    show_img_count = 16
    # generate and write pictures to tensorboard

    ings_pred = (torch.sigmoid(output[:, ings_start_idx:]) > 0.5).cpu().numpy()
    ings_truth = data["ingredients"].numpy()
    meta = [
        {
            "truth": {
                "ings": [
                    ingredient_names[inx].replace(",", "")
                    for inx in ing_truth.nonzero()[0]
                ]
            },
            "pred": {
                "ings": [
                    ingredient_names[inx].replace(",", "")
                    for inx in ing_pred.nonzero()[0]
                ]
            },
            "fname": fname,
        }
        for ing_truth, ing_pred, fname in zip(ings_truth, ings_pred, data["fname"])
    ]

    for pred_inx, pred_key in enumerate(prediction_keys):
        kcal_cpu = data[pred_key]

        truth, pred = (
            kcal_cpu.squeeze(-1).numpy(),
            output[:, pred_inx : pred_inx + 1].cpu().squeeze(-1).numpy(),
        )
        pred_key_p = "carbs" if pred_key == "carbohydrates" else pred_key
        for i, (t, p) in enumerate(zip(truth, pred)):
            meta[i]["truth"][pred_key_p] = t.astype(float)
            meta[i]["pred"][pred_key_p] = p.astype(float)

    pred_arrs = ["" for _ in meta]
    for i, m in enumerate(meta):
        text = ""
        for k in m["truth"]:
            t = m["truth"][k]
            p = m["pred"][k]
            if k == "ings":
                tru_str = ", ".join(t)
                pred_str = ", ".join(p)
                text += f"\n\nings truth: {tru_str}\nings pred: {pred_str}"
            elif k == "kcal":
                text += f"\ntruth: {t:.0f}kcal, pred: {p:.0f}kcal"
            else:
                text += f"\ntruth: {t:.0f}g {k}, pred: {p:.0f}g {k}"
        pred_arrs[i] = text
    images_cpu = image.view(-1, 3, 224, 224)[:show_img_count].cpu().numpy()
    pred_arrs = [s.strip() for s in pred_arrs]

    # write images to out dir
    imgdir = Path(logdir) / "val_images"
    write_imgs_to_dir(batch_idx, imgdir, images_cpu, pred_arrs, meta)

    if writer:
        # write images to tensorboard
        images_cpu = put_text(images_cpu, pred_arrs,)
        writer.add_images("val examples", images_cpu, global_step=batch_idx)

def MyLoader(datadir: str, ds: str, batch_size: int):
    # ds = train, test, val
    dataset = FoodDataset(calories_file=datadir / f"{ds}.json", image_dir=datadir / ds,)
    return DataLoader(
        dataset, batch_size=batch_size, shuffle=True, num_workers=12, pin_memory=True
    )

def get_args(predict: bool):
    print("ARGS:", sys.argv)
    parser = argparse.ArgumentParser()
    if not predict:
        parser.add_argument("--runname", help="name this experiment", required=True)
        parser.add_argument(
            "--datadir",
            help="input data dir generated by data/split.py (contains e.g. train.json and train/",
            required=True,
        )
        parser.add_argument(
            "--test", required=True, choices=["train", "train+test", "test"]
        )
    else:
        parser.add_argument("--input-file", help="input jpg", required=True)
    parser.add_argument(
        "--train-type",
        required=True,
        choices=["kcal", "kcal+nut", "kcal+nut+topings",],
    )
    parser.add_argument(
        "--bce-weight",
        required=True,
        type=int,
        help="scale of the contribution of the ingredients loss (called gamma in the paper). For equal weighing, set this to 400 for per 100g, 2000 for per recipe, 800 for per portion",
    )
    parser.add_argument(
        "--model",
        required=True,
        choices=[
            "resnet50",
            "resnet101",
            "resnet152",
            "densenet121",
            "densenet201",
            "resnext50_32x4d",
        ],
    )
    parser.add_argument(
        "--weights",
        required=False,
        help="continue training (/testing) from these model weights",
    )
    parser.add_argument(
        "--no-predict-portion-size",
        dest="predict_portion_size",
        required=False,
        default=True,
        action="store_false",
    )

    return parser.parse_args()

class Model:
    def __init__(self, args):
        self.args = args
        self.batch_size = 50

        training_type = args.train_type
        predict_portion_size = args.predict_portion_size

        # regression settings
        num_top_ingredients = 100

        def criterion_rel_error(pred, truth):
            # https://en.wikipedia.org/wiki/Approximation_error
            ret = torch.abs(1 - pred / truth)
            ret[torch.isnan(ret)] = 0  # if truth = 0 relative error is undefined
            return torch.mean(ret)

        ings_start_idx = 4
        if predict_portion_size:
            ings_start_idx = 5

        # multi-task loss
        def loss_top_ingredients(pred, data):
            from torch.nn.functional import (
                smooth_l1_loss,
                binary_cross_entropy_with_logits,
            )

            # todo: loop over enumerate(prediction_keys) here
            l1 = smooth_l1_loss(pred[:, 0:1], data["kcal"])
            l1 += smooth_l1_loss(pred[:, 1:2], data["protein"])
            l1 += smooth_l1_loss(pred[:, 2:3], data["fat"])
            l1 += smooth_l1_loss(pred[:, 3:4], data["carbohydrates"])
            if predict_portion_size:
                l1 += smooth_l1_loss(pred[:, 4:5], data["mass_per_portion"])
            if training_type == "kcal+nut+topings":
                # todo: adjust the 400 factor to 2000 if per recipe etc
                bce = (
                    binary_cross_entropy_with_logits(
                        pred[:, ings_start_idx:], data["ingredients"]
                    )
                    * args.bce_weight
                )
                if (
                    random.random() < 0.02
                ):  # debug checking if contribution to loss is similar
                    print(
                        "l1 vs bce weight (should be around the same)",
                        float(l1),
                        float(bce),
                    )
                return l1 + bce
            return l1

        loss_fns = {}

        prediction_keys = ["kcal"]

        self.num_output_neurons = 1

        loss_fns["loss"] = lambda pred, data: nn.functional.smooth_l1_loss(
            pred, data["kcal"]
        )
        loss_fns["l1_kcal"] = lambda pred, data: nn.functional.l1_loss(
            pred, data["kcal"]
        )
        loss_fns["rel_error_kcal"] = lambda pred, data: criterion_rel_error(
            pred, data["kcal"]
        )

        if training_type.startswith("kcal+nut"):
            prediction_keys = ["kcal", "protein", "fat", "carbohydrates"]
            if predict_portion_size:
                prediction_keys += ["mass_per_portion"]
            self.num_output_neurons = len(prediction_keys)
            loss_fns["loss"] = loss_top_ingredients
            from torch.nn.functional import l1_loss

            def mk_loss(i, k, fn):
                def fuck_python(pred, data):
                    return fn(pred[:, i : (i + 1)], data[k])

                return fuck_python

            for i, k in enumerate(prediction_keys):
                loss_fns[f"l1_{k}"] = mk_loss(i, k, l1_loss)
                loss_fns[f"rel_error_{k}"] = mk_loss(i, k, criterion_rel_error)
            i = 999
        if training_type == "kcal+nut+topings":
            self.num_output_neurons += num_top_ingredients

        devname = "cuda:0" if torch.cuda.is_available() else "cpu"
        print("device: ", devname)
        self.device = torch.device(devname)
        self.loss_fns = loss_fns
        self.ings_start_idx = ings_start_idx
        self.prediction_keys = prediction_keys

        self.model = PretrainedModel(
            self.num_output_neurons, pytorch_model=args.model, all_layers_trainable=True
        )  #

        if args.weights:
            print(f"Loading model weights from {args.weights}")
            self.model.load(args.weights)
        print("model:", self.model.name)
        self.net = self.model.to(self.device)
        print(self.net)

    def train(self, epochs):
        net = self.net
        args = self.args

        validate_every = 200
        validate_batches = 100

        logdir = (
            "runs/"
            + datetime.datetime.now()
            .replace(microsecond=0)
            .isoformat()
            .replace(":", ".")
            + "-"
            + args.runname
        )
        writer = SummaryWriter(logdir)
        print(f"tensorboard logdir: {writer.log_dir}")

        datadir = Path(args.datadir)
        train_loader = MyLoader(datadir, "train", self.batch_size)
        val_loader = MyLoader(datadir, "val", self.batch_size)

        optimizer = optim.Adam(net.parameters())

        trainable_params, total_params = count_parameters(net)
        print(f"Parameters: {trainable_params} trainable, {total_params} total")
        running_losses = defaultdict(list)
        batch_idx = -1

        for epoch in range(1, epochs + 1):
            if epoch == 3:
                for param in net.parameters():
                    param.requires_grad = True
                trainable_params, total_params = count_parameters(net)
                print("all params unfreezed")
                print(f"Parameters: {trainable_params} trainable, {total_params} total")

            for epoch_batch_idx, data in enumerate(train_loader, 0):
                batch_idx += 1
                # print("batch idx", batch_idx)
                # if batch_idx > 100:
                #   return
                # print("is pinned", data["image"].is_pinned())
                image_ongpu = data["image"].to(self.device)
                optimizer.zero_grad()

                outputs = net(image_ongpu)

                # print("out", outputs.shape)

                target_data = {
                    k: v.to(self.device)
                    for k, v in data.items()
                    if k not in ["image", "fname"]
                }

                for loss_name, loss_fn in self.loss_fns.items():
                    loss_value = loss_fn(outputs, target_data)
                    if loss_name == "loss":
                        loss_value.backward()
                        optimizer.step()
                    # technically, this is not 100% correct because it assumes all batches are the same size
                    running_losses[loss_name].append(float(loss_value.item()))

                if batch_idx % validate_every == 0:
                    write_losses(
                        writer=writer,
                        running_losses=running_losses,
                        epoch=epoch,
                        batch_idx=batch_idx,
                        epoch_batch_idx=epoch_batch_idx,
                    )
                    running_losses = defaultdict(list)

                if batch_idx % validate_every == 0:
                    val_error = defaultdict(list)
                    # validation loop
                    with torch.no_grad():
                        for val_batch_inx, data in enumerate(
                            islice(val_loader, validate_batches)
                        ):
                            image = data["image"].to(self.device)

                            target_data = {
                                k: v.to(self.device)
                                for k, v in data.items()
                                if k not in ["image", "fname"]
                            }

                            output = net(image)
                            for loss_name, loss_fn in self.loss_fns.items():
                                val_error[loss_name].append(
                                    float(loss_fn(output, target_data).item())
                                )

                            if val_batch_inx == 0:
                                # only run this on last batch from val loop
                                draw_val_images(
                                    writer=writer,
                                    output=output,
                                    ings_start_idx=self.ings_start_idx,
                                    data=data,
                                    image=image,
                                    device=self.device,
                                    prediction_keys=self.prediction_keys,
                                    ingredient_names=train_loader.dataset.ingredient_names,
                                    logdir=logdir,
                                    epoch=epoch,
                                    batch_idx=batch_idx,
                                    epoch_batch_idx=epoch_batch_idx,
                                )
                    write_losses(
                        writer=writer,
                        running_losses=val_error,
                        epoch=epoch,
                        batch_idx=batch_idx,
                        epoch_batch_idx=epoch_batch_idx,
                        prefix="val_",
                    )

            self.model.save(net, f"epoch-{epoch:03d}", logdir)

        writer.close()
        do_test = "test" in args.test
        do_train = "train" in args.test

        if do_test:

            if not do_train:
                self.model.load(args.weights)

            test_loader = MyLoader(datadir, "test", self.batch_size)

            with torch.no_grad():
                model.model.eval()
                for epoch_batch_idx, data in enumerate(test_loader, 0):
                    image_ongpu = data["image"].to(self.device)

                    outputs = net(image_ongpu)

                    target_data = {
                        k: v.to(self.device) for k, v in data.items() if k != "image"
                    }

                    for loss_name, loss_fn in self.loss_fns.items():
                        loss_value = loss_fn(outputs, target_data)
                        running_losses[loss_name].append(float(loss_value.item()))

                        print(loss_value)

if __name__ == "__main__":
    with torch.autograd.detect_anomaly():
        args = get_args(False)
        epochs = 40

        if args.test == "test":
            epochs = 0
        Model(args).train(epochs=epochs)

{'smooth_l1_loss': 'torch.nn.functional.smooth_l1_loss', 'binary_cross_entropy_with_logits': 'torch.nn.functional.binary_cross_entropy_with_logits', 'l1_loss': 'torch.nn.functional.l1_loss'}


import cv2
import numpy as np
dictionary = cv2.aruco.getPredefinedDictionary(cv2.aruco.DICT_4X4_250)

cap = cv2.VideoCapture(3)
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 1920)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 1080)

curr_x = 400
curr_y = 70 + 50
while True:
    ret, img = cap.read()
    print(img)
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    res = cv2.aruco.detectMarkers(gray, dictionary)
    if res is not None:
        if 83 in res[1]:
            index = np.where(res[1] == 83)[0][0]
            x1 = (res[0][index][0][0][0] + res[0][index][0][2][0])//2
            y1 = (res[0][index][0][0][1] + res[0][index][0][2][1])//2
            cv2.circle(img, (int(x1), int(y1)), 10, (0, 0, 255), -1)
    cv2.imshow("asd.png", img)
    if cv2.waitKey(25) & 0xFF == ord('s'):
        with open('r.txt', 'a') as fd:
            fd.write(f"{curr_x} {curr_y} {x1} {y1}")
            fd.write('\n')
        curr_x += 50
        if curr_x >= 2800:
            curr_x = 200
            curr_y += 50
        if curr_y >= 2000:
            print("Finish")


#!/usr/bin/env python3

import math

import pandas as pd
from nucleodiver.params import Params
from nucleodiver.utils import read_vcf, time_stamp

pm = Params('nucleodiver')
args = pm.set_options()

from multiprocessing import Pool

class NucleoDiver_2(object):

    def __init__(self, args):
        self.args = args
        self.wind = self.args.window
        self.step = self.args.step
        self.output_1 = '{}_mutations.tsv'.format(args.name)
        self.output_2 = '{}_pie_values.tsv'.format(args.name)
        self.output_3 = '{}_list.tsv'.format(args.name)

        #Prepare chromosome information
        self.fai_data = []
        with open(self.args.fai, 'r') as f:
            for row in f:
                row = row.strip()
                self.fai_data.append(row.split('\t'))
        fai_col = ['chr', 'len', 'A', 'B', 'C']
        self.fai_data = pd.DataFrame(self.fai_data, columns=fai_col)
        self.fai_data['len'] = self.fai_data['len'].astype(int)

        #Read vcf file
        vcf_list = read_vcf(args.vcf)
        self.vcf_header = vcf_list[0]
        self.vcf_col = vcf_list[1]
        self.vcf_col[0] = 'CHROM'
        self.data = pd.DataFrame(vcf_list[2], columns=self.vcf_col)
        self.data['POS'] = self.data['POS'].astype(int)
        self.newdata = self.data.iloc[:, 0:9]

        #Making set of regions to focus.
        #[chr01, 1, 100000], [chr01, 10001, 110000]...
        self.loc = []
        for i in range(len(self.fai_data['chr'])):
            for j in range(math.ceil(self.fai_data['len'][i]/self.step)):
                c = self.fai_data['chr'][i]
                sta = 1 + self.step * j
                end = sta + self.wind - 1
                self.loc.append([c, sta, end])

    def run(self):
        #Make new DataFrame easy to parse.
        for i in range(9, len(self.data.columns)):
            newcol = []
            for j in range(len(self.data)):
                temp = self.data.iat[j, i]
                #ex. temp = "1/1:0,5:5:15:203,15,0"

                geno = temp.split(':')[0]
                #ex. geno = "1/1"

                hap = geno.split('/')
                #ex. hap = ["1", "1"]
                if len(hap) != 2:
                    newcol.append(-1)

                #Homo --> 0,1,2.... Hetero or missing --> -1
                check_list = (str.isdigit(hap[0])
                              and str.isdigit(hap[1])
                              and hap[0] == hap[1])
                if check_list:
                    newcol.append(int(hap[0]))
                else:
                    newcol.append(-1)
            self.newdata['{}'.format(self.data.columns[i])] = newcol
        print(time_stamp(), 'Extracting variants data has done.', flush=True)

        #Making list of 2 varieties set
        #If 100 varieties, 100*99/2=4550 sets
        set = []
        count = 0
        for i in range(9, len(self.vcf_col)-1):
            for j in range(i+1, len(self.vcf_col)):
                set.append([count, i, j])
                count = count + 1
        #set[0] = ID, set[1]=Column number of variety 1, set[2]=that of 2.

        with Pool(self.args.cpu) as p:
            results = p.map(self.parallel, set)

        #result is the list of [id, list of mutation number, list of pie value,
        #                       names of 2 varieties]
        #sort by id
        results.sort(reverse=False, key=lambda x:x[0])

        df_1 = pd.DataFrame(self.loc, columns=['chr','start','end'])
        df_2 = pd.DataFrame(self.loc, columns=['chr','start','end'])
        list_3 = self.vcf_col[9:]
        for i in range(len(results)):
            res = results[i]
            df_1['{}'.format(res[3])] = res[1]
            df_2['{}'.format(res[3])] = res[2]

        df_1.to_csv(self.output_1, sep='\t')
        df_2.to_csv(self.output_2, sep='\t')
        with open(self.output_3, 'w') as o:
            for h in list_3:
                o.write('{}\n'.format(h))

    def parallel(self, set):
        id = set[0]
        n1 = set[1]
        n2 = set[2]
        name_of_set = '{}_{}'.format(self.vcf_col[n1], self.vcf_col[n2])
        print(time_stamp(), 'Processing data of No. {}.'.format(id), flush=True)

        #Make DataFrame with 0 or 1 data according to mutation or not
        mut_data = self.newdata.iloc[:, 0:2] #only CHROM and POS
        mutcol = []
        for i in range(len(self.newdata)):
            val_1 = self.newdata.iat[i, n1]
            val_2 = self.newdata.iat[i, n2]

            check_list = (val_1 != -1
                          and val_2 != -1
                          and val_1 != val_2)
            if check_list:
                mutcol.append(1)
            else:
                mutcol.append(0)
        mut_data['mut'] = mutcol

        #Calculate mutation number and pie value
        mutations = []
        pie = []
        for i in range(len(self.loc)):
            temp_chr = self.loc[i][0]
            temp_sta = self.loc[i][1]
            temp_end = self.loc[i][2]
            sub_data = mut_data.query('CHROM == @temp_chr '
                                      'and POS >= @temp_sta '
                                      'and POS <= @temp_end')
            mut = sub_data['mut'].sum()
            mutations.append(mut)
            pie.append(mut / self.wind)

        return [id, mutations, pie, name_of_set]

def main():
    print(time_stamp(), 'nucleodiver started.', flush=True)

    prog = NucleoDiver_2(args)
    prog.run()
    print(time_stamp(), 'nucleodiver successfully finished.\n', flush=True)

if __name__ == '__main__':
    main()


from rest_framework.response import Response
from rest_framework.decorators import api_view
from .serializers import TaskSerializer
from .models import Task

# testing without db connection
person = {"p1":"tamim"}

@api_view(["GET","POST"])
def getData(request):
    global person
    if request.method == "GET":
        # person = {"name":'Tamim'}
        return Response(person)
    if request.method == "POST":
        data = request.data
        person = data
        return Response(f"person : {person}")

# testing with db connection
@api_view(["GET","POST",'DELETE'])
def taskList(request):
    if request.method == "GET":
        tasks = Task.objects.all()
        serializers = TaskSerializer(tasks,many = True)
        return Response(serializers.data)
    if request.method == "POST":
        serializers = TaskSerializer(data=request.data)
        if serializers.is_valid():
            serializers.save()
            return Response("Successfully Added")
        return Response("Product Add Failed")
    if request.method == "DELETE":
        task = Task.objects.get(id=request.data.id)
        task.delete()
        return Response("Item successfully deleted")
    # print(serializers)
    # return Response("hello world")