#! /usr/bin/env python3

# scr_common.py
"""Defines for common methods shared across scripts"""

import os, sys

if 'scrjob' not in sys.path:
  sys.path.insert(0, '/'.join(os.path.realpath(__file__).split('/')[:-2]))
  import scrjob

import argparse, inspect
from subprocess import Popen, PIPE
import shlex
from scrjob import scr_const

def tracefunction(frame, event, arg):
  """This method provides a hook for tracing python calls

  Usage:  sys.settrace(scr_common.tracefunction)
  Prints: filename:function:linenum -> event

  The method below will be called on events as scripts execute.
  The print message in the try block attempts to print the filename.
  If the filename could not be obtained, an exception is thrown.

  This output is very verbose. Filters could be added for event type,
  and print messages could be suppressed when the filename is not known.
  """
  try:
    print(
        inspect.getfile(frame).split('/')[-1] + ':' +
        str(frame.f_code.co_name) + '():' + str(frame.f_lineno) + ' -> ' +
        str(event))
  except:
    print(
        str(frame.f_code.co_name) + '():' + str(frame.f_lineno) + ' -> ' +
        str(event))

def interpolate_variables(varstr):
  """This method will expand a string, including paths

  This method allows interpretation of path strings such as:
  ~ . ../ ../../

  Environment variables in the string will also be expanded.
  The input need not be a path.
  """
  if varstr is None or len(varstr) == 0:
    return ''
  # replace ~ and . symbols from front of path
  if varstr[0] == '~':
    varstr = '$HOME' + varstr[1:]
  elif varstr.startswith('..'):
    topdir = '/'.join(os.getcwd().split('/')[:-1])
    varstr = varstr[3:]
    while varstr.startswith('..'):
      topdir = '/'.join(topdir.split('/')[:-1])
      varstr = varstr[3:]
    varstr = topdir + '/' + varstr
  elif varstr[0] == '.':
    varstr = os.getcwd() + varstr[1:]
  if varstr[-1] == '/' and len(varstr) > 1:
    varstr = varstr[:-1]
  return os.path.expandvars(varstr)

def scr_prefix():
  """This method will return the prefix for SCR

  If the environment variable is not set,
  it will return the current working directory.
  Allows the environment variable to contain relative paths by returning
  the string after passing through interpolate_variables
  """
  prefix = os.environ.get('SCR_PREFIX')
  if prefix is None:
    return os.getcwd()
  # tack on current working dir if needed
  # don't resolve symlinks
  # don't worry about missing parts, the calling script calling might create it
  return interpolate_variables(prefix)

def runproc(argv, wait=True, getstdout=False, getstderr=False, verbose=False, shell=False):
  """This method will execute a command using subprocess.Popen

  Required argument
  -----------------
  argv        a string or a list, the command to be ran

  Returns
  -------
  tuple       2 values are returned as a tuple, the values depend on optional parameters
              When getting output (waiting, default):
                The first return value is the output, the second is the returncode
                If only getting stdout OR stderr, the first element will be a string
                If getting stdout AND stderr, the first element will be a list,
                where the first element of the list is stdout and the second is stderr
                The return code will be an integer
              When not getting output, when wait = False:
                When wait is False, this method immediately returns after calling Popen.
                The Popen object will be both the first and second return values
              On ERROR:
                On error this method returns None, None
                This has an exception if getstderr is True, then the first return value
                will contain stderr in the expected position, and the second value will be None

  Optional parameters
  -------------------
    All optional parameters are True/False boolean values
    The parameters getstdout and getstderr require waiting for the program to complete,
    and should not be used in combination with wait=True

  wait        If wait is False this method returns the Popen object as both elements of the tuple
  getstdout   If getstdout is True then this method will include stdout in the first return value
  getstderr   If getstderr is True then this method will include stderr in the first return value
  verbose     If verbose is True then this method will print the command to be executed
  shell       If shell is True, then argv is transformed into -> 'bash -c ' + shlex.quote(argv)

  The shell option is passed into the Popen call, and is supposed to prepend commands (?) with
    '/bin/sh -c', although when I tried simply using shell=True for 'which pdsh' I had errors.
  """
  if shell:
    if type(argv) is list:
      argv = ' '.join(argv)
    # following the security recommendation from subprocess.Popen:
    # turn the command into a shlex quoted string
    argv = 'bash -c ' + shlex.quote(argv)
  # allow caller to pass command as a string, as in:
  #   "ls -lt" rather than ["ls", "-lt"]
  elif type(argv) is str:
    argv = shlex.split(argv)

  if len(argv) < 1:
    return None, None
  try:
    # if verbose, print the command we will run to stdout
    if verbose:
      if type(argv) is list:
        print(" ".join(argv))
      else:
        print(argv)

    runproc = Popen(argv,
                    bufsize=1,
                    stdin=None,
                    stdout=PIPE,
                    stderr=PIPE,
                    shell=shell,
                    universal_newlines=True)
    if wait == False:
      return runproc, runproc
    if getstdout == True and getstderr == True:
      output = runproc.communicate()
      return output, runproc.returncode
    if getstdout == True:
      output = runproc.communicate()[0]
      return output, runproc.returncode
    if getstderr == True:
      output = runproc.communicate()[1]
      return output, runproc.returncode
    runproc.communicate()
    return None, runproc.returncode
  except Exception as e:
    print('runproc: ERROR: ' + str(e))
    if getstdout == True and getstderr == True:
      return ['', str(e)], None
    if getstdout == True:
      return '', None
    if getstderr == True:
      return str(e), None
    return None, None

# pipeproc works as runproc above, except argvs is a list of argv lists
# the first subprocess is opened and from there stdout is chained to stdin
# values returned (returncode/pid/stdout/stderr) will be from the final process
def pipeproc(argvs, wait=True, getstdout=False, getstderr=False):
  """This method is an extension of the above runproc method

  This is essentially duplicated code from the runproc.
   ### TODO : This functionality could be put into runproc, increasing
    the complexity slightly, or it could be kept separate.

  This method will loop through search argv in a list of argvs.
  The output of each previous 'runproc' will be piped to the next as stdin.
  Any return values will come from the result of the final Popen command
  """
  if len(argvs) < 1:
    return None, None
  if len(argvs) == 1:
    return runproc(argvs[0], wait, getstdout, getstderr)
  try:
    # split command into list if given as string
    cmd = argvs[0]
    if type(cmd) is str:
      cmd = shlex.split(cmd)

    nextprog = Popen(cmd,
                     bufsize=1,
                     stdin=None,
                     stdout=PIPE,
                     stderr=PIPE,
                     universal_newlines=True)

    for i in range(1, len(argvs)):
      # split command into list if given as string
      cmd = argvs[i]
      if type(cmd) is str:
        cmd = shlex.split(cmd)

      pipeprog = Popen(cmd,
                       stdin=nextprog.stdout,
                       stdout=PIPE,
                       stderr=PIPE,
                       bufsize=1,
                       universal_newlines=True)
      nextprog.stdout.close()
      nextprog = pipeprog
    if wait == False:
      return nextprog, nextprog.pid
    if getstdout == True and getstderr == True:
      output = nextprog.communicate()
      return output, nextprog.returncode
    if getstdout == True:
      output = nextprog.communicate()[0]
      return output, nextprog.returncode
    if getstderr == True:
      output = nextprog.communicate()[1]
      return output, nextprog.returncode
    return None, nextprog.returncode
  except Exception as e:
    print('pipeproc: ERROR: ' + str(e))
    if getstdout == True and getstderr == True:
      return ['', str(e)], 1
    if getstdout == True:
      return '', 1
    if getstderr == True:
      return str(e), 1
    return None, 1

def choose_bindir():
  """Determine appropriate location of binaries.

  Testing using "make test" or "make check" must operate based on the contents
  of the build directory.

  If the script is being run from the install directory, then return X_BINDIR
  as defined by cmake so installed scripts use installed binaries.  Otherwise,
  determine the appropriate build directory.
  """
  # Needed to find binaries in build dir when testing
  parent_of_this_module = '/'.join(os.path.realpath(__file__).split('/')[:-1])
  if scr_const.X_BINDIR in parent_of_this_module:
      bindir = scr_const.X_BINDIR  # path to install bin directory
  else:
      bindir = os.path.join(scr_const.CMAKE_BINARY_DIR + "/src/")

  return bindir

def log(bindir=None,
        prefix=None,
        username=None,
        jobname=None,
        jobid=None,
        start=None,
        event_type=None,
        event_note=None,
        event_dset=None,
        event_name=None,
        event_start=None,
        event_secs=None):
  """This method is a wrapper for commong logging operations

  Logging operations call the scr_log_event program with formatted arguments.

  This method formats an argv, then calls the scr_log_event method.

  Parameters
  ----------
  All input parameters are optional, with each just extending the log entry
  Input parameters should (with one exception) be string values

  event_note   may be a string or a dictionary, if a dictionary is passed then
               the remaining arguments are duplicated
               This allows several runproc calls without rebuilding argv each time
  """
  if prefix is None:
    prefix = scr_prefix()
    #print('log: prefix is required')
  if bindir is None:
    bindir = scr_const.X_BINDIR
  argv = [bindir + '/scr_log_event', '-p', prefix]
  if username is not None:
    argv.extend(['-u', username])
  if jobname is not None:
    argv.extend(['-j', jobname])
  if jobid is not None:
    argv.extend(['-i', jobid])
  if start is not None:
    argv.extend(['-s', start])
  if event_type is not None:
    argv.extend(['-T', event_type])
  if event_dset is not None:
    argv.extend(['-D', event_dset])
  if event_name is not None:
    argv.extend(['-n', event_name])
  if event_start is not None:
    argv.extend(['-S', event_start])
  if event_secs is not None:
    argv.extend(['-L', event_secs])
  if type(event_note) is dict:
    argv.extend(['-N', ''])
    lastarg = len(argv) - 1
    for key in event_note:
      argv[lastarg] = key + ':' + event_note[key]
      runproc(argv=argv)
  else:
    if event_note is not None:
      argv.extend(['-N', event_note])
    runproc(argv=argv)

if __name__ == '__main__':
  """This script allows being called as a standalone script

  This is meant for testing purposes, to directly call scr_common methods.
  """
  parser = argparse.ArgumentParser(add_help=False,
                                   argument_default=argparse.SUPPRESS,
                                   prog='scr_common')
  parser.add_argument('-h',
                      '--help',
                      action='store_true',
                      help='Show this help message and exit.')
  parser.add_argument('--interpolate',
                      metavar='<variable>',
                      type=str,
                      help='Interpolate a variable string.')
  parser.add_argument('--prefix',
                      action='store_true',
                      help='Print the SCR prefix.')
  parser.add_argument('--runproc',
                      nargs=argparse.REMAINDER,
                      help='Launch process with arguments')
  parser.add_argument(
      '--pipeproc',
      nargs=argparse.REMAINDER,
      help=
      'Launch processes and pipe output to other processes. (separate processes with a colon)'
  )
  parser.add_argument(
      '--log',
      nargs='+',
      metavar='<option=value>',
      help=
      'Create a log entry, available options: bindir, prefix, username, jobname, jobid, start, event_type, event_note, event_dset, event_name, event_start, event_secs'
  )
  args = vars(parser.parse_args())
  if 'help' in args:
    parser.print_help()
    sys.exit(1)
  if 'interpolate' in args:
    print('interpolate_variables(' + args['interpolate'] + ')')
    print('  -> ' + str(interpolate_variables(args['interpolate'])))
  if 'prefix' in args:
    print('scr_prefix()')
    print('  -> ' + str(scr_prefix()))
  if 'runproc' in args:
    print('runproc(' + ' '.join(args['runproc']) + ')')
    out, returncode = runproc(argv=args['runproc'],
                              getstdout=True,
                              getstderr=True)
    print('  process returned with code ' + str(returncode))
    print('  stdout:')
    print(out[0])
    print('  stderr:')
    print(out[1])
  if 'pipeproc' in args:
    printstr = 'pipeproc( '
    argvs = []
    argvs.append([])
    i = 0
    for arg in args['pipeproc']:
      if arg == ':':
        i += 1
        argvs.append([])
        printstr += '| '
      else:
        argvs[i].append(arg)
        printstr += arg + ' '
    print(printstr + ')')
    out, returncode = pipeproc(argvs=argvs, getstdout=True, getstderr=True)
    print('  final process returned with code ' + str(returncode))
    if out is not None:
      print('  stdout:')
      print(out[0])
      print('  stderr:')
      print(out[1])
  if 'log' in args:
    bindir, prefix, username, jobname = None, None, None, None
    jobid, start, event_type, event_note = None, None, None, None
    event_dset, event_name, event_start, event_secs = None, None, None, None
    printstr = 'log('
    for keyvalpair in args['log']:
      if '=' not in keyvalpair:
        continue
      vals = keyvalpair.split('=')
      if vals[0] == 'bindir':
        bindir = vals[1]
      elif vals[0] == 'prefix':
        prefix = vals[1]
      elif vals[0] == 'username':
        username = vals[1]
      elif vals[0] == 'jobname':
        jobname = vals[1]
      elif vals[0] == 'jobid':
        jobid = vals[1]
      elif vals[0] == 'start':
        start = vals[1]
      elif vals[0] == 'event_type':
        event_type = vals[1]
      elif vals[0] == 'event_note':
        event_note = vals[1]
      elif vals[0] == 'event_dset':
        event_dset = vals[1]
      elif vals[0] == 'event_name':
        event_name = vals[1]
      elif vals[0] == 'event_start':
        event_start = vals[1]
      elif vals[0] == 'event_secs':
        event_secs = vals[1]
      else:
        continue
      printstr += keyvalpair + ', '
    if printstr[-1] == '(':
      print('Nothing to log, see \'--help\' for available value pairs.')
      sys.exit(1)
    print(printstr[:-2] + ')')
    log(bindir=bindir,
        prefix=prefix,
        username=username,
        jobname=jobname,
        jobid=jobid,
        start=start,
        event_type=event_type,
        event_note=event_note,
        event_dset=event_dset,
        event_name=event_name,
        event_start=event_start,
        event_secs=event_secs)


import os
from copy import deepcopy

import numpy as np
import torch
import torch.nn as nn
import gym
import logging

from torch.distributions.categorical import Categorical
import torch.nn.functional as F

from rllib.memory.replay_buffer import ReplayBuffer
from rllib.model.dqn_actor_critic import ActorCritic

class DDPG:

    def __init__(self, ac_model: ActorCritic, gamma=1.0):
        self._ac_model = ac_model
        self._ac_target_model = deepcopy(ac_model)
        for para in self._ac_target_model.parameters():
            para.requires_grad = False
        self.gamma = gamma

    def get_action(self, obs, noise_scale=0.0):
        obs = torch.as_tensor(obs, dtype=torch.float32)
        action = self._ac_model.act(obs).numpy()
        action += noise_scale * (np.random.rand(self._ac_model.action_dim) - 0.5)
        return np.clip(action, -self._ac_model.act_limit, self._ac_model.act_limit)

    def compute_pi_loss(self, data) -> torch.Tensor:
        obs = data['obs']
        q = self._ac_model.q(obs, self._ac_model.pi(obs))
        # 因为优化器是梯度下降，所以取负
        return -q.mean()

    def compute_q_loss(self, data):
        obs, action, reward, obs2, done = data['obs'], data['act'], data['rew'], data['obs2'], data['done']
        q = self._ac_model.q(obs, action)

        with torch.no_grad():
            target = self._ac_target_model.q(obs2, self._ac_target_model.pi(obs2))
            # TODO: done
            target = reward + self.gamma * target

        loss = torch.nn.functional.mse_loss(q, target)
        return loss

def train(epoch=1, gamma=1.0, polyak=0.995, noise_scale=0.1, batch_size=100, render=False,
          env_name='Pendulum-v0', model_path='./model/ddpg/'):
    env = gym.make(env_name)
    model_file = os.path.join(model_path, env_name + '.model')
    observation_space, action_space = env.observation_space, env.action_space
    steps = 1000

    ac = ActorCritic(env.observation_space, env.action_space)

    if os.path.exists(model_file):
        state_dict = torch.load(model_file)
        ac.load_state_dict(state_dict)
        logging.info('Load model successfully')

    agent = DDPG(ac, gamma=gamma)
    pi_optimizer = torch.optim.Adam(ac.pi.parameters())
    q_optimizer = torch.optim.Adam(ac.q.parameters())

    buffer = ReplayBuffer(observation_space.shape[0], action_space.shape[0], size=int(1e6))

    def update(data):
        q_optimizer.zero_grad()
        loss_q = agent.compute_q_loss(data)
        loss_q.backward()
        q_optimizer.step()

        for para in ac.q.parameters():
            para.requires_grad = False

        pi_optimizer.zero_grad()
        loss_pi = agent.compute_pi_loss(data)
        loss_pi.backward()
        pi_optimizer.step()

        for para in ac.q.parameters():
            para.requires_grad = True

        with torch.no_grad():
            for para, para_target in zip(agent._ac_model.parameters(), agent._ac_target_model.parameters()):
                para_target.data.mul_(polyak)
                para_target.data.add_((1 - polyak) * para.data)

        return loss_q, loss_pi

    def one_epoch():

        obs, done = env.reset(), False
        rsum, rlen = 0.0, 0

        for t in range(steps):
            if render:
                env.render()

            # TODO: start step
            # 逆时针是正的
            action = agent.get_action(obs, noise_scale=noise_scale)
            obs_next, reward, done, _ = env.step(action)
            rsum += reward
            rlen += 1
            buffer.store(obs, action, reward, obs_next, done)
            obs = obs_next

            if done:
                logging.info(f"Policy_loss:{loss_pi:.3} Value_loss:{loss_q:.3} Reward:{reward} Return:{rsum} Length:{rlen}")
                obs, rsum, rlen = env.reset(), 0.0, 0

            # TODO: update after & update_every
            loss_q, loss_pi = update(buffer.sample_batch(batch_size))
            # logging.info("")

        logging.info(f"Policy_loss:{loss_pi:.3} Value_loss:{loss_q:.3} Reward:{reward} Return:{rsum} Length:{rlen}")

    for epoch_n in range(epoch):
        logging.info(f"Epoch {epoch_n}:")
        one_epoch()

    env.close()
    if not os.path.exists(model_path):
        os.makedirs(model_path)
    torch.save(ac.state_dict(), model_file)
    logging.info("Dump model")

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    train(gamma=0.99, polyak=0.995, noise_scale=2.0, render=True, env_name='MountainCarContinuous-v0')


# -*- coding: utf-8 -*-

from setuptools import find_packages, setup

__description__ = 'Api REST Python Flask'
__long_description__ = 'This is an api for SW Universe B2W chalenge'

__author__ = 'Anderson Picollo'
__author_email__ = 'andersonpicollo@gmail.com'

testing_extras = [
    'pytest',
    'pytest-cov',
]

setup(
    name='swdesafio',
    author=__author__,
    author_email=__author_email__,
    packages=find_packages(),

    description=__description__,
    long_description=__long_description__,
    url='https://github.com/andersonpicollo/swdesafio',
    keywords='API, REST, Flask, MongoDB',
    include_package_data=True,
    zip_safe=False,
    setup_requires=['pytest-runner'],
    tests_require=['pytest'],
    extras_require={
        'testing': testing_extras,
    },
)


"""
This file defines actions, i.e. functions the URLs are mapped into
The @action(path) decorator exposed the function at URL:

    http://127.0.0.1:8000/{app_name}/{path}

If app_name == '_default' then simply

    http://127.0.0.1:8000/{path}

If path == 'index' it can be omitted:

    http://127.0.0.1:8000/

The path follows the bottlepy syntax.

@action.uses('generic.html')  indicates that the action uses the generic.html template
@action.uses(session)         indicates that the action uses the session
@action.uses(db)              indicates that the action uses the db
@action.uses(T)               indicates that the action uses the i18n & pluralization
@action.uses(auth.user)       indicates that the action requires a logged in user
@action.uses(auth)            indicates that the action requires the auth object

session, db, T, auth, and tempates are examples of Fixtures.
Warning: Fixtures MUST be declared with @action.uses({fixtures}) else your app will result in undefined behavior
"""

import uuid

from py4web import action, request, abort, redirect, URL, Field
from py4web.utils.form import Form, FormStyleBulma
from py4web.utils.url_signer import URLSigner

from yatl.helpers import A
from . common import db, session, T, cache, auth, signed_url
from . models import get_user_email

url_signer = URLSigner(session)

# The auth.user below forces login.
@action('index')
@action.uses('index.html', url_signer, auth.user)
def index():
    return dict(
        # This is an example of a signed URL for the callback.
        get_posts_url = URL('get_posts', signer=url_signer),
        add_post_url = URL('add_post', signer=url_signer),
        delete_post_url = URL('delete_post', signer=url_signer),
        thumb_post_url = URL('thumb_post', signer=url_signer),
        get_thumbs_url = URL('get_thumbs', signer=url_signer),
        get_username_url = URL('get_username', signer=url_signer),
        # Add other callbacks here.
        user_email = get_user_email(),
        username = auth.current_user.get('first_name') + " " + auth.current_user.get("last_name")
    )

@action('get_posts')
@action.uses(url_signer.verify(), auth.user, db)
def get_posts():
    # Complete.
    posts = db(db.post).select(orderby=~db.post.ts).as_list()
    for p in posts:
        # Get post author
        r = db(db.auth_user.email == p['user_email']).select().first()
        p['author'] = r.first_name + " " + r.last_name if r is not None else "Unknown"

        # Get whether post was liked or disliked by current user
        r = db((db.thumb.user_email == auth.current_user.get("email")) & (db.thumb.post_id == p['id'])).select()
        if r:
            r = r.first()
        p['liked'] = bool(r and r['rating'] > 0)
        p['disliked'] = bool(r and r['rating'] < 0)
        p['liked_by'] = ''
        p['disliked_by'] = ''
        p['show_likers'] = False
        p['cache_likes'] = False
    return dict(posts=posts)

@action('add_post', method="POST")
@action.uses(url_signer.verify(), auth.user, db)
def add_post():
    id = db.post.insert(
        post_text = request.json.get('new_post_content')
    )
    db.thumb.insert(
        post_id = id
    )
    return dict(id = id)

@action('delete_post', method="POST")
@action.uses(url_signer.verify(), auth.user, db)
def delete_post():
    id = request.json.get('id')
    if id is not None:
        r = db(db.post.id == id)
        if r and r.select().first()['user_email'] == auth.current_user.get('email'):
            r.delete()
            db(db.thumb.post_id == id).delete()
    return {}

@action('thumb_post', method="POST")
@action.uses(url_signer.verify(), auth.user, db)
def thumb_post():
    rating = request.json.get('rating')
    if rating < -1 or rating > 1:
        return {}
    id = request.json.get('id')
    if id is not None:
        r = db((db.thumb.post_id == id) & (db.thumb.user_email == auth.current_user.get('email')))
        if r:
            r.delete()
        db.thumb.insert(
            post_id = id,
            user_email = auth.current_user.get("email"),
            rating = rating
        )
    return {}

@action('get_thumbs', method="POST")
@action.uses(url_signer.verify(), auth.user, db)
def get_thumbs():
    id = request.json.get('id')
    rows = db(db.thumb.post_id == id).select()
    liked_list = []
    disliked_list = []
    for r in rows:
        user = db(db.auth_user.email == r.user_email).select().first()
        if r.rating > 0:
            liked_list.append(user.first_name + " " + user.last_name if user else "Unknown")
        if r.rating < 0:
            disliked_list.append(user.first_name + " " + user.last_name if user else "Unknown")
    liked_by = "Liked by " + ','.join(liked_list) if len(liked_list) else ""
    disliked_by = "Disliked by " + ','.join(disliked_list) if len(disliked_list) else ""
    return dict(liked_by = liked_by, disliked_by = disliked_by)

@action('get_username', method="POST")
@action.uses(url_signer.verify(), auth.user, db)
def get_username():
    r = db(db.auth_user.email == auth.current_user.get("email")).select().first()
    return dict(username=(r.first_name + " " + r.last_name if r is not None else "Unknown"))


from .single import Attention
from ..modules import dy_model

import dynet as dy
import numpy as np

import math

@dy_model
class Attention:
    """
    Compute 'Scaled Dot Product Attention
    """

    def __call__(self, query, key, value, mask=None, dropout=None):

        tensor_mask = dy.inputTensor(mask, batched=True)

        sentence_length = query.dim()[0][0]
        batch_size = query.dim()[1]
        h = query.dim()[0][2]
        remaining_dim = query.dim()[0][1]

        query = dy.reshape(query, (sentence_length, remaining_dim * h), batch_size=batch_size)
        value = dy.reshape(value, (sentence_length, remaining_dim * h), batch_size=batch_size)
        key_transpose = dy.reshape(key, (remaining_dim * h, sentence_length), batch_size=batch_size)

        scores = (query * key_transpose) / math.sqrt(query.dim()[1])

        sentence_length = mask.shape[0]

        mask_arr = [ [1 for i in range(sentence_length)]  for j in range(sentence_length)]

        if mask is not None:
            for i in range(sentence_length):
                for j in range(sentence_length):
                    mask_arr[i][j] = np.multiply(mask[i], mask[j])

        mask_arr_np = np.array(mask_arr)
        mask_arr_tensor = dy.inputTensor(mask_arr_np, batched=True)

        scores = dy.cmult(scores, mask_arr_tensor)

        p_attn = dy.softmax(scores)

        return p_attn * value, p_attn

        # scores = torch.matmul(query, key.transpose(-2, -1)) \
        #          / math.sqrt(query.size(-1))

        # if mask is not None:
        #     scores = scores.masked_fill(mask == 0, -1e9)

        # p_attn = F.softmax(scores, dim=-1) # (5, 100)

        # if dropout is not None:
        #     p_attn = dropout(p_attn)

        # (5, 5) * ((5, 200), 100)
        # return torch.matmul(p_attn, value), p_attn

@dy_model
class MultiHeadedAttention:
    """
    Take in model size and number of heads.
    """

    def __init__(self, model, h, d_model, dropout=0.1):
        #super().__init__()
        assert d_model % h == 0

        pc = model.add_subcollection()

        # We assume d_v always equals d_k
        self.d_k = d_model // h
        self.h = h

        self.linear_layers = [pc.add_parameters((d_model, d_model)) for _ in range(3)]
        self.output_linear = pc.add_parameters((d_model, d_model))
        #self.linear_layers = nn.ModuleList([nn.Linear(d_model, d_model) for _ in range(3)])
        #self.output_linear = nn.Linear(d_model, d_model)

        self.attention = Attention()
        # self.attention = Attention()

        #self.dropout = nn.Dropout(p=dropout)

    def __call__(self, query, key, value, mask, drop):

        if type(query).__name__ == 'list':
            batch_size = query[0].dim()[1] #100

            sentence_length = len(query)

            np_query = np.array([q.npvalue() for q in query])
            query = dy.inputTensor(np_query, batched=True) # ((5, 200), 100)

        sentence_length = query.dim()[0][0]
        token_dim = query.dim()[0][1]
        batch_size = query.dim()[1]

        key = query
        value = query

        arr = []
        for l, x in zip(self.linear_layers, (query, key, value)):
            test = x * l

            curr_shape = test.dim()
            total_dim = curr_shape[0][0] * curr_shape[0][1] * curr_shape[1]

            test = dy.reshape(test, (total_dim // (batch_size * self.h * self.d_k), self.h, self.d_k), batch_size=batch_size)

            test = dy.transpose(test, [0, 2, 1])

            arr.append(test)

        query, key, value = arr[0], arr[1], arr[2]

        # 2) Apply attention on all the projected vectors in batch.
        x, attn = self.attention(query, key, value, mask=mask, dropout=drop)

        res = x * self.output_linear
        print(res.dim())
        return res

        # 1) Do all the linear projections in batch from d_model => h x d_k
        #query, key, value = [l(x).view(batch_size, -1, self.h, self.d_k).transpose(1, 2)
        #                     for l, x in zip(self.linear_layers, (query, key, value))]

        # 2) Apply attention on all the projected vectors in batch.
        #x, attn = self.attention(query, key, value, mask=mask, dropout=self.dropout)

        # 3) "Concat" using a view and apply a final linear.
        #x = x.transpose(1, 2).contiguous().view(batch_size, -1, self.h * self.d_k)

        #return self.output_linear(x)

class MultiLayerMultiHeadAttention:
    def __init__(self, model, h, d_model, num_layers=1, dropout=0.1):
        self.num_layers = num_layers
        self.layers = [MultiHeadedAttention(model, h, d_model) for _ in range(self.num_layers)]
        # final layer has to be dimension 800

        pc = model.add_subcollection()
        self.output_linear = pc.add_parameters((d_model, 800))

        # Save Variable
        self.pc = pc
        self.spec = (h, d_model, num_layers)

    def __call__(self, query, key, value, mask, drop):
        for i in range(self.num_layers):
            curr_layer = self.layers[i]
            query = curr_layer(query, key, value, mask, drop)

            key = query
            value = query

        return query * self.output_linear

    @staticmethod
    def from_spec(spec, model):
        """Create and return a new instane with the needed parameters.

        It is one of the prerequisites for Dynet save/load method.
        """

        #cfg, vocabulary = spec
        #return GraphNNDecoder(model, cfg, vocabulary)

        h, d_model, num_layers = spec

        #token_repre, encoder, decoder = spec

        return MultiLayerMultiHeadAttention(model, h, d_model, num_layers=num_layers)

    def param_collection(self):
        """Return a :code:`dynet.ParameterCollection` object with the parameters.

        It is one of the prerequisites for Dynet save/load method.
        """
        return self.pc


import sys
import warnings
warnings.filterwarnings('ignore',category=FutureWarning)
from collections import Counter
from src.get_masks.evaluate_quality_masks import Quality_masks
from src.classifiers.classifier_all import All_classifier, evaluate_all
from src.ToT.table_of_truth import ToT
from src.data_classifier.Generator_proba_classifier import Genrator_data_prob_classifier
from src.get_masks.get_masks import Get_masks
from src.nn.nn_model_ref import NN_Model_Ref
from src.data_cipher.create_data import Create_data_binary
from src.utils.initialisation_run import init_all_for_run, init_cipher
from src.utils.config import Config
import argparse
from src.utils.utils import str2bool, two_args_str_int, two_args_str_float, str2list, transform_input_type, str2hexa, \
    transform_input_type2
import numpy as np
import pandas as pd
#--------------------------------------------------------------------------------------------------------------------------------------------------------------------------
# initiate the parser

config = Config()
parser = argparse.ArgumentParser()

parser.add_argument("--seed", default=config.general.seed, type=two_args_str_int, choices=[i for i in range(100)])
parser.add_argument("--device", default=config.general.device, type=two_args_str_int, choices=[i for i in range(100)])
parser.add_argument("--logs_tensorboard", default=config.general.logs_tensorboard)
parser.add_argument("--models_path", default=config.general.models_path)
parser.add_argument("--models_path_load", default=config.general.models_path_load)
parser.add_argument("--cipher", default=config.general.cipher, choices=["speck", "simon", "aes228", "aes224", "simeck", "gimli"])
parser.add_argument("--nombre_round_eval", default=config.general.nombre_round_eval, type=two_args_str_int)
parser.add_argument("--inputs_type", default=config.general.inputs_type, type=transform_input_type)
parser.add_argument("--word_size", default=config.general.word_size, type=two_args_str_int)
parser.add_argument("--alpha", default=config.general.alpha, type=two_args_str_int)
parser.add_argument("--beta", default=config.general.beta, type=two_args_str_int)
parser.add_argument("--type_create_data", default=config.general.type_create_data, choices=["normal", "real_difference"])

parser.add_argument("--diff", default=config.train_nn.diff, type=str2hexa)
parser.add_argument("--retain_model_gohr_ref", default=config.train_nn.retain_model_gohr_ref, type=str2bool)
parser.add_argument("--load_special", default=config.train_nn.load_special, type=str2bool)
parser.add_argument("--finetunning", default=config.train_nn.finetunning, type=str2bool)
parser.add_argument("--model_finetunne", default=config.train_nn.model_finetunne, choices=["baseline", "cnn_attention", "multihead", "deepset", "baseline_bin"])
parser.add_argument("--load_nn_path", default=config.train_nn.load_nn_path)
parser.add_argument("--countinuous_learning", default=config.train_nn.countinuous_learning, type=str2bool)
parser.add_argument("--curriculum_learning", default=config.train_nn.curriculum_learning, type=str2bool)
parser.add_argument("--nbre_epoch_per_stage", default=config.train_nn.nbre_epoch_per_stage, type=two_args_str_int)
parser.add_argument("--a_bit", default=config.train_nn.a_bit, type=two_args_str_int)
parser.add_argument("--type_model", default=config.train_nn.type_model, choices=["baseline_real", "baseline", "cnn_attention", "multihead", "deepset", "baseline_bin", "baseline_bin_v2"])
parser.add_argument("--nbre_sample_train", default=config.train_nn.nbre_sample_train, type=two_args_str_int)
parser.add_argument("--nbre_sample_eval", default=config.train_nn.nbre_sample_eval, type=two_args_str_int)
parser.add_argument("--num_epochs", default=config.train_nn.num_epochs, type=two_args_str_int)
parser.add_argument("--batch_size", default=config.train_nn.batch_size, type=two_args_str_int)
parser.add_argument("--loss_type", default=config.train_nn.loss_type, choices=["BCE", "MSE", "SmoothL1Loss", "CrossEntropyLoss", "F1"])
parser.add_argument("--lr_nn", default=config.train_nn.lr_nn, type=two_args_str_float)
parser.add_argument("--weight_decay_nn", default=config.train_nn.weight_decay_nn, type=two_args_str_float)
parser.add_argument("--momentum_nn", default=config.train_nn.momentum_nn, type=two_args_str_float)
parser.add_argument("--base_lr", default=config.train_nn.base_lr, type=two_args_str_float)
parser.add_argument("--max_lr", default=config.train_nn.max_lr, type=two_args_str_float)
parser.add_argument("--demicycle_1", default=config.train_nn.demicycle_1, type=two_args_str_int)
parser.add_argument("--optimizer_type", default=config.train_nn.optimizer_type, choices=["Adam", "AdamW", "SGD"])
parser.add_argument("--scheduler_type", default=config.train_nn.scheduler_type, choices=["CyclicLR", "None"])
parser.add_argument("--numLayers", default=config.train_nn.numLayers, type=two_args_str_int)
parser.add_argument("--limit", default=config.train_nn.limit, type=two_args_str_int)
parser.add_argument("--kstime", default=config.train_nn.kstime, type=two_args_str_int)
parser.add_argument("--out_channel0", default=config.train_nn.out_channel0, type=two_args_str_int)
parser.add_argument("--out_channel1", default=config.train_nn.out_channel1, type=two_args_str_int)
parser.add_argument("--hidden1", default=config.train_nn.hidden1, type=two_args_str_int)
parser.add_argument("--hidden2", default=config.train_nn.hidden2, type=two_args_str_int)
parser.add_argument("--kernel_size0", default=config.train_nn.kernel_size0, type=two_args_str_int)
parser.add_argument("--kernel_size1", default=config.train_nn.kernel_size1, type=two_args_str_int)
parser.add_argument("--num_workers", default=config.train_nn.num_workers, type=two_args_str_int)
parser.add_argument("--clip_grad_norm", default=config.train_nn.clip_grad_norm, type=two_args_str_float)
parser.add_argument("--end_after_training", default=config.train_nn.end_after_training, type=str2bool)

parser.add_argument("--load_masks", default=config.getting_masks.load_masks, type=str2bool)
parser.add_argument("--file_mask", default=config.getting_masks.file_mask)
parser.add_argument("--nbre_max_masks_load", default=config.getting_masks.nbre_max_masks_load, type=two_args_str_int)
parser.add_argument("--nbre_generate_data_train_val", default=config.getting_masks.nbre_generate_data_train_val, type=two_args_str_int)
parser.add_argument("--nbre_necessaire_val_SV", default=config.getting_masks.nbre_necessaire_val_SV, type=two_args_str_int)
parser.add_argument("--nbre_max_batch", default=config.getting_masks.nbre_max_batch, type=two_args_str_int)
parser.add_argument("--liste_segmentation_prediction", default=config.getting_masks.liste_segmentation_prediction)
parser.add_argument("--liste_methode_extraction", default=config.getting_masks.liste_methode_extraction, type=transform_input_type)
parser.add_argument("--liste_methode_selection", default=config.getting_masks.liste_methode_selection, type=transform_input_type)
parser.add_argument("--hamming_weigth", default=config.getting_masks.hamming_weigth, type=transform_input_type2)
parser.add_argument("--thr_value", default=config.getting_masks.thr_value, type=str2list)
parser.add_argument("--research_new_masks", default=config.getting_masks.research_new_masks, type=str2bool)
parser.add_argument("--save_fig_plot_feature_before_mask", default=config.getting_masks.save_fig_plot_feature_before_mask, type=str2bool)
parser.add_argument("--end_after_step2", default=config.getting_masks.end_after_step2, type=str2bool)

parser.add_argument("--create_new_data_for_ToT", default=config.make_ToT.create_new_data_for_ToT, type=str2bool)
parser.add_argument("--create_ToT_with_only_sample_from_cipher", default=config.make_ToT.create_ToT_with_only_sample_from_cipher, type=str2bool)
parser.add_argument("--nbre_sample_create_ToT", default=config.make_ToT.nbre_sample_create_ToT, type=two_args_str_int)

parser.add_argument("--create_new_data_for_classifier", default=config.make_data_classifier.create_new_data_for_classifier, type=str2bool)
parser.add_argument("--nbre_sample_train_classifier", default=config.make_data_classifier.nbre_sample_train_classifier, type=two_args_str_int)
parser.add_argument("--nbre_sample_val_classifier", default=config.make_data_classifier.nbre_sample_val_classifier, type=two_args_str_int)

parser.add_argument("--retrain_nn_ref", default=config.compare_classifer.retrain_nn_ref, type=str2bool)
parser.add_argument("--num_epch_2", default=config.compare_classifer.num_epch_2, type=two_args_str_int)
parser.add_argument("--batch_size_2", default=config.compare_classifer.batch_size_2, type=two_args_str_int)
parser.add_argument("--classifiers_ours", default=config.compare_classifer.classifiers_ours, type=str2list)
parser.add_argument("--retrain_with_import_features", default=config.compare_classifer.retrain_with_import_features, type=str2bool)
parser.add_argument("--keep_number_most_impactfull", default=config.compare_classifer.keep_number_most_impactfull, type=two_args_str_int)
parser.add_argument("--num_epch_our", default=config.compare_classifer.num_epch_our, type=two_args_str_int)
parser.add_argument("--batch_size_our", default=config.compare_classifer.batch_size_our, type=two_args_str_int)
parser.add_argument("--alpha_test", default=config.compare_classifer.alpha_test, type=two_args_str_float)
parser.add_argument("--quality_of_masks", default=config.compare_classifer.quality_of_masks, type=str2bool)
parser.add_argument("--end_after_step4", default=config.compare_classifer.end_after_step4, type=str2bool)
parser.add_argument("--eval_nn_ref", default=config.compare_classifer.eval_nn_ref, type=str2bool)
parser.add_argument("--compute_independance_feature", default=config.compare_classifer.compute_independance_feature, type=str2bool)
parser.add_argument("--save_data_proba", default=config.compare_classifer.save_data_proba, type=str2bool)

args = parser.parse_args()

#--------------------------------------------------------------------------------------------------------------------------------------------------------------------------
#
print("---" * 100)
writer, device, rng, path_save_model, path_save_model_train, name_input = init_all_for_run(args)

print("LOAD CIPHER")
print()
cipher = init_cipher(args)
creator_data_binary = Create_data_binary(args, cipher, rng)

#--------------------------------------------------------------------------------------------------------------------------------------------------------------------------
print("---" * 100)
print("STEP 1 : LOAD/ TRAIN NN REF")
print()
print("COUNTINUOUS LEARNING: "+ str(args.countinuous_learning) +  " | CURRICULUM LEARNING: " +  str(args.curriculum_learning) + " | MODEL: " + str(args.type_model))
print()

"""nombre_round_eval = args.nombre_round_eval
args.nombre_round_eval = nombre_round_eval - 2
nn_model_ref2 = NN_Model_Ref(args, writer, device, rng, path_save_model, cipher, creator_data_binary, path_save_model_train)
nn_model_ref2.epochs = 10
nn_model_ref2.train_general(name_input)
args.nombre_round_eval = nombre_round_eval - 1
nn_model_ref3 = NN_Model_Ref(args, writer, device, rng, path_save_model, cipher, creator_data_binary, path_save_model_train)
nn_model_ref3.epochs = 10
nn_model_ref3.net = nn_model_ref2.net
nn_model_ref3.train_general(name_input)
args.nombre_round_eval = nombre_round_eval
nn_model_ref = NN_Model_Ref(args, writer, device, rng, path_save_model, cipher, creator_data_binary, path_save_model_train)
nn_model_ref.net = nn_model_ref3.net"""

nn_model_ref = NN_Model_Ref(args, writer, device, rng, path_save_model, cipher, creator_data_binary, path_save_model_train)

if args.retain_model_gohr_ref:
    nn_model_ref.train_general(name_input)
else:
    #nn_model_ref.load_nn()
    try:
        if args.finetunning:
            nn_model_ref.load_nn()
            nn_model_ref.train_from_scractch(name_input + "fine-tune")
        #nn_model_ref.eval(["val"])
        else:
            nn_model_ref.load_nn()
    except:
        print("ERROR")
        print("NO MODEL AVALAIBLE FOR THIS CONFIG")
        print("CHANGE ARGUMENT retain_model_gohr_ref")
        print()
        sys.exit(1)

if args.create_new_data_for_ToT and args.create_new_data_for_classifier:
    del nn_model_ref.X_train_nn_binaire, nn_model_ref.X_val_nn_binaire, nn_model_ref.Y_train_nn_binaire, nn_model_ref.Y_val_nn_binaire
    del nn_model_ref.c0l_train_nn, nn_model_ref.c0l_val_nn, nn_model_ref.c0r_train_nn, nn_model_ref.c0r_val_nn
    del nn_model_ref.c1l_train_nn, nn_model_ref.c1l_val_nn, nn_model_ref.c1r_train_nn, nn_model_ref.c1r_val_nn

print("STEP 1 : DONE")
print("---" * 100)
if args.end_after_training:
    sys.exit(1)

#--------------------------------------------------------------------------------------------------------------------------------------------------------------------------
print("STEP 2 : GET MASKS")
print()
print("LOAD MASKS: "+ str(args.load_masks) +  " | RESEARCH NEW: " +  str(args.research_new_masks) + " | MODEL: " + str(args.liste_segmentation_prediction) +" " + str(args.liste_methode_extraction)+" " + str(args.liste_methode_selection)+" " + str(args.hamming_weigth)+" " + str(args.thr_value))
print()
get_masks_gen = Get_masks(args, nn_model_ref.net, path_save_model, rng, creator_data_binary, device)
if args.research_new_masks:
    get_masks_gen.start_step()
    get_masks_gen.save_masks(path_save_model)
    del get_masks_gen.X_deltaout_train, get_masks_gen.X_eval, get_masks_gen.Y_tf, get_masks_gen.Y_eval

print("STEP 2 : DONE")
print("---" * 100)
if args.end_after_step2:
    sys.exit(1)

#--------------------------------------------------------------------------------------------------------------------------------------------------------------------------
print("STEP 3 : MAKE M-ODT")
print()
print("NEW DATA: "+ str(args.create_new_data_for_ToT) +  " | PURE ToT: " +  str(args.create_ToT_with_only_sample_from_cipher) )
print()

table_of_truth = ToT(args, nn_model_ref.net, path_save_model, rng, creator_data_binary, device, get_masks_gen.masks, nn_model_ref)
table_of_truth.create_DDT()

del table_of_truth.c0l_create_ToT, table_of_truth.c0r_create_ToT
del table_of_truth.c1l_create_ToT, table_of_truth.c1r_create_ToT

print("STEP 3 : DONE")
print("---" * 100)
#--------------------------------------------------------------------------------------------------------------------------------------------------------------------------
print("STEP 4 : CREATE DATA PROBA AND CLASSIFY")
print()
print("NEW DATA: "+ str(args.create_new_data_for_classifier))
print()

generator_data = Genrator_data_prob_classifier(args, nn_model_ref.net, path_save_model, rng, creator_data_binary, device, get_masks_gen.masks, nn_model_ref)
generator_data.create_data_g(table_of_truth)
#EVALUATE GOHR NN ON NEW DATASET
nn_model_ref.X_train_nn_binaire = generator_data.X_bin_train
nn_model_ref.X_val_nn_binaire = generator_data.X_bin_val
nn_model_ref.Y_train_nn_binaire = generator_data.Y_create_proba_train
nn_model_ref.Y_val_nn_binaire = generator_data.Y_create_proba_val

if args.eval_nn_ref:
    nn_model_ref.eval_all(["train", "val"])
all_clfs = All_classifier(args, path_save_model, generator_data, get_masks_gen, nn_model_ref, table_of_truth)
#all_clfs.X_train_proba = np.concatenate((all_clfs.X_train_proba, X_feat_temp), axis = 1)
#all_clfs.X_eval_proba =  np.concatenate((all_clfs.X_eval_proba, X_feat_temp_val), axis = 1)
all_clfs.classify_all()

if args.quality_of_masks:
    qm = Quality_masks(args, path_save_model, generator_data, get_masks_gen, nn_model_ref, table_of_truth, all_clfs)
    qm.start_all()
else:
    qm = None

print("STEP 4 : DONE")
print("---" * 100)
if args.end_after_step4:
    sys.exit(1)
#--------------------------------------------------------------------------------------------------------------------------------------------------------------------------
print("STEP 5: START EVALUATE ClASSIFIERS")
print()

evaluate_all(all_clfs, generator_data, nn_model_ref, table_of_truth, qm, path_save_model)


import operator
from itertools import islice

import contexttimer
import pytest
from mypy_boto3_sqs import Client as SQSClient

from tests.test_queue.robot import Command, CommandReceiver, ReceiverAndSender

def test_pause(receiver_and_sender: ReceiverAndSender):
    """Wait a bit if the queue is empty."""
    receiver, _sender = receiver_and_sender

    assert receiver.iteration_timeout == 3

    with contexttimer.Timer() as timer:
        receiver._pause_while_iterating_over_queue()

        elapsed_time = timer.elapsed

    assert round(elapsed_time) <= receiver.iteration_timeout

def test_iterate_imperative(receiver_and_sender: ReceiverAndSender):
    """The `for` loop construct can iterate by messages in queue."""
    receiver, sender = receiver_and_sender

    sent_commands = [Command.RIGHT, Command.FORWARD, Command.LEFT, Command.JUMP]

    sender.send_many(sent_commands)

    received_commands = []
    for message in receiver:
        received_commands.append(message.value)

        receiver.acknowledge(message)

        if message.value == Command.JUMP:
            break

    assert received_commands == sent_commands

def test_iterate_functional(receiver_and_sender: ReceiverAndSender):
    """The `for` loop construct can iterate by messages in queue."""
    receiver, sender = receiver_and_sender

    sent_commands = [Command.RIGHT, Command.FORWARD, Command.LEFT, Command.JUMP]

    sender.send_many(sent_commands)

    # We have to limit the number of messages we are interested in,
    # or .receive() will hang forever.
    messages = islice(receiver, len(sent_commands))

    # Now let's acknowledge those messages.
    messages = map(receiver.acknowledge, messages)

    # And let's see what they are.
    received_commands = list(map(operator.attrgetter('value'), messages))

    assert received_commands == sent_commands

class NotPausingReceiver(CommandReceiver):
    """Receiver with overridden pause method."""

    def _pause_while_iterating_over_queue(self) -> None:
        """Do not pause, raise instead."""
        raise ValueError('foo!')

def test_empty_queue(mock_sqs_client: SQSClient):
    """Test empty queue."""
    sqs_queue_url = mock_sqs_client.create_queue(
        QueueName='robot_commands',
        Attributes={
            'VisibilityTimeout': '3',
        },
    )['QueueUrl']

    receiver = NotPausingReceiver(url=sqs_queue_url)

    with pytest.raises(ValueError):
        for sqs_message in receiver:
            assert sqs_message is not None


# -*- coding: utf-8 -*-
import scrapy
from scrapy_redis.spiders import RedisSpider
from urllib import  parse

class BookspiderSpider(RedisSpider, scrapy.Spider):
    name = 'book_spider'
    redis_key = "book_spider:start_urls"
    # start_urls = ['https://www.barnesandnoble.com/h/books/browse']

    def parse(self, response):
        superbject_scrope = response.xpath('//section[@class="links-container browse clearer"]/ul/li/ul/li')
        for item in superbject_scrope:
            subject_url = item.xpath('./a/@href').extract_first()
            subject_url = parse.urljoin(response.url, subject_url)
            subject = item.xpath('./a/text()').extract_first()

            yield scrapy.Request(url=subject_url,
                                 callback=self.parse_subject,
                                 meta={'subject':subject},
                                 dont_filter=True,
                                 )

    def parse_subject(self, response):
        """解析分类"""
        scopes = response.xpath('//ul[@class="lists lists--unstyled"]/li')
        for item in scopes:
            secondary_subject = item.xpath('./a/text()').extract_first()
            secondary_subject_url = item.xpath('./a/@href').extract_first()
            secondary_subject_url = parse.urljoin(response.url, secondary_subject_url)
            response.meta.update(secondary_subject=secondary_subject)
            yield scrapy.Request(url=secondary_subject_url,
                                 callback=self.parse_page_info,
                                 meta=response.meta,
                                 dont_filter=True)

    def parse_page_info(self, response):
        """解析详情页"""
        current_page_num = response.xpath('//li[@class="pagination__active"]/span/text()').extract_first()
        response.meta.update(current_page_num=current_page_num)
        book_urls = response.xpath('//a[@class="pImageLink "]/@href').extract()
        book_urls = [parse.urljoin(response.url, href) for href in book_urls]
        for book_url in book_urls:
            yield scrapy.Request(url=book_url,
                                 callback=self.parse_detail,
                                 meta=response.meta,
                                 dont_filter=False,)

        next_page_link = response.xpath('//a[@class="next-button"]/@href').extract_first()
        if next_page_link:
            yield scrapy.Request(url=parse.urljoin(response.url, next_page_link),
                                 callback=self.parse_page_info,
                                 meta=response.meta,
                                 dont_filter=True,)

    def parse_detail(self, response):
        """解析book 详情"""
        result = {}
        title = response.xpath('//h1[@class="pdp-header-title "]/text()').extract_first()
        author = response.xpath('//span[@id="key-contributors"]/a/text()').extract_first()
        avg_rating = response.xpath('//div[@class="bv_avgRating_component_container notranslate"]/text()').extract_first()
        reviews_num = response.xpath('//div[@class="bv_numReviews_text"]/text()').extract_first()
        if reviews_num:
            reviews_num = reviews_num.strip('(').strip(')')
        price = response.xpath('//span[@id="pdp-cur-price"]/text()').extract_first()
        old_price = response.xpath('//s[@class="old-price"]/text()').extract_first()
        if old_price:
            old_price=old_price.strip('$')
        isbn = response.xpath('//th[contains(.,"ISBN")]/parent::*/td/text()').extract_first()
        publisher = response.xpath('//th[contains(.,"Publisher")]/parent::*/td/a/span/text()').extract_first()
        pages= response.xpath('//th[contains(.,"Pages")]/parent::*/td/text()').extract_first()
        sales_rank = response.xpath('//th[contains(.,"Sales rank")]/parent::*/td/text()').extract_first()
        pub_date = response.xpath('//th[contains(.,"Publication date")]/parent::*/td/text()').extract_first()
        product_dimensions = response.xpath('//th[contains(.,"Product dimensions")]/parent::*/td/text()').extract_first()

        result.update(subject=response.meta['subject'],
                      secondary_subject = response.meta['secondary_subject'],
                      current_page_num = response.meta['current_page_num'],
                      title = title,
                      author=author,
                      avg_rating=avg_rating,
                      reviews_num=reviews_num,
                      price=price,
                      old_price=old_price,
                      isbn=isbn,
                      publisher=publisher,
                      pages=pages,
                      sales_rank=sales_rank,
                      pub_date=pub_date,
                      product_dimensions=product_dimensions,
                      url=response.url,

                      )
        yield result


import matplotlib.pyplot as plt

def plot_df(df, title, size):
    df.plot(kind='bar', stacked=False, width=1, logy=True, figsize=size, legend=False)
    plt.title(title)
    plt.legend(loc='center left', bbox_to_anchor=(1, 0.5), prop={'size': 8})
    plt.tight_layout()
    plt.tick_params(axis='both', labelsize=8)
    plt.show()

def plot_3d(data, title, x_axis, y_axis, z_axis, size):
    fig = plt.figure(figsize=size)
    ax = fig.gca(projection='3d')
    index = 0
    shapes = ['o', 'v', 's', 'x', '*', 'D', 'P', '^', '1', 'p', '+', 'X', '<', '>']
    colours = ['b', 'g', 'r', 'c', 'y', 'k', 'm', 'tab:brown']
    for name, row in data.iterrows():
        ax.scatter(row[x_axis], row[y_axis], row[z_axis], label=name, c=colours[index % len(colours)], marker=shapes[index // len(colours)])
        index += 1
    ax.set_xlabel(x_axis, fontsize=8)
    ax.set_ylabel(y_axis, fontsize=8)
    ax.set_zlabel(z_axis, fontsize=8)
    ax.tick_params(axis='both', labelsize=8)
    fig.suptitle(title)
    plt.legend(loc='center left', bbox_to_anchor=(1, 0.5), prop={'size': 8}, ncol=2)
    plt.tight_layout()
    plt.show()


from sqlalchemy import or_
from flask import Blueprint, request, jsonify, current_app, send_from_directory

from core.database import db
from core.anunciantes.models import Anunciante
from core.categorias.models import Categoria
from core.image_upload import handle_image_upload

anunciantes_blueprint = Blueprint("anunciantes", __name__)

@anunciantes_blueprint.route('/', methods=['GET', 'POST'])
def index_anunciantes():
    if request.method == 'POST':
        anunciante = save_advertiser_profile(request.json)
        return jsonify(anunciante.serialize)
    id_categoria = int(request.args.get('id_categoria', '0'))
    if id_categoria == 0:
        anunciantes = Anunciante.query.all()
    else:
        anunciantes = Anunciante.query.filter_by(id_categoria=id_categoria)
    return jsonify([anunciante.serialize for anunciante in anunciantes])

@anunciantes_blueprint.route('/fotos', methods=["GET", "POST"])
def image_upload():
    return handle_image_upload(request,
                               save_path=current_app.config['ADVERTISER_MEDIA_PATH'],
                               success_path='.get_advertiser_image')

@anunciantes_blueprint.route('/foto/<filename>')
def get_advertiser_image(filename):
    return send_from_directory(current_app.config['ADVERTISER_MEDIA_PATH'], filename)

@anunciantes_blueprint.route('/search')
def search_advertiser():
    search_query = '%' + request.args.get('q') + '%'
    q = db.session.query(Anunciante) \
        .outerjoin(Categoria, Anunciante.id_categoria==Categoria.id_categoria) \
        .filter(or_(Anunciante.nome_fantasia.like(search_query), Categoria.descricao.like(search_query)))
    return jsonify([anunciante.serialize for anunciante in q])

def save_advertiser_profile(json):
    new_advertiser = False
    advertiser = Anunciante.query.filter_by(guid_anunciante=json["guid_anunciante"]).first()
    if not advertiser:
        advertiser = Anunciante()
        new_advertiser = True
    advertiser.guid_anunciante = json["guid_anunciante"]
    advertiser.nome_fantasia = json["nome_fantasia"]
    advertiser.logradouro = json["logradouro"]
    advertiser.numero = json["numero"]
    advertiser.bairro = json["bairro"]
    advertiser.telefone = json["telefone"]
    advertiser.celular = json["celular"]
    advertiser.email = json["email"]
    advertiser.id_cidade = json['id_cidade']
    advertiser.id_categoria = json["id_categoria"]
    advertiser.picture_file = json.get("picture_file", None)
    if new_advertiser:
        db.session.add(advertiser)
    db.session.commit()
    return advertiser


__author__ = 'root'
from dbservices import servicechecklogs
from opsviewapi.models import ServiceCheckLog
from exc.exceptions import ServiceCheckLogError
from django.utils import timezone
import logging

logger = logging.getLogger(__name__)

def write_service_check_log(log_obj=None, time_stamp=None, device_info=None, description=None, status=None,
                            device_name=None, error_code=None):
    if time_stamp is None:
        time_stamp = timezone.now()

    if log_obj is None:
        log_obj = ServiceCheckLog(time_stamp=time_stamp, device_info=device_info, description=description,
                                  status=status, device_name=device_name, error_code=error_code)

    try:
        servicechecklogs.insert_update_log(log_obj=log_obj)
    except ServiceCheckLogError as ex:
        logger.warning(ex)

def delete_service_check_log(dev_name):
    try:
        servicechecklogs.delete_log(device_name=dev_name)
    except ServiceCheckLogError as ex:
        logger.warning(ex)


import os
from tensorflow import io, newaxis, float32
from tensorflow import image as img
import numpy as np
from Anchor_generator import kmeans
import datetime

max_param = 20
width, height = 608, 608

def get_path(directory_path, selection):
    """
    parameters
    -----------
    directory_path : TRAIN_DIR, TEST_DIR or VALID_DIR
    selection : ".txt" or ".jpg"

    output
    ------
    list of string of path to each indiviual jpg or txt file.
    """
    path_list = list()
    for filename in os.listdir(directory_path):
        # Select only items finishing with the selection (.txt or .jpg)
        if filename.endswith(selection):
            path_list.append(f"{directory_path}/{filename}")
    path_list.sort()
    return path_list

def load_labels(labels_path):
    with open(labels_path, 'r') as labels_file:
        labels = labels_file.read().splitlines()
        for i in range(len(labels)):
            labels[i] = labels[i].replace(" ", "")
    return labels

def load_image(image_path):
    """
    Convert all image into array of shape (1, width, height, 3)
    """
    image = io.read_file(image_path)
    image = img.decode_image(image, channels=3)
    image = img.convert_image_dtype(image, float32)

    image = img.resize(image, (width, height))
    image = image[newaxis, :]
    return image

def get_boxes(annotations_path, training=True):
    """ Get boxes[labels, x, y, width, height]"""
    boxes = np.loadtxt(annotations_path, delimiter=" ", dtype='float32')
    # Add a dimension if the image only has 1 boxes and therefore is (5, )
    if boxes.ndim == 1:
        boxes = np.expand_dims(boxes, 0)
    # add a batch dimension and multiple boxes filled with zero to get the final shape of (1, max_param, 5)
    elif training:
        zero_array = np.zeros((max_param, 5))
        boxes = np.expand_dims(np.concatenate((boxes, zero_array))[:max_param], 0)
        return np.roll(boxes, -1, axis=2)
    return np.roll(boxes, -1, axis=1)

def anchors_generator(boxes_path, cluster):
    box_wh = np.zeros((1, 2))
    for i in range(len(boxes_path)):
        bbox = get_boxes(boxes_path[i], False)[..., 2:4]
        if bbox.shape[1] != 0:
            box_wh = np.append(box_wh, bbox, axis=0)

    bbox = list()
    for box in box_wh[1:]:  # [1:] so we dont select the row of zero
        ymin = box[1] / 2.
        xmin = box[0] / 2.
        ymax = -ymin
        xmax = -xmin

        bbox.append([ymin, xmin, ymax, xmax])
    bbox = np.array(bbox)

    result = kmeans(bbox, k=cluster)

    anchors = []
    for box in result:
        width = box[1] * 2
        height = box[0] * 2

        anchors.append([width, height])

    return np.array(anchors)

def save_anchors(anchors):
    filename = datetime.datetime.now()
    anchors_path = 'Data/anchors'

    np.savetxt(f"{anchors_path}/{filename.strftime('%H:%M %d %B %Y') + '.txt'}", anchors, fmt='%f')

def read_anchors(selection=None):
    anchors_path = 'Data/anchors'
    last_anchors = os.listdir(anchors_path)[-1]
    print(last_anchors)
    if selection:
        anchors = np.loadtxt(f'{anchors_path}/{selection}', dtype='float32')
        return anchors
    else:
        anchors = np.loadtxt(f'{anchors_path}/{last_anchors}', dtype='float32')
        return anchors


# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from django.conf import settings

class CorsHeaderMiddleware(object):
    """Add a CORS header if running in debug mode."""

    def process_response(self, request, response):
        if settings.DEBUG:
            response['Access-Control-Allow-Origin'] = '*'
        return response


import numpy as np
import matplotlib.pyplot as plt

root = 'model6_r_0p1_to_1_500_nolensing_no_temp_no_scalar_label_rescale_10_do_label_shift_yes_rescale_0p1_0p1_no_pretrain_at_epoch_59'
plot_dir = './plots/' + root + '/'
plot_data_dir = plot_dir + 'plot_data/'
pred_means = np.genfromtxt(plot_data_dir + 'pred_means.dat')
pred_stds = np.genfromtxt(plot_data_dir + 'pred_stds.dat')
true_values = np.genfromtxt(plot_data_dir + 'true_values.dat')
len_same_values = np.genfromtxt(plot_data_dir + 'len_same_values.dat')
predictions_all = np.genfromtxt(plot_data_dir + 'predictions_all.dat')

r = np.zeros(int(np.sum(len_same_values)))
ii = 0
for i in np.arange(true_values.shape[0]):
    for j in  np.arange(len_same_values[i]):
        r[ii] = true_values[i] # might change to true_values[i,:] if n_params >1
        ii = ii + 1
print(r.shape)
print('r', r)
print(true_values[0],true_values[1], true_values[-1])
print('r[:55]', r[:55])
print('len_same_values', len_same_values)
r_pred = predictions_all
print('r_pred', r_pred)
err = r - r_pred
abs_err = np.abs(err)
print('err', err)

#r = np.array([0.12, 0.13, 0.2, 0.25, 0.3,0.4,0.5,0.6,0.7,0.8,0.9,1.0])+0.05
#err = r/10.0
#err[1] = 0.1
#r_pred = r + r/10.0
#print('r, err', r, err)

bins = np.linspace(0.1,1.0,10)
print('bins = ', bins)
ind_bin = np.digitize(r,bins,right=False)
bin_center = (bins[1:]+bins[:-1])/2.0
print('bin center = ', bin_center)
print('ind_bin', ind_bin)
print('heyehye')

err_binned = np.zeros(bin_center.size)
for i in np.arange(1, bin_center.size+1):
    err_binned[i-1] = np.mean(abs_err[np.where(ind_bin==i)])
    if err_binned[i-1] == np.nan:
        err_binned[i-1] = 0.0
    print('i = ', i)
    print('np.where(ind_bin==i)', np.where(ind_bin==i))
    print('abs_err[np.where(ind_bin==i)]', abs_err[np.where(ind_bin==i)])
    print('err_binned', err_binned)

print('\n Making figures now')

fig, ax = plt.subplots()
ax.plot(r, err, 'x')
ax.set_xlabel(r'$r$')
ax.set_ylabel('error on r')
plotname = plot_dir + 'plot_r_error_vs_r.pdf'
plt.savefig(plotname)
print('Saved figure: ', plotname)

fig, ax = plt.subplots()
ax.plot(bin_center, err_binned, '.')
ax.set_xlabel(r'$r$')
ax.set_ylabel('binned error on r')
plotname = plot_dir + 'plot_binned_r_error_vs_r.pdf'
plt.savefig(plotname)
print('Saved figure: ', plotname)

fig, ax = plt.subplots()
ax.plot(r, r, 'r-')
ax.plot(r, r_pred, 'bx') #no error bars yet
ax.set_xlabel(r'$r$')
ax.set_ylabel(r'$r_{\mathrm{pred}}$')
plotname = plot_dir + 'plot_r_pred_vs_r_no_error_bar.pdf'
plt.savefig(plotname)
print('Saved figure: ', plotname)


"""
===================
Demo Axis Direction
===================

"""

import numpy as np
import matplotlib.pyplot as plt
import mpl_toolkits.axisartist.angle_helper as angle_helper
import mpl_toolkits.axisartist.grid_finder as grid_finder
from matplotlib.projections import PolarAxes
from matplotlib.transforms import Affine2D

import mpl_toolkits.axisartist as axisartist

from mpl_toolkits.axisartist.grid_helper_curvelinear import \
    GridHelperCurveLinear

# nodebox section
if __name__ == '__builtin__':
    # were in nodebox
    import os
    import tempfile
    W = 800
    inset = 20
    size(W, 600)
    plt.cla()
    plt.clf()
    plt.close('all')
    def tempimage():
        fob = tempfile.NamedTemporaryFile(mode='w+b', suffix='.png', delete=False)
        fname = fob.name
        fob.close()
        return fname
    imgx = 20
    imgy = 0
    def pltshow(plt, dpi=150):
        global imgx, imgy
        temppath = tempimage()
        plt.savefig(temppath, dpi=dpi)
        dx,dy = imagesize(temppath)
        w = min(W,dx)
        image(temppath,imgx,imgy,width=w)
        imgy = imgy + dy + 20
        os.remove(temppath)
        size(W, HEIGHT+dy+40)
else:
    def pltshow(mplpyplot):
        mplpyplot.show()
# nodebox section end

def setup_axes(fig, rect):
    """
    polar projection, but in a rectangular box.
    """

    # see demo_curvelinear_grid.py for details
    tr = Affine2D().scale(np.pi/180., 1.) + PolarAxes.PolarTransform()

    extreme_finder = angle_helper.ExtremeFinderCycle(20, 20,
                                                     lon_cycle=360,
                                                     lat_cycle=None,
                                                     lon_minmax=None,
                                                     lat_minmax=(0, np.inf),
                                                     )

    grid_locator1 = angle_helper.LocatorDMS(12)
    grid_locator2 = grid_finder.MaxNLocator(5)

    tick_formatter1 = angle_helper.FormatterDMS()

    grid_helper = GridHelperCurveLinear(tr,
                                        extreme_finder=extreme_finder,
                                        grid_locator1=grid_locator1,
                                        grid_locator2=grid_locator2,
                                        tick_formatter1=tick_formatter1
                                        )

    ax1 = axisartist.Subplot(fig, rect, grid_helper=grid_helper)
    ax1.axis[:].toggle(ticklabels=False)

    fig.add_subplot(ax1)

    ax1.set_aspect(1.)
    ax1.set_xlim(-5, 12)
    ax1.set_ylim(-5, 10)

    return ax1

def add_floating_axis1(ax1):
    ax1.axis["lat"] = axis = ax1.new_floating_axis(0, 30)
    axis.label.set_text(r"$\theta = 30^{\circ}$")
    axis.label.set_visible(True)

    return axis

def add_floating_axis2(ax1):
    ax1.axis["lon"] = axis = ax1.new_floating_axis(1, 6)
    axis.label.set_text(r"$r = 6$")
    axis.label.set_visible(True)

    return axis

fig = plt.figure(1, figsize=(8, 4))
fig.clf()
fig.subplots_adjust(left=0.01, right=0.99, bottom=0.01, top=0.99,
                    wspace=0.01, hspace=0.01)

for i, d in enumerate(["bottom", "left", "top", "right"]):
    ax1 = setup_axes(fig, rect=241++i)
    axis = add_floating_axis1(ax1)
    axis.set_axis_direction(d)
    ax1.annotate(d, (0, 1), (5, -5),
                 xycoords="axes fraction", textcoords="offset points",
                 va="top", ha="left")

for i, d in enumerate(["bottom", "left", "top", "right"]):
    ax1 = setup_axes(fig, rect=245++i)
    axis = add_floating_axis2(ax1)
    axis.set_axis_direction(d)
    ax1.annotate(d, (0, 1), (5, -5),
                 xycoords="axes fraction", textcoords="offset points",
                 va="top", ha="left")

pltshow(plt)


#!/usr/bin/env python

"""
Classes for writing seqs or snps to popular data formats
like VCF, PHY, NEXUS, or HDF5, while also optionally
combining haplotypes into diploid base calls.
"""

import os
from typing import Optional, Iterable

import numpy as np
import pandas as pd

# import ipcoal
from ipcoal.io.genos import Genos
from ipcoal.io.transformer import Transformer
from ipcoal.io.vcf import VCF
from ipcoal.utils.utils import IpcoalError

NEXHEADER = """#nexus
begin data;
  dimensions ntax={} nchar={};
  format datatype=DNA missing=N gap=- interleave=yes;
  matrix\n
"""

class Writer:
    """
    Writer class object to write ipcoal seqs in a variety of formats.

    Parameters
    ----------
    seqs (ndarray)
        A .seqs array from ipcoal of dimensions (nloci, ntaxa, nsites).
        The data for the ntaxa is ordered by their names alphanumerically.
    names (list)
        A list of the taxon names ordered alphanumerically.
    """
    def __init__(self, model):
        # both are already ordered alphanumerically
        self.seqs = model.seqs.copy()
        self.names = model.alpha_ordered_names.copy()
        self.alleles = model.alleles.copy()
        self.outdir = None
        self.outfile = None
        self.idxs = None
        self.ancestral_seq = None
        if model.ancestral_seq is not None:
            self.ancestral_seq = model.ancestral_seq.copy()

    def _subset_loci(self, idxs):
        """
        If datasets is dim=3 then subset loci by idxs argument.
        """
        # subselect the linkage groups to write (default is to select all)
        if idxs is not None:
            # ensure it is iterable
            if isinstance(idxs, int):
                self.idxs = [self.idxs]
            else:
                self.idxs = idxs

            # check that idxs exist
            for loc in np.array(self.idxs):
                if loc not in range(self.seqs.shape[0]):
                    raise IpcoalError(
                        "idx {} is not in the data set".format(loc))

            # subset self.seqs to selected loci
            idxs = np.array(sorted(self.idxs))
            self.seqs = self.seqs[idxs]
        else:
            self.idxs = range(self.seqs.shape[0])

    def _transform_seqs(self, diploid):
        """
        Transform seqs from int type to str type. Also optionally combine
        haploid samples into diploids and use IUPAC ambiguity codes to
        represent hetero sites.
        """
        txf = Transformer(
            self.seqs,
            self.names,
            alleles=self.alleles,
            diploid=diploid,
        )
        txf.transform_seqs()
        self.seqs = txf.seqs
        self.names = txf.names

    def write_loci_to_phylip(
        self,
        outdir,
        idxs=None,
        name_prefix=None,
        name_suffix=None,
        diploid=False,
        quiet=False,
        ):
        """
        Write all seq data for each locus to a separate phylip file in
        a shared directory with each locus named by locus index. If
        you want to write only a subset of loci to file you can list
        their index

        Parameters:
        -----------
        outdir (str):
            A directory in which to write all the phylip files. It will be
            created if it does not yet exist. Default is "./ipcoal_loci/".
        idxs (list):
            Numeric indices of the rows (loci) to be written to file.
            Default=None meaning that all loci will be written to file.
        ...
        """
        # bail out and complain if sim_snps() or sim_trees().
        if self.seqs is None:
            raise IpcoalError("cannot write 'loci' for sim_trees() result.")

        if self.seqs.ndim == 2:
            raise IpcoalError("cannot write 'loci' for sim_snps() result.")

        # make outdir if it does not yet exist
        self.outdir = os.path.realpath(os.path.expanduser(outdir))
        if not os.path.exists(self.outdir):
            os.makedirs(self.outdir)

        # set names parts to empty string if None
        name_suffix = ("" if name_suffix is None else name_suffix)
        name_prefix = ("" if name_prefix is None else name_prefix)

        # self.seqs is reduced to the loci in idxs, and self.idxs is set.
        self._subset_loci(idxs)

        # self.seqs and self.names are reduced to diploid samples
        self._transform_seqs(diploid)

        # iterate over loci writing each one individually
        for loc in self.idxs:

            # get locus as bytes
            arr = self.seqs[loc]

            # open file handle numbered unless user
            fhandle = os.path.join(
                self.outdir,
                "{}{}{}.phy".format(name_prefix, loc, name_suffix),
            )

            # build list of line strings
            phystring = self.build_phystring_from_loc(arr)

            # write to file
            with open(fhandle, 'w') as out:
                out.write(phystring)

        # report
        if not quiet:
            print("wrote {} loci ({} x {}bp) to {}/[...].phy".format(
                len(self.idxs), self.seqs.shape[1], self.seqs.shape[2],
                self.outdir.rstrip("/")
                ),
            )

    def write_concat_to_phylip(
        self,
        outdir:Optional[str]="./",
        name:Optional[str]=None,
        idxs:Optional[Iterable[int]]=None,
        diploid:bool=False,
        quiet:bool=False,
        ):
        """
        Write all seq data (loci or snps) concated to a single phylip file.

        Parameters:
        -----------
        outfile (str):
            The name/path of the outfile to write.
        idxs (list):
            A list of locus indices to subselect which will be concatenated.
        """
        # reshape SNPs array to be like loci
        if self.seqs.ndim == 2:
            self.seqs = self.seqs.T.reshape(
                self.seqs.shape[1], self.seqs.shape[0], 1)

        # subset selected loci
        self._subset_loci(idxs)

        # transform data to string type and ploidy
        self._transform_seqs(diploid)

        # concatenate sequences
        arr = np.concatenate(self.seqs, axis=1)

        # build list of line strings
        phystring = self.build_phystring_from_loc(arr)

        # return result as a string
        if not name:
            return phystring

        # create a directory if it doesn't exist
        outdir = os.path.realpath(os.path.expanduser(outdir))
        if not os.path.exists(outdir):
            os.makedirs(outdir)
        outfile = os.path.join(outdir, name.rstrip(".phy") + ".phy")

        # write to file
        with open(outfile, 'w') as out:
            out.write(phystring)

        # report
        if not quiet:
            print("wrote concat locus ({} x {}bp) to {}"
                  .format(arr.shape[0], arr.shape[1], outfile))
        return None

    def write_concat_to_nexus(
        self,
        outdir="./",
        name=None,
        idxs=None,
        diploid=False,
        quiet=False,
        ):
        """
        Write all seq data (loci or snps) concated to a single phylip file.

        Parameters:
        -----------
        outfile (str):
            The name/path of the outfile to write.
        idxs (list):
            A list of locus indices to subselect which will be concatenated.
        """
        # create a directory if it doesn't exist
        # reshape SNPs array to be like loci
        if self.seqs.ndim == 2:
            self.seqs = self.seqs.T.reshape(
                self.seqs.shape[1], self.seqs.shape[0], 1)

        # subset selected loci
        self._subset_loci(idxs)

        # transform data to string type and ploidy
        self._transform_seqs(diploid)

        # concatenate sequences
        arr = np.concatenate(self.seqs, axis=1)

        # build list of line strings
        nexstring = self.build_nexstring_from_loc(arr)

        # return result as a string
        if not name:
            return nexstring

        # create a directory if it doesn't exist
        outdir = os.path.realpath(os.path.expanduser(outdir))
        if not os.path.exists(outdir):
            os.makedirs(outdir)
        outfile = os.path.join(outdir, name.rstrip(".nex") + ".nex")

        # write to file
        with open(outfile, 'w') as out:
            out.write(nexstring)

        # report
        if not quiet:
            print("wrote concat locus ({} x {}bp) to {}"
                  .format(arr.shape[0], arr.shape[1], outfile))
        return None

    def write_loci_to_hdf5(self, name, outdir, diploid, quiet):
        """
        Optional writing option to write output to HDF5 database format
        used by ipyrad analysis toolkit. This will check that you have
        the h5py package installed and raise an exception if it is missing.

        The .snps.hdf5 output file can be used in ipa.tetrad,
        ipa.window_extracter, ipa.treeslider, etc.
        """
        # if non-dependency h5py is not installed then raise exception.
        try:
            import h5py
        except ImportError as err:
            raise ImportError(
                "Writing to HDF5 format requires the additional dependency "
                "h5py which you can install with the following command:\n "
                "  conda install h5py -c conda-forge \n"
                "After installing you will need to restart your notebook."
            ) from err

        # get seqs as bytes
        txf = Transformer(self.seqs, self.names, self.alleles, diploid)
        txf.transform_seqs()

        # open h5py database handle
        if name is None:
            name = "test"
        if outdir is None:
            outdir = "."
        outdir = os.path.realpath(os.path.expanduser(outdir))
        h5file = os.path.join(outdir, name + ".seqs.hdf5")
        with h5py.File(h5file, 'w') as io5:

            # write the concatenated seqs bytes array to 'seqs'
            io5.create_dataset(
                "phy",
                data=np.concatenate(txf.seqs, 1).view(np.uint8)
            )

            # write the phymap array
            nloci = txf.seqs.shape[0]
            loclen = txf.seqs.shape[2]
            phymap = io5.create_dataset(
                "phymap", shape=(nloci, 5), dtype=np.int64)
            phymap[:, 0] = range(1, nloci + 1)  # 1-indexed
            phymap[:, 1] = range(0, nloci * loclen, loclen)
            phymap[:, 2] = phymap[:, 1] + loclen
            phymap[:, 3] = 0
            phymap[:, 4] = phymap[:, 1] + loclen

            # placeholders for now
            io5.create_dataset("scaffold_lengths", data=np.repeat(loclen, nloci))
            io5.create_dataset("scaffold_names", data=(
                ['loc-{}'.format(i).encode() for i in range(1, nloci + 1)]))

            # meta info stored to phymap
            phymap.attrs["columns"] = (b'chroms', b'phy0', b'phy1', b'pos0', b'pos1')
            phymap.attrs["phynames"] = [i.encode() for i in txf.names]
            phymap.attrs["reference"] = 'ipcoal-simulation'

        # report
        if not quiet:
            print("wrote {} loci to {}".format(nloci, h5file))

    def write_snps_to_hdf5(self, name, outdir, diploid, quiet):
        """
        Writes the snps array data to snps, snpsmap, and genos arrays
        in HDF5 database format for use in ipa.
        """
        # if non-dependency h5py is not installed then raise exception.
        try:
            import h5py
        except ImportError as err:
            raise ImportError(
                "Writing to HDF5 format requires the additional dependency "
                "'h5py' which you can install with the following command:\n "
                "  conda install h5py -c conda-forge \n"
                "After installing you will need to restart your notebook."
            ) from err

        # reshape SNPs to be like loci.
        if self.seqs.ndim == 2:
            self.seqs = self.seqs.T.reshape(
                self.seqs.shape[1], self.seqs.shape[0], 1)
            self.ancestral_seq = self.ancestral_seq.reshape(
                self.ancestral_seq.size, 1)

        # get seqs as bytes (with optional diploid collapsing)
        txf = Transformer(self.seqs, self.names, self.alleles, diploid)
        tarr = np.concatenate(txf.seqs, axis=1)

        # get indices of variable sites (while allowing missing data)
        arr = np.concatenate(self.seqs, axis=1)
        marr = np.ma.array(data=arr, mask=(arr == 9))
        common = marr.mean(axis=0).round().astype(int)
        varsites = np.where(np.any(marr != common, axis=0).data)[0]
        nsites = varsites.size

        # get genos as string array [0|0, 0|1, 1|1, ...]
        genos = Genos(
            arr,
            np.concatenate(self.ancestral_seq),
            varsites,
            txf.dindex_map,
        )
        _, gmat = genos.get_alts_and_genos_matrix()

        # get snpsmap (chrom, locidx, etc.)
        smap = np.zeros((nsites, 5), dtype=np.uint32)
        gidx = 0
        for loc in range(self.seqs.shape[0]):

            # mask array to select only variable sites (while allow missing)
            larr = np.ma.array(self.seqs[loc], mask=(self.seqs[loc] == 9))
            lcom = larr.mean(axis=0).round().astype(int)
            lvar = np.where(np.any(larr != lcom, axis=0).data)[0]
            lidx = 0

            # enter variants to snpmap
            for snppos in lvar:
                smap[gidx] = loc + 1, lidx, snppos + 1, 0, gidx + 1
                lidx += 1
                gidx += 1

        # open h5py database handle
        if name is None:
            name = "test"
        if outdir is None:
            outdir = "."
        outdir = os.path.realpath(os.path.expanduser(outdir))
        h5file = os.path.join(outdir, name + ".snps.hdf5")

        # write datasets to database
        with h5py.File(h5file, 'w') as io5:

            # write the concatenated seqs as bytes->uint8 to snps
            snps = io5.create_dataset(
                name="snps",
                data=tarr[:, varsites].view(np.uint8),
            )

            # write snpsmap [chrom1, locsnpidx0, locsnppos0, 0, snpidx1]
            snpsmap = io5.create_dataset("snpsmap", data=smap)

            # genotype calls (derived/ancestral) compared to known ancestral
            io5.create_dataset(name="genos", data=gmat)

            # placeholders for now
            io5.create_dataset(name="psuedoref", shape=(nsites, 2))

            # meta info stored to phymap
            snpsmap.attrs["columns"] = (
                b'locus', b'locidx', b'locpos', b'scaf', b'scafpos')
            snps.attrs['names'] = [i.encode() for i in txf.names]

        # report
        if not quiet:
            print("wrote {} SNPs to {}".format(nsites, h5file))

    def write_vcf(self, name=None, outdir=None, diploid=None, bgzip=False, fill_missing_alleles=True, quiet=False):
        """
        Passes data to VCF object for conversion and writes resulting table
        to CSV.

        Parameters
        ----------
        name (str):
            Prefix name for output file. Will write {name}.vcf.
        outdir (str):
            The directory to write the output file to.
        diploid (bool):
            Combine haploid samples into diploid genotypes.
        bgzip (bool):
            Call bgzip to block compress the file (writes as .vcf.gz).
        fill_missing_alleles (bool):
            If there is missing data this will fill diploid missing alleles.
            e.g., the call (0|.) will be written as (0|0). This is meant to
            emulate real data where we often do not know the other allele
            is missing (also, some software tools do not accept basecalls
            with one missing allele, such as vcftools).
        """
        # reshape SNPs array to be like loci
        if self.seqs.ndim == 2:
            self.seqs = self.seqs.T.reshape(
                self.seqs.shape[1], self.seqs.shape[0], 1)
            self.ancestral_seq = self.ancestral_seq.reshape(
                self.ancestral_seq.size, 1)

        # make into genotype calls relative to reference
        vcf = VCF(
            self.seqs,
            self.names,
            diploid,
            self.ancestral_seq,
            fill_missing_alleles,
        )

        # return dataframe if no filename
        if name is None:
            # concatenate all chunks and return as full dataframe
            fullv = pd.concat(vcf.vcf_chunk_generator(), axis=0)
            fullv.reset_index(drop=True, inplace=True)
            return fullv

        # create a directory if it doesn't exist
        outdir = (outdir if outdir else "./")
        outdir = os.path.realpath(os.path.expanduser(outdir))
        if not os.path.exists(outdir):
            os.makedirs(outdir)

        # get output filepath
        outfile = os.path.join(outdir, name.rstrip(".vcf") + ".vcf")

        # write to filepath and record stats while doing it.
        nchroms = 0
        nsnps = 0
        with open(outfile, 'wt') as vout:
            vout.write(vcf.get_header())
            for vchunk in vcf.vcf_chunk_generator():
                vout.write(vchunk.to_csv(header=False, index=False, sep="\t"))
                nsnps += vchunk.shape[0]
                nchroms += vchunk.CHROM.unique().shape[0]

        # call bgzip from tabix so that bcftools can be used on VCF.
        # this will not work with normal gzip compression.
        if bgzip:
            import subprocess
            subprocess.run(["bgzip", "-f", outfile], check=True)
            outfile = outfile.rstrip(".gz") + ".gz"

        # report
        if not quiet:
            print("wrote {} SNPs across {} linkage blocks to {}"
                .format(nsnps, nchroms, outfile))
        return None

    def build_phystring_from_loc(self, arr):
        """
        Builds phylip format string with 10-spaced names.

        3 200
        aaaaaaaaaa ATCTCTACAT...
        bbbbbbb    ATCTCTACAT...
        cccccccc   ATCACTACAT...
        """
        loclist = ["{} {}".format(arr.shape[0], arr.shape[1])]
        for row in range(arr.shape[0]):
            line = "{:<10} {}".format(
                self.names[row], b"".join(arr[row]).decode())
            loclist.append(line)
        return "\n".join(loclist)

    def build_nexstring_from_loc(self, arr):
        """
        Builds nexus format string
        """
        # write the header
        lines = []
        lines.append(
            NEXHEADER.format(arr.shape[0], arr.shape[1])
        )

        # grab a big block of data
        for block in range(0, arr.shape[1], 100):
            # store interleaved seqs 100 chars with longname+2 before
            stop = min(block + 100, arr.shape[1])
            for idx, name in enumerate(self.names):

                # py2/3 compat --> b"TGCGGG..."
                seqdat = arr[idx, block:stop]
                lines.append(
                    "  {}\t{}\n".format(
                        name,
                        "".join([base_.decode() for base_ in seqdat])
                        )
                    )
            lines.append("\n")
        lines.append("\t;\nend;")
        return "".join(lines)

    # def write_seqs_as_fasta(self, loc, path):
    #     fastaseq = deepcopy(self.seqs[loc]).astype(str)
    #     fasta = []
    #     for idx, name in enumerate(self.names):
    #         fasta.append(('>' + name + '\n'))
    #         fasta.append("".join(fastaseq[idx])+'\n')

    #     with open(path, 'w') as file:
    #         for line in fasta:
    #             file.write(line)

if __name__ == "__main__":

    import h5py
    import ipcoal
    import toytree

    TREE = toytree.rtree.unittree(6, 1e6)
    MOD = ipcoal.Model(TREE, Ne=1e6, nsamples=2)
    MOD.sim_loci(10, 100)
    MOD.apply_missing_mask(coverage=0.5)

    # test vcf writing to file, and to diploid and haploid dataframes
    WRITER = Writer(MOD.seqs, MOD.alpha_ordered_names, MOD.ancestral_seq)
    df = WRITER.write_vcf(
        name='test',
        outdir='/tmp/',
        diploid=True,
        bgzip=True,
        quiet=False,
    )
    df = WRITER.write_vcf(diploid=True)
    print(df.head())
    df = WRITER.write_vcf(diploid=False)
    print(df.head())

    # test writing loci to hdf5 and show snpsmap
    WRITER.write_loci_to_hdf5(
        name="test",
        outdir="/tmp",
        diploid=True,
        quiet=False,
    )
    with h5py.File("/tmp/test.seqs.hdf5", 'r') as io5:
        print(io5["phymap"][:5])

    WRITER.write_snps_to_hdf5(
        name="test",
        outdir="/tmp",
        diploid=True,
        quiet=False,
    )
    with h5py.File("/tmp/test.snps.hdf5", 'r') as io5:
        print(io5["snps"].shape)
        print(io5["snpsmap"][:20])


import base64
import json
import time
import cv2
import requests

server_config = {
    "camera_id": 0,
    "push_server": "http://localhost:5000"
}

def draw_picture(src_picture, results):
    for result in results:
        label, confidence, top, left, bottom, right = result
        src_picture = cv2.putText(
            src_picture,
            "{0}: {1}".format(label, confidence),
            (left, bottom),
            cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 10
        )
        src_picture = cv2.rectangle(
            src_picture, (left, top), (right, bottom), (255, 0, 0), 10)
    return src_picture

def send_request(src_picture, results, draw=True):
    if draw:
        src_picture = draw_picture(src_picture, results)
    try:
        retval, buffer = cv2.imencode('.jpg', src_picture)
        jpg_as_text = base64.b64encode(buffer)
        requests.post(
            server_config['push_server'],
            data={
                "machine": server_config["camera_id"],
                "timestamp": time.time(),
                "picture": jpg_as_text,
                "box": json.dumps(results)
            }
        )
        print("Detection sent to {0}".format(server_config["push_server"]))
    except requests.exceptions.ConnectionError:
        print("Unable to connect to {0}".format(server_config["push_server"]))


from typing import List

class Node:
    def __init__(self, val=None, children=None):
        self.val = val
        self.children = children

def levelOrder(self, root: 'Node') -> List[List[int]]:
    if root:
        result = [root.val]
        lvl = [root]
        while lvl:
            c_level_nodes = []
            c_level_vals = []
            parents = lvl.pop()
            for p in parents:
                for child in p.children:
                    c_level_nodes.append(child)
                    c_level_vals.append(child.val)
            lvl = c_level_nodes
            result.append(c_level_vals)
        return result
    return []