from typing import List
import typing

from PyQt5 import QtGui
from PyQt5.QtCore import QEvent, QObject, QSignalBlocker, QStringListModel
from ime.ui.ui_access_control_list import Ui_AccessControlList
from PyQt5.QtWidgets import QMessageBox, QWidget, QLineEdit

class AccessControlList(QWidget):

    def __init__(self, parent = None):
        super().__init__(parent)
        self.ui = Ui_AccessControlList()
        self.ui.setupUi(self)
        #Libby to add line edit
        #self.ui.input_box = QLineEdit(self)

        #Libby Initialize the flag to False
        self.item_added = False

        self.ui.btnAdd.clicked.connect(self.handle_insert_new)
        self.ui.btnDelete.clicked.connect(self.handle_remove)
        # To monitor focus out events and deselect
        # self.ui.aclList.installEventFilter(self)
        self.ui.overrideCheckBox.installEventFilter(self)
        self.ui.overrideCheckBox.setVisible(False)
        self.ui.overrideCheckBox.toggled.connect(self.handle_override_changed)
        self.has_inheritance = False

    def set_has_inheritance(self, has_inheritance: bool):
        self.has_inheritance = has_inheritance
        self.ui.overrideCheckBox.setVisible(has_inheritance)
        if not has_inheritance:
            self.ui.aclList.setEnabled(True)
        elif getattr(self, 'list_model', None) is None:
            self.ui.aclList.setDisabled(True)
            return
        elif self.list_model.rowCount() > 0:
            # TODO Need to differentiate no vs empty state
            self.ui.overrideCheckBox.setChecked(True)
            self.ui.aclList.setEnabled(True)
        else:
            self.ui.aclList.setDisabled(True)
            self.ui.overrideCheckBox.setChecked(False)

    def handle_override_changed(self):
        checked = self.ui.overrideCheckBox.isChecked()
        list_count = self.list_model.rowCount()
        if not checked and list_count > 0:
            msg = QMessageBox()
            msg.setWindowTitle("Use inherited value instead?")
            msg.setText("Are you sure you want to remove all users/groups from this field?")
            msg.setInformativeText("Inherited values will apply instead.")
            msg.setStandardButtons(typing.cast(QMessageBox.StandardButtons, QMessageBox.StandardButton.Ok | QMessageBox.StandardButton.Cancel))
            res = msg.exec()
            if res == QMessageBox.StandardButton.Cancel:
                with QSignalBlocker(self.ui.overrideCheckBox):
                    # Undo the check.
                    self.ui.overrideCheckBox.setChecked(True)
            else:
                # Remove all items in list
                self.list_model.removeRows(0, self.list_model.rowCount())
        # Finally, set to enable to disable the list
        self.ui.aclList.setEnabled(self.ui.overrideCheckBox.isChecked())

    def handle_insert_new(self):
        if getattr(self, 'list_model', None) is None:
            return
        ### Libby to add text value to the listmodel

        checked = self.ui.overrideCheckBox.isChecked()
        if checked and (not self.item_added):
            row = self.list_model.rowCount() # get the number of rows in model --> int
            self.list_model.insertRows(row,1) # Enable add one or more row.
            model_idx = self.list_model.index(row)
            self.ui.aclList.setCurrentIndex(model_idx)
            self.ui.aclList.edit(model_idx)
            # Set the flag to True after an item has been added
            self.item_added = True

        # If an item has been added, remove any empty or null strings from the model
        elif checked and (self.item_added):
            self.remove_empty_strings(self.list_model)
            row = self.list_model.rowCount() # get the number of rows in model --> int
            self.list_model.insertRows(row,1) # Enable add one or more row.
            model_idx = self.list_model.index(row)
            self.ui.aclList.setCurrentIndex(model_idx)
            self.ui.aclList.edit(model_idx)
            self.item_added = True

    def handle_remove(self):
        if getattr(self, 'list_model', None) is None:
            return
        idx_list = self.ui.aclList.selectedIndexes()
        rows_to_remove = [idx.row() for idx in idx_list]
        # Reverse sort the rows to remove, so we're not affected
        # by row index changes.
        rows_to_remove.sort(reverse=True)
        for row in rows_to_remove:
            self.list_model.removeRow(row)

    def set_list(self, ac_list: List[str]):
        self.list_model = QStringListModel(self) # create stringlistmodel object
        self.list_model.setStringList(ac_list)  # assign values to the model
        self.ui.aclList.setModel(self.list_model) # connect view and model
        # Reapply in case this list has value
        self.set_has_inheritance(self.has_inheritance)

    def remove_empty_strings(self,model):
        # Get the string list from the model
        string_list = model.stringList()

        # Iterate through the string list and remove any empty or null strings
        for i in reversed(range(len(string_list))):
            if string_list[i] == '' or string_list[i] is None:
                model.removeRows(i, 1)


from django.shortcuts import render, redirect

from .forms import CreateForm
from .models import Products

def create_product(request):
    if request.method == 'POST':
        form = CreateForm(request.POST)
        if form.is_valid():
            form.save()
            return redirect('/list')
    else:
        form = CreateForm()

    return render(request, 'add_product.html', {'form': form})

def product_list(request):
    products = Products.objects.all()

    if request.method == 'POST':
        product_id = request.POST.get('product_id')
        if product_id:
            product_to_delete = Products.objects.get(pk=product_id)
            product_to_delete.delete()
            return redirect('product_list')

    return render(request, 'list.html', {'products': products})


from typing import List, Tuple

# Takes an integer `num`, a number of bits `n_b`, and a length l, and returns a tuple (d0, ..., d_{l-1})
# such that num = d_0 + d_1 * 2**(n_b) + d_2 * 2**(2n_b) + ... + d_{l-1}*2**((l-1)*n_b)
# Each d_i is a positive integer of less than n_b bits
def split(num: int, num_bits_shift: int = 128, length: int=3) -> List[int]:
    a = []
    for _ in range(length):
        a.append( num & ((1 << num_bits_shift) - 1) )
        num = num >> num_bits_shift
    return tuple(a)

# Takes an integer `num`, a number of bits `n_b`, and a length l, and returns a tuple (d0, ..., d_{l-1})
# of overlapping sections of num of the type used by Uint384_expand
def split2(num: int, num_bits_shift: int = 64, length: int=7) -> List[int]:
    a = []
    num = num << num_bits_shift
    for _ in range(length):
        a.append( num & ((1 << 2*num_bits_shift) - 1) )
        num = num >> num_bits_shift
    return tuple(a)

# The inverse operation of `split`: takes a tuple z of integers (d0, ..., d_{l-1}) and returns
# d_0 + d_1 * 2**(n_b) + d_2 * 2**(2n_b) + ... + d_{l-1}*2**((l-1)*n_b)
# Note that d_i can be larger than 2**128 here (but in this repo d_i is always less than 2**128)
def pack(z: Tuple[int], num_bits_shift: int = 128) -> int:
    limbs = (limb for limb in z)
    return sum(limb << (num_bits_shift * i) for i, limb in enumerate(limbs))

# Prime p with which to test the field arithmetic functions
elliptic_curve_field_modulus = 4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559787


import collections
import copy
import os

from tensorflow.python.eager import def_function
from tensorflow.python.keras import backend as K
from tensorflow.python.keras import losses
from tensorflow.python.keras import optimizer_v1
from tensorflow.python.keras import optimizers
from tensorflow.python.keras.engine import base_layer_utils
from tensorflow.python.keras.utils import generic_utils
from tensorflow.python.keras.utils import version_utils
from tensorflow.python.keras.utils.io_utils import ask_to_proceed_with_overwrite
from tensorflow.python.platform import tf_logging as logging
from tensorflow.python.util import nest

def extract_model_metrics(model):
  """Convert metrics from a Keras model `compile` API to dictionary.

  This is used for converting Keras models to Estimators and SavedModels.

  Args:
    model: A `tf.keras.Model` object.

  Returns:
    Dictionary mapping metric names to metric instances. May return `None` if
    the model does not contain any metrics.
  """
  if getattr(model, '_compile_metrics', None):
    # TODO(psv/kathywu): use this implementation in model to estimator flow.
    # We are not using model.metrics here because we want to exclude the metrics
    # added using `add_metric` API.
    return {m.name: m for m in model._compile_metric_functions}  # pylint: disable=protected-access
  return None

def model_input_signature(model, keep_original_batch_size=False):
  """Inspect model to get its input signature.

  The model's input signature is a list with a single (possibly-nested) object.
  This is due to the Keras-enforced restriction that tensor inputs must be
  passed in as the first argument.

  For example, a model with input {'feature1': <Tensor>, 'feature2': <Tensor>}
  will have input signature: [{'feature1': TensorSpec, 'feature2': TensorSpec}]

  Args:
    model: Keras Model object.
    keep_original_batch_size: A boolean indicating whether we want to keep using
      the original batch size or set it to None. Default is `False`, which means
      that the batch dim of the returned input signature will always be set to
      `None`.

  Returns:
    A list containing either a single TensorSpec or an object with nested
    TensorSpecs. This list does not contain the `training` argument.
  """
  input_specs = model._get_save_spec(dynamic_batch=not keep_original_batch_size)  # pylint: disable=protected-access
  if input_specs is None:
    return None
  input_specs = _enforce_names_consistency(input_specs)
  # Return a list with a single element as the model's input signature.
  if isinstance(input_specs,
                collections.abc.Sequence) and len(input_specs) == 1:
    # Note that the isinstance check filters out single-element dictionaries,
    # which should also be wrapped as a single-element list.
    return input_specs
  else:
    return [input_specs]

def raise_model_input_error(model):
  raise ValueError(
      'Model {} cannot be saved because the input shapes have not been '
      'set. Usually, input shapes are automatically determined from calling'
      ' `.fit()` or `.predict()`. To manually set the shapes, call '
      '`model.build(input_shape)`.'.format(model))

def trace_model_call(model, input_signature=None):
  """Trace the model call to create a tf.function for exporting a Keras model.

  Args:
    model: A Keras model.
    input_signature: optional, a list of tf.TensorSpec objects specifying the
      inputs to the model.

  Returns:
    A tf.function wrapping the model's call function with input signatures set.

  Raises:
    ValueError: if input signature cannot be inferred from the model.
  """
  if input_signature is None:
    if isinstance(model.call, def_function.Function):
      input_signature = model.call.input_signature

  if input_signature is None:
    input_signature = model_input_signature(model)

  if input_signature is None:
    raise_model_input_error(model)

  @def_function.function(input_signature=input_signature)
  def _wrapped_model(*args):
    """A concrete tf.function that wraps the model's call function."""
    # When given a single input, Keras models will call the model on the tensor
    # rather than a list consisting of the single tensor.
    inputs = args[0] if len(input_signature) == 1 else list(args)

    with base_layer_utils.call_context().enter(
        model, inputs=inputs, build_graph=False, training=False, saving=True):
      outputs = model(inputs, training=False)

    # Outputs always has to be a flat dict.
    output_names = model.output_names  # Functional Model.
    if output_names is None:  # Subclassed Model.
      from tensorflow.python.keras.engine import compile_utils  # pylint: disable=g-import-not-at-top
      output_names = compile_utils.create_pseudo_output_names(outputs)
    outputs = nest.flatten(outputs)
    return {name: output for name, output in zip(output_names, outputs)}

  return _wrapped_model

def model_metadata(model, include_optimizer=True, require_config=True):
  """Returns a dictionary containing the model metadata."""
  from tensorflow.python.keras import __version__ as keras_version  # pylint: disable=g-import-not-at-top
  from tensorflow.python.keras.optimizer_v2 import optimizer_v2  # pylint: disable=g-import-not-at-top

  model_config = {'class_name': model.__class__.__name__}
  try:
    model_config['config'] = model.get_config()
  except NotImplementedError as e:
    if require_config:
      raise e

  metadata = dict(
      keras_version=str(keras_version),
      backend=K.backend(),
      model_config=model_config)
  if model.optimizer and include_optimizer:
    if isinstance(model.optimizer, optimizer_v1.TFOptimizer):
      logging.warning(
          'TensorFlow optimizers do not '
          'make it possible to access '
          'optimizer attributes or optimizer state '
          'after instantiation. '
          'As a result, we cannot save the optimizer '
          'as part of the model save file. '
          'You will have to compile your model again after loading it. '
          'Prefer using a Keras optimizer instead '
          '(see keras.io/optimizers).')
    elif model._compile_was_called:  # pylint: disable=protected-access
      training_config = model._get_compile_args(user_metrics=False)  # pylint: disable=protected-access
      training_config.pop('optimizer', None)  # Handled separately.
      metadata['training_config'] = _serialize_nested_config(training_config)
      if isinstance(model.optimizer, optimizer_v2.RestoredOptimizer):
        raise NotImplementedError(
            'As of now, Optimizers loaded from SavedModel cannot be saved. '
            'If you\'re calling `model.save` or `tf.keras.models.save_model`,'
            ' please set the `include_optimizer` option to `False`. For '
            '`tf.saved_model.save`, delete the optimizer from the model.')
      else:
        optimizer_config = {
            'class_name':
                generic_utils.get_registered_name(model.optimizer.__class__),
            'config':
                model.optimizer.get_config()
        }
      metadata['training_config']['optimizer_config'] = optimizer_config
  return metadata

def should_overwrite(filepath, overwrite):
  """Returns whether the filepath should be overwritten."""
  # If file exists and should not be overwritten.
  if not overwrite and os.path.isfile(filepath):
    return ask_to_proceed_with_overwrite(filepath)
  return True

def compile_args_from_training_config(training_config, custom_objects=None):
  """Return model.compile arguments from training config."""
  if custom_objects is None:
    custom_objects = {}

  with generic_utils.CustomObjectScope(custom_objects):
    optimizer_config = training_config['optimizer_config']
    optimizer = optimizers.deserialize(optimizer_config)

    # Recover losses.
    loss = None
    loss_config = training_config.get('loss', None)
    if loss_config is not None:
      loss = _deserialize_nested_config(losses.deserialize, loss_config)

    # Recover metrics.
    metrics = None
    metrics_config = training_config.get('metrics', None)
    if metrics_config is not None:
      metrics = _deserialize_nested_config(_deserialize_metric, metrics_config)

    # Recover weighted metrics.
    weighted_metrics = None
    weighted_metrics_config = training_config.get('weighted_metrics', None)
    if weighted_metrics_config is not None:
      weighted_metrics = _deserialize_nested_config(_deserialize_metric,
                                                    weighted_metrics_config)

    sample_weight_mode = training_config['sample_weight_mode'] if hasattr(
        training_config, 'sample_weight_mode') else None
    loss_weights = training_config['loss_weights']

  return dict(
      optimizer=optimizer,
      loss=loss,
      metrics=metrics,
      weighted_metrics=weighted_metrics,
      loss_weights=loss_weights,
      sample_weight_mode=sample_weight_mode)

def _deserialize_nested_config(deserialize_fn, config):
  """Deserializes arbitrary Keras `config` using `deserialize_fn`."""

  def _is_single_object(obj):
    if isinstance(obj, dict) and 'class_name' in obj:
      return True  # Serialized Keras object.
    if isinstance(obj, str):
      return True  # Serialized function or string.
    return False

  if config is None:
    return None
  if _is_single_object(config):
    return deserialize_fn(config)
  elif isinstance(config, dict):
    return {
        k: _deserialize_nested_config(deserialize_fn, v)
        for k, v in config.items()
    }
  elif isinstance(config, (tuple, list)):
    return [_deserialize_nested_config(deserialize_fn, obj) for obj in config]

  raise ValueError('Saved configuration not understood.')

def _serialize_nested_config(config):
  """Serialized a nested structure of Keras objects."""

  def _serialize_fn(obj):
    if callable(obj):
      return generic_utils.serialize_keras_object(obj)
    return obj

  return nest.map_structure(_serialize_fn, config)

def _deserialize_metric(metric_config):
  """Deserialize metrics, leaving special strings untouched."""
  from tensorflow.python.keras import metrics as metrics_module  # pylint:disable=g-import-not-at-top
  if metric_config in ['accuracy', 'acc', 'crossentropy', 'ce']:
    # Do not deserialize accuracy and cross-entropy strings as we have special
    # case handling for these in compile, based on model output shape.
    return metric_config
  return metrics_module.deserialize(metric_config)

def _enforce_names_consistency(specs):
  """Enforces that either all specs have names or none do."""

  def _has_name(spec):
    return hasattr(spec, 'name') and spec.name is not None

  def _clear_name(spec):
    spec = copy.deepcopy(spec)
    if hasattr(spec, 'name'):
      spec._name = None  # pylint:disable=protected-access
    return spec

  flat_specs = nest.flatten(specs)
  name_inconsistency = (
      any(_has_name(s) for s in flat_specs) and
      not all(_has_name(s) for s in flat_specs))

  if name_inconsistency:
    specs = nest.map_structure(_clear_name, specs)
  return specs

def try_build_compiled_arguments(model):
  if (not version_utils.is_v1_layer_or_model(model) and
      model.outputs is not None):
    try:
      if not model.compiled_loss.built:
        model.compiled_loss.build(model.outputs)
      if not model.compiled_metrics.built:
        model.compiled_metrics.build(model.outputs, model.outputs)
    except:  # pylint: disable=bare-except
      logging.warning(
          'Compiled the loaded model, but the compiled metrics have yet to '
          'be built. `model.compile_metrics` will be empty until you train '
          'or evaluate the model.')

def is_hdf5_filepath(filepath):
  return (filepath.endswith('.h5') or filepath.endswith('.keras') or
          filepath.endswith('.hdf5'))


#!/usr/bin/env python3

from functools import wraps
from pprint import pprint
import base64
import hashlib
import json
import random
import requests
import sys
import readline
import signal
import socketio
import sys
import time
import yaml

from eventador.query import EventadorQuery
from eventador import parser

# configure readline support for input
readline.parse_and_bind('tab: complete')
readline.parse_and_bind('set editing-mode vi')

print("Loading configuration from config.yaml.. ", end='')
try:
    with open('config.yaml') as f:
        config = yaml.load(f, Loader=yaml.SafeLoader)
except Exception as ex:
    print("failed to load config: {}".format(ex))
    sys.exit(1)
print("done.")

def query_repl(e):
    try:
        if e.selected_cluster:
            i = input('Eventador SQL[{}]> '.format(e.selected_cluster_name))
        else:
            i = input('Eventador SQL> ')
    except (EOFError, KeyboardInterrupt):
        print("\nGoodbye.")
        sys.exit(1)

    p.parse(i)

def ev_help():
    """simple help screen."""
    h = """
------------------------------------------------------------------------------
Commands:

show tables
show clusters
set cluster <clusterName>

Or enter any valid SQL query.  eg.

SELECT * FROM airplanes;
------------------------------------------------------------------------------
        """

    return h

# Initializing REPL loop

BASE_URL = config['endpoint']['urlbase']
WSS_URL = config['endpoint']['urlbase']

ev = EventadorQuery(BASE_URL, WSS_URL)

try:
    ev.login(config['auth']['username'], config['auth']['password'])
except NoCSRFTokenFound as ex:
    print("Login failed - no CSRF token retrieved.")
    sys.exit(1)

p = parser.Parser(ev)

print("Welcome to Eventador.  Enter a command, type 'help' for more information,"
      "or press ctrl-d to exit.")

while True:
    try:
        query_repl(p)
    except Exception as ex:
        print("Failed: {}".format(ex))


import timings
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np

# plotting params
props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)

#
# 1. Cache vs. NoCache
#
ranks = range(1,13,1)
ms = [100]
bs = [2]
cs = [True,False]
ns = [200]
algs = ['vector']
df = timings.time_mist(ms=ms, ns=ns, bs=bs, rs=ranks, cs=cs, q=200, algs=algs)
r = df[df['cache'] == True]['r']
cache = df[df['cache'] == True]['time'] * 1e6
nocache = df[df['cache'] == False]['time'] * 1e6
fig,ax = plt.subplots()
ax.plot(r,cache,label="cache")
ax.plot(r,nocache,label="nocache")
ax.text(0.20, 0.90, "m=%s,b=%s,n=%s,\nalg=%s,q=200" %(str(ms),str(bs),str(ns), str(algs)), transform=ax.transAxes, fontsize=10, verticalalignment='top', bbox=props)
ax.legend()
ax.set_ylabel('us')
ax.set_xlabel('threads')
plt.savefig("scaling1.tif")

#
# 2. Thread scaling, nocache
#
ranks = range(1,13,1)
ms = [10000]
bs = [2]
cs = [False]
ns = [100]
algs = ['vector']
df = timings.time_mist(ms=ms, ns=ns, bs=bs, rs=ranks, cs=cs, q=200, algs=algs)
r = df['r']
t = df['time'] * 1e3
fig,ax = plt.subplots()
ax.plot(r,t)
ax.text(0.20, 0.90, "m=%s,b=%s,n=%s,cache=%s\nalg=%s,q=200" %(str(ms),str(bs),str(ns), str(cs), str(algs)), transform=ax.transAxes, fontsize=10, verticalalignment='top', bbox=props)
ax.legend()
ax.set_ylabel('ms')
ax.set_xlabel('threads')
plt.savefig("scaling2.tif")

#
# 3. Thread scaling d3, cache
#
df = timings.time_mist(ms=[1000], ns=[100], bs=[2], ds=[3], rs=range(1,21,2), cs=[True], q=10, algs=['vector'], progress=True)
ranks = range(1,13,1)
ms = [1000]
bs = [2]
ds = [3]
cs = [True]
ns = [100]
algs = ['vector']
df = timings.time_mist(ms=ms, ns=ns, bs=bs, ds=ds, rs=ranks, cs=cs, q=200, algs=algs)
r = df['r']
t = df['time']
fig,ax = plt.subplots()
ax.plot(r,t)
ax.text(0.20, 0.90, "m=%s,b=%s,n=%s,d=%s\ncache=%s,alg=%s,q=200" %(str(ms),str(bs),str(ns), str(ds), str(cs), str(algs)), transform=ax.transAxes, fontsize=10, verticalalignment='top', bbox=props)
ax.legend()
ax.set_ylabel('s')
ax.set_xlabel('threads')
plt.savefig("scaling3.tif")

#
# 4. N scaling at different thread counts
#
ranks = [2,4,6]
ms=[100]
bs=[2]
cs=[False]
ns=[50,100,150,200,250,300]
algs=['vector']
df = timings.time_mist(ms=ms, ns=ns, bs=bs, rs=ranks, cs=cs, q=100, algs=algs)
n = ns
fig,ax = plt.subplots()
for rank in ranks:
    t = df[df['r'] == rank]['time'] * 1e6
    ax.plot(n,t,label="%d threads" % rank)
ax.text(0.40, 0.90, "m=%s,b=%s,cache=%s,\nalg=%s" %(str(ms),str(bs),str(cs), str(algs)),
        transform=ax.transAxes, fontsize=10, verticalalignment='top', bbox=props)
ax.legend()
ax.set_ylabel('us')
ax.set_xlabel('n')
plt.savefig("scaling4.tif")

#
# 5. Thread scaling at different N counts
#
ranks = range(10,130,10)
ms = [1000]
bs = [2]
cs = [False]
ns = range(100,1000,200)
algs = ['vector']
df = timings.time_mist(ms=ms, ns=ns, bs=bs, rs=ranks, cs=cs, q=200, algs=algs)
fig,ax = plt.subplots()
for n in ns:
    t = df[df['n'] == n]['time'] * 1e6
    ax.plot(ranks,t,label="n=%d" % n)
ax.text(0.20, 0.90, "m=%s,b=%s,cache=%s,\nalg=%s,q=200" %(str(ms),str(bs),str(cs), str(algs)), transform=ax.transAxes, fontsize=10, verticalalignment='top', bbox=props)
ax.legend()
ax.set_ylabel('us')
ax.set_xlabel('threads')
plt.savefig("scaling5.tif")

#
# 6. Thread scaling at different M counts
#
ranks = range(1,121,20) # salyut
ranks = range(1,13,1)   # galas155335
ms = range(100,2000,200)
bs = [2]
cs = [False]
ns = [300]
algs = ['vector']
df = timings.time_mist(ms=ms, ns=ns, bs=bs, rs=ranks, cs=cs, q=200, algs=algs)
fig,ax = plt.subplots()
for m in ms:
    t = df[df['m'] == m]['time'] * 1e6
    ax.plot(ranks,t,label="m=%d" % m)
ax.text(0.20, 0.90, "n=%s,b=%s,cache=%s,\nalg=%s,q=200" %(str(ns),str(bs),str(cs), str(algs)), transform=ax.transAxes, fontsize=10, verticalalignment='top', bbox=props)
ax.legend()
ax.set_ylabel('us')
ax.set_xlabel('threads')
plt.savefig("scaling6.tif")

#
# 7. Demonstrate Amdahl's Law
#
ranks = range(1,121,10)
ms = [1000]
bs = [2]
cs = [False]
ns = [300]
algs = ['vector']
df = timings.time_mist(ms=ms, ns=ns, bs=bs, rs=ranks, cs=cs, q=200, algs=algs)
fig,ax = plt.subplots()
t = df['time'] * 1e6
speedup = t[0] / t
ax.plot(ranks, speedup)
props = dict(boxstyle = 'round', facecolor = 'wheat', alpha = 0.5)
ax.text(0.20, 0.10, "m=%s,b=%s,n=%s,cache=%s\nalg=%s,q=200" %(str(ms),str(bs),str(ns), str(cs), str(algs)), transform=ax.transAxes, fontsize=10, verticalalignment='top', bbox=props)
ax.set_ylabel('speedup')
ax.set_xlabel('threads')
plt.savefig("scaling7.tif")


import os
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
from tensorflow.keras.layers import Conv2D, Activation, UpSampling2D, Input, Add
from tensorflow.keras.models import Model
from keras.applications.efficientnet import EfficientNetB0

def GCN_module(features, k=5):

  y_a = Conv2D(filters=21, kernel_size=(1,k), padding='same', use_bias=False)(features)
  y_a = Conv2D(filters=21, kernel_size=(k,1), padding='same', use_bias=False)(y_a)

  y_b = Conv2D(filters=21, kernel_size=(k,1), padding='same', use_bias=False)(features)
  y_b = Conv2D(filters=21, kernel_size=(1,k), padding='same', use_bias=False)(y_b)

  y = Add()([y_a, y_b])

  return y

def BR(features):

  y = Conv2D(filters=21, kernel_size=(3,3), padding='same', use_bias=False)(features)
  y = Activation('relu')(y)
  y = Conv2D(filters=21, kernel_size=(3,3), padding='same', use_bias=False)(y)

  y = Add()([y, features])

  return y

def GCN(inputs, classes=1):

    inputs = Input(inputs)

    base_model = EfficientNetB0(weights='imagenet', include_top=False, input_tensor=inputs)
    res_2 = base_model.get_layer('block3a_expand_activation').output         # 4x >> block3a_expand_activation
    res_3 = base_model.get_layer('block4a_expand_activation').output         # 8x >> block4a_expand_activation
    res_4 = base_model.get_layer('block6a_expand_activation').output         # 16x >> block6a_expand_activation
    res_5 = base_model.get_layer('top_activation').output                    # 32x >> top_activation

    gcn_2 = GCN_module(res_2, 7)
    gcn_3 = GCN_module(res_3, 7)
    gcn_4 = GCN_module(res_4, 7)
    gcn_5 = GCN_module(res_5, 7)

    bn_2 = BR(gcn_2)
    bn_3 = BR(gcn_3)
    bn_4 = BR(gcn_4)
    bn_5 = BR(gcn_5)

    bn_4_a = UpSampling2D(size=2, interpolation='bilinear')(bn_5)
    bn_4_a = Add()([bn_4, bn_4_a])
    bn_4_a = BR(bn_4_a)
    bn_3_a = UpSampling2D(size=2, interpolation='bilinear')(bn_4_a)
    bn_3_a = Add()([bn_3, bn_3_a])
    bn_3_a = BR(bn_3_a)
    bn_2_a = UpSampling2D(size=2, interpolation='bilinear')(bn_3_a)
    bn_2_a = Add()([bn_2, bn_2_a])
    bn_2_a = BR(bn_2_a)
    bn_1_a = UpSampling2D(size=2, interpolation='bilinear')(bn_2_a)
    bn_1_a = BR(bn_1_a)
    bn_0_a = UpSampling2D(size=2, interpolation='bilinear')(bn_1_a)
    bn_0_a = BR(bn_0_a)

    """ Outputs """
    x = Conv2D(classes, (1, 1), name='output_layer')(bn_0_a)

    if classes == 1:
      x = Activation('sigmoid')(x)
    else:
      x = Activation('softmax')(x)

    model = Model(inputs=inputs, outputs=x)

    return model

def main():

    model = GCN(inputs=(512,512,3), classes=10)
    model.summary()

if __name__== '__main__':

    main()


import sys
import os
import datetime

#Roughly (very) 1 year per hour on wcoss2

#convert to a python script that will compare original and fcst N days later
#--------------- Utility Functions --------------------------------
def parse_8digits(tag):
  tmp = int(tag)
  (yy,mm,dd) = (int(int(tmp)/10000),int((int(tmp)%10000)/100),int(tmp)%100)
  tag_out = datetime.date(int(yy), int(mm), int(dd))
  return tag_out
#------------------------------------------------

obsdir = os.environ['OBSDIR']
fixdir = os.environ['FIXDIR']
exdir  = os.environ['EXDIR']

day   = datetime.timedelta(1)
start = parse_8digits(sys.argv[1])
end   = datetime.date.today()
lead  = int(sys.argv[2])
dt    = day*lead

for i in range(0,4*365+1):
  fdate = start + dt
  if (fdate >= end):
      break

  sname = obsdir+"/cleaned/s."+start.strftime("%Y%j")+".beta"
  fname = obsdir+"/cleaned/s."+fdate.strftime("%Y%j")+".beta"
  oname = "persist/nic_v_nic."+"{:d}".format(lead)+"/score.s."+start.strftime("%Y%j")
  if (os.path.exists(sname) and os.path.exists(fname) ):
    if (not os.path.exists(oname) ):
      os.system("$EXDIR/cscore_edge $FIXDIR/seaice_alldist.bin "+sname+" "+fname+" 50.0 > "+oname)
  else:
    if (not os.path.exists(sname)):
      print("missing ",sname)
    if (not os.path.exists(fname)):
      print("missing ", fname)
    print(flush=True)

  sname = obsdir+"/cleaned/n."+start.strftime("%Y%j")+".beta"
  fname = obsdir+"/cleaned/n."+fdate.strftime("%Y%j")+".beta"
  #oname = "score.n."+start.strftime("%Y%j")
  oname = "persist/nic_v_nic."+"{:d}".format(lead)+"/score.n."+start.strftime("%Y%j")
  if (os.path.exists(sname) and os.path.exists(fname) ):
    if (not os.path.exists(oname) ):
      os.system("$EXDIR/cscore_edge $FIXDIR/seaice_alldist.bin "+sname+" "+fname+" 50.0 > "+oname)
  else:
    if (not os.path.exists(sname)):
      print("missing ",sname)
    if (not os.path.exists(fname)):
      print("missing ", fname)
    print(flush=True)

  start += day


#!/usr/bin/env python

'''
Advent of Code 2022 - Day 11: Monkey In The Middle (Part 1)
https://adventofcode.com/2022/day/11
'''

from collections import OrderedDict, deque
import time
import re
from dataclasses import dataclass

@dataclass
class Monkey:
    number: int
    items: deque
    operation: str
    divisor: int
    true_throw_to: int
    false_throw_to: int
    num_inspected: int = 0

    def add_item(self, item):
        self.items.append(item)

    def inspect(self, monkeys: dict):
        while self.items:
            item = self.items.popleft()
            self.num_inspected += 1
            # Evaluate the string operation as an expression
            def f(old): return eval(self.operation)
            worry_level = f(item)  # Change the worry level
            worry_level = worry_level // 3  # Floor division
            throw_to = self.true_throw_to if worry_level % self.divisor == 0 else self.false_throw_to
            monkeys[throw_to].add_item(worry_level)  # Throw item to monkey

def create_monkey(note: re.Match) -> Monkey:
    monkey_no = int(note.group(1))
    items = note.group(2).split(',')
    items = [int(item) for item in items]  # Convert items to integers
    operation = note.group(3)
    divisor = int(note.group(4))
    true = int(note.group(5))
    false = int(note.group(6))

    return Monkey(number=monkey_no, items=deque(
        items), operation=operation, divisor=divisor, true_throw_to=true, false_throw_to=false)

def main():

    filename = input("What is the input file name? ")

    try:
        with open(filename, "r") as file:

            start = time.time()

            monkeys = {}

            # Read the notes
            note = ''
            for line in file:
                # Process note if a blank line
                if line == '\n':
                    # Generated regex using regex101.com (https://regex101.com/r/hLTCaG/1)
                    result = re.search(
                        "^Monkey (\d){1}:\n^\s+Starting items: ([\d,\s]+)\n^\s+Operation: new = (.+)\n^\s+Test: divisible by (\d+)\n^\s+If true: throw to monkey (\d+)\n^\s+If false: throw to monkey (\d)+$", note, re.MULTILINE)
                    if not result:
                        print("Could not find an observation note for a monkey")
                        continue

                    monkey = create_monkey(result)
                    monkeys[monkey.number] = monkey
                    note = ''  # reset to blank
                    continue

                note += line

            # Add the last monkey note
            result = re.search(
                "^Monkey (\d){1}:\n^\s+Starting items: ([\d,\s]+)\n^\s+Operation: new = (.+)\n^\s+Test: divisible by (\d+)\n^\s+If true: throw to monkey (\d+)\n^\s+If false: throw to monkey (\d)+$", note, re.MULTILINE)
            if result:
                monkey = create_monkey(result)
                monkeys[monkey.number] = monkey

            # Sort the monkeys into ascending order
            monkeys = OrderedDict(sorted(monkeys.items()))

            for round in range(20):  # Inspect over 20 rounds
                for _, monkey in monkeys.items():
                    monkey.inspect(monkeys)

            num_inspected = []
            for _, monkey in monkeys.items():
                num_inspected.append(monkey.num_inspected)
                print(
                    f"Monkey {monkey.number} inspected items {monkey.num_inspected} times.")

            num_inspected = sorted(num_inspected, reverse=True)
            print(
                f"The level of monkey business: {num_inspected[0] * num_inspected[1]}")

        end = time.time()
        print(f"Execution time in seconds: {end - start}\n")

    except FileNotFoundError:
        print(f"No such file or directory: '{filename}'")

if __name__ == "__main__":
    main()


from typing import Tuple

from common.data_models.base import Mode, AtomicUpdate
from common.data_models.image import Image
from common.data_models.labs import Laboratory
from common.data_models.measures import Value
from common.data_models.patient import Patient
from common.data_models.protocol import Protocol
from common.data_models.referrals import Referral

@Patient.watcher.callback('intake', 'complaint', mask=Mode.mask_new)
async def handle_complaint_change(new: Patient, new_value: str):
    protocol_config = await AtomicUpdate.get_connection().get_protocol_config()
    new.protocol = Protocol(active=new_value in protocol_config, items=protocol_config.get(new_value, []))

@Patient.watcher.callback('measures', '.*', mask=Mode.mask_new)
async def handle_measures_change(new: 'Patient', path: Tuple[str], new_value: Value):
    await new.protocol.match_protocol(key=f'measure-{path[-1]}', value=new_value.value, at=new_value.at)

@Patient.watcher.callback('imaging', '.*', mask=Mode.mask_new)
async def handle_imaging_change_protocol(new: 'Patient', new_value: Image):
    await new.protocol.match_protocol(
        key=f'imaging-{new_value.title}',
        value=new_value.status_text,
        at=new_value.updated_at
    )

@Patient.watcher.callback('labs', '.*', 'results', '.*', mask=Mode.mask_new)
async def handle_labs_change_protocol(new: 'Patient', new_value: Laboratory):
    await new.protocol.match_protocol(
        key=f'lab-{new_value.test_type_id}',
        value=f'{new_value.result} {new_value.units}' if new_value.result_at else 'הוזמן',
        at=new_value.result_at if new_value.result_at else new_value.ordered_at
    )

@Patient.watcher.callback('referrals', '.*', mask=Mode.mask_new)
async def handle_referrals_change_protocol(new: 'Patient', new_value: Referral):
    await new.protocol.match_protocol(
        key=f'referral-{new_value.to}',
        value=f'הפנייה נסגרה' if new_value.completed_at else 'הפנייה פתוחה',
        at=new_value.completed_at if new_value.completed_at else new_value.at
    )


import cv2
import numpy as np
import random

def applyAffineTransform(src, srcTri, dstTri, size) :
    warpMat = cv2.getAffineTransform( np.float32(srcTri), np.float32(dstTri) )
    dst = cv2.warpAffine( src, warpMat, (size[0], size[1]), None, flags=random.randrange(1,5), borderMode=cv2.BORDER_REFLECT_101 )

    return dst

def rectContains(rect, point) :
    if point[0] < rect[0] :
        return False
    elif point[1] < rect[1] :
        return False
    elif point[0] > rect[0] + rect[2] :
        return False
    elif point[1] > rect[1] + rect[3] :
        return False
    return True

def calculateDelaunayTriangles(rect, points):
    subdiv = cv2.Subdiv2D(rect)

    for p in points:
        subdiv.insert(p)

    triangleList = subdiv.getTriangleList()
    delaunayTri = []
    pt = []
    for t in triangleList:
        pt.append((t[0], t[1]))
        pt.append((t[2], t[3]))
        pt.append((t[4], t[5]))

        pt1 = (t[0], t[1])
        pt2 = (t[2], t[3])
        pt3 = (t[4], t[5])

        if rectContains(rect, pt1) and rectContains(rect, pt2) and rectContains(rect, pt3):
            ind = []
            for j in range(0, 3):
                for k in range(0, len(points)):
                    if(abs(pt[j][0] - points[k][0]) < 1.0 and abs(pt[j][1] - points[k][1]) < 1.0):
                        ind.append(k)
            if len(ind) == 3:
                delaunayTri.append((ind[0], ind[1], ind[2]))

        pt = []

    return delaunayTri, subdiv

def warpTriangle(img1, img2, t1, t2) :
    r1 = cv2.boundingRect(np.float32([t1]))
    r2 = cv2.boundingRect(np.float32([t2]))

    t1Rect = []
    t2Rect = []
    t2RectInt = []

    for i in range(0, 3):
        t1Rect.append(((t1[i][0] - r1[0]),(t1[i][1] - r1[1])))
        t2Rect.append(((t2[i][0] - r2[0]),(t2[i][1] - r2[1])))
        t2RectInt.append(((t2[i][0] - r2[0]),(t2[i][1] - r2[1])))

    mask = np.zeros((r2[3], r2[2], 3), dtype = np.float32)
    cv2.fillConvexPoly(mask, np.int32(t2RectInt), (1.0, 1.0, 1.0), 16, 0)

    size = (r2[2], r2[3])
    img1Rect = img1[r1[1]:r1[1] + r1[3], r1[0]:r1[0] + r1[2]]
    img2Rect = applyAffineTransform(img1Rect, t1Rect, t2Rect, size)
    img2Rect = img2Rect * mask

    img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]] = img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]] * ( (1.0, 1.0, 1.0) - mask )
    img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]] = img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]] + img2Rect

def cvt_annot_to_pt(annot):

    pt_list = []
    for x, y in annot:
        pt_list.append((int(x), int(y)))
    return pt_list

def filter_keyid_annot(src_annot, filter_ids):
    """ src_annot: a list of (keypoint_id, x, y)
        filter_ids: a list of [keypoint_ids] to be remained
    """
    new_list = []
    for item in src_annot:
        if item[0] in filter_ids:
            new_list.append(item)

    return new_list

def draw_face_landmark(img, annot):

    for x, y in annot:
        cv2.circle(img, (int(x),int(y)), 3, (0,0,255), -1)

def calc_new_keypoints(annot):
    """ Caculate additional 11 keypoints from existing 21 keypoints.
        For visual representation of new key points, please refer to the reference report.
        annot:
    """
    # assert len(annot) == 21, "To add new key points, existing annotation should be 21, but {}.".format(len(annot))

    b1_annot = []
    b1_annot.append(np.zeros((2,), np.int32))   # add dummy array to make annot list to 1 base index.
    b1_annot.extend(annot)

    # below eyeblows
    i22 = (b1_annot[2] + b1_annot[8])/2     #left
    b1_annot.append(i22)
    i23 = (b1_annot[5] + b1_annot[11])/2    #right
    b1_annot.append(i23)

    # temples
    i24 = b1_annot[7] + (b1_annot[7] - b1_annot[9])/2   #left
    b1_annot.append(i24)
    i25 = b1_annot[12]+ (b1_annot[12] - b1_annot[10])/2     #right
    b1_annot.append(i25)

    # the ridge of the nose
    i26 = (b1_annot[9] + b1_annot[10] + b1_annot[15])/3
    b1_annot.append(i26)

    #cheeks
    i27 = (b1_annot[13] + b1_annot[14])/2     #left
    b1_annot.append(i27)
    i28 = (b1_annot[16] + b1_annot[17])/2    #right
    b1_annot.append(i28)

    #philtrum
    i29 = (b1_annot[15] + b1_annot[19])/2
    b1_annot.append(i29)

    #upper chin
    i30 = (b1_annot[19] + b1_annot[21])/2
    b1_annot.append(i30)

    #below eyes
    i31 = (b1_annot[7] * 2 + b1_annot[15])/3     #left
    b1_annot.append(i31)
    i32 = (b1_annot[12] * 2 + b1_annot[15])/3    #right
    b1_annot.append(i32)

    #cheek bones
    i33 = (b1_annot[27] + b1_annot[31])/2   #left
    b1_annot.append(i33)
    i34 = (b1_annot[28] + b1_annot[32])/2   #right
    b1_annot.append(i34)

    #below nose
    i35 = (b1_annot[14] + b1_annot[18])/2   #left
    b1_annot.append(i35)
    i36 = (b1_annot[16] + b1_annot[20])/2   #right
    b1_annot.append(i36)

    #below lips
    r1 = (b1_annot[18] + b1_annot[21])/2    #left reference
    r2 = (b1_annot[20] + b1_annot[21])/2    #right reference
    i37 = (r1+r2)/2 + 3*(r1-r2)/4           #left
    b1_annot.append(i37)
    i38 = (r1+r2)/2 + 3*(r2-r1)/4           #right
    b1_annot.append(i38)

    #sides of lips
    i39 = b1_annot[18] + (b1_annot[18] - b1_annot[19])/3
    b1_annot.append(i39)
    i40 = b1_annot[20] + (b1_annot[20] - b1_annot[19])/3
    b1_annot.append(i40)

    assert len(b1_annot) - 1 == 40, "the length of new annot list should be 32 but {}".format(len(b1_annot) - 1)

    return b1_annot[1:]

def getBoundaryMask(mask):
    kernel = cv2.getStructuringElement(cv2.MORPH_RECT,(7,7))
    new_mask = cv2.morphologyEx(mask, cv2.MORPH_GRADIENT, kernel)
    return new_mask

def faceswap(img1, face_rect1, annot1, img2, face_rect2, annot2, style, rect_crop, resolution):
    """ crop img1's face and merge into img2's face region
        face1 and face2 have the identical keypoints.
    """
    img1Warped = np.copy(img2);

    # Read array of corresponding points
    points1 = cvt_annot_to_pt(annot1)
    points2 = cvt_annot_to_pt(annot2)

    # # Find convex hull
    hull1 = []
    hull2 = []
    hullIndex = cv2.convexHull(np.array(points2), returnPoints = False)
    for i in range(len(hullIndex)):
        hull1.append(points1[int(hullIndex[i])])
        hull2.append(points2[int(hullIndex[i])])
    # # print('hull1: {}, hull2: {}'.format(hull1, hull2))
    # # Find delanauy traingulation for convex hull points
    # dt, subdiv = calculateDelaunayTriangles(face_rect2, hull2)

    # if len(dt) == 0:
    #     raise Exception()

    # # Apply affine transformation to Delaunay triangles
    # for i in range(len(dt)):
    #     t1, t2 = [], []
    #     #get points for img1, img2 corresponding to the triangles
    #     for j in range(3):
    #         t1.append(hull1[dt[i][j]])
    #         t2.append(hull2[dt[i][j]])

    #     warpTriangle(img1, img1Warped, t1, t2)

    dt, subdiv = calculateDelaunayTriangles(face_rect2, points2)
    # Apply affine transformation to Delaunay triangles
    for i in range(len(dt)):
        t1, t2 = [], []
        #get points for img1, img2 corresponding to the triangles
        for j in range(3):
            t1.append(points1[dt[i][j]])
            t2.append(points2[dt[i][j]])

        warpTriangle(img1, img1Warped, t1, t2)

    hull8U = []
    for i in range(0, len(hull2)):
        hull8U.append((hull2[i][0], hull2[i][1]))

    mask = np.zeros(img2.shape, dtype = img2.dtype)
    # img_h, img_w = img2.shape[0:2]
    # mask_1ch = np.zeros((img_h, img_w, 1), dtype = np.uint8)

    cv2.fillConvexPoly(mask, np.int32(hull8U), (255, 255, 255))
    # cv2.fillConvexPoly(mask_1ch, np.int32(hull8U), (255, 255, 255))

    # x, y, w, h = face_rect2
    # cx = int(x + 0.5 * w)
    # cy = int(y + 0.5 * h)
    # maxwh = max(w, h)
    # half = int(0.5 * (1 + 2 * arg.pad) * maxwh)
    # x = max(0, cx - half)
    # y = max(0, cy - half)
    # w = 2 * half
    # h = 2 * half

    # img1Warped = img1Warped[y:y+h, x:x+w, :]
    # img1Warped = cv2.resize(img1Warped, (arg.resize, arg.resize), cv2.INTER_CUBIC)
    top, left, bottom, right = rect_crop
    mask = mask[top:bottom, left:right, :]
    mask = cv2.resize(mask, (resolution, resolution), cv2.INTER_NEAREST)
    img1Warped = img1Warped[top:bottom, left:right, :]
    img1Warped = cv2.resize(img1Warped, (resolution, resolution), cv2.INTER_CUBIC)
    resize_ratio_x = resolution / (right - left)
    resize_ratio_y = resolution / (bottom - top)

    if style == 'warp':
        # output = img1Warped
        output = cv2.colorChange(np.uint8(img1Warped), mask, random.uniform(0.5, 2.5), random.uniform(0.5, 2.5), random.uniform(0.5, 2.5))
    elif style == 'smoothwarp':
        output = cv2.colorChange(np.uint8(img1Warped), mask, random.uniform(0.5, 2.5), random.uniform(0.5, 2.5), random.uniform(0.5, 2.5))
        output_blur = cv2.GaussianBlur(output, (7,7), random.uniform(0.8, 2.0))
        b_mask = getBoundaryMask(mask)
        b_mask = b_mask.astype(np.bool)
        output[b_mask] = output_blur[b_mask]

    elif style == 'seamless':
        hx, hy, hw, hh = cv2.boundingRect(np.float32([hull2]))
        hull_top = max(top, hy)
        hull_left = max(left, hx)
        hull_bottom = min(bottom, hy + hh)
        hull_right = min(right, hx + hw)
        center_hull = (int((hull_left + hull_right) / 2), int((hull_top + hull_bottom) / 2))
        # center_hull = (int(hx + hw / 2), int(hy + hh / 2))

        img2 = img2[top:bottom, left:right, :]
        img2 = cv2.resize(img2, (resolution, resolution), cv2.INTER_CUBIC)

        chx, chy = center_hull
        chx = int((chx - left) * resize_ratio_x)
        chy = int((chy - top) * resize_ratio_y)
        center = (chx, chy)

        output = cv2.seamlessClone(np.uint8(img1Warped), img2, mask, center, cv2.NORMAL_CLONE)

    # for debugging - KJH
    # temp_annot=[]
    # for x,y in annot2:
    #     r_x = int((x - left) * resize_ratio_x)
    #     r_y = int((y - top) * resize_ratio_y)
    #     temp_annot.append((r_x,r_y))
    # draw_face_landmark(output, temp_annot)

    # mask = getBoundaryMask(mask)

    return output, mask

if __name__ == '__main__':
    import sys; sys.path.append('../')
    from imdb import UMDFaces
    from dataset import DatasetSyn
    from util import get_config, Logger
    import datetime
    import argparse
    import os
    from glob import glob
    from tqdm import trange

    now = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
    parser = argparse.ArgumentParser(description='Description')
    parser.add_argument('--tag', default="faceswap", type=str, help='Experiment tag')
    parser.add_argument('--name', type=str, default='UMDFaces')
    parser.add_argument('--num_img', type=int, default=300)
    parser.add_argument('--clear_cache', action='store_true')
    parser.add_argument('--preset', type=str, default='gan')
    parser.add_argument('--set', type=str, default=None, nargs=argparse.REMAINDER)
    args = parser.parse_args()
    cfg = get_config(args, now, src_dir=os.path.join(os.getcwd(), '../'))

    print('>> Loading Logger...')
    logger = Logger(cfg, None)

    class_name = args.name.replace('-', '_')
    imdb = eval(class_name)(cfg, args.name, logger=logger)
    dset = DatasetSyn(cfg, imdb, 'train', 'fake')
    # dset.swap_style = ['seamless']
    dset.swap_style = { 'warp': 0.0, 'smoothwarp': 1.0, 'seamless': 0.0 }

    check_path = os.path.join(cfg.test_path, 'check_faceswap', args.name)
    os.makedirs(check_path, exist_ok=True)

    import cv2
    li = list(range(len(imdb.imgpath)))
    import random
    # print('>> Shuffling...')
    # random.shuffle(li)

    print('>> Removing existing files...')
    for filename in glob(os.path.join(check_path, '*.jpg')):
        os.remove(filename)

    dset.reset()

    print('>> Start!')
    for i in trange(args.num_img):
        idx = li[i]
        idx_src = idx
        idx_dst = dset._get_idx_dst(idx_src)
        crop_rect = dset._crop(idx_dst)

        imgpath = os.path.join(cfg.data_path, imdb.imgpath[idx_dst])
        bbox_original = imdb.bbox_original[idx]
        bbox_min = imdb.bbox_min[idx]
        bbox_max = imdb.bbox_max[idx]
        img = cv2.imread(imgpath)
        h, w, c = img.shape
        # thickness = max(5, 5 * h // 500)
        thickness = 1

        imgpath_src = os.path.join(cfg.data_path, imdb.imgpath[idx_src])
        img_src = cv2.imread(imgpath_src)
        ann_src = imdb.points[idx_src]
        new_ann_src = calc_new_keypoints(ann_src)
        draw_face_landmark(img_src, new_ann_src)
        # img = cv2.rectangle(
        #     img,
        #     (bbox_original[1], bbox_original[0]),
        #     (bbox_original[3], bbox_original[2]),
        #     (0, 0, 255), thickness) # Red

        # img = cv2.rectangle(
        #     img,
        #     (bbox_min[1], bbox_min[0]),
        #     (bbox_min[3], bbox_min[2]),
        #     (255, 0, 0), thickness) # Blue

        # img = cv2.rectangle(
        #     img,
        #     (bbox_max[1], bbox_max[0]),
        #     (bbox_max[3], bbox_max[2]),
        #     (255, 0, 0), thickness) # Blue

        img = cv2.rectangle(
            img,
            (crop_rect[1], crop_rect[0]),
            (crop_rect[3], crop_rect[2]),
            (0, 0, 255), thickness) # Red

        ann_dst = imdb.points[idx_dst]
        new_ann_dst = calc_new_keypoints(ann_dst)
        draw_face_landmark(img, new_ann_dst)

        imgpath = imgpath.replace(cfg.data_path + '/', '')
        imgpath = imgpath.replace(args.name + '/', '')
        imgpath = imgpath.replace('.jpg', '')
        imgpath = imgpath.replace('.png', '')
        imgname = imgpath.replace('/', '+')

        save_img_name = os.path.join(check_path, '{:03d}_{}_img.jpg'.format(i, imgname))
        save_face_name = os.path.join(check_path, '{:03d}_{}_face.jpg'.format(i, imgname))
        save_mask_name = os.path.join(check_path, '{:03d}_{}_mask.jpg'.format(i, imgname))

        # cv2.imwrite(save_img_name, img)

        save_img_name_src = save_img_name.replace('img.jpg', 'srcimg.jpg')
        #cv2.imwrite(save_img_name_src, img_src)

        # face, mask = dset._random_swap(idx_src, idx_dst, crop_rect)
        face = dset._random_swap(idx_src, idx_dst, crop_rect)

        # cv2.imwrite(save_face_name, face)
        # cv2.imwrite(save_mask_name, mask)


from setuptools import setup, find_packages

with open("README.md", "r") as readme_file:
    long_description = readme_file.read()

setup(
    name="whatsapp-wordlcloud-generator",
    version="1.1.0",
    description="Creates a wordcloud with the most used words in a whatsapp conversation",
    long_description=long_description,
    long_description_content_type="text/markdown",
    author="Angelo Mosquera",
    license="MIT",
    keywords="whatsapp wordcloud generator",
    packages=find_packages(),
    install_requires=["wordcloud", "pillow", "matplotlib", "numpy", "BeautifulSoup4"],
    python_requires="~=3.5"
)


import torch.nn as nn
from utils.layers.gen_conv_layer import GenDownsampling, GenUpsampling

"""
    EncoderDecoderGenerator:
        Generator that does use the encoder-decoder architecture explained in the paper.
    Inputs:
        >> in_chs: (int)
"""
class EncoderDecoderGenerator(nn.Module):
    def __init__(self, in_chs,) -> None:
        super(EncoderDecoderGenerator, self).__init__()
        self.model = nn.Sequential(
            # Encoder
            ## Following the paper, batch norm not applied to the first layer
            nn.Conv2d(in_chs, 64, kernel_size=4, stride=2, padding=1,padding_mode='reflect'),
            nn.LeakyReLU(0.2),
            GenDownsampling(64,  128, padding_value=1,),
            GenDownsampling(128, 256, padding_value=1,),
            GenDownsampling(256, 512, padding_value=1,),
            GenDownsampling(512, 512, padding_value=1,),
            GenDownsampling(512, 512, padding_value=1,),
            GenDownsampling(512, 512, padding_value=1,),
            GenDownsampling(512, 512, padding_value=1,),

            # Decoder
            ## Following the paper, last layer followed by tanh
            GenUpsampling(512, 512, padding_value=1, dropout=True),
            GenUpsampling(512, 512, padding_value=1, dropout=True),
            GenUpsampling(512, 512, padding_value=1, dropout=True),
            GenUpsampling(512, 512, padding_value=1, dropout=False),
            GenUpsampling(512, 256, padding_value=1, dropout=False),
            GenUpsampling(256, 128, padding_value=1, dropout=False),
            GenUpsampling(128, 64, padding_value=1, dropout=False),
            nn.ConvTranspose2d(64, in_chs, kernel_size=4, stride=2, padding=1,),
            nn.Tanh(),
        )
        return

    def forward(self, x):
        _, _, height, width = x.size()
        if (height, width) != (256, 256):
            raise Exception(f'The input images need to be 256x256, and it has been fed an image of size {height}x{width}')
        return self.model(x)


import numpy as np

import pandas as pd

import gc

import time

from contextlib import contextmanager

from lightgbm import LGBMClassifier

from sklearn.metrics import roc_auc_score

from sklearn.model_selection import KFold, StratifiedKFold

from sklearn.preprocessing import LabelEncoder

import matplotlib.pyplot as plt

import seaborn as sns

import warnings

warnings.simplefilter(action='ignore', category=FutureWarning)
"""

Input should be a dataframe (df) and the function reduces data sizes of the df

Note: If you have problems with this function try to discard Float16 part as

there might be errors because of this data type:

https://github.com/pandas-dev/pandas/issues/9220

"""

def reduce_mem_usage(df, verbose=True):

    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']

    start_mem = df.memory_usage().sum() / 1024**2

    for col in df.columns:

        col_type = df[col].dtypes

        if col_type in numerics:

            c_min = df[col].min()

            c_max = df[col].max()

            if str(col_type)[:3] == 'int':

                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:

                    df[col] = df[col].astype(np.int8)

                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:

                    df[col] = df[col].astype(np.int16)

                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:

                    df[col] = df[col].astype(np.int32)

                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:

                    df[col] = df[col].astype(np.int64)

            else:

                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:

                    df[col] = df[col].astype(np.float16)

                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:

                    df[col] = df[col].astype(np.float32)

                else:

                    df[col] = df[col].astype(np.float64)

    end_mem = df.memory_usage().sum() / 1024**2

    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))

    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))

    return df
@contextmanager

def timer(title):

    t0 = time.time()

    yield

    print("{} - done in {:.0f}s".format(title, time.time() - t0))

def one_hot_encoder(df, nan_as_category = True):

    original_columns = list(df.columns) # col names as string in a list

    categorical_columns = [col for col in df.columns if df[col].dtype == 'object'] #categorical col names

    df = pd.get_dummies(df, columns = categorical_columns, dummy_na = nan_as_category) #creating dummies

    new_columns = [c for c in df.columns if c not in original_columns] #new col names

    return df, new_columns

def label_encoder(df):

    # Create a label encoder object

    le = LabelEncoder()

    le_count = 0

    # Iterate through the columns

    for col in df:

        if df[col].dtype == 'object':

            # If 2 or fewer unique categories

            if len(list(df[col].unique())) <= 2:

                # Train on the training data

                le.fit(df[col])

                # Transform both training and testing data

                df[col] = le.transform(df[col])

                # Keep track of how many columns were label encoded

                le_count += 1

    print('%d columns were label encoded.' % le_count)

    return df
# Preprocess application_train.csv and application_test.csv

def application_train_test(num_rows = None, nan_as_category = False):

    df = pd.read_csv("../input/home-credit-default-risk/application_train.csv", nrows = num_rows)

    test_df = pd.read_csv("../input/home-credit-default-risk/application_test.csv", nrows = num_rows)

    df = df.append(test_df).reset_index()

    del df["index"]

    df = df[df['CODE_GENDER'] != 'XNA']

    df['DAYS_EMPLOYED'].replace(365243, np.nan, inplace= True)

    df['APP_NEW_AGE'] = df['DAYS_BIRTH'] / (- 365.25)

    # Some simple new features (percentages)

    df['APP_NEW_DAYS_EMPLOYED_PERC'] = df['DAYS_EMPLOYED'] / df['DAYS_BIRTH']

    df['APP_NEW_INCOME_CREDIT_PERC'] = df['AMT_INCOME_TOTAL'] / df['AMT_CREDIT']

    df['APP_NEW_INCOME_PER_PERSON'] = df['AMT_INCOME_TOTAL'] / df['CNT_FAM_MEMBERS']

    df['APP_NEW_ANNUITY_INCOME_PERC'] = df['AMT_ANNUITY'] / df['AMT_INCOME_TOTAL']

    df['APP_NEW_PAYMENT_RATE'] = df['AMT_ANNUITY'] / df['AMT_CREDIT']

    df["APP_NEW_CREDIT_GOODS_PRICE_RATIO"] = df["AMT_CREDIT"] / df["AMT_GOODS_PRICE"]

    df["APP_NEW_CREDIT_DOWNPAYMENT"] = df["AMT_GOODS_PRICE"] - df["AMT_CREDIT"]

    df.loc[(df.AMT_INCOME_TOTAL < 30000),'APP_NEW_INCOME_BAND'] = 1

    df.loc[(df.AMT_INCOME_TOTAL >= 30000)&(df.AMT_INCOME_TOTAL < 65000),'APP_NEW_INCOME_BAND'] = 2

    df.loc[(df.AMT_INCOME_TOTAL >= 65000)&(df.AMT_INCOME_TOTAL < 95000),'APP_NEW_INCOME_BAND'] = 3

    df.loc[(df.AMT_INCOME_TOTAL >= 95000)&(df.AMT_INCOME_TOTAL < 130000),'APP_NEW_INCOME_BAND'] = 4

    df.loc[(df.AMT_INCOME_TOTAL >= 130000)&(df.AMT_INCOME_TOTAL < 160000),'APP_NEW_INCOME_BAND'] = 5

    df.loc[(df.AMT_INCOME_TOTAL >= 160000)&(df.AMT_INCOME_TOTAL < 190000),'APP_NEW_INCOME_BAND'] = 6

    df.loc[(df.AMT_INCOME_TOTAL >= 190000)&(df.AMT_INCOME_TOTAL < 220000),'APP_NEW_INCOME_BAND'] = 7

    df.loc[(df.AMT_INCOME_TOTAL >= 220000)&(df.AMT_INCOME_TOTAL < 275000),'APP_NEW_INCOME_BAND'] = 8

    df.loc[(df.AMT_INCOME_TOTAL >= 275000)&(df.AMT_INCOME_TOTAL < 325000),'APP_NEW_INCOME_BAND'] = 9

    df.loc[(df.AMT_INCOME_TOTAL >= 325000),'APP_NEW_INCOME_BAND'] = 10

    APP_NEW_AGE_CAT = pd.Series(["Young", "Adult 1","Adult 2","Adult 3", "Adult 4"], dtype = "object")

    df["APP_NEW_AGE_CAT"] = APP_NEW_AGE_CAT

    df.loc[(df["APP_NEW_AGE"] > 20.0) & (df["APP_NEW_AGE"] <= 30.0), "APP_NEW_AGE_CAT"] = APP_NEW_AGE_CAT[0]

    df.loc[(df["APP_NEW_AGE"] > 30.0) & (df["APP_NEW_AGE"] <= 40.0), "APP_NEW_AGE_CAT"] = APP_NEW_AGE_CAT[1]

    df.loc[(df["APP_NEW_AGE"] > 40.0) & (df["APP_NEW_AGE"] <= 50.0), "APP_NEW_AGE_CAT"] = APP_NEW_AGE_CAT[2]

    df.loc[(df["APP_NEW_AGE"] > 50.0) & (df["APP_NEW_AGE"] <= 60.0), "APP_NEW_AGE_CAT"] = APP_NEW_AGE_CAT[3]

    df.loc[df["APP_NEW_AGE"] > 60 ,"APP_NEW_AGE_CAT"] = APP_NEW_AGE_CAT[4]

    df["APP_NEW_AGE_DAYS_EMP"] = df["DAYS_EMPLOYED"] / (- 365.25)

    df["APP_NEW_AGE_WORK_PERCENT"] = (df["APP_NEW_AGE_DAYS_EMP"] / df['APP_NEW_AGE']) * 100

    df['APP_NEW_CREDIT_INCOME_PERCENT'] = df['AMT_CREDIT'] / df['AMT_INCOME_TOTAL']

    df['APP_NEW_ANNUITY_INCOME_PERCENT'] = df['AMT_ANNUITY'] / df['AMT_INCOME_TOTAL']

    df['APP_NEW_DAYS_EMPLOYED_PERCENT'] = df['DAYS_EMPLOYED'] / df['DAYS_BIRTH']

    df['APP_NEW_INCOME_PER_PERSON'] = df['AMT_INCOME_TOTAL'] / df['CNT_FAM_MEMBERS']

    df['APP_NEW_INCOME_CREDIT_PERC'] = df['AMT_INCOME_TOTAL'] / df['AMT_CREDIT']

    df['APP_NEW_PAYMENT_RATE'] = df['AMT_ANNUITY'] / df['AMT_CREDIT']

    df['APP_NEW_AMT_PAY_YEAR'] = df['AMT_CREDIT'] / df['AMT_ANNUITY']

    df['APP_NEW_AGE_PAYOFF'] = df['APP_NEW_AGE'] + df['APP_NEW_AMT_PAY_YEAR']

    df['APP_NEW_AMT_DIFF_CREDIT_GOODS'] = df['AMT_CREDIT'] - df['AMT_GOODS_PRICE']

    df['APP_NEW_AMT_CREDIT_GOODS_PERC'] = ((df['AMT_GOODS_PRICE'] / df['AMT_CREDIT']) * 100)

    df['APP_NEW_CNT_ADULT'] = df['CNT_FAM_MEMBERS'] - df['CNT_CHILDREN']

    df['APP_NEW_EXT_SOURCES_MEAN'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].mean(axis=1)

    df['APP_NEW_EXT_SOURCES_MAX'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].max(axis=1)

    df['APP_NEW_EXT_SOURCES_SUM'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].sum(axis=1)

    df = label_encoder(df)

    # Categorical features with One-Hot encode

    df, cat_cols = one_hot_encoder(df, nan_as_category) #returns: encoded new df and cat col names

    del test_df

    gc.collect()

    return df
# Preprocess bureau.csv and bureau_balance.csv

def bureau_and_balance(num_rows = None, nan_as_category = True):

    bureau = pd.read_csv('../input/home-credit-default-risk/bureau.csv', nrows = num_rows)

    bureau_balance = pd.read_csv('../input/home-credit-default-risk/bureau_balance.csv', nrows = num_rows)

    # Bureau balance: Perform aggregations and merge with bureau.csv

    def _status_to_int(status):

        if status in ['X', 'C']:

            return 0

        if pd.isnull(status):

            return np.nan

        return int(status)

    bureau_balance['NEW_BUREAU_BALANCE_DPD_LEVEL'] = bureau_balance['STATUS'].apply(_status_to_int)

    bureau_balance['NEW_BUREAU_BALANCE_STATUS_UNKNOWN'] = (bureau_balance['STATUS'] == 'X').astype(int)

    bureau_balance["MONTHS_BALANCE"] = (-1*bureau_balance["MONTHS_BALANCE"])+1

    bb_aggregations = {'MONTHS_BALANCE': ["max"],

                    'NEW_BUREAU_BALANCE_DPD_LEVEL':['sum', 'mean', 'max', 'std', 'skew'],

                    'NEW_BUREAU_BALANCE_STATUS_UNKNOWN':['sum', 'mean']}

    bb_agg = bureau_balance.groupby('SK_ID_BUREAU').agg(bb_aggregations)

    bb_agg.columns = pd.Index([e[0] + "_" + e[1].upper() for e in bb_agg.columns.tolist()])

    bureau = bureau.join(bb_agg, how='left', on='SK_ID_BUREAU')

    bureau.drop(['SK_ID_BUREAU'], axis=1, inplace= True)

    del bureau_balance, bb_agg

    gc.collect()

     # Bureau new features

    bureau.drop(["AMT_CREDIT_SUM_LIMIT","AMT_CREDIT_SUM_OVERDUE","CREDIT_DAY_OVERDUE","AMT_CREDIT_SUM_OVERDUE"],axis=1,inplace=True)

    bureau['BUREAU_CREDIT_TYPE_CONSUMER'] = (bureau['CREDIT_TYPE'] == 'Consumer credit').astype(int)

    bureau['BUREAU_CREDIT_TYPE_CAR'] = (bureau['CREDIT_TYPE'] == 'Car loan').astype(int)

    bureau['BUREAU_CREDIT_TYPE_MORTGAGE'] = (bureau['CREDIT_TYPE'] == 'Mortgage').astype(int)

    bureau['BUREAU_CREDIT_TYPE_CREDIT_CARD'] = (bureau['CREDIT_TYPE'] == 'Credit card').astype(int)

    bureau['BUREAU_CREDIT_TYPE_OTHER'] = (~(bureau['CREDIT_TYPE'].isin(['Consumer credit', 'Car loan', 'Mortgage', 'Credit card']))).astype(int)

    bureau['BUREAU_UNUSUAL_CURRENCY'] = (~(bureau['CREDIT_CURRENCY'] == 'currency 1')).astype(int)

    bureau['NEW_PAYMENT_RATE_SUM'] = bureau['AMT_ANNUITY'] / bureau['AMT_CREDIT_SUM']

    bureau['NEW_PAYMENT_RATE_SUM_DEBT'] = bureau['AMT_ANNUITY'] / bureau['AMT_CREDIT_SUM_DEBT']

    bureau['NEW_PAYMENT_RATE_AMT_CREDIT_MAX_OVERDUE'] = bureau['AMT_ANNUITY'] / bureau['AMT_CREDIT_MAX_OVERDUE']

    bureau.drop(["CREDIT_TYPE","CREDIT_CURRENCY"],axis=1,inplace=True)

    bureau.loc[(bureau["CREDIT_ACTIVE"]=="Active")&(bureau["DAYS_CREDIT_ENDDATE"]>-42056.001)&(bureau["DAYS_CREDIT_ENDDATE"]<=-1773.0),"BUREAU_NEW_ACTIVE_EARLY_PAYERS_SCORE"] = 1

    bureau.loc[(bureau["CREDIT_ACTIVE"]=="Active")&(bureau["DAYS_CREDIT_ENDDATE"]>-1773.0)&(bureau["DAYS_CREDIT_ENDDATE"]<=-1273.0),"BUREAU_NEW_ACTIVE_EARLY_PAYERS_SCORE"] = 2

    bureau.loc[(bureau["CREDIT_ACTIVE"]=="Active")&(bureau["DAYS_CREDIT_ENDDATE"]>-1273.0)&(bureau["DAYS_CREDIT_ENDDATE"]<=-777.0),"BUREAU_NEW_ACTIVE_EARLY_PAYERS_SCORE"] = 3

    bureau.loc[(bureau["CREDIT_ACTIVE"]=="Active")&(bureau["DAYS_CREDIT_ENDDATE"]>-777.0)&(bureau["DAYS_CREDIT_ENDDATE"]<=-474.0),"BUREAU_NEW_ACTIVE_EARLY_PAYERS_SCORE"] = 4

    bureau.loc[(bureau["CREDIT_ACTIVE"]=="Active")&(bureau["DAYS_CREDIT_ENDDATE"]>-474.0)&(bureau["DAYS_CREDIT_ENDDATE"]<=-268.0),"BUREAU_NEW_ACTIVE_EARLY_PAYERS_SCORE"] = 5

    bureau.loc[(bureau["CREDIT_ACTIVE"]=="Active")&(bureau["DAYS_CREDIT_ENDDATE"]>-268.0)&(bureau["DAYS_CREDIT_ENDDATE"]<=-133.0),"BUREAU_NEW_ACTIVE_EARLY_PAYERS_SCORE"] = 6

    bureau.loc[(bureau["CREDIT_ACTIVE"]=="Active")&(bureau["DAYS_CREDIT_ENDDATE"]>-133.0)&(bureau["DAYS_CREDIT_ENDDATE"]<=-38.0),"BUREAU_NEW_ACTIVE_EARLY_PAYERS_SCORE"] = 7

    bureau.loc[(bureau["CREDIT_ACTIVE"]=="Active")&(bureau["DAYS_CREDIT_ENDDATE"]>-38.0)&(bureau["DAYS_CREDIT_ENDDATE"]<=-1.0),"BUREAU_NEW_ACTIVE_EARLY_PAYERS_SCORE"] = 8

    bureau.loc[(bureau["CREDIT_ACTIVE"]=="Closed")&(bureau["DAYS_ENDDATE_FACT"]<bureau["DAYS_CREDIT_ENDDATE"])&(bureau["DAYS_ENDDATE_FACT"]>-42023.001)&(bureau["DAYS_ENDDATE_FACT"]<=-2145.0),"BUREAU_NEW_CLOSED_EARLY_PAYERS_SCORE"] = 1

    bureau.loc[(bureau["CREDIT_ACTIVE"]=="Closed")&(bureau["DAYS_ENDDATE_FACT"]<bureau["DAYS_CREDIT_ENDDATE"])&(bureau["DAYS_ENDDATE_FACT"]>-2145.0)&(bureau["DAYS_ENDDATE_FACT"]<=-1598.0),"BUREAU_NEW_CLOSED_EARLY_PAYERS_SCORE"] = 2

    bureau.loc[(bureau["CREDIT_ACTIVE"]=="Closed")&(bureau["DAYS_ENDDATE_FACT"]<bureau["DAYS_CREDIT_ENDDATE"])&(bureau["DAYS_ENDDATE_FACT"]>-1598.0)&(bureau["DAYS_ENDDATE_FACT"]<=-1191.0),"BUREAU_NEW_CLOSED_EARLY_PAYERS_SCORE"] = 3

    bureau.loc[(bureau["CREDIT_ACTIVE"]=="Closed")&(bureau["DAYS_ENDDATE_FACT"]<bureau["DAYS_CREDIT_ENDDATE"])&(bureau["DAYS_ENDDATE_FACT"]>-1191.0)&(bureau["DAYS_ENDDATE_FACT"]<=-902.),"BUREAU_NEW_CLOSED_EARLY_PAYERS_SCORE"] = 4

    bureau.loc[(bureau["CREDIT_ACTIVE"]=="Closed")&(bureau["DAYS_ENDDATE_FACT"]<bureau["DAYS_CREDIT_ENDDATE"])&(bureau["DAYS_ENDDATE_FACT"]>-902.)&(bureau["DAYS_ENDDATE_FACT"]<=-656.0),"BUREAU_NEW_CLOSED_EARLY_PAYERS_SCORE"] = 5

    bureau.loc[(bureau["CREDIT_ACTIVE"]=="Closed")&(bureau["DAYS_ENDDATE_FACT"]<bureau["DAYS_CREDIT_ENDDATE"])&(bureau["DAYS_ENDDATE_FACT"]>-656.0)&(bureau["DAYS_ENDDATE_FACT"]<=-439.0),"BUREAU_NEW_CLOSED_EARLY_PAYERS_SCORE"] = 6

    bureau.loc[(bureau["CREDIT_ACTIVE"]=="Closed")&(bureau["DAYS_ENDDATE_FACT"]<bureau["DAYS_CREDIT_ENDDATE"])&(bureau["DAYS_ENDDATE_FACT"]>-439.0)&(bureau["DAYS_ENDDATE_FACT"]<=-229.0),"BUREAU_NEW_CLOSED_EARLY_PAYERS_SCORE"] = 7

    bureau.loc[(bureau["CREDIT_ACTIVE"]=="Closed")&(bureau["DAYS_ENDDATE_FACT"]<bureau["DAYS_CREDIT_ENDDATE"])&(bureau["DAYS_ENDDATE_FACT"]>-229.0)&(bureau["DAYS_ENDDATE_FACT"]<=0.0),"BUREAU_NEW_CLOSED_EARLY_PAYERS_SCORE"] = 8

    bureau.loc[(bureau["CREDIT_ACTIVE"]=="Closed")&(bureau["DAYS_ENDDATE_FACT"]>bureau["DAYS_CREDIT_ENDDATE"])&(bureau["DAYS_ENDDATE_FACT"]>-2876.001)&(bureau["DAYS_ENDDATE_FACT"]<=-2046.0),"BUREAU_NEW_CLOSED_LATE_PAYERS_SCORE"] = 1

    bureau.loc[(bureau["CREDIT_ACTIVE"]=="Closed")&(bureau["DAYS_ENDDATE_FACT"]>bureau["DAYS_CREDIT_ENDDATE"])&(bureau["DAYS_ENDDATE_FACT"]>-2046.0)&(bureau["DAYS_ENDDATE_FACT"]<=-1501.0),"BUREAU_NEW_CLOSED_LATE_PAYERS_SCORE"] = 2

    bureau.loc[(bureau["CREDIT_ACTIVE"]=="Closed")&(bureau["DAYS_ENDDATE_FACT"]>bureau["DAYS_CREDIT_ENDDATE"])&(bureau["DAYS_ENDDATE_FACT"]>-1501.0)&(bureau["DAYS_ENDDATE_FACT"]<=-1182.0),"BUREAU_NEW_CLOSED_LATE_PAYERS_SCORE"] = 3

    bureau.loc[(bureau["CREDIT_ACTIVE"]=="Closed")&(bureau["DAYS_ENDDATE_FACT"]>bureau["DAYS_CREDIT_ENDDATE"])&(bureau["DAYS_ENDDATE_FACT"]>-1182.0)&(bureau["DAYS_ENDDATE_FACT"]<=-1012.0),"BUREAU_NEW_CLOSED_LATE_PAYERS_SCORE"] = 4

    bureau.loc[(bureau["CREDIT_ACTIVE"]=="Closed")&(bureau["DAYS_ENDDATE_FACT"]>bureau["DAYS_CREDIT_ENDDATE"])&(bureau["DAYS_ENDDATE_FACT"]>-1012.0)&(bureau["DAYS_ENDDATE_FACT"]<=-878.0),"BUREAU_NEW_CLOSED_LATE_PAYERS_SCORE"] = 5

    bureau.loc[(bureau["CREDIT_ACTIVE"]=="Closed")&(bureau["DAYS_ENDDATE_FACT"]>bureau["DAYS_CREDIT_ENDDATE"])&(bureau["DAYS_ENDDATE_FACT"]>-878.0)&(bureau["DAYS_ENDDATE_FACT"]<=-615.0),"BUREAU_NEW_CLOSED_LATE_PAYERS_SCORE"] = 6

    bureau.loc[(bureau["CREDIT_ACTIVE"]=="Closed")&(bureau["DAYS_ENDDATE_FACT"]>bureau["DAYS_CREDIT_ENDDATE"])&(bureau["DAYS_ENDDATE_FACT"]>-615.0)&(bureau["DAYS_ENDDATE_FACT"]<=-307.0),"BUREAU_NEW_CLOSED_LATE_PAYERS_SCORE"] = 7

    bureau.loc[(bureau["CREDIT_ACTIVE"]=="Closed")&(bureau["DAYS_ENDDATE_FACT"]>bureau["DAYS_CREDIT_ENDDATE"])&(bureau["DAYS_ENDDATE_FACT"]>-307.0)&(bureau["DAYS_ENDDATE_FACT"]<=0.0),"BUREAU_NEW_CLOSED_LATE_PAYERS_SCORE"] = 8

    bureau['BUREAU_NEW_DAYS_CREDIT_SCORE'] = pd.qcut(bureau['DAYS_CREDIT'],10,labels=[10.0,9.0,8.0,7.0,6.0,5.0,4.0,3.0,2.0,1.0])

    bureau.loc[(bureau["CREDIT_ACTIVE"]=="Active")&(bureau["BUREAU_NEW_DAYS_CREDIT_SCORE"] == 10),"BUREAU_NEW_LAST_ACTIVE_DAYS_CREDIT"] = 1

    # Bureau and bureau_balance numeric features

    num_aggregations = {

        "DAYS_CREDIT": ['min', 'max', 'mean', 'var', 'sum'],

        'DAYS_CREDIT_ENDDATE': ['min', 'max', 'mean'],

        'DAYS_CREDIT_UPDATE': ['mean'],

        'AMT_CREDIT_SUM': ['max', 'mean', 'sum'],

        'AMT_CREDIT_SUM_DEBT': ['max', 'mean', 'sum'],

        'AMT_CREDIT_MAX_OVERDUE': ['mean'],

        'DAYS_ENDDATE_FACT': ['mean', 'sum'],

        'AMT_ANNUITY': ['min', 'max', 'mean', 'var', 'sum'],

        'CNT_CREDIT_PROLONG': ['sum','std'],

        'MONTHS_BALANCE_MAX': ['max'],

        "NEW_BUREAU_BALANCE_DPD_LEVEL_SUM" :['max',"sum"],

        "NEW_BUREAU_BALANCE_DPD_LEVEL_MEAN" :['max',"sum","mean"],

        "NEW_BUREAU_BALANCE_DPD_LEVEL_MAX" :['max',"sum"],

        "NEW_BUREAU_BALANCE_DPD_LEVEL_STD" :['max',"sum","std"],

        "NEW_BUREAU_BALANCE_DPD_LEVEL_SKEW" :['max',"sum","skew"],

        "NEW_BUREAU_BALANCE_STATUS_UNKNOWN_SUM" :['max',"sum"],

        "NEW_BUREAU_BALANCE_STATUS_UNKNOWN_MEAN" :['max',"sum","mean"],

        'BUREAU_CREDIT_TYPE_CONSUMER': ['mean', 'sum'],

        'BUREAU_CREDIT_TYPE_CAR': ['mean', 'sum'],

        'BUREAU_CREDIT_TYPE_MORTGAGE': ['mean', 'sum'],

        'BUREAU_CREDIT_TYPE_CREDIT_CARD': ['mean', 'sum'],

        'BUREAU_CREDIT_TYPE_OTHER': ['mean', 'sum'],

        'BUREAU_UNUSUAL_CURRENCY': ['mean', 'sum'],

        'NEW_PAYMENT_RATE_SUM':['max',"mean","sum"],

        'NEW_PAYMENT_RATE_SUM_DEBT':['max',"mean","sum"],

        'NEW_PAYMENT_RATE_AMT_CREDIT_MAX_OVERDUE':['max',"mean","sum",'min'],

        'BUREAU_NEW_ACTIVE_EARLY_PAYERS_SCORE':['mean','sum','max','min'],

        'BUREAU_NEW_CLOSED_EARLY_PAYERS_SCORE':['mean','sum','max','min'],

        'BUREAU_NEW_CLOSED_LATE_PAYERS_SCORE':['mean','sum','max','min'],

    }

    # Bureau and bureau_balance categorical features

    bureau, bureau_cat = one_hot_encoder(bureau, nan_as_category)

    cat_aggregations = {}

    for cat in bureau_cat: cat_aggregations[cat] = ['mean']

    bureau_agg = bureau.groupby('SK_ID_CURR').agg({**num_aggregations, **cat_aggregations})

    bureau_agg.columns = pd.Index(['BURO_' + e[0] + "_" + e[1].upper() for e in bureau_agg.columns.tolist()])

    # Bureau: Active credits - using only numerical aggregations

    active = bureau[bureau['CREDIT_ACTIVE_Active'] == 1]

    active_agg = active.groupby('SK_ID_CURR').agg(num_aggregations)

    active_agg.columns = pd.Index(['ACTIVE_' + e[0] + "_" + e[1].upper() for e in active_agg.columns.tolist()])

    bureau_agg = bureau_agg.join(active_agg, how='left', on='SK_ID_CURR')

    del active, active_agg

    gc.collect()

    # Bureau: Closed credits - using only numerical aggregations

    closed = bureau[bureau['CREDIT_ACTIVE_Closed'] == 1]

    closed_agg = closed.groupby('SK_ID_CURR').agg(num_aggregations)

    closed_agg.columns = pd.Index(['CLOSED_' + e[0] + "_" + e[1].upper() for e in closed_agg.columns.tolist()])

    bureau_agg = bureau_agg.join(closed_agg, how='left', on='SK_ID_CURR')

    del closed, closed_agg, bureau

    gc.collect()

    return bureau_agg
# Preprocess previous_applications.csv

def previous_applications(num_rows = None, nan_as_category = True):

    df = pd.read_csv("../input/home-credit-default-risk/previous_application.csv", nrows = num_rows)

    df.replace(365243,np.nan,inplace = True)

    df.replace("XNA",np.nan,inplace = True)

    df['NEW_RETURN_DAY'] = df['DAYS_DECISION'] + df['CNT_PAYMENT'] * 30

    df['NEW_DAYS_TERMINATION_diff'] = df['DAYS_TERMINATION'] - df['NEW_RETURN_DAY']

    df['NEW_AMT_DOWN_PAYMENT_rate'] = df['AMT_DOWN_PAYMENT'] / (df['AMT_CREDIT'] + 0.01)

    df['NEW_AMT_SPEND_TO_PRODUCT'] = df['AMT_GOODS_PRICE'] / df['AMT_CREDIT']

    df['NEW_DAYS_DUE']=df['DAYS_FIRST_DUE'] - df['DAYS_LAST_DUE_1ST_VERSION']

    df['NEW_APP_CREDIT_PERC'] = df['AMT_APPLICATION'] / df['AMT_CREDIT']

    df["NAME_PAYMENT_TYPE"].replace(["Non-cash from your account","Cashless from the account of the employer"],np.nan,inplace=True)

    a = ["Channel of corporate sales","Car dealer"]

    df["CHANNEL_TYPE"].replace(a,"Others_Type",inplace=True)

    b = ['Family', 'Spouse, partner', 'Children', 'Other_B', 'Other_A', 'Group of people']

    df["NAME_TYPE_SUITE"] = df["NAME_TYPE_SUITE"].replace(b, 'not_alone')

    df["WEEKDAY_APPR_PROCESS_START"] = df["WEEKDAY_APPR_PROCESS_START"].replace(['MONDAY','TUESDAY', 'WEDNESDAY','THURSDAY','FRIDAY'], 'WEEK_DAY')

    df["WEEKDAY_APPR_PROCESS_START"] = df["WEEKDAY_APPR_PROCESS_START"].replace(['SATURDAY', 'SUNDAY'], 'WEEKEND')

    a = ['Auto technology', 'Jewelry', 'MLM partners', 'Tourism']

    df["NAME_SELLER_INDUSTRY"] = df["NAME_SELLER_INDUSTRY"].replace(a, 'Other_Ind')

    a = ['Auto Accessories', 'Jewelry', 'Homewares', 'Medical Supplies', 'Vehicles', 'Sport and Leisure','Gardening', 'Other', 'Office Appliances', 'Tourism', 'Medicine', 'Direct Sales', 'Fitness', 'Additional Service','Education', 'Weapon', 'Insurance', 'House Construction', 'Animals']

    df["NAME_GOODS_CATEGORY"] = df["NAME_GOODS_CATEGORY"].replace(a, 'Other_Cat')

    a = ['Buying a used car','Building a house or an annex','Everyday expenses','Medicine','Payments on other loans','Education','Journey', 'Purchase of electronic equipment','Buying a new car','Wedding / gift / holiday','Buying a home','Car repairs','Furniture','Buying a holiday home / land', 'Business development','Gasification / water supply','Buying a garage','Hobby','Money for a third person','Refusal to name the goal','Urgent needs','Other']

    df['NAME_CASH_LOAN_PURPOSE']= df['NAME_CASH_LOAN_PURPOSE'].replace(a,'Others')

    df["NAME_PORTFOLIO"].replace("cars",np.nan,inplace=True)

    a = [8,9,10,11,12,13,14,15,16,17]

    df["HOUR_APPR_PROCESS_START"] = df["HOUR_APPR_PROCESS_START"].replace(a, 'Working_Hours')

    b = [18,19,20,21,22,23,0,1,2,3,4,5,6,7]

    df["HOUR_APPR_PROCESS_START"] = df["HOUR_APPR_PROCESS_START"].replace(b, 'Off_Hours')

    drops = ["RATE_INTEREST_PRIMARY","RATE_INTEREST_PRIVILEGED","FLAG_LAST_APPL_PER_CONTRACT","NFLAG_LAST_APPL_IN_DAY","NAME_PRODUCT_TYPE","SELLERPLACE_AREA"]

    df.drop(drops,inplace=True,axis=1)

    df["NFLAG_INSURED_ON_APPROVAL"] = df["NFLAG_INSURED_ON_APPROVAL"].astype("object")

    cat_features = list(df.select_dtypes(['object']).columns)

    df = pd.get_dummies(df, columns= cat_features, dummy_na= True,drop_first=True)

    agg1 = {'SK_ID_CURR': ['size'],

    'AMT_ANNUITY': ['max', 'min', 'mean','std', 'sum'],

    'AMT_APPLICATION':['max', 'min', 'mean','std', 'sum'],

    'AMT_CREDIT':['max', 'min', 'mean','std', 'sum'],

    'AMT_DOWN_PAYMENT': ['max', 'min', 'mean','std', 'sum'],

    'AMT_GOODS_PRICE': ['max', 'min', 'mean','std', 'sum'],

    'RATE_DOWN_PAYMENT': ['max', 'min', 'mean','std'],

    'DAYS_DECISION': ['max', 'min', 'mean', 'sum'],

    'CNT_PAYMENT': ['max', 'min', 'mean','std', 'sum'],

    'DAYS_FIRST_DRAWING': ['max','mean', 'sum'],

    'DAYS_FIRST_DUE': ['max', 'min', 'mean', 'sum'],

    'DAYS_LAST_DUE_1ST_VERSION': ['max', 'min', 'mean', 'sum'],

    'DAYS_LAST_DUE': ['max', 'min', 'mean', 'sum'],

    'DAYS_TERMINATION': ['max', 'min', 'mean','std', 'sum'],

    'NEW_RETURN_DAY': ['max', 'min', 'mean','std', 'sum'],

    'NEW_DAYS_TERMINATION_diff': ['max', 'mean','std', 'sum'],

    'NEW_AMT_DOWN_PAYMENT_rate': ['max','mean','std'],

    'NEW_AMT_SPEND_TO_PRODUCT': ['max', 'mean','std', 'sum'],

    'NEW_APP_CREDIT_PERC': ['max', 'min', 'mean'],

    'NAME_CONTRACT_TYPE_Consumer loans': ['max', 'min','sum'],

    'NAME_CONTRACT_TYPE_Revolving loans': ['max', 'min','sum'],

    'NAME_CONTRACT_TYPE_nan': ['max', 'min','sum'],

    'WEEKDAY_APPR_PROCESS_START_WEEK_DAY': ['max', 'min', 'sum'],

    'WEEKDAY_APPR_PROCESS_START_nan': ['max', 'sum'],

    'HOUR_APPR_PROCESS_START_Working_Hours': ['max', 'min', 'sum'],

    'HOUR_APPR_PROCESS_START_nan': ['max','sum'],

    'NAME_CASH_LOAN_PURPOSE_Repairs': ['max', 'min', 'sum'],

    'NAME_CASH_LOAN_PURPOSE_XAP': ['max', 'sum'],

    'NAME_CASH_LOAN_PURPOSE_nan': ['max', 'sum'],

    'NAME_CONTRACT_STATUS_Canceled': ['max','sum'],

    'NAME_CONTRACT_STATUS_Refused': ['max','sum'],

    'NAME_CONTRACT_STATUS_Unused offer': ['max','sum'],

    'NAME_CONTRACT_STATUS_nan': ['max', 'sum'],

    'NAME_PAYMENT_TYPE_nan': ['max', 'sum'],

    'CODE_REJECT_REASON_HC': ['max', 'min','sum'],

    'CODE_REJECT_REASON_LIMIT': ['max', 'min','sum'],

    'CODE_REJECT_REASON_SCO': ['max', 'min','sum'],

    'CODE_REJECT_REASON_SCOFR': ['max', 'sum'],

    'CODE_REJECT_REASON_VERIF': ['max',  'sum'],

    'CODE_REJECT_REASON_XAP': ['max',  'sum'],

    'CODE_REJECT_REASON_nan': ['max','sum'],

    'NAME_TYPE_SUITE_not_alone': ['max', 'min','sum'],

    'NAME_TYPE_SUITE_nan': ['max', 'min', 'sum'],

    'NAME_CLIENT_TYPE_Refreshed': ['max', 'min','sum'],

    'NAME_CLIENT_TYPE_Repeater': ['max', 'min', 'sum'],

    'NAME_CLIENT_TYPE_nan': ['max', 'min','sum'],

    'NAME_GOODS_CATEGORY_Clothing and Accessories': ['max', 'min', 'sum'],

    'NAME_GOODS_CATEGORY_Computers': ['max', 'min','sum'],

    'NAME_GOODS_CATEGORY_Construction Materials': ['max', 'min', 'sum'],

    'NAME_GOODS_CATEGORY_Consumer Electronics': ['max', 'min', 'sum'],

    'NAME_GOODS_CATEGORY_Furniture': ['max', 'min', 'sum'],

    'NAME_GOODS_CATEGORY_Mobile': ['max', 'min', 'sum'],

    'NAME_GOODS_CATEGORY_Other_Cat': ['max', 'min', 'sum'],

    'NAME_GOODS_CATEGORY_Photo / Cinema Equipment': ['max', 'min', 'sum'],

    'NAME_GOODS_CATEGORY_nan': ['max', 'min', 'sum'],

    'NAME_PORTFOLIO_Cars': ['max', 'min', 'sum'],

    'NAME_PORTFOLIO_Cash': ['max', 'min', 'sum'],

    'NAME_PORTFOLIO_POS': ['max', 'min','sum'],

    'NAME_PORTFOLIO_nan': ['max', 'min', 'sum'],

    'CHANNEL_TYPE_Contact center': ['max', 'min', 'sum'],

    'CHANNEL_TYPE_Country-wide': ['max', 'min', 'sum'],

    'CHANNEL_TYPE_Credit and cash offices': ['max', 'min', 'sum'],

    'CHANNEL_TYPE_Others_Type': ['max', 'min', 'sum'],

    'CHANNEL_TYPE_Regional / Local': ['max', 'min','sum'],

    'CHANNEL_TYPE_Stone': ['max', 'min','sum'],

    'CHANNEL_TYPE_nan': ['max', 'min', 'sum'],

    'NAME_SELLER_INDUSTRY_Connectivity': ['max', 'min','sum'],

    'NAME_SELLER_INDUSTRY_Construction': ['max', 'min', 'sum'],

    'NAME_SELLER_INDUSTRY_Consumer electronics': ['max', 'min',  'sum'],

    'NAME_SELLER_INDUSTRY_Furniture': ['max', 'min', 'sum'],

    'NAME_SELLER_INDUSTRY_Industry': ['max', 'min', 'sum'],

    'NAME_SELLER_INDUSTRY_Other_Ind': ['max', 'min','sum'],

    'NAME_SELLER_INDUSTRY_nan': ['max', 'min','sum'],

    'NAME_YIELD_GROUP_low_action': ['max', 'min', 'sum'],

    'NAME_YIELD_GROUP_low_normal': ['max', 'min', 'sum'],

    'NAME_YIELD_GROUP_middle': ['max', 'min','sum'],

    'NAME_YIELD_GROUP_nan': ['max', 'min','sum'],

    'PRODUCT_COMBINATION_Card X-Sell': ['max', 'min', 'sum'],

    'PRODUCT_COMBINATION_Cash': ['max', 'min', 'sum'],

    'PRODUCT_COMBINATION_Cash Street: high': ['max', 'min', 'sum'],

    'PRODUCT_COMBINATION_Cash Street: low': ['max', 'min','sum'],

    'PRODUCT_COMBINATION_Cash Street: middle': ['max', 'min','sum'],

    'PRODUCT_COMBINATION_Cash X-Sell: high': ['max', 'min','sum'],

    'PRODUCT_COMBINATION_Cash X-Sell: low': ['max', 'min','sum'],

    'PRODUCT_COMBINATION_Cash X-Sell: middle': ['max', 'min','sum'],

    'PRODUCT_COMBINATION_POS household with interest': ['max', 'min','sum'],

    'PRODUCT_COMBINATION_POS household without interest': ['max', 'min','sum'],

    'PRODUCT_COMBINATION_POS industry with interest': ['max', 'min','sum'],

    'PRODUCT_COMBINATION_POS industry without interest': ['max', 'min','sum'],

    'PRODUCT_COMBINATION_POS mobile with interest': ['max', 'min','sum'],

    'PRODUCT_COMBINATION_POS mobile without interest': ['max', 'min','sum'],

    'PRODUCT_COMBINATION_POS other with interest': ['max', 'min','sum'],

    'PRODUCT_COMBINATION_POS others without interest': ['max', 'min','sum'],

    'PRODUCT_COMBINATION_nan': ['max', 'min','sum'],

    'NFLAG_INSURED_ON_APPROVAL_1.0': ['max', 'min','sum'],

    'NFLAG_INSURED_ON_APPROVAL_nan': ['max', 'min','sum']}

    df = df.groupby(['SK_ID_CURR']).agg(agg1)

    df.columns = pd.Index(['PREV_' + e[0] + "_" + e[1].upper() for e in df.columns.tolist()])

    return df
# Preprocess POS_CASH_balance.csv

def pos_cash(num_rows = None, nan_as_category = True):

    df=pd.read_csv('../input/home-credit-default-risk/POS_CASH_balance.csv',nrows = num_rows)

    df['NEW_ADJOURNMENT']=df['SK_DPD']-df['SK_DPD_DEF']

    b = ["Demand","Returned to the store","Approved","Amortized debt","Canceled","XNA"]

    df["NAME_CONTRACT_STATUS"].replace(b, 'Others',inplace=True)

    cat_features = list(df.select_dtypes(['object']).columns)

    df = pd.get_dummies(df, columns= cat_features, dummy_na= True)

    agg={

    'MONTHS_BALANCE': ['max'],

    'SK_DPD': ['max', 'mean',"std"],

    'SK_DPD_DEF': ['max', 'mean',"std"],

    'CNT_INSTALMENT':['min','mean','max'],

    'CNT_INSTALMENT_FUTURE':['mean','min','max'],

    'SK_ID_CURR':['max','size'],

    'NEW_ADJOURNMENT':['max','mean',"std"],

    'NAME_CONTRACT_STATUS_Active':['sum'],

    'NAME_CONTRACT_STATUS_Completed':['sum'],

    'NAME_CONTRACT_STATUS_Signed':['sum'],

    'NAME_CONTRACT_STATUS_Others':['sum']

    }

    pos_agg = df.groupby(['SK_ID_PREV']).agg(agg)

    pos_agg.columns = pd.Index([e[0] + "_" + e[1].upper() for e in pos_agg.columns.tolist()])

    pos_agg["NEW_PAID_MONTH"] = pos_agg["CNT_INSTALMENT_MAX"] - pos_agg["CNT_INSTALMENT_FUTURE_MIN"]

    agg2={

        "MONTHS_BALANCE_MAX":["max","mean"],

        "SK_DPD_MAX":["max","mean"],

        "SK_DPD_MEAN":["max","mean"],

        "SK_DPD_STD":["max","mean","min","std"],

        "SK_DPD_DEF_MAX":["max","mean"],

        "SK_DPD_DEF_MEAN":["max","mean"],

        "SK_DPD_DEF_STD":["max","mean"],

        "CNT_INSTALMENT_MIN":["max","mean","min"],

        "CNT_INSTALMENT_MEAN":["max","mean"],

        "CNT_INSTALMENT_MAX":["max","mean"],

        "CNT_INSTALMENT_FUTURE_MEAN":["max","mean"],

        "CNT_INSTALMENT_FUTURE_MIN":["max","mean"],

        "CNT_INSTALMENT_FUTURE_MAX":["max","mean"],

        "SK_ID_CURR_MAX":["max"],

        "SK_ID_CURR_SIZE":["max","min"],

        "NEW_ADJOURNMENT_MAX":["max","mean"],

        "NEW_ADJOURNMENT_MEAN":["max","mean"],

        "NEW_ADJOURNMENT_STD":["max","mean","min"],

        "NAME_CONTRACT_STATUS_Active_SUM":["max","sum"],

        'NAME_CONTRACT_STATUS_Signed_SUM':["max","sum"],

        'NAME_CONTRACT_STATUS_Completed_SUM':["max","sum"],

        'NAME_CONTRACT_STATUS_Others_SUM':["max","sum"]

    }

    pos_agg2 = pos_agg.groupby(["SK_ID_CURR_MAX"]).agg(agg2)

    pos_agg2.index.names = ['SK_ID_CURR']

    pos_agg2.columns = pd.Index(["POS" + "_" + e[0] + "_" + e[1].upper() for e in pos_agg2.columns.tolist()])

    del df

    gc.collect()

    return pos_agg2
# Preprocess installments_payments.csv

def installments_payments(num_rows=None, nan_as_category = True):

    pd.options.mode.chained_assignment = None

    df = pd.read_csv("../input/home-credit-default-risk/installments_payments.csv", nrows = num_rows)

    df["NEW_DELAY"] = df["DAYS_INSTALMENT"] - df["DAYS_ENTRY_PAYMENT"]

    df['NEW_FLAG_DELAY'] = df['NEW_DELAY'].apply(lambda x : 1 if x < 0 else 0)

    df['NEW_RATIO_DELAY'] = df[['SK_ID_PREV','NEW_FLAG_DELAY']].groupby('SK_ID_PREV')['NEW_FLAG_DELAY'].transform(lambda x : x.sum() / x.count())

    df["NEW_PAYMENT_DIFF"] = df["AMT_INSTALMENT"] - df["AMT_PAYMENT"]

    df["IP_NEW_PAYMENT_INSTALMENT_DIFF"] = df["AMT_PAYMENT"] - df["AMT_INSTALMENT"]

    df["NUM_INSTALMENT_VERSION"] = df["NUM_INSTALMENT_VERSION"].astype("object")

    df[(df["NUM_INSTALMENT_VERSION"] != 1) & (df["NUM_INSTALMENT_VERSION"] != 0) & (df["NUM_INSTALMENT_VERSION"] != 2) & (df["NUM_INSTALMENT_VERSION"] != 3)]['NUM_INSTALMENT_VERSION'] = 4

    cat_features = list(df.select_dtypes(['object']).columns)

    df = pd.get_dummies(df, columns= cat_features,drop_first=True)

    agg1 = {'SK_ID_CURR': ['count','max'],

           'NEW_DELAY': ['max', 'min', 'mean','std', 'sum'],

           'NUM_INSTALMENT_NUMBER':['min','max'],

           'DAYS_INSTALMENT':['max','min','std'],

           'NEW_PAYMENT_DIFF': ['max', 'mean', 'std', 'min','sum'],

           'IP_NEW_PAYMENT_INSTALMENT_DIFF': ['max', 'mean', 'std', 'min','sum'],

           'AMT_INSTALMENT': ['max', 'mean', 'sum', 'min', 'std'],

           'AMT_PAYMENT': ['min', 'max', 'mean', 'sum', 'std'],

           'DAYS_ENTRY_PAYMENT': ['max', 'min', 'std'],

           "NUM_INSTALMENT_VERSION_1.0":["sum"],

           "NUM_INSTALMENT_VERSION_2.0":["sum"],

           "NUM_INSTALMENT_VERSION_3.0":["sum"],

           "NUM_INSTALMENT_VERSION_4.0":["sum"]

            }

    Installments_agg = df.groupby(['SK_ID_PREV']).agg(agg1)

    Installments_agg.columns = pd.Index([e[0] + "_" + e[1].upper() for e in Installments_agg.columns.tolist()])

    del df

    gc.collect()

    return Installments_agg
# Preprocess credit_card_balance.csv

def credit_card_balance(num_rows = None, nan_as_category = True):

    cc = pd.read_csv('../input/home-credit-default-risk/credit_card_balance.csv', nrows = num_rows)

    cc, cat_cols = one_hot_encoder(cc, nan_as_category= True)

    # General aggregations

    cc.drop(['SK_ID_PREV'], axis= 1, inplace = True)

    cc['number_of_instalments'] = cc.groupby(

    by=['SK_ID_CURR'])['CNT_INSTALMENT_MATURE_CUM'].agg('max').reset_index()[

    'CNT_INSTALMENT_MATURE_CUM']

    cc['AMT_DRAWINGS_ATM_CURRENT'][cc['AMT_DRAWINGS_ATM_CURRENT'] < 0] = np.nan

    cc['AMT_DRAWINGS_CURRENT'][cc['AMT_DRAWINGS_CURRENT'] < 0] = np.nan

    cc_agg = cc.groupby('SK_ID_CURR').agg(['min', 'max', 'mean', 'sum', 'var'])

    cc_agg.columns = pd.Index(['CC_' + e[0] + "_" + e[1].upper() for e in cc_agg.columns.tolist()])

    # Count credit card lines

    cc_agg['CC_COUNT'] = cc.groupby('SK_ID_CURR').size()

    del cc

    gc.collect()

    return cc_agg
# LightGBM GBDT with KFold or Stratified KFold

# Parameters from Tilii kernel: https://www.kaggle.com/tilii7/olivier-lightgbm-parameters-by-bayesian-opt/code

def kfold_lightgbm(df, num_folds, stratified = False, debug= False):

    import re

    df = df.rename(columns = lambda x:re.sub('[^A-Za-z0-9_]+', '', x))

    # Divide in training/validation and test data

    train_df = df[df['TARGET'].notnull()]

    test_df = df[df['TARGET'].isnull()]

    print("Starting LightGBM. Train shape: {}, test shape: {}".format(train_df.shape, test_df.shape))

    del df

    gc.collect()

    # Cross validation model

    if stratified:

        folds = StratifiedKFold(n_splits= num_folds, shuffle=True, random_state=1001)

    else:

        folds = KFold(n_splits= num_folds, shuffle=True, random_state=1001)

    # Create arrays and dataframes to store results

    oof_preds = np.zeros(train_df.shape[0])

    sub_preds = np.zeros(test_df.shape[0])

    feature_importance_df = pd.DataFrame()

    feats = [f for f in train_df.columns if f not in ['TARGET','SK_ID_CURR','SK_ID_BUREAU','SK_ID_PREV','index']]

    for n_fold, (train_idx, valid_idx) in enumerate(folds.split(train_df[feats], train_df['TARGET'])):

        train_x, train_y = train_df[feats].iloc[train_idx], train_df['TARGET'].iloc[train_idx]

        valid_x, valid_y = train_df[feats].iloc[valid_idx], train_df['TARGET'].iloc[valid_idx]

        # LightGBM parameters found by Bayesian optimization

        clf = LGBMClassifier(

            njobs = -1,

            n_estimators=10000,

            learning_rate=0.02,

            num_leaves=34,

            colsample_bytree=0.9497036,

            subsample=0.8715623,

            max_depth=8,

            reg_alpha=0.041545473,

            reg_lambda=0.0735294,

            min_split_gain=0.0222415,

            min_child_weight=39.3259775,

            silent=-1,

            verbose=-1, )

        clf.fit(train_x, train_y, eval_set = [(train_x, train_y), (valid_x, valid_y)],

            eval_metric = 'auc', verbose = 300, early_stopping_rounds = 200)

        oof_preds[valid_idx] = clf.predict_proba(valid_x, num_iteration=clf.best_iteration_)[:, 1]

        sub_preds += clf.predict_proba(test_df[feats], num_iteration=clf.best_iteration_)[:, 1] / folds.n_splits

        fold_importance_df = pd.DataFrame()

        fold_importance_df["feature"] = feats

        fold_importance_df["importance"] = clf.feature_importances_

        fold_importance_df["fold"] = n_fold + 1

        feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)

        print('Fold %2d AUC : %.6f' % (n_fold + 1, roc_auc_score(valid_y, oof_preds[valid_idx])))

        del clf, train_x, train_y, valid_x, valid_y

        gc.collect()

    print('Full AUC score %.6f' % roc_auc_score(train_df['TARGET'], oof_preds))

    # Write submission file and plot feature importance

    if not debug:

        test_df['TARGET'] = sub_preds

        test_df[['SK_ID_CURR', 'TARGET']].to_csv(submission_file_name, index= False)

    display_importances(feature_importance_df)

    return feature_importance_df

# Display/plot feature importance

def display_importances(feature_importance_df_):

    cols = feature_importance_df_[["feature", "importance"]].groupby("feature").mean().sort_values(by="importance", ascending=False)[:40].index

    best_features = feature_importance_df_.loc[feature_importance_df_.feature.isin(cols)]

    plt.figure(figsize=(8, 10))

    sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False))

    plt.title('LightGBM Features (avg over folds)')

    plt.tight_layout()

    plt.savefig('lgbm_importances01.png')
def main(debug = False):

    num_rows = 10000 if debug else None

    df = application_train_test(num_rows)

    with timer("Process bureau and bureau_balance"):

        bureau = bureau_and_balance(num_rows)

        print("Bureau df shape:", bureau.shape)

        df = df.join(bureau, how='left', on='SK_ID_CURR')

        del bureau

        gc.collect()

    with timer("Process previous_applications"):

        prev = previous_applications(num_rows)

        print("Previous applications df shape:", prev.shape)

        df = df.join(prev, how='left', on='SK_ID_CURR')

        del prev

        gc.collect()

    with timer("Process POS-CASH balance"):

        pos = pos_cash(num_rows)

        print("Pos-cash balance df shape:", pos.shape)

        df = df.join(pos, how='left', on='SK_ID_CURR')

        del pos

        gc.collect()

    with timer("Process installments payments"):

        ins = installments_payments(num_rows)

        print("Installments payments df shape:", ins.shape)

        df = df.join(ins, how='left', on='SK_ID_CURR')

        del ins

        gc.collect()

    with timer("Process credit card balance"):

        cc = credit_card_balance(num_rows)

        print("Credit card balance df shape:", cc.shape)

        df = df.join(cc, how='left', on='SK_ID_CURR')

        del cc

        gc.collect()

    with timer("Run LightGBM with kfold"):

        feat_importance = kfold_lightgbm(df, num_folds= 10, stratified= False, debug= debug)

if __name__ == "__main__":

    submission_file_name = "submission.csv"

    with timer("Full model run"):

        main()


# Considering a list of positive integers where each item represents the maximum steps you can take
# return the minimum amount of steps to get from index 0 to index size - 1

from random import randint
from typing import List

from data_structures.queue import Queue

def get_least_amount_of_steps(steps: List[int]) -> List[int]:
    return breadth_first_search(steps, 0, len(steps) - 1)

def breadth_first_search(graph: List[int], source_vertex: int, target_vertex: int) -> List[int]:
    vertices_to_visit = Queue()
    vertices_to_visit.enqueue(source_vertex)

    distances = {source_vertex: 0}

    while not vertices_to_visit.is_empty():
        current_vertex = vertices_to_visit.dequeue()
        for neighbor in range(1, graph[current_vertex] + 1):
            neighbor_index = current_vertex + neighbor
            if neighbor_index < len(graph):
                if neighbor_index not in distances:
                    distances[neighbor_index] = distances[current_vertex] + 1
                    if neighbor_index == target_vertex:
                        return distances[target_vertex]
                    vertices_to_visit.enqueue(neighbor_index)

if __name__ == '__main__':
    max_steps = 5
    max_nodes = 10
    steps = [randint(1, max_steps) for i in range(max_nodes)]
    print(steps)
    print(get_least_amount_of_steps(steps))


import time

from libvirt import libvirtError
from libvirttestapi.src import sharedmod
from libvirttestapi.utils import process, utils

required_params = ('portdev',)
optional_params = {}

def check_ebtables_rule(portdev, logger):
    cmd = "ebtables -t nat -L"
    ret = process.run(cmd, shell=True, ignore_status=True)
    if ret.exit_status:
        logger.error("%s failed: %s." % (cmd, ret.stderr))
        return False
    if portdev in ret.stdout:
        logger.error("%s still exist in ebtables rule." % portdev)
        return False
    else:
        logger.info("%s don't exist in ebtables rule." % portdev)
    return True

def nwfilterbind_delete(params):
    logger = params['logger']
    portdev = params['portdev']

    if not utils.version_compare("libvirt-python", 4, 5, 0, logger):
        logger.info("Current libvirt-python don't support nwfilterbind.delete().")
        return 0
    try:
        conn = sharedmod.libvirtobj['conn']
        nwfilterbind = conn.nwfilterBindingLookupByPortDev(portdev)
        nwfilterbind.delete()
        time.sleep(3)
        nwfilterbind_list = conn.listAllNWFilterBindings()
        if nwfilterbind in nwfilterbind_list:
            logger.error("nwfilterbind still exist.")
            return 1
        if not check_ebtables_rule(portdev, logger):
            logger.error("FAIL: check ebtables rule failed.")
            return 1
        else:
            logger.info("PASS: check ebtables rule successful.")

    except libvirtError as e:
        logger.error("API error message: %s" % e.get_error_message())
        return 1

    return 0


from typing import List, Optional

import click

import clinica.pydra.engine_utils as pydra_utils
from clinica.pipelines import cli_param
from clinica.pipelines.engine import clinica_pipeline

pipeline_name = "pydra-t1-volume-register-dartel"

@clinica_pipeline
@click.command(name=pipeline_name, hidden=True)
@cli_param.argument.bids_directory
@cli_param.argument.caps_directory
@cli_param.argument.group_label
@cli_param.option_group.common_pipelines_options
# @cli_param.option.subjects_sessions_tsv
# @cli_param.option.working_directory
@cli_param.option.n_procs
@cli_param.option_group.advanced_pipeline_options
@cli_param.option.dartel_tissues
def cli(
    bids_directory: str,
    caps_directory: str,
    group_label: str,
    dartel_tissues: List[int] = (1, 2, 3),
    n_procs: Optional[int] = None,
) -> None:
    """Inter-subject registration using an existing Dartel template (Pydra engine).

    Parameters
    ----------
    bids_directory : str
        Path to the input BIDS directory.

    caps_directory : str
        Path to the CAPS directory.

    group_label : str
        User-defined identifier to target a specific group of subjects.
        For this pipeline, it is associated to the DARTEL template that was created
        when running the t1-volume pipeline.

    dartel_tissues : List[int], optional
        Indices for selecting the tissues. Possible values are:
            - 1: "graymatter"
            - 2: "whitematter"
            - 3: "csf"
            - 4: "bone"
            - 5: "softtissue"
            - 6: "background"

        Default=[1, 2, 3].

    n_procs : int, optional
        Number of processes to use.

    Notes
    -----
    https://aramislab.paris.inria.fr/clinica/docs/public/latest/Pipelines/T1_Volume/
    """
    import clinica.pydra.t1_volume.register_dartel.pipeline as pydra_t1vol_rd

    parameters = {"group_label": group_label, "dartel_tissues": dartel_tissues}

    t1_volume_tissue_segmentation_pipeline = pydra_t1vol_rd.t1volume_register_dartel(
        name="t1-volume-tissue-segmentation-pydra",
        input_dir=bids_directory,
        output_dir=caps_directory,
        parameters=parameters,
    )
    pydra_utils.run(t1_volume_tissue_segmentation_pipeline, n_procs=n_procs)