import math
import six

import tensorflow as tf

from object_detection.core import box_list
from object_detection.core import box_list_ops
from object_detection.core import standard_fields as fields
from object_detection.utils import static_shape

def expanded_shape(orig_shape, start_dim, num_dims):
  """Inserts multiple ones into a shape vector.

  Inserts an all-1 vector of length num_dims at position start_dim into a shape.
  Can be combined with tf.reshape to generalize tf.expand_dims.

  Args:
    orig_shape: the shape into which the all-1 vector is added (int32 vector)
    start_dim: insertion position (int scalar)
    num_dims: length of the inserted all-1 vector (int scalar)
  Returns:
    An int32 vector of length tf.size(orig_shape) + num_dims.
  """
  with tf.name_scope('ExpandedShape'):
    start_dim = tf.expand_dims(start_dim, 0)  # scalar to rank-1
    before = tf.slice(orig_shape, [0], start_dim)
    add_shape = tf.ones(tf.reshape(num_dims, [1]), dtype=tf.int32)
    after = tf.slice(orig_shape, start_dim, [-1])
    new_shape = tf.concat([before, add_shape, after], 0)
    return new_shape

def normalized_to_image_coordinates(normalized_boxes, image_shape,
                                    parallel_iterations=32):
  """Converts a batch of boxes from normal to image coordinates.

  Args:
    normalized_boxes: a float32 tensor of shape [None, num_boxes, 4] in
      normalized coordinates.
    image_shape: a float32 tensor of shape [4] containing the image shape.
    parallel_iterations: parallelism for the map_fn op.

  Returns:
    absolute_boxes: a float32 tensor of shape [None, num_boxes, 4] containg the
      boxes in image coordinates.
  """
  def _to_absolute_coordinates(normalized_boxes):
    return box_list_ops.to_absolute_coordinates(
        box_list.BoxList(normalized_boxes),
        image_shape[1], image_shape[2], check_range=False).get()

  absolute_boxes = tf.map_fn(
      _to_absolute_coordinates,
      elems=(normalized_boxes),
      dtype=tf.float32,
      parallel_iterations=parallel_iterations,
      back_prop=True)
  return absolute_boxes

def meshgrid(x, y):
  """Tiles the contents of x and y into a pair of grids.

  Multidimensional analog of numpy.meshgrid, giving the same behavior if x and y
  are vectors. Generally, this will give:

  xgrid(i1, ..., i_m, j_1, ..., j_n) = x(j_1, ..., j_n)
  ygrid(i1, ..., i_m, j_1, ..., j_n) = y(i_1, ..., i_m)

  Keep in mind that the order of the arguments and outputs is reverse relative
  to the order of the indices they go into, done for compatibility with numpy.
  The output tensors have the same shapes.  Specifically:

  xgrid.get_shape() = y.get_shape().concatenate(x.get_shape())
  ygrid.get_shape() = y.get_shape().concatenate(x.get_shape())

  Args:
    x: A tensor of arbitrary shape and rank. xgrid will contain these values
       varying in its last dimensions.
    y: A tensor of arbitrary shape and rank. ygrid will contain these values
       varying in its first dimensions.
  Returns:
    A tuple of tensors (xgrid, ygrid).
  """
  with tf.name_scope('Meshgrid'):
    x = tf.convert_to_tensor(x)
    y = tf.convert_to_tensor(y)
    x_exp_shape = expanded_shape(tf.shape(x), 0, tf.rank(y))
    y_exp_shape = expanded_shape(tf.shape(y), tf.rank(y), tf.rank(x))

    xgrid = tf.tile(tf.reshape(x, x_exp_shape), y_exp_shape)
    ygrid = tf.tile(tf.reshape(y, y_exp_shape), x_exp_shape)
    new_shape = y.get_shape().concatenate(x.get_shape())
    xgrid.set_shape(new_shape)
    ygrid.set_shape(new_shape)

    return xgrid, ygrid

def pad_to_multiple(tensor, multiple):
  """Returns the tensor zero padded to the specified multiple.

  Appends 0s to the end of the first and second dimension (height and width) of
  the tensor until both dimensions are a multiple of the input argument
  'multiple'. E.g. given an input tensor of shape [1, 3, 5, 1] and an input
  multiple of 4, PadToMultiple will append 0s so that the resulting tensor will
  be of shape [1, 4, 8, 1].

  Args:
    tensor: rank 4 float32 tensor, where
            tensor -> [batch_size, height, width, channels].
    multiple: the multiple to pad to.

  Returns:
    padded_tensor: the tensor zero padded to the specified multiple.
  """
  tensor_shape = tensor.get_shape()
  batch_size = static_shape.get_batch_size(tensor_shape)
  tensor_height = static_shape.get_height(tensor_shape)
  tensor_width = static_shape.get_width(tensor_shape)
  tensor_depth = static_shape.get_depth(tensor_shape)

  if batch_size is None:
    batch_size = tf.shape(tensor)[0]

  if tensor_height is None:
    tensor_height = tf.shape(tensor)[1]
    padded_tensor_height = tf.to_int32(
        tf.ceil(tf.to_float(tensor_height) / tf.to_float(multiple))) * multiple
  else:
    padded_tensor_height = int(
        math.ceil(float(tensor_height) / multiple) * multiple)

  if tensor_width is None:
    tensor_width = tf.shape(tensor)[2]
    padded_tensor_width = tf.to_int32(
        tf.ceil(tf.to_float(tensor_width) / tf.to_float(multiple))) * multiple
  else:
    padded_tensor_width = int(
        math.ceil(float(tensor_width) / multiple) * multiple)

  if tensor_depth is None:
    tensor_depth = tf.shape(tensor)[3]

  # Use tf.concat instead of tf.pad to preserve static shape
  height_pad = tf.zeros([
      batch_size, padded_tensor_height - tensor_height, tensor_width,
      tensor_depth
  ])
  padded_tensor = tf.concat([tensor, height_pad], 1)
  width_pad = tf.zeros([
      batch_size, padded_tensor_height, padded_tensor_width - tensor_width,
      tensor_depth
  ])
  padded_tensor = tf.concat([padded_tensor, width_pad], 2)

  return padded_tensor

def padded_one_hot_encoding(indices, depth, left_pad):
  """Returns a zero padded one-hot tensor.

  This function converts a sparse representation of indices (e.g., [4]) to a
  zero padded one-hot representation (e.g., [0, 0, 0, 0, 1] with depth = 4 and
  left_pad = 1). If `indices` is empty, the result will simply be a tensor of
  shape (0, depth + left_pad). If depth = 0, then this function just returns
  `None`.

  Args:
    indices: an integer tensor of shape [num_indices].
    depth: depth for the one-hot tensor (integer).
    left_pad: number of zeros to left pad the one-hot tensor with (integer).

  Returns:
    padded_onehot: a tensor with shape (num_indices, depth + left_pad). Returns
      `None` if the depth is zero.

  Raises:
    ValueError: if `indices` does not have rank 1 or if `left_pad` or `depth are
      either negative or non-integers.

  TODO: add runtime checks for depth and indices.
  """
  if depth < 0 or not isinstance(depth, (int, long) if six.PY2 else int):
    raise ValueError('`depth` must be a non-negative integer.')
  if left_pad < 0 or not isinstance(left_pad, (int, long) if six.PY2 else int):
    raise ValueError('`left_pad` must be a non-negative integer.')
  if depth == 0:
    return None
  if len(indices.get_shape().as_list()) != 1:
    raise ValueError('`indices` must have rank 1')

  def one_hot_and_pad():
    one_hot = tf.cast(tf.one_hot(tf.cast(indices, tf.int64), depth,
                                 on_value=1, off_value=0), tf.float32)
    return tf.pad(one_hot, [[0, 0], [left_pad, 0]], mode='CONSTANT')
  result = tf.cond(tf.greater(tf.size(indices), 0), one_hot_and_pad,
                   lambda: tf.zeros((depth + left_pad, 0)))
  return tf.reshape(result, [-1, depth + left_pad])

def dense_to_sparse_boxes(dense_locations, dense_num_boxes, num_classes):
  """Converts bounding boxes from dense to sparse form.

  Args:
    dense_locations:  a [max_num_boxes, 4] tensor in which only the first k rows
      are valid bounding box location coordinates, where k is the sum of
      elements in dense_num_boxes.
    dense_num_boxes: a [max_num_classes] tensor indicating the counts of
       various bounding box classes e.g. [1, 0, 0, 2] means that the first
       bounding box is of class 0 and the second and third bounding boxes are
       of class 3. The sum of elements in this tensor is the number of valid
       bounding boxes.
    num_classes: number of classes

  Returns:
    box_locations: a [num_boxes, 4] tensor containing only valid bounding
       boxes (i.e. the first num_boxes rows of dense_locations)
    box_classes: a [num_boxes] tensor containing the classes of each bounding
       box (e.g. dense_num_boxes = [1, 0, 0, 2] => box_classes = [0, 3, 3]
  """

  num_valid_boxes = tf.reduce_sum(dense_num_boxes)
  box_locations = tf.slice(dense_locations,
                           tf.constant([0, 0]), tf.stack([num_valid_boxes, 4]))
  tiled_classes = [tf.tile([i], tf.expand_dims(dense_num_boxes[i], 0))
                   for i in range(num_classes)]
  box_classes = tf.concat(tiled_classes, 0)
  box_locations.set_shape([None, 4])
  return box_locations, box_classes

def indices_to_dense_vector(indices,
                            size,
                            indices_value=1.,
                            default_value=0,
                            dtype=tf.float32):
  """Creates dense vector with indices set to specific value and rest to zeros.

  This function exists because it is unclear if it is safe to use
    tf.sparse_to_dense(indices, [size], 1, validate_indices=False)
  with indices which are not ordered.
  This function accepts a dynamic size (e.g. tf.shape(tensor)[0])

  Args:
    indices: 1d Tensor with integer indices which are to be set to
        indices_values.
    size: scalar with size (integer) of output Tensor.
    indices_value: values of elements specified by indices in the output vector
    default_value: values of other elements in the output vector.
    dtype: data type.

  Returns:
    dense 1D Tensor of shape [size] with indices set to indices_values and the
        rest set to default_value.
  """
  size = tf.to_int32(size)
  zeros = tf.ones([size], dtype=dtype) * default_value
  values = tf.ones_like(indices, dtype=dtype) * indices_value

  return tf.dynamic_stitch([tf.range(size), tf.to_int32(indices)],
                           [zeros, values])

def retain_groundtruth(tensor_dict, valid_indices):
  """Retains groundtruth by valid indices.

  Args:
    tensor_dict: a dictionary of following groundtruth tensors -
      fields.InputDataFields.groundtruth_boxes
      fields.InputDataFields.groundtruth_classes
      fields.InputDataFields.groundtruth_is_crowd
      fields.InputDataFields.groundtruth_area
      fields.InputDataFields.groundtruth_label_types
      fields.InputDataFields.groundtruth_difficult
    valid_indices: a tensor with valid indices for the box-level groundtruth.

  Returns:
    a dictionary of tensors containing only the groundtruth for valid_indices.

  Raises:
    ValueError: If the shape of valid_indices is invalid.
    ValueError: field fields.InputDataFields.groundtruth_boxes is
      not present in tensor_dict.
  """
  input_shape = valid_indices.get_shape().as_list()
  if not (len(input_shape) == 1 or
          (len(input_shape) == 2 and input_shape[1] == 1)):
    raise ValueError('The shape of valid_indices is invalid.')
  valid_indices = tf.reshape(valid_indices, [-1])
  valid_dict = {}
  if fields.InputDataFields.groundtruth_boxes in tensor_dict:
    # Prevents reshape failure when num_boxes is 0.
    num_boxes = tf.maximum(tf.shape(
        tensor_dict[fields.InputDataFields.groundtruth_boxes])[0], 1)
    for key in tensor_dict:
      if key in [fields.InputDataFields.groundtruth_boxes,
                 fields.InputDataFields.groundtruth_classes]:
        valid_dict[key] = tf.gather(tensor_dict[key], valid_indices)
      # Input decoder returns empty tensor when these fields are not provided.
      # Needs to reshape into [num_boxes, -1] for tf.gather() to work.
      elif key in [fields.InputDataFields.groundtruth_is_crowd,
                   fields.InputDataFields.groundtruth_area,
                   fields.InputDataFields.groundtruth_difficult,
                   fields.InputDataFields.groundtruth_label_types]:
        valid_dict[key] = tf.reshape(
            tf.gather(tf.reshape(tensor_dict[key], [num_boxes, -1]),
                      valid_indices), [-1])
      # Fields that are not associated with boxes.
      else:
        valid_dict[key] = tensor_dict[key]
  else:
    raise ValueError('%s not present in input tensor dict.' % (
        fields.InputDataFields.groundtruth_boxes))
  return valid_dict

def retain_groundtruth_with_positive_classes(tensor_dict):
  """Retains only groundtruth with positive class ids.

  Args:
    tensor_dict: a dictionary of following groundtruth tensors -
      fields.InputDataFields.groundtruth_boxes
      fields.InputDataFields.groundtruth_classes
      fields.InputDataFields.groundtruth_is_crowd
      fields.InputDataFields.groundtruth_area
      fields.InputDataFields.groundtruth_label_types
      fields.InputDataFields.groundtruth_difficult

  Returns:
    a dictionary of tensors containing only the groundtruth with positive
    classes.

  Raises:
    ValueError: If groundtruth_classes tensor is not in tensor_dict.
  """
  if fields.InputDataFields.groundtruth_classes not in tensor_dict:
    raise ValueError('`groundtruth classes` not in tensor_dict.')
  keep_indices = tf.where(tf.greater(
      tensor_dict[fields.InputDataFields.groundtruth_classes], 0))
  return retain_groundtruth(tensor_dict, keep_indices)

def filter_groundtruth_with_nan_box_coordinates(tensor_dict):
  """Filters out groundtruth with no bounding boxes.

  Args:
    tensor_dict: a dictionary of following groundtruth tensors -
      fields.InputDataFields.groundtruth_boxes
      fields.InputDataFields.groundtruth_classes
      fields.InputDataFields.groundtruth_is_crowd
      fields.InputDataFields.groundtruth_area
      fields.InputDataFields.groundtruth_label_types

  Returns:
    a dictionary of tensors containing only the groundtruth that have bounding
    boxes.
  """
  groundtruth_boxes = tensor_dict[fields.InputDataFields.groundtruth_boxes]
  nan_indicator_vector = tf.greater(tf.reduce_sum(tf.to_int32(
      tf.is_nan(groundtruth_boxes)), reduction_indices=[1]), 0)
  valid_indicator_vector = tf.logical_not(nan_indicator_vector)
  valid_indices = tf.where(valid_indicator_vector)

  return retain_groundtruth(tensor_dict, valid_indices)

def normalize_to_target(inputs,
                        target_norm_value,
                        dim,
                        epsilon=1e-7,
                        trainable=True,
                        scope='NormalizeToTarget',
                        summarize=True):
  """L2 normalizes the inputs across the specified dimension to a target norm.

  This op implements the L2 Normalization layer introduced in
  Liu, Wei, et al. "SSD: Single Shot MultiBox Detector."
  and Liu, Wei, Andrew Rabinovich, and Alexander C. Berg.
  "Parsenet: Looking wider to see better." and is useful for bringing
  activations from multiple layers in a convnet to a standard scale.

  Note that the rank of `inputs` must be known and the dimension to which
  normalization is to be applied should be statically defined.

  TODO: Add option to scale by L2 norm of the entire input.

  Args:
    inputs: A `Tensor` of arbitrary size.
    target_norm_value: A float value that specifies an initial target norm or
      a list of floats (whose length must be equal to the depth along the
      dimension to be normalized) specifying a per-dimension multiplier
      after normalization.
    dim: The dimension along which the input is normalized.
    epsilon: A small value to add to the inputs to avoid dividing by zero.
    trainable: Whether the norm is trainable or not
    scope: Optional scope for variable_scope.
    summarize: Whether or not to add a tensorflow summary for the op.

  Returns:
    The input tensor normalized to the specified target norm.

  Raises:
    ValueError: If dim is smaller than the number of dimensions in 'inputs'.
    ValueError: If target_norm_value is not a float or a list of floats with
      length equal to the depth along the dimension to be normalized.
  """
  with tf.variable_scope(scope, 'NormalizeToTarget', [inputs]):
    if not inputs.get_shape():
      raise ValueError('The input rank must be known.')
    input_shape = inputs.get_shape().as_list()
    input_rank = len(input_shape)
    if dim < 0 or dim >= input_rank:
      raise ValueError(
          'dim must be non-negative but smaller than the input rank.')
    if not input_shape[dim]:
      raise ValueError('input shape should be statically defined along '
                       'the specified dimension.')
    depth = input_shape[dim]
    if not (isinstance(target_norm_value, float) or
            (isinstance(target_norm_value, list) and
             len(target_norm_value) == depth) and
            all([isinstance(val, float) for val in target_norm_value])):
      raise ValueError('target_norm_value must be a float or a list of floats '
                       'with length equal to the depth along the dimension to '
                       'be normalized.')
    if isinstance(target_norm_value, float):
      initial_norm = depth * [target_norm_value]
    else:
      initial_norm = target_norm_value
    target_norm = tf.contrib.framework.model_variable(
        name='weights', dtype=tf.float32,
        initializer=tf.constant(initial_norm, dtype=tf.float32),
        trainable=trainable)
    if summarize:
      mean = tf.reduce_mean(target_norm)
      mean = tf.Print(mean, ['NormalizeToTarget:', mean])
      tf.summary.scalar(tf.get_variable_scope().name, mean)
    lengths = epsilon + tf.sqrt(tf.reduce_sum(tf.square(inputs), dim, True))
    mult_shape = input_rank*[1]
    mult_shape[dim] = depth
    return tf.reshape(target_norm, mult_shape) * tf.truediv(inputs, lengths)

def position_sensitive_crop_regions(image,
                                    boxes,
                                    box_ind,
                                    crop_size,
                                    num_spatial_bins,
                                    global_pool,
                                    extrapolation_value=None):
  """Position-sensitive crop and pool rectangular regions from a feature grid.

  The output crops are split into `spatial_bins_y` vertical bins
  and `spatial_bins_x` horizontal bins. For each intersection of a vertical
  and a horizontal bin the output values are gathered by performing
  `tf.image.crop_and_resize` (bilinear resampling) on a a separate subset of
  channels of the image. This reduces `depth` by a factor of
  `(spatial_bins_y * spatial_bins_x)`.

  When global_pool is True, this function implements a differentiable version
  of position-sensitive RoI pooling used in
  [R-FCN detection system](https://arxiv.org/abs/1605.06409).

  When global_pool is False, this function implements a differentiable version
  of position-sensitive assembling operation used in
  [instance FCN](https://arxiv.org/abs/1603.08678).

  Args:
    image: A `Tensor`. Must be one of the following types: `uint8`, `int8`,
      `int16`, `int32`, `int64`, `half`, `float32`, `float64`.
      A 4-D tensor of shape `[batch, image_height, image_width, depth]`.
      Both `image_height` and `image_width` need to be positive.
    boxes: A `Tensor` of type `float32`.
      A 2-D tensor of shape `[num_boxes, 4]`. The `i`-th row of the tensor
      specifies the coordinates of a box in the `box_ind[i]` image and is
      specified in normalized coordinates `[y1, x1, y2, x2]`. A normalized
      coordinate value of `y` is mapped to the image coordinate at
      `y * (image_height - 1)`, so as the `[0, 1]` interval of normalized image
      height is mapped to `[0, image_height - 1] in image height coordinates.
      We do allow y1 > y2, in which case the sampled crop is an up-down flipped
      version of the original image. The width dimension is treated similarly.
      Normalized coordinates outside the `[0, 1]` range are allowed, in which
      case we use `extrapolation_value` to extrapolate the input image values.
    box_ind:  A `Tensor` of type `int32`.
      A 1-D tensor of shape `[num_boxes]` with int32 values in `[0, batch)`.
      The value of `box_ind[i]` specifies the image that the `i`-th box refers
      to.
    crop_size: A list of two integers `[crop_height, crop_width]`. All
      cropped image patches are resized to this size. The aspect ratio of the
      image content is not preserved. Both `crop_height` and `crop_width` need
      to be positive.
    num_spatial_bins: A list of two integers `[spatial_bins_y, spatial_bins_x]`.
      Represents the number of position-sensitive bins in y and x directions.
      Both values should be >= 1. `crop_height` should be divisible by
      `spatial_bins_y`, and similarly for width.
      The number of image channels should be divisible by
      (spatial_bins_y * spatial_bins_x).
      Suggested value from R-FCN paper: [3, 3].
    global_pool: A boolean variable.
      If True, we perform average global pooling on the features assembled from
        the position-sensitive score maps.
      If False, we keep the position-pooled features without global pooling
        over the spatial coordinates.
      Note that using global_pool=True is equivalent to but more efficient than
        running the function with global_pool=False and then performing global
        average pooling.
    extrapolation_value: An optional `float`. Defaults to `0`.
      Value used for extrapolation, when applicable.
  Returns:
    position_sensitive_features: A 4-D tensor of shape
      `[num_boxes, K, K, crop_channels]`,
      where `crop_channels = depth / (spatial_bins_y * spatial_bins_x)`,
      where K = 1 when global_pool is True (Average-pooled cropped regions),
      and K = crop_size when global_pool is False.
  Raises:
    ValueError: Raised in four situations:
      `num_spatial_bins` is not >= 1;
      `num_spatial_bins` does not divide `crop_size`;
      `(spatial_bins_y*spatial_bins_x)` does not divide `depth`;
      `bin_crop_size` is not square when global_pool=False due to the
        constraint in function space_to_depth.
  """
  total_bins = 1
  bin_crop_size = []

  for (num_bins, crop_dim) in zip(num_spatial_bins, crop_size):
    if num_bins < 1:
      raise ValueError('num_spatial_bins should be >= 1')

    if crop_dim % num_bins != 0:
      raise ValueError('crop_size should be divisible by num_spatial_bins')

    total_bins *= num_bins
    bin_crop_size.append(crop_dim // num_bins)

  if not global_pool and bin_crop_size[0] != bin_crop_size[1]:
    raise ValueError('Only support square bin crop size for now.')

  ymin, xmin, ymax, xmax = tf.unstack(boxes, axis=1)
  spatial_bins_y, spatial_bins_x = num_spatial_bins

  # Split each box into spatial_bins_y * spatial_bins_x bins.
  position_sensitive_boxes = []
  for bin_y in range(spatial_bins_y):
    step_y = (ymax - ymin) / spatial_bins_y
    for bin_x in range(spatial_bins_x):
      step_x = (xmax - xmin) / spatial_bins_x
      box_coordinates = [ymin + bin_y * step_y,
                         xmin + bin_x * step_x,
                         ymin + (bin_y + 1) * step_y,
                         xmin + (bin_x + 1) * step_x,
                        ]
      position_sensitive_boxes.append(tf.stack(box_coordinates, axis=1))

  image_splits = tf.split(value=image, num_or_size_splits=total_bins, axis=3)

  image_crops = []
  for (split, box) in zip(image_splits, position_sensitive_boxes):
    crop = tf.image.crop_and_resize(split, box, box_ind, bin_crop_size,
                                    extrapolation_value=extrapolation_value)
    image_crops.append(crop)

  if global_pool:
    # Average over all bins.
    position_sensitive_features = tf.add_n(image_crops) / len(image_crops)
    # Then average over spatial positions within the bins.
    position_sensitive_features = tf.reduce_mean(
        position_sensitive_features, [1, 2], keep_dims=True)
  else:
    # Reorder height/width to depth channel.
    block_size = bin_crop_size[0]
    if block_size >= 2:
      image_crops = [tf.space_to_depth(
          crop, block_size=block_size) for crop in image_crops]

    # Pack image_crops so that first dimension is for position-senstive boxes.
    position_sensitive_features = tf.stack(image_crops, axis=0)

    # Unroll the position-sensitive boxes to spatial positions.
    position_sensitive_features = tf.squeeze(
        tf.batch_to_space_nd(position_sensitive_features,
                             block_shape=[1] + num_spatial_bins,
                             crops=tf.zeros((3, 2), dtype=tf.int32)),
        squeeze_dims=[0])

    # Reorder back the depth channel.
    if block_size >= 2:
      position_sensitive_features = tf.depth_to_space(
          position_sensitive_features, block_size=block_size)

  return position_sensitive_features

def reframe_box_masks_to_image_masks(box_masks, boxes, image_height,
                                     image_width):
  """Transforms the box masks back to full image masks.

  Embeds masks in bounding boxes of larger masks whose shapes correspond to
  image shape.

  Args:
    box_masks: A tf.float32 tensor of size [num_masks, mask_height, mask_width].
    boxes: A tf.float32 tensor of size [num_masks, 4] containing the box
           corners. Row i contains [ymin, xmin, ymax, xmax] of the box
           corresponding to mask i. Note that the box corners are in
           normalized coordinates.
    image_height: Image height. The output mask will have the same height as
                  the image height.
    image_width: Image width. The output mask will have the same width as the
                 image width.

  Returns:
    A tf.float32 tensor of size [num_masks, image_height, image_width].
  """
  # TODO: Make this a public function.
  def transform_boxes_relative_to_boxes(boxes, reference_boxes):
    boxes = tf.reshape(boxes, [-1, 2, 2])
    min_corner = tf.expand_dims(reference_boxes[:, 0:2], 1)
    max_corner = tf.expand_dims(reference_boxes[:, 2:4], 1)
    transformed_boxes = (boxes - min_corner) / (max_corner - min_corner)
    return tf.reshape(transformed_boxes, [-1, 4])

  box_masks = tf.expand_dims(box_masks, axis=3)
  num_boxes = tf.shape(box_masks)[0]
  unit_boxes = tf.concat(
      [tf.zeros([num_boxes, 2]), tf.ones([num_boxes, 2])], axis=1)
  reverse_boxes = transform_boxes_relative_to_boxes(unit_boxes, boxes)
  image_masks = tf.image.crop_and_resize(image=box_masks,
                                         boxes=reverse_boxes,
                                         box_ind=tf.range(num_boxes),
                                         crop_size=[image_height, image_width],
                                         extrapolation_value=0.0)
  return tf.squeeze(image_masks, axis=3)


from decimal import *
import time
from typing import List
from datetime import datetime
import json
from itertools import groupby
from dateutil.parser import parse

def _ElementJsonToOSMObject(jsonDict):
    """
    Parses an OpenStreetMap object into an :class:`OSMNode`, :class:`OSMWay` or :class:`OSMRelation`.

    Parameters
    ----------

    jsonDict : dict
        The dict object representing one of the OpenStreetMap objects

    Returns
    -------
    :class:`OSMNode`, :class:`OSMWay` or :class:`OSMRelation`

    """
    if jsonDict['type'] == 'node':
        return OSMNode(jsonDict['id'], jsonDict['type'], jsonDict['lat'], jsonDict['lon'], jsonDict.get('tags'))
    elif jsonDict['type'] == 'way':
        return OSMWay(jsonDict['id'], jsonDict['type'], jsonDict['nodes'], jsonDict.get('tags'))
    elif jsonDict['type'] == 'relation':
        return OSMRelation(jsonDict['id'], jsonDict['type'], _JsonMemberListToOSMRelationMember(jsonDict['members']), jsonDict.get('tags'))
    else:
            raise Exception("OSM Element type (%s) not implemented!" % jsonDict['type'])

def _JsonMemberListToOSMRelationMember(self, jsonMemberList):
    """
    Parses an OSMRelationMember dict into an OSMRelationMember object.

    Parameters
    ----------

    jsonMemberList : dict
        The dict object representing an :class:`OSMRelationMember`.

    Returns
    -------
    An arrays containing each parsed :class:`OSMRelationMember`.

    """
    members = []
    for member in jsonMemberList:
        members.append(OSMRelationMember(member['type'], member['ref'], member['role']))
    return members

class OSMObject(object):
    """Base class for other OSM objects:
        Attribute Name: default value
        version: 0
        visible: True
        timestamp: None
        changeset: 0
        user: '' #user's name
        uid: 0 #user's id
        """

    _optDefAttributes={"version": 0, "visible": True, "timestamp": None, "changeset": 0,
        "user": "", "uid": 0}

    ##  'tags' attribute is set as an empty dictionary so that it's possible to try
    ##  to look up for a 'name' tag even when an (faulty) object doesn't have any tags
    def __init__(self, id: int, type:str, tags={}, **kwargs):
        self.id = id
        self.type = type
        self.tags = tags
        for kw in OSMObject._optDefAttributes:
            setattr(self, kw, kwargs.get('kw') if kw in kwargs else OSMObject._optDefAttributes.get('kw'))

class OSMNode(OSMObject):
    """
    Class used as an OpenStreetMap Node object wrapper

    fields:
        - lat : Decimal
            Represents this OSMNode latitude
        - lon : Decimal
            Represents this OSMNode longitude
        - tags : dict
            Each :class:`OSMNode`, :class:`OSMWay` and :class:`OSMRelation` can
            contain an arbitratry number of tags, each of which corresponds to
            some attribute of that OSM object (e.g. an OSMNode could
            have a tag 'highway' with the value 'traffic_signals').
    """
    def __init__(self, id: int, type: str, lat: Decimal, lon: Decimal, tags={}):
        super().__init__(id, type, tags)
        self.lat = lat
        self.lon = lon

class OSMWay(OSMObject):
    """
    Class used as an OpenStreetMap Way object wrapper.
    In this abstraction an OSMWay is only considered to be a set
    of nodes contained in the field nodes.

    fields:
        - nodes : List[int]
            The set of nodes contained in this OSMWay. An OSMWay
            is used to represent usually linear features such
            as avenues and streets.
        - tags : dict
            See tags in :class:`OSMNode`
    """
    def __init__(self, id: int, type,  nodes: List[int], tags={}):
        super().__init__(id, type, tags)
        self.nodes = nodes

class OSMRelation(OSMObject):
    """
        A OpenStreetMap relation represents 'belongs to' relations (i.e. :class:`OSMNode` objects belonging
        to the same :class:`OSMWay` or a collection of :class:`OSMWay` objects that belongs to the same
        OSMRelation which represets for example the boundaries of some region.)

        fields:
            - id : int
                Represents the relation id as informed by OpenStreetMap database
            - type : str
                It can be anything for example 'boundary'.
            - members : List[OSMObject]
                An OSMRelation can be composed by any number and type of objects
                (e.g. an OSMRelation with type 'route' can contain different
                OSMWays that compose the route of a bus).
    """
    def __init__(self, id: int, type, members: List[OSMObject], tags={}):
        super().__init__(id, type, tags)
        self.members = members

class OSM3S:
    """Timestamp and copyright info"""
    def __init__(self, timestamp_osm_base = None, copyright = ""):
        if not timestamp_osm_base is None:
            self.timestamp_osm_base = parse(timestamp_osm_base)
        self.copyright = copyright

    def JsonToOSM3S(jsonString):
        """Parses an string to an OSM3S object"""
        return DictToOSM3S(json.loads(jsonString));

    def DictToOSM3S(jsonDict):
        """Parses an dict object into an OSM3S object"""
        return OSM3S(jsonDict['timestamp_osm_base'], jsonDict['copyright'])

class OSMRelationMember:
    """An element that 'belogs' to another (i.e. An :class:`OSMNode` that belongs to an :class:`OSMWay`)"""
    def __init__(self, type, ref, role):
        self.type = type
        self.ref = ref
        self.role = role

class OSMResult:
    """Represents a query result to an OpenStreetMap server"""
    def __init__(self, version: float, generator: str, osm3s: OSM3S, elements: list = []):
        self.version = version
        self.generator = generator
        self.osm3s = osm3s
        self.Nodes = {}
        self.Ways = {}
        self.Relations = {}
        for element in elements:
            osmObject = _ElementJsonToOSMObject(element)
            if type(osmObject) is OSMNode: self.Nodes[osmObject.id] = osmObject
            elif type(osmObject) is OSMWay:
                self.Ways[osmObject.id] = osmObject
            elif type(osmObject) is OSMRelation:
                self.Relations[osmObject.id] = osmObject

    def fromJsonString(jsonString):
        """Parses a json string into a OSMResult object"""
        return OSMResult.fromJsonDict(json.loads(jsonString))

    def fromJsonDict(jsonDict):
        """Parses a dict into a OSMResult"""
        version = jsonDict['version']
        generator = jsonDict['generator']
        osm3sDict = OSM3S.DictToOSM3S(jsonDict['osm3s'])
        return OSMResult(version, generator, osm3sDict, jsonDict['elements'])


# -*- coding: utf-8 -*-

import os
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
import argparse
import time
from tqdm import tqdm, trange
import evaluation as eva
from processor import UbuntuProcessor
from STMDataLoader import build_corpus_dataloader, build_corpus_tokenizer, build_corpus_embedding
from models import STM

import numpy as np
import pickle
import torch
import torch.optim as optim
from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
import matplotlib.pyplot as plt
plt.interactive(False)
plt.figure(figsize=(20,30))
import logging
logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
                    datefmt = '%m/%d/%Y %H:%M:%S',
                    level = logging.INFO)
logger = logging.getLogger(__name__)

def plot_grad_flow(named_parameters):
    ave_grads = []
    layers = []
    for n, p in named_parameters:
        if(p.requires_grad) and ("bias" not in n):
            if p.grad is not None:
                layers.append(n)
                if n != 'embedding.weight':
                    ave_grads.append(p.grad.abs().mean())
                else:
                    ave_grads.append(p.grad.abs().sum() / (p.grad !=0).sum().float())

    plt.plot(ave_grads, alpha=0.3, color="b")
    plt.hlines(0, 0, len(ave_grads)+1, linewidth=1, color="k" )
    plt.xticks(range(0,len(ave_grads), 1), layers, rotation="vertical")
    plt.xlim(xmin=0, xmax=len(ave_grads))
    plt.xlabel("Layers")
    plt.ylabel("average gradient")
    plt.title("Gradient flow")
    plt.grid(True)
    plt.savefig('./tmp/gradient.png')

def main():

    parser = argparse.ArgumentParser(description='Spatio-Temporal Matching Network')
    parser.add_argument('--task', type=str, default='Response Selection',
                        help='task name')
    parser.add_argument('--model', type=str, default='STM',
                        help='model name')
    parser.add_argument('--encoder_type', type=str, default='GRU',
                        help='encoder:[GRU, LSTM, SRU, Transoformer]')
    parser.add_argument('--vocab_size', type=int, default=30000,
                        help='vocabulary size')
    parser.add_argument('--max_turns_num', type=int, default=9,
                        help='the max turn number in dialogue context')
    parser.add_argument('--max_options_num', type=int, default=100,
                        help='the max turn number in dialogue context')
    parser.add_argument('--max_seq_len', type=int, default=50,
                        help='the max length of the input sequence')

    parser.add_argument('--emb_dim', type=int, default=300,
                        help='size of word embeddings')
    parser.add_argument('--rnn_layers', type=int, default=3,
                        help='the number of rnn layers for feature extraction')
    parser.add_argument('--mem_dim', type=int, default=150,
                        help='hidden memory size')

    parser.add_argument('--lr', type=float, default=0.001,
                        help='initial learning rate')
    parser.add_argument('--epochs', type=int, default=5,
                        help='upper epoch limit')
    parser.add_argument('--batch_size', type=int, default=10, metavar='N',
                        help='batch size')
    parser.add_argument('--dropoutP', type=float, default=0.2,
                        help='dropout ratio')
    parser.add_argument('--seed', type=int, default=1111,
                        help='random seed')
    parser.add_argument('--cuda', type=bool, default=True,
                        help='use CUDA')

    parser.add_argument('--init_checkpoint', type=str, default=None,
                        help='The initial checkpoint')
    parser.add_argument('--save_path', type=str, default='./tmp/',
                        help='The initial checkpoint')
    parser.add_argument('--cache_path', type=str, default='./cache/',
                        help='The initial checkpoint')
    parser.add_argument('--pretrain_embedding', type=str, default='/hdd/lujunyu/dataset/glove/glove.42B.300d.txt',
                        help='The pretraining embedding')

    parser.add_argument('--do_train', type=bool, default=True,
                        help='training or not')
    parser.add_argument('--do_eval', type=bool, default=True,
                        help='evaluate or not')
    parser.add_argument('--do_test', type=bool, default=True,
                        help='test or not')

    args = parser.parse_args()
    if args.cuda:
        device = torch.device("cuda")
    else:
        device = torch.device("cpu")
    logger.info("Using device %s", device)

    torch.manual_seed(args.seed)
    if torch.cuda.is_available():
        if not args.cuda:
            print("WARNING: You have a CUDA device, so you should probably run with --cuda")
        else:
            torch.cuda.manual_seed(args.seed)

    ### Build dataloader
    processor = UbuntuProcessor()
    train_examples = processor.get_train_examples()
    eval_examples = processor.get_dev_examples()
    test_examples = processor.get_test_examples()

    if args.cache_path and os.path.exists(os.path.join(args.cache_path, 'cache_tokenizer.pkl')):
        with open(os.path.join(args.cache_path, 'cache_tokenizer.pkl'), 'rb') as ff:
            tokenizer = pickle.load(ff)
    else:
        tokenizer = build_corpus_tokenizer(train_examples, args.vocab_size)
        with open(os.path.join(args.cache_path, 'cache_tokenizer.pkl'), 'wb') as ff:
            pickle.dump(tokenizer, ff)

    if args.cache_path and os.path.exists(os.path.join(args.cache_path, 'cache_dataset.pt')):
        with open(os.path.join(args.cache_path, 'cache_dataset.pt'), 'rb') as f:
            train_dataset, eval_dataset, test_dataset = pickle.load(f)
    else:
        train_dataset = build_corpus_dataloader(train_examples, args.max_turns_num, args.max_seq_len,tokenizer)
        eval_dataset = build_corpus_dataloader(eval_examples, args.max_turns_num, args.max_seq_len, tokenizer)
        test_dataset = build_corpus_dataloader(test_examples, args.max_turns_num, args.max_seq_len, tokenizer)

        with open(os.path.join(args.cache_path, 'cache_dataset.pt'), 'wb') as f:
            pickle.dump((train_dataset, eval_dataset, test_dataset), f)

    train_dataloader = DataLoader(train_dataset, sampler=RandomSampler(train_dataset), batch_size=args.batch_size)
    eval_dataloader = DataLoader(eval_dataset, sampler=SequentialSampler(eval_dataset), batch_size=args.batch_size)
    test_dataloader = DataLoader(test_dataset, sampler=SequentialSampler(test_dataset), batch_size=args.batch_size)

    ### Build pretrained embedding
    if args.cache_path and os.path.exists(os.path.join(args.cache_path, 'cache_embedding.pt')):
        with open(os.path.join(args.cache_path, 'cache_embedding.pt'), 'rb') as f:
            pretrained_embedding = pickle.load(f)
    else:
        if args.pretrain_embedding:
            pretrained_embedding, vocab = build_corpus_embedding(args.vocab_size, args.emb_dim, args.pretrain_embedding, tokenizer)
        else:
            pretrained_embedding = None
            vocab = []

        with open(os.path.join(args.cache_path, 'cache_embedding.pt'), 'wb') as f:
            pickle.dump(pretrained_embedding, f)

        with open(os.path.join(args.cache_path, 'cache_vocab.txt'), 'w') as f:
            for word in vocab:
                f.write(word+'\n')

    ### Build model
    model = eval(args.model)(args, pretrained_embedding)
    if args.init_checkpoint is not None:
        model.bert.load_state_dict(torch.load(args.init_checkpoint, map_location='cpu'), strict=False)
    model.to(device)

    for name, param in model.named_parameters():
        if param.requires_grad:
            # print(name, param.data)
            print(name, param.data.size())

    parameters = filter(lambda p: p.requires_grad, model.parameters())
    optimizer = optim.Adamax(parameters, lr=args.lr)

    ### training
    save_interval = len(train_dataset) // args.batch_size
    print_step = max(1., save_interval // 10)
    if args.do_train:
        model.train()
        for epoch_id in range(int(args.epochs)):
            tr_loss = 0.0
            for step, batch in enumerate(train_dataloader):
                batch = tuple(t.to(device) for t in batch)
                contexts_ids, candidate_ids, label_ids = batch
                loss, _ = model(contexts_ids, candidate_ids, label_ids)
                loss.backward()

                if step % 20 == 0 and step != 0:
                    plot_grad_flow(model.named_parameters())

                optimizer.step()
                optimizer.zero_grad()

                tr_loss += float(loss.data)

                if step % print_step == 0:
                    cur_loss = tr_loss / print_step if step != 0 else tr_loss
                    logger.info("processed: [{:3.2f}".format(step / save_interval) +
                          "] lr: [{:3.5f}".format(optimizer.param_groups[0]['lr']) + "] loss: [{:3.5f}".format(cur_loss) + "]")
                    tr_loss = 0.0

            # Evaluate at the end of every epoch
            if args.do_eval:
                model.eval()
                eval_loss = 0
                best_result = [0, 0, 0, 0, 0, 0, 0]
                eval_score_file = open(os.path.join(args.save_path,'eval_score.txt'), 'w')

                for nb_eval_steps, batch in enumerate(eval_dataloader):
                    batch = tuple(t.to(device) for t in batch)
                    contexts_ids, candidate_ids, label_ids = batch
                    with torch.no_grad():
                        tmp_eval_loss, logits = model(contexts_ids, candidate_ids, label_ids)

                    logits = logits.detach().cpu().numpy()
                    label_ids = label_ids.cpu().numpy()

                    for i in range(args.batch_size):
                        for j in range(args.max_options_num):
                            if label_ids[i] == j:
                                eval_score_file.write('{:d}_{:d}\t{:2.5f}\t{:d}\n'.format(i, j, logits[i][j], 1))
                            else:
                                eval_score_file.write('{:d}_{:d}\t{:2.5f}\t{:d}\n'.format(i, j, logits[i][j], 0))

                    eval_loss += tmp_eval_loss.mean().item()

                eval_score_file.close()

                eval_loss /= (nb_eval_steps + 1)

                # write evaluation result
                eval_result = eva.evaluate(os.path.join(args.save_path,'eval_score.txt'))
                eval_result_file_path = os.path.join(args.save_path, 'eval_result.txt')
                with open(eval_result_file_path, 'w') as out_file:
                    for p_at in eval_result:
                        out_file.write(str(p_at) + '\n')

                if eval_result[0] > best_result[0]:
                    save_path = os.path.join(args.save_path, 'model_best.pt')
                    best_result = eval_result
                    torch.save([model, optimizer], save_path)
                    logger.info('eval loss: %2.4f' % eval_loss)
                    logger.info("best result: %2.4f" % best_result[0])
                    logger.info("succ saving model in %s" % save_path)

                model.train()

    if args.do_test:
        model, _ = torch.load(os.path.join(args.save_path, 'model_best.pt'))
        model.eval()
        test_loss = 0
        test_score_file = open(os.path.join(args.save_path, 'test_score.txt'), 'w')

        for nb_test_steps, batch in enumerate(test_dataloader):
            batch = tuple(t.to(device) for t in batch)
            contexts_ids, candidate_ids, label_ids = batch
            with torch.no_grad():
                tmp_test_loss, logits = model(contexts_ids, candidate_ids, label_ids)

            logits = logits.detach().cpu().numpy()
            label_ids = label_ids.cpu().numpy()

            for i in range(args.batch_size):
                for j in range(args.max_options_num):
                    if label_ids[i] == j:
                        test_score_file.write('{:d}_{:d}\t{:2.5f}\t{:d}\n'.format(i, j, logits[i][j], 1))
                    else:
                        test_score_file.write('{:d}_{:d}\t{:2.5f}\t{:d}\n'.format(i, j, logits[i][j], 0))

            test_loss += tmp_test_loss.mean().item()

        test_score_file.close()

        test_loss /= (nb_eval_steps + 1)

        # write evaluation result
        test_result = eva.evaluate(os.path.join(args.save_path, 'test_score.txt'))
        test_result_file_path = os.path.join(args.save_path, 'test_result.txt')
        with open(test_result_file_path, 'w') as out_file:
            for p_at in test_result:
                out_file.write(str(p_at) + '\n')

if __name__ == '__main__':
    main()


from setuptools import setup

package_name = 'rover_waypoint'

setup(
    name=package_name,
    version='0.0.0',
    packages=[package_name],
    data_files=[
        ('share/ament_index/resource_index/packages',
            ['resource/' + package_name]),
        ('share/' + package_name, ['package.xml']),
    ],
    install_requires=['setuptools'],
    zip_safe=True,
    maintainer='root',
    maintainer_email='devansh@umich.edu',
    description='TODO: Package description',
    license='TODO: License declaration',
    tests_require=['pytest'],
    entry_points={
        'console_scripts': [
            'hello_node = rover_waypoint.hello:main',
            'move_forward = rover_waypoint.move_rover_forward:main',
            'joystick_safety = rover_waypoint.safety:main',
            'move_waypoint = rover_waypoint.move_waypoint:main'
        ],
    },
)


import numpy as np
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.utils import check_X_y
from sklearn.linear_model import LogisticRegression
from scipy.sparse import csr_matrix
from scipy.sparse import lil_matrix

class SelfTraining(BaseEstimator, TransformerMixin):
    def __init__(self, learner=LogisticRegression(), iterations_nb=5, pool_size=100, training_way=0, replacement=True, random_state=None):
        self.learner = learner
        self.iterations_nb = iterations_nb
        self.pool_size = pool_size
        self.training_way = training_way #0: Hard, 1: Soft, 2: Soft multi
        self.replacement = replacement
        self.random_state = random_state
        self._labeled_data_size = -1
        self._unlabeled_data_size = -1
        self._categories_nb = -1
        self._accessible_index = None

    def get_params(self, deep=False): # deep=False due to a little issue in Scikit-Learn
        return {"learner": self.learner,
            "iterations_nb": self.iterations_nb,
            "pool_size": self.pool_size,
            "training_way": self.training_way,
            "replacement": self.replacement,
            "random_state": self.random_state}

    def set_params(self, **parameters):
        for parameter, value in parameters.items():
            setattr(self, parameter, value)

        return self

    def __x_y_preparation(self, X, y):
        X, y = check_X_y(X, y, accept_sparse=['csr', 'csc', 'coo'])

        self._categories_nb = len(np.unique(y))

        l_index = np.argwhere(y != -1)
        u_index = np.argwhere(y == -1)

        self._labeled_data_size = len(np.argwhere(y != -1))
        self._unlabeled_data_size = len(np.argwhere(y == -1))

        X_new_shape = ((self._labeled_data_size + self._categories_nb * self._unlabeled_data_size), X.shape[1])
        y_new_shape = ((self._labeled_data_size + self._categories_nb * self._unlabeled_data_size),)
        X_new = lil_matrix(X_new_shape, dtype=np.float64)
        y_new = np.zeros(y_new_shape)

        i_l = 0
        i_u = 0
        for i, y_row in enumerate(y):
            X_row = X[i]
            y_row = y_row

            # label
            if i in l_index:
                X_new[i_l] = X_row
                y_new[i_l] = y_row
                i_l += 1

            # unlabel
            elif i in u_index:
                # we need all possible category for each unlabeled data
                for cat in range(self._categories_nb):
                    local_i = (self._labeled_data_size) + (cat * self._unlabeled_data_size) + i_u
                    X_new[local_i] = X_row
                    y_new[local_i] = cat
                i_u += 1

        self._accessible_index = np.array(range(self._labeled_data_size, self._labeled_data_size + self._unlabeled_data_size))

        if not self.replacement and self._unlabeled_data_size < (self.iterations_nb * self.pool_size):
            print("Warning: Not enough unlabeled data for {} iterations with pool size of {} WITHOUT replacement!".format(self.iterations_nb, self.pool_size))
            self.iterations_nb = self._unlabeled_data_size // self.pool_size
            print("Number of iterations has been automaticly reduced to {}.".format(self.iterations_nb))

        return X_new, y_new

    def __m_step(self, X, y, weights):
        self.learner.fit(X, y, sample_weight=weights)

        return self

    def __e_step(self, X, weights):

        if not getattr(self.learner, "decision_function", None):
            raise ValueError('No decision_function in model.')

        selected_sample = np.random.choice(self._accessible_index, self.pool_size, replace=False)

        # Without replacement: we only keep labeled data as basis at each new step
        if not self.replacement:
            weights = np.array(self._labeled_data_size * [1] + self._categories_nb * self._unlabeled_data_size * [0], dtype=np.float64)

        for i in selected_sample:

            scores = self.learner.decision_function(X[i])
            predicted_cat = scores.argmax(axis=1)[0]

            # Computing weights
            new_weights = np.zeros(self._categories_nb)
            if self.training_way == 0: # Hard
                new_weights[predicted_cat] = 1

            elif self.training_way == 1: # Soft
                probs = np.exp(scores) / np.sum(np.exp(scores))
                new_weights[predicted_cat] = probs[0][predicted_cat] # only the best weight, not any others

            elif self.training_way == 2: # Soft multi
                probs = np.exp(scores) / np.sum(np.exp(scores))
                new_weights = probs[0] # all weights

            # Updating weights in each category for X[i]
            for cat in range(self._categories_nb):
                weights[i + cat * self._unlabeled_data_size] = new_weights[cat]

        # Without replacement: we remove used unlabeled data points index
        if not self.replacement:
            self._accessible_index = np.setdiff1d(self._accessible_index, selected_sample)

        return weights

    def fit(self, X, y):
        np.random.seed(self.random_state)

        X, y = self.__x_y_preparation(X, y)

        weights = np.array(self._labeled_data_size * [1] + self._categories_nb * self._unlabeled_data_size * [0], dtype=np.float64)

        self.__m_step(X, y, weights)
        for _ in range(self.iterations_nb):
            weights = self.__e_step(X, weights)
            self.__m_step(X, y, weights)

        return self

    def predict(self, X):
        return self.learner.predict(X)

    def decision_function(self, X):
        return self.learner.decision_function(X)

    def score(self, X, y):
        return self.learner.score(X, y)

if __name__ == '__main__':
    pass


# encoding: utf-8

import pytest

from web.contentment.taxonomy import Taxonomy

@pytest.fixture(scope='module')
def connection(request):
	from mongoengine import connect
	from .fixture import apply

	connection = connect('testing')
	apply()

	request.addfinalizer(lambda: connection.drop_database('testing'))

	return connection

@pytest.fixture
def root(connection):
	from web.component.asset.model import Asset
	return Asset.objects.get(path='/careers.illicohodes.com')

@pytest.fixture
def models(connection):
	from collections import namedtuple

	Models = namedtuple('Models', 'parent child1 child2')
	TModel.objects.delete()
	return Models(
		TModel.objects.create(name='parent', path='/'),
		TModel.objects.create(name='child1', path='1'),
		TModel.objects.create(name='child2', path='2'),
	)

class TModel(Taxonomy):
	pass

@pytest.mark.usefixtures('connection')
class TestTaxonomy:
	def test_siblings(self, root):
		assert root.children[1].siblings.count() == 5

	def test_next(self, root):
		subroot = root.children[0].children[1]
		nexts = list(root.children[0].children[2:4])
		assert subroot.nextAll.count() == 2
		assert list(subroot.nextAll) == nexts
		c = subroot
		count = 0
		while True:
			c = c.next
			if c is None:
				break
			count += 1
		assert count == 2

	def test_prev(self, root):
		subroot = root.children[0].children[2]
		prevs = list(root.children[0].children[0:2])
		assert subroot.prevAll.count() == 2
		assert list(subroot.prevAll) == prevs
		c = subroot
		count = 0
		while True:
			c = c.prev
			if c is None:
				break
			count += 1
		assert count == 2

	def test_insert_detach(self, models):
		assert models.parent.children.count() == 0
		assert models.child1.parents == []

		models.parent.insert(0, models.child1)
		models.parent.insert(1, models.child2)
		assert list(models.parent.children) == [models.child1, models.child2]

		models.child2.detach()
		assert list(models.parent.children) == [models.child1]

		models.parent.insert(0, models.child2)
		assert list(models.parent.children) == [models.child2, models.child1]

		models.child1.append(models.child2)
		assert list(models.parent.children) == [models.child1] and list(models.child1.children) == [models.child2]
		assert list(models.parent.contents) == [models.child1, models.child2]

		models.child1.empty()
		assert list(models.parent.contents) == [models.child1]
		assert TModel.objects.count() == 2

	def test_append_prepend(self, models):
		models.parent.append(models.child1)
		assert list(models.parent.children) == [models.child1]
		models.parent.append(models.child2)
		assert list(models.parent.children) == [models.child1, models.child2]

		models.parent.prepend(models.child2)
		assert list(models.parent.children) == [models.child2, models.child1]

	def test_after_before(self, models):
		models.parent.append(models.child1)
		models.child1.after(models.child2)
		assert list(models.parent.children) == [models.child1, models.child2]

		models.child1.before(models.child2)
		assert list(models.parent.children) == [models.child2, models.child1]

	def test_pivoted(self, models):
		models.child1.appendTo(models.parent)
		assert list(models.parent.children) == [models.child1]
		models.child2.prependTo(models.parent)
		assert list(models.parent.children) == [models.child2, models.child1]

		models.child1.insertBefore(models.child2)
		assert list(models.parent.children) == [models.child1, models.child2]
		models.child1.insertAfter(models.child2)
		assert list(models.parent.children) == [models.child2, models.child1]

	def test_extend(self, models):
		parent2 = TModel.objects.create(name='par2', path='/P2')
		models.child1.appendTo(models.parent)
		models.child2.appendTo(parent2)

		models.parent.extend(parent2)
		assert list(models.parent.children) == [models.child1, models.child2]

	def test_replace(self, models):
		replacer = TModel.objects.create(name='replacer', path='r')
		models.child1.appendTo(models.parent)
		models.child1.reload(-1)
		assert TModel.objects.count() == 4
		replacer.replace(models.child1)
		assert TModel.objects.count() == 3
		assert list(models.parent.children) == [replacer.reload(-1)]

		models.child2.replaceWith(replacer)
		replacer.reload(-1)
		assert models.parent.children.count() == 0
		assert replacer.name == 'child2'


# coding: utf-8

from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger

def get_paginator(request, object_list, num_per_page):
    paginator = Paginator(object_list, num_per_page)
    page = request.GET.get('page')
    try:
        ret = paginator.page(page)
    except PageNotAnInteger:
        # If page is not an integer, deliver first page.
        ret = paginator.page(1)
    except EmptyPage:
        # If page is out of range (e.g. 9999), deliver last page of results.
        ret = paginator.page(paginator.num_pages)
    return ret


import os
from Models import get_model
from torch.nn.parallel import DistributedDataParallel as DDP
import torch
from Models import get_model
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import matplotlib.ticker as ticker
import math
from espnet_local.nets.pytorch_backend.nets_utils import make_non_pad_mask
import kaldiio
from load_data import read_text
from load_data import Load_audio_and_text_data
from torch.utils.data import DataLoader
from my_collate import my_collate

def predict_result(opt):

    loader = kaldiio.load_scp(opt.scp_path)
    read_in = read_text(checkpoint=opt.bert_checkpoint, bert_embedding_length=opt.bert_embedding_length)
    _, test_dataset = read_in.read_text_and_label(train_path=None, test_path=opt.test_text_path)
    test_data = Load_audio_and_text_data(dataset=test_dataset, bert_embedding_length=opt.bert_embedding_length,
                                        loader=loader)
    opt.test = DataLoader(dataset=test_data, batch_size=opt.batchsize, shuffle=False, collate_fn=my_collate,
                         num_workers=10)

    model = get_model(opt)
    model.load_state_dict(torch.load(opt.model_path), strict=True)
    model = torch.nn.DataParallel(model, device_ids=opt.device_ids)
    model = model.cuda()

    model.eval()
    with torch.no_grad():
        for i, batch in enumerate(opt.test):
            attention_mask, input_ids, padded_labels, org_len, mfcc, source, id = batch
            attention_mask = attention_mask.cuda()
            input_ids = input_ids.cuda()
            padded_labels = padded_labels.cuda()
            org_len = org_len.cuda()
            mfcc = mfcc.cuda()

            mfcc_len = [mfcc[i].size(0) for i in range(0, mfcc.size(0))]
            src_mask = make_non_pad_mask(mfcc_len).unsqueeze(-2).cuda()

            preds, attention_map = model(mfcc=mfcc, input_ids=input_ids, attention_mask=attention_mask,
                                         mfcc_mask=None, src_mask=src_mask)
            temp_mask = torch.ones(attention_mask.size()).cuda() - attention_mask
            masked_pre = preds.masked_fill(temp_mask.unsqueeze(-1).bool(), value=torch.tensor(0))
            masked_pred_label = torch.argmax(torch.softmax(input=masked_pre, dim=-1), dim=-1)

            padded_labels = padded_labels.cpu().numpy().tolist()
            masked_pred_label = masked_pred_label.cpu().numpy().tolist()
            if opt.save_pred_res:
                with open(opt.test_pred_save_path, 'a') as f:
                    for m in range(0, len(masked_pred_label)):
                        f.write(id[m] + '\n')
                        f.write(source[m] + '\n')
                        f.write('label: ' + str(padded_labels[m][1:org_len[m]-1]) + '\n')
                        f.write('pred: ' + str(masked_pred_label[m][1:org_len[m]-1]) + '\n')
            if opt.save_attention_map:
                for k in range(0, attention_map.size(0)):
                    attention_map_1 = attention_map[k]
                    print(attention_map_1.size())
                    input_ids = [i for i in range(0, 512)]
                    mfcc_mask = [i for i in range(0, attention_map_1.size(-1))]

                    for j in range(0, 8):
                        h = attention_map_1[j].cpu().detach().numpy()
                        df = pd.DataFrame(h, columns=mfcc_mask, index=input_ids)
                        fig = plt.figure()

                        ax = fig.add_subplot(111)

                        cax = ax.matshow(df, interpolation='nearest', cmap='GnBu')
                        fig.colorbar(cax)

                        tick_spacing = df.index.size/5
                        ax.xaxis.set_major_locator(ticker.MultipleLocator(tick_spacing))
                        ax.yaxis.set_major_locator(ticker.MultipleLocator(tick_spacing))

                        ax.set_xticklabels([''] + list(df.columns))
                        ax.set_yticklabels([''] + list(df.index))

                        name = 'batch_' + str(i) + 'text_' + str(k) + '_head_' + str(j) + 'png'
                        plt.savefig(os.path.join(opt.attention_map_save_path, name), bbox_inches='tight')


"""
Pydantic Model for Staff API's
"""
import json
import re
import regex
from typing import List, Optional
from typing_extensions import Literal
from pydantic import BaseModel, constr, Extra, validator
from schemas.schema_examples import (BASIC_STAFF_EXAMPLE, FULL_STAFF_EXAMPLE,
                                     PROFILE_FIELDS)
from common.utils.schema_validator import BaseConfigModel

with open("./data/profile_fields.json", "r", encoding="utf-8") as f:
  fields = json.load(f)
  pronouns = fields["pronouns"]
  workdays = fields["workdays"]
  timezones = fields["timezones"]

PRONOUNS = Literal[tuple(pronouns)]
WORKDAYS = Literal[tuple(workdays)]
TIMEZONES = Literal[tuple(timezones)]

class AvailabilityModel(BaseModel):
  """Availability Pydantic Model"""
  day_of_week: List[WORKDAYS]
  start_time: str
  end_time: str

class BasicStaffModel(BaseConfigModel):
  """Staff Skeleton Pydantic Model"""
  first_name: str
  last_name: str
  email: constr(
      min_length=7,
      max_length=128,
      regex=r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,7}\b")
  preferred_name: Optional[str] = None
  bio: Optional[str] = None
  pronoun: Optional[PRONOUNS]
  phone_number: Optional[str] = None
  shared_inboxes: Optional[str] = None
  timezone: Optional[TIMEZONES]
  office_hours: Optional[List[AvailabilityModel]] = []
  photo_url: Optional[str] = None
  calendly_url: Optional[str] = None

  # pylint: disable=no-self-argument
  @validator("preferred_name")
  def validate_preferred_name(cls, v):
    assert v == "" or v is None or re.match(
      r"[a-zA-Z0-9`!#&*%_[\]{}\\;:'\,.\?\s-]+$", v),\
      "Invalid format for preferred_name"
    return v
  @validator("first_name")
  def first_name_regex(cls,value):
    result = regex.fullmatch(r"[\D\p{L}\p{N}]+$", value)
    if len(value)<=60 and result:
      return value
    raise ValueError("Invalid first name format")
  @validator("last_name")
  def last_name_regex(cls,value):
    result = regex.fullmatch(r"[\D\p{L}\p{N}\s]+$", value)
    if len(value)<=60 and result:
      return value
    raise ValueError("Invalid last name format")

  class Config():
    orm_mode = True
    schema_extra = {"example": BASIC_STAFF_EXAMPLE}

class FullStaffModel(BasicStaffModel):
  """Staff Model with uuid, created and updated time"""
  uuid: str
  created_time: str
  last_modified_time: str

class StaffSearchResponseModel(BaseModel):
  """Staff Search Response Pydantic Model"""
  success: bool = True
  message: str = "Successfully fetched the Staffs"
  data: List[FullStaffModel]

  class Config():
    orm_mode = True
    schema_extra = {
        "example": {
            "success": True,
            "message": "Successfully fetched the Staffs",
            "data": [FULL_STAFF_EXAMPLE]
        }
    }

class UpdateStaffModel(BaseConfigModel):
  """Update Staff Pydantic Model"""
  first_name: Optional[str] = None
  last_name: Optional[str] = None
  preferred_name: Optional[constr(
      max_length=60,
      regex=r"[a-zA-Z0-9`!#&*%_[\]{}\\;:'\,.\?\s-]+$")] = None
  bio: Optional[str] = ""
  pronoun: Optional[PRONOUNS]
  phone_number: Optional[str] = ""
  shared_inboxes: Optional[str] = ""
  timezone: Optional[TIMEZONES]
  office_hours: Optional[List[AvailabilityModel]] = []
  photo_url: Optional[str]
  calendly_url: Optional[str]
  email : Optional[constr(
      min_length=7,
      max_length=128,
      regex=r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,7}\b")]

  # pylint: disable=no-self-argument
  @validator("first_name")
  def first_name_regex(cls,value):
    result = regex.fullmatch(r"[\D\p{L}\p{N}]+$", value)
    if len(value)<=60 and result:
      return value
    raise ValueError("Invalid first name format")
  @validator("last_name")
  def last_name_regex(cls,value):
    result = regex.fullmatch(r"[\D\p{L}\p{N}\s]+$", value)
    if len(value)<=60 and result:
      return value
    raise ValueError("Invalid last name format")
  class Config:
    extra = Extra.forbid

class GetStaffResponseModel(BaseModel):
  """Get Staff Response Pydantic Model"""
  success: Optional[bool] = True
  message: Optional[str] = "Successfully fetched the Staff"
  data: Optional[FullStaffModel]

  class Config():
    orm_mode = True
    schema_extra = {
        "example": {
            "success": True,
            "message": "Successfully fetched the Staff",
            "data": FULL_STAFF_EXAMPLE
        }
    }

class PostStaffResponseModel(BaseModel):
  """Create Staff Response Pydantic Model"""
  success: Optional[bool] = True
  message: Optional[str] = "Successfully created the Staff"
  data: FullStaffModel

  class Config():
    orm_mode = True
    schema_extra = {
        "example": {
            "success": True,
            "message": "Successfully created the Staff",
            "data": FULL_STAFF_EXAMPLE
        }
    }

class UpdateStaffResponseModel(BaseModel):
  """Update Staff Response Pydantic Model"""
  success: Optional[bool] = True
  message: Optional[str] = "Successfully updated the Staff"
  data: Optional[FullStaffModel]

  class Config():
    orm_mode = True
    schema_extra = {
        "example": {
            "success": True,
            "message": "Successfully updated the Staff",
            "data": FULL_STAFF_EXAMPLE
        }
    }

class DeleteStaffReponseModel(BaseModel):
  """Delete Staff Pydantic Model"""
  success: Optional[bool] = True
  message: Optional[str] = "Successfully deleted the Staff"

  class Config():
    orm_mode = True
    schema_extra = {
        "example": {
            "success": True,
            "message": "Successfully deleted the Staff"
        }
    }

class TotalCountResponseModel(BaseModel):
  records: Optional[List[FullStaffModel]]
  total_count: int

class AllStaffResponseModel(BaseModel):
  """Get All Staff Response Pydantic Model"""
  success: Optional[bool] = True
  message: Optional[str] = "Data fetched successfully"
  data: Optional[TotalCountResponseModel]

  class Config():
    orm_mode = True
    schema_extra = {
        "example": {
            "success": True,
            "message": "Data fetched successfully",
            "data": {
                      "records":[FULL_STAFF_EXAMPLE],
                      "total_count": 50
                    }
        }
    }

class StaffImportJsonResponse(BaseModel):
  """Staff Import Json Response Pydantic Model"""
  success: Optional[bool] = True
  message: Optional[str] = "Successfully created the Staffs"
  data: Optional[List[str]]

  class Config():
    orm_mode = True
    schema_extra = {
        "example": {
            "success": True,
            "message": "Successfully created the Staffs",
            "data": [
                "44qxEpc35pVMb6AkZGbi", "00MPqUhCbyPe1BcevQDr",
                "lQRzcrRuDpJ9IoW8bCHu"
            ]
        }
    }

class ProfileFieldsModel(BaseModel):
  pronouns: Optional[list]
  workdays: Optional[list]
  timezones: Optional[list]

class ProfileFieldsResponseModel(BaseModel):
  success: Optional[bool] = True
  message: Optional[str] = "Successfully fetched the possible options for"\
                            " pronouns, workdays, timezones"
  data: Optional[ProfileFieldsModel]

  class Config():
    orm_mode = True
    schema_extra = {
      "example": {
        "success": True,
        "message": "Successfully fetched the possible options for"
                    " pronouns, workdays, timezones",
        "data": PROFILE_FIELDS
      }
    }


from django.urls import path
from . import views

urlpatterns = [
    path('type/',views.ManageFeeTypeListView, name="class_fee_list"),
    path('type/create/',views.ManageFeeTypeCreateView, name="class_fee_create"),
    path('type/edit/<fee_type_code>/', views.ManageFeeTypeEditView, name = 'class_fee_edit'),
    path('type/delete/<fee_type_code>/', views.ManageFeeTypeDeleteView, name = 'class_fee_delete'),

    path('def/',views.ManageFeeDefListView, name="fee_def_list"),
    path('def/create/',views.ManageFeeDefCreateView, name="fee_def_create"),
    path('def/edit/<fee_def_code>/', views.ManageFeeDefEditView, name = 'fee_def_edit'),
    path('def/delete/<fee_def_code>/', views.ManageFeeDefDeleteView, name = 'fee_def_delete'),

    path('reg/',views.ManageFeeRegisterListView, name="fee_reg_list"),
    path('reg/print',views.ManageFeeRegisterPrintView, name="fee_reg_print"),
    path('reg/create/',views.ManageFeeRegisterCreateView, name="fee_reg_create"),
    path('reg/create/all/',views.ManageFeeRegisterCreateToAllView, name="fee_reg_create_to_all"),
    path('reg/detail/<fee_reg_id>/',views.ManageFeeRegisterDetailView, name="fee_reg_detail"),
    path('reg/edit/<fee_reg_id>/', views.ManageFeeRegisterEditView, name = 'fee_reg_edit'),
    path('reg/delete/<fee_reg_id>/', views.ManageFeeRegisterDeleteView, name = 'fee_reg_delete'),
]


# -*- coding: utf-8 -*-
#
# pyramid documentation build configuration file, created by
# sphinx-quickstart on Wed Jul 16 13:18:14 2008.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# The contents of this file are pickled, so don't put values in the namespace
# that aren't pickleable (module imports are okay, they're removed automatically).
#
# All configuration values have a default value; values that are commented out
# serve to show the default value.

import sys, os
import datetime
import inspect
import warnings
import pkg_resources
import pylons_sphinx_themes

warnings.simplefilter('ignore', DeprecationWarning)

# skip raw nodes
from sphinx.writers.text import TextTranslator
from sphinx.writers.latex import LaTeXTranslator

from docutils import nodes
from docutils import utils

def raw(*arg):
    raise nodes.SkipNode
TextTranslator.visit_raw = raw

# make sure :app:`Pyramid` doesn't mess up LaTeX rendering
def nothing(*arg):
    pass
LaTeXTranslator.visit_inline = nothing
LaTeXTranslator.depart_inline = nothing

book = os.environ.get('BOOK')

# If your extensions are in another directory, add it here. If the directory
# is relative to the documentation root, use os.path.abspath to make it
# absolute, like shown here.
parent = os.path.dirname(os.path.dirname(__file__))
sys.path.append(os.path.abspath(parent))
wd = os.getcwd()
os.chdir(parent)
os.system('%s setup.py test -q' % sys.executable)
os.chdir(wd)

for item in os.listdir(parent):
    if item.endswith('.egg'):
        sys.path.append(os.path.join(parent, item))

# General configuration
# ---------------------

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest']

# Add any paths that contain templates here, relative to this directory.
#templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The master toctree document.
master_doc = 'index'

# General substitutions.
project = 'Pyramid SimpleForm library'
copyright = '%s, Dan Jacob' % datetime.datetime.now().year

# The default replacements for |version| and |release|, also used in various
# other places throughout the built documents.
#
# The short X.Y version.
version = pkg_resources.get_distribution('pyramid_simpleform').version
# The full version, including alpha/beta/rc tags.
release = version

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
today_fmt = '%B %d, %Y'

# List of documents that shouldn't be included in the build.
#unused_docs = []

# List of directories, relative to source directories, that shouldn't be searched
# for source files.
#exclude_dirs = []

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
add_module_names = False

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
#pygments_style = book and 'bw' or 'tango'
if book:
    pygments_style = 'bw'

# The default language to highlight source code in.
#highlight_language = 'guess'

# Options for HTML output
# -----------------------

# Add and use Pyramid theme
sys.path.append(os.path.abspath('_themes'))
html_theme = 'pyramid'
html_theme_path = pylons_sphinx_themes.get_html_themes_path()

# The style sheet to use for HTML and HTML Help pages. A file of that name
# must exist either in Sphinx' static/ path, or in one of the custom paths
# given in html_static_path.
#html_style = 'pyramid.css'

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
html_title = 'Simple form library pyramid_simpleform v%s' % release

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = 'Home'

# The name of an image file (within the static path) to place at the top of
# the sidebar.
#html_logo = '_static/pyramid.png'

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = '_static/pyramid.ico'

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
#html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_use_modindex = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, the reST sources are included in the HTML build as _sources/<name>.
#html_copy_source = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = ''

# Output file base name for HTML help builder.
htmlhelp_basename = 'pyramid'

# Options for LaTeX output
# ------------------------

# The paper size ('letter' or 'a4').
latex_paper_size = 'letter'

# The font size ('10pt', '11pt' or '12pt').
latex_font_size = '10pt'

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, document class [howto/manual]).
latex_documents = [
  ('index', 'pyramid_simpleform.tex',
   'pyramid\_simpleform',
   'Dan Jacob', 'manual'),
    ]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = '_static/pylons_small.png'

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
latex_toplevel_sectioning = 'part'

# Additional stuff for the LaTeX preamble.
#latex_preamble = ''

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
latex_domain_indices = False

## Say, for a moment that you have a twoside document that needs a 3cm
## inner margin to allow for binding and at least two centimetres the
## rest of the way around. You've been using the a4wide package up until
## now, because you like the amount of text it places on the
## page. Perhaps try something like this in your preamble:

## \usepackage[bindingoffset=1cm,textheight=22cm,hdivide={2cm,*,2cm},vdivide={*,22cm,*}]{geometry}

## _PREAMBLE = r"""\usepackage[bindingoffset=0.45in,textheight=7.25in,hdivide={0.5in,*,0.75in},vdivide={1in,7.25in,1in},papersize={7.5in,9.25in}]{geometry}"""

_PREAMBLE = r"""
\usepackage[]{geometry}
\geometry{bindingoffset=0.45in,textheight=7.25in,hdivide={0.5in,*,0.75in},vdivide={1in,7.25in,1in},papersize={7.5in,9.25in}}
\hypersetup{
    colorlinks=true,
    linkcolor=black,
    citecolor=black,
    filecolor=black,
    urlcolor=black
}
\fvset{frame=single,xleftmargin=9pt,numbersep=4pt}

\pagestyle{fancy}

% header and footer styles
\renewcommand{\chaptermark}[1]%
  {\markboth{\MakeUppercase{\thechapter.\ #1}}{}
  }
\renewcommand{\sectionmark}[1]%
  {\markright{\MakeUppercase{\thesection.\ #1}}
  }

% defaults for fancy style
\renewcommand{\headrulewidth}{0pt}
\renewcommand{\footrulewidth}{0pt}
\fancyhf{}
\fancyfoot[C]{\thepage}

% plain style
\fancypagestyle{plain}{
  \renewcommand{\headrulewidth}{0pt} % ho header line
  \renewcommand{\footrulewidth}{0pt}% no footer line
  \fancyhf{} % empty header and footer
  \fancyfoot[C]{\thepage}
}

% title page styles
\makeatletter
\def\@subtitle{\relax}
\newcommand{\subtitle}[1]{\gdef\@subtitle{#1}}
\renewcommand{\maketitle}{
  \begin{titlepage}
    {\rm\Huge\@title\par}
    {\em\large\py@release\releaseinfo\par}
    \if\@subtitle\relax\else\large\@subtitle\par\fi
    {\large\@author\par}
  \end{titlepage}
}
\makeatother

% Redefine link and title colors
\definecolor{TitleColor}{rgb}{0,0,0}
\definecolor{InnerLinkColor}{rgb}{0.208,0.374,0.486}
\definecolor{OuterLinkColor}{rgb}{0.216,0.439,0.388}
% Redefine these colors to something not white if you want to have colored
% background and border for code examples.
\definecolor{VerbatimColor}{rgb}{1,1,1}
\definecolor{VerbatimBorderColor}{rgb}{1,1,1}

\makeatletter
\renewcommand{\py@noticestart@warning}{\py@heavybox}
\renewcommand{\py@noticeend@warning}{\py@endheavybox}
\renewcommand{\py@noticestart@note}{\py@heavybox}
\renewcommand{\py@noticeend@note}{\py@endheavybox}
\makeatother

% icons in note and warning boxes
\usepackage{ifthen}
% Keep a copy of the original notice environment
\let\origbeginnotice\notice
\let\origendnotice\endnotice

% Redefine the notice environment so we can add our own code to it
\renewenvironment{notice}[2]{%
  \origbeginnotice{#1}{}% equivalent to original \begin{notice}{#1}{#2}
  % load graphics
  \ifthenelse{\equal{#1}{warning}}{\includegraphics{latex-warning.png}}{}
  \ifthenelse{\equal{#1}{note}}{\includegraphics{latex-note.png}}{}
  % etc.
}{%
  \origendnotice% equivalent to original \end{notice}
}

% try to prevent code-block boxes from splitting across pages
\sloppy
\widowpenalty=300
\clubpenalty=300
\setlength{\parskip}{3ex plus 2ex minus 2ex}

% suppress page numbers on pages showing part title
\makeatletter
\let\sv@endpart\@endpart
\def\@endpart{\thispagestyle{empty}\sv@endpart}
\makeatother

% prevent page numbers in TOC (reset to fancy by frontmatter directive)
\pagestyle{empty}
"""

latex_elements = {
    'preamble': _PREAMBLE,
    'wrapperclass':'book',
    'date':'',
    'releasename':'Version',
    'title':r'pyramid_simpleform',
#    'pointsize':'12pt', # uncomment for 12pt version
}

# secnumdepth counter reset to 2 causes numbering in related matter;
# reset to -1 causes chapters to not be numbered, reset to -2 causes
# parts to not be numbered.

#part	      -1
#chapter       0
#section       1
#subsection    2
#subsubsection 3
#paragraph     4
#subparagraph  5

def frontmatter(name, arguments, options, content, lineno,
                content_offset, block_text, state, state_machine):
    return [nodes.raw(
        '',
        r"""
\frontmatter
% prevent part/chapter/section numbering
\setcounter{secnumdepth}{-2}
% suppress headers
\pagestyle{plain}
% reset page counter
\setcounter{page}{1}
% suppress first toc pagenum
\addtocontents{toc}{\protect\thispagestyle{empty}}
""",
        format='latex')]

def mainmatter(name, arguments, options, content, lineno,
               content_offset, block_text, state, state_machine):
    return [nodes.raw(
        '',
        r"""
\mainmatter
% allow part/chapter/section numbering
\setcounter{secnumdepth}{2}
% get headers back
\pagestyle{fancy}
\fancyhf{}
\renewcommand{\headrulewidth}{0.5pt}
\renewcommand{\footrulewidth}{0pt}
\fancyfoot[C]{\thepage}
\fancyhead[RO]{\rightmark}
\fancyhead[LE]{\leftmark}
""",
        format='latex')]

def backmatter(name, arguments, options, content, lineno,
              content_offset, block_text, state, state_machine):
    return [nodes.raw('', '\\backmatter\n\\setcounter{secnumdepth}{-1}\n',
                      format='latex')]

def app_role(role, rawtext, text, lineno, inliner, options={}, content=[]):
    """custom role for :app: marker, does nothing in particular except allow
    :app:`Pyramid` to work (for later search and replace)."""
    if 'class' in options:
        assert 'classes' not in options
        options['classes'] = options['class']
        del options['class']
    return [nodes.inline(rawtext, utils.unescape(text), **options)], []

def setup(app):
    app.add_role('app', app_role)
    app.add_directive('frontmatter', frontmatter, 1, (0, 0, 0))
    app.add_directive('mainmatter', mainmatter, 1, (0, 0, 0))
    app.add_directive('backmatter', backmatter, 1, (0, 0, 0))
    app.connect('autodoc-process-signature', resig)

def resig(app, what, name, obj, options, signature, return_annotation):
    """ Allow for preservation of ``@action_method`` decorated methods
    in configurator """
    docobj = getattr(obj, '__docobj__', None)
    if docobj is not None:
        argspec = inspect.getargspec(docobj)
        if argspec[0] and argspec[0][0] in ('cls', 'self'):
            del argspec[0][0]
        signature = inspect.formatargspec(*argspec)
    return signature, return_annotation

# turn off all line numbers in latex formatting

## from pygments.formatters import LatexFormatter
## from sphinx.highlighting import PygmentsBridge

## class NoLinenosLatexFormatter(LatexFormatter):
##     def __init__(self, **options):
##         LatexFormatter.__init__(self, **options)
##         self.linenos = False

## PygmentsBridge.latex_formatter = NoLinenosLatexFormatter

# -- Options for Epub output ---------------------------------------------------

# Bibliographic Dublin Core info.
epub_title = 'pyramid_simpleform'
epub_author = 'Dan Jacob'
epub_copyright = '2010-2011'

# The language of the text. It defaults to the language option
# or en if the language is not set.
epub_language = 'en'

# The scheme of the identifier. Typical schemes are ISBN or URL.
epub_scheme = 'ISBN'

# The unique identifier of the text. This can be a ISBN number
# or the project homepage.

# A unique identification for the text.
epub_uid = 'pyramid_simpleform'

# HTML files that should be inserted before the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_pre_files = []

# HTML files shat should be inserted after the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_post_files = []

# A list of files that should not be packed into the epub file.
#epub_exclude_files = []

# The depth of the table of contents in toc.ncx.
epub_tocdepth = 3


import os
import sys
from utils import *
from datetime import datetime

import telegram
from telegram.ext import (Updater, CommandHandler, MessageHandler, Filters,
                          ConversationHandler)

def start(update, context):
    update.message.reply_text("ברוכ/ה הבא/ה לרכבוט!")
    update.message.reply_text("הכנס תחנת מוצא:")
    return CHOOSE_ORIGIN

def get_depart_station(update, context):
    user = update.message.from_user
    input_text = update.message.text
    logger.info("User {}, departure input: {}".format(user, input_text))
    # TODO - Validate input and fuzzy lookup
    context.user_data['depart_station'], score = get_fuzzy_station_name(update.message.text)
    update.message.reply_text("תחנת מוצא: {}".format(context.user_data['depart_station']))
    update.message.reply_text("הכנס/י תחנת יעד:")
    return CHOOSE_DEST

def get_dest_station(update, context):
    user = update.message.from_user
    logger.info("User {}, destination input: {}".format(user, update.message.text))
    # TODO - Validate input and fuzzy lookup
    context.user_data['dest_station'], score = get_fuzzy_station_name(update.message.text)
    update.message.reply_text("תחנת יעד: {}".format(context.user_data['dest_station']))

    if 'depart_station' in context.user_data and 'dest_station' in context.user_data:
        depart_station = context.user_data['depart_station']
        dest_station = context.user_data['dest_station']
        if depart_station == dest_station:
            update.message.reply_text("אנא הכנס/י תחנת מוצא ויעד שונות.")
            update.message.reply_text("הכנס תחנת מוצא:")
            return CHOOSE_ORIGIN
    else:
        update.message.reply_text("נתונים חסרים. אנא התחל/י מחדש")
        return ConversationHandler.END

    update.message.reply_text("עכשיו?", reply_markup=telegram.ReplyKeyboardMarkup([['לא', 'כן']]))

    return CHOOSE_TIME

def get_choose_time(update, context):
    user = update.message.from_user
    context.user_data['time_keyboard_input'] = update.message.text
    logger.info("User {}, keyboard 'now' input: {}".format(user, context.user_data['time_keyboard_input']))
    if context.user_data['time_keyboard_input'] == "כן":
        update.message.reply_text("מחפש רכבות קרובות...", reply_markup=telegram.ReplyKeyboardRemove())
        timestamp = datetime.now()
        get_route(update, context, timestamp)
        update.message.reply_text("איך עוד אפשר לעזור?",
                                  reply_markup=telegram.ReplyKeyboardMarkup([['סיימתי', 'חיפוש חדש', 'צור תזכורת']]))
        return PAST_ROUTE
    elif context.user_data['time_keyboard_input'] == "לא":
        update.message.reply_text("מתי?",
                                  reply_markup=telegram.ReplyKeyboardMarkup([['בתאריך...', 'מחר', 'היום']]))
    else:
        update.message.reply_text("לא הצלחתי להבין.", reply_markup=telegram.ReplyKeyboardRemove())
        update.message.reply_text("לחפש רכבות לעכשיו?", reply_markup=telegram.ReplyKeyboardMarkup([['לא', 'כן']]))

        return CHOOSE_TIME
    return PARSE_DAY

def get_parsed_day(update, context):
    user = update.message.from_user
    context.user_data['day_keyboard_input'] = update.message.text
    logger.info("User {}, parsed day input: {}".format(user, context.user_data['day_keyboard_input']))
    if context.user_data['day_keyboard_input'] == "היום":
        context.user_data['date_input'] = datetime.now()
    elif context.user_data['day_keyboard_input'] == "מחר":
        context.user_data['date_input'] = datetime.now() + timedelta(hours=24)
    elif context.user_data['day_keyboard_input'] == "בתאריך...":
        update.message.reply_text("הכנס תאריך:", reply_markup=telegram.ReplyKeyboardRemove())
        return CUSTOM_DAY
    else:
        update.message.reply_text("לא הצלחתי להבין.", reply_markup=telegram.ReplyKeyboardRemove())
        update.message.reply_text("בחר חלון תאריכים לחיפוש רכבות:",
                                  reply_markup=telegram.ReplyKeyboardMarkup([['בתאריך...', 'מחר', 'היום']]))
        return PARSE_DAY

    update.message.reply_text("באיזו שעה?", reply_markup=telegram.ReplyKeyboardRemove())

    return PARSE_HOUR

def get_custom_day(update, context):
    user = update.message.from_user
    manual_day_input = update.message.text
    logger.info("User {}, manual day input: {}".format(user, manual_day_input))
    found_time = parse(manual_day_input)
    if found_time is None or found_time.year < CUR_YEAR:
        update.message.reply_text("לא הצלחתי להבין.", reply_markup=telegram.ReplyKeyboardRemove())
        update.message.reply_text("בחר חלון תאריכים לחיפוש רכבות:",
                                  reply_markup=telegram.ReplyKeyboardMarkup([['בתאריך...', 'מחר', 'היום']]))
        return PARSE_DAY
    context.user_data['date_input'] = found_time
    update.message.reply_text("אוקיי, ה-{}".format(context.user_data['date_input'].strftime("%d/%m/%Y")),
                              reply_markup=telegram.ReplyKeyboardRemove())
    update.message.reply_text("באיזו שעה?", reply_markup=telegram.ReplyKeyboardRemove())

    return PARSE_HOUR

def get_parsed_hour(update, context):
    user = update.message.from_user
    logger.info("User {}, parsed time input: {}".format(user, update.message.text))
    context.user_data['time_input'] = update.message.text
    context.user_data['time_input'] = get_fuzzy_time(context.user_data['time_input'])
    if context.user_data['time_input'] is None:
        update.message.reply_text("לא הצלחתי להבין מתי. נסה/י שוב.")
        return PARSE_HOUR
    context.user_data['timestamp_input'] = datetime.combine(context.user_data['date_input'].date(),
                                                            context.user_data['time_input'].time())
    update.message.reply_text("מחפש רכבות עבור: {}"
                              .format(context.user_data['timestamp_input'].strftime('%d/%m/%y %H:%M')))
    if (datetime.now() - context.user_data['timestamp_input']).total_seconds() > 30:
        update.message.reply_text("לא ניתן לקבל לוחות זמנים עבור רכבות עבר. הכנס/י תאריך ושעה אחרים.")
        update.message.reply_text("מתי?", reply_markup=telegram.ReplyKeyboardMarkup([['בתאריך...', 'מחר', 'היום']]))
        return PARSE_DAY
    get_route(update, context, context.user_data['timestamp_input'])
    update.message.reply_text("איך עוד אפשר לעזור?",
                              reply_markup=telegram.ReplyKeyboardMarkup([['סיימתי', 'חיפוש חדש', 'צור תזכורת']]))
    return PAST_ROUTE

def past_route(update, context):
    user = update.message.from_user
    context.user_data['conv_end_message'] = update.message.text
    logger.info("User {}, past route input: {}".format(user, context.user_data['conv_end_message']))
    if context.user_data['conv_end_message'] == 'סיימתי':
        return happy_end(update, context)
    elif context.user_data['conv_end_message'] == 'חיפוש חדש':
        update.message.reply_text("הכנס תחנת מוצא:", reply_markup=telegram.ReplyKeyboardRemove())
        return CHOOSE_ORIGIN
    elif context.user_data['conv_end_message'] == 'צור תזכורת':
        update.message.reply_text("באיזו שעה להתריע?", reply_markup=telegram.ReplyKeyboardRemove())
        return TIME_SCHEDULE
    else:
        update.message.reply_text("לא הצלחתי להבין.")
        update.message.reply_text("איך עוד אפשר לעזור?",
                                  reply_markup=telegram.ReplyKeyboardMarkup([['סיימתי', 'חיפוש חדש', 'צור תזכורת']]))
        return PAST_ROUTE

def get_time_schedule(update, context):
    user = update.message.from_user
    sched_time_message = update.message.text
    logger.info("User {}, sched time input: {}".format(user, sched_time_message))
    found_time = get_fuzzy_time(sched_time_message)
    if found_time is None:
        update.message.reply_text("לא הצלחתי להבין מתי. נסה/י להזין שעה בפורמט כמו '08:30':")
        return TIME_SCHEDULE
    context.user_data['time_schedule'] = found_time.time()
    update.message.reply_text("אוקיי, ב{}".format(context.user_data['time_schedule'].strftime("%H:%M")))
    update.message.reply_text("באיזה ימים? הכנס/י רשימה מהצורה 'א,ב,ג...'.")
    return DAY_SCHEDULE

def get_day_schedule(update, context):
    user = update.message.from_user
    sched_day_message = update.message.text
    try:
        sanitized_days = [DAYS_DICT[x] for x in list(sched_day_message.replace(" ", "").replace(",", ""))]
        if len(sanitized_days) == 0:
            raise ValueError("לא נמצאו ימים")
    except:
        update.message.reply_text("לא הצלחתי להבין את טווח הימים. נסה/י להזין שוב ללא גרשיים או פיסוק מיותר.")
        return DAY_SCHEDULE

    # Pack needed job data in context.user_data
    update.message.reply_text("אתריע בימים: {}".format([REV_DAYS_DICT[x] for x in sanitized_days]))
    context.user_data['chat_id'] = update.effective_chat.id
    context.user_data['days'] = sanitized_days
    found_time = context.user_data['time_schedule']
    logger.info("User {}, adding schedule at time {}, days: {}".format(user, found_time, sanitized_days))
    context.job_queue.run_daily(notify, found_time, days=tuple(sanitized_days), context=context)
    update.message.reply_text("התראה נוספה בהצלחה.")
    return happy_end(update, context)

def timeout(update, context):
    user = context.effective_user
    chat_id = context.message.chat_id
    logger.info("User {} timed out".format(user))
    update.send_message(chat_id=chat_id, text="עקב חוסר פעילות נותקת מהמערכת.", reply_markup=telegram.ReplyKeyboardRemove())
    update.send_message(chat_id=chat_id, text="להתחלה לחצ/י על /start", reply_markup=telegram.ReplyKeyboardRemove())
    return ConversationHandler.END

def cancel(update, context):
    user = update.message.from_user
    logger.info("User {} canceled the conversation.".format(user))
    return happy_end(update, context)

def happy_end(update, context):
    chat_id = update.effective_chat.id
    context.bot.send_message(chat_id=chat_id, text="רכבוט שמח לעזור!", reply_markup=telegram.ReplyKeyboardRemove())
    context.bot.send_message(chat_id=chat_id, text="להתחלה לחצ/י על /start", reply_markup=telegram.ReplyKeyboardRemove())
    return ConversationHandler.END

def error(update, context):
    """Log Errors caused by Updates."""
    logger.warning('Update {} caused error {}'.format(update, context.error))

def main():

    # Get the dispatcher to register handlers
    dp = updater.dispatcher

    # Add conversation handler with states
    conv_handler = ConversationHandler(
        entry_points=[CommandHandler('start', start)],

        states={
            CHOOSE_ORIGIN: [MessageHandler(Filters.text, get_depart_station)],
            CHOOSE_DEST: [MessageHandler(Filters.text, get_dest_station)],
            CHOOSE_TIME: [MessageHandler(Filters.text, get_choose_time)],
            PARSE_DAY: [MessageHandler(Filters.text, get_parsed_day)],
            CUSTOM_DAY: [MessageHandler(Filters.text, get_custom_day)],
            PARSE_HOUR: [MessageHandler(Filters.text, get_parsed_hour)],
            PAST_ROUTE: [MessageHandler(Filters.text, past_route)],
            TIME_SCHEDULE: [MessageHandler(Filters.text, get_time_schedule)],
            DAY_SCHEDULE: [MessageHandler(Filters.text, get_day_schedule, pass_job_queue=True)],
            ConversationHandler.TIMEOUT: [MessageHandler(None, timeout)]
        },

        fallbacks=[CommandHandler('cancel', cancel)],

        conversation_timeout=CONVERSATION_TIMEOUT
    )

    dp.add_handler(conv_handler)

    # log all errors
    dp.add_error_handler(error)

    # Start the Bot
    updater.start_polling()

    # Run the bot until you press Ctrl-C or the process receives SIGINT,
    # SIGTERM or SIGABRT. This should be used most of the time, since
    # start_polling() is non-blocking and will stop the bot gracefully.
    updater.idle()

if __name__ == '__main__':

    mode = os.getenv("MODE")
    TOKEN = os.getenv("TOKEN")
    logger.info("Starting bot")

    updater = Updater(TOKEN, use_context=True)

    if mode == "dev":
        logger.info("Dev mode")
        updater.start_polling()

        # app_url = "https://localhost/"
        # wakeup_wrapper(app_url)

    elif mode == "prod":
        logger.info("Production mode")

        PORT = int(os.environ.get("PORT", "8443"))
        HEROKU_APP_NAME = os.environ.get("HEROKU_APP_NAME")
        # Code from https://github.com/python-telegram-bot/python-telegram-bot/wiki/Webhooks#heroku
        updater.start_webhook(listen="0.0.0.0",
                              port=PORT,
                              url_path=TOKEN)
        updater.bot.set_webhook("https://{}.herokuapp.com/{}".format(HEROKU_APP_NAME, TOKEN))

        # app_url = "https://{}.herokuapp.com/".format(HEROKU_APP_NAME)
        # wakeup_wrapper("127.0.0.1")

    else:
        logger.error("No MODE specified!")
        sys.exit(1)

    main()