"""
PUT /groups/<group name>
    Updates the membership list for the group. The body of the request should
    be a JSON list describing the group's members.

expected:
    200 created - group populated
    400 bad request - missing params (probably!)
    409 conflict - you made a PUT with at least one
                   user that doesn't exist
                   great job

"""

import requests

class RunTest:
    def __init__(self, host, port, check):
        self.host = host
        self.port = port
        self.check = check
        self.check_message = None

        self.good_post_template = {"message": "group created successfully",
                                   "response": {}
                                   }
        self.test_groups = {'good': [{'first_name': 'the',
                                      'last_name': 'good',
                                      'userid': 'blondie'}],
                            'bad': [{'first_name': 'the',
                                     'last_name': 'bad',
                                     'userid': 'angeleyes'}],
                            'ugly': [{'first_name': 'the',
                                      'last_name': 'ugly',
                                      'userid': 'tuco'},
                                     {'first_name': 'the',
                                      'last_name': 'ugly',
                                      'userid': 'joel'}]}

    def setup_test_groups(self):
        base_url = '%s:%s/groups' % (self.host, self.port)
        for group in self.test_groups.keys():
            group_url = '%s/%s' % (base_url, group)
            r = requests.delete(group_url)
            if r.status_code not in [404, 204]:
                return False
            data = {'groupname': group}
            r = requests.post(base_url, data=data)
            if r.status_code != 201:
                return False
            r = requests.get(group_url)
            if r.status_code != 200 or r.json() != []:
                return False
        return True

    def setup_test_users(self):
        base_url = '%s:%s/users' % (self.host, self.port)
        for group_users in self.test_groups.values():
            for user in group_users:
                user_url = '%s/%s' % (base_url, user['userid'])
                r = requests.delete(user_url)
                if r.status_code not in [404, 204]:
                    return False
                r = requests.post(base_url, data=user)
                if r.status_code != 201:
                    return False
        return True

    def check_400(self, resp):
        if resp.status_code == 400:
            self.check_message = 'OK - expected 400 received'
            return True
        self.check_message = 'FAIL - unexpected status code (%s)' % resp.status_code
        return False

    def check_404(self, resp):
        if resp.status_code != 404:
            self.check_message = 'FAIL - unexpected status code (%s)' % resp.status_code
        self.check_message = 'OK - got expected 404'
        return True

    def check_bad_users_in_list(self, resp):
        if resp.status_code != 409:
            self.check_message = 'FAIL - unexpected status code (%s)' % resp.status_code
        self.check_message = 'OK - expected 409 with message: %s' % resp.json()['unprocessable_userids']
        return True

    def test(self):
        base_url = '%s:%s/' % (self.host, self.port)
        # method = 'PUT'
        resource = 'groups'
        full_url = '%s%s' % (base_url, resource)

        if self.check == 'setup_groups':
            setup_res = self.setup_test_groups()
            if setup_res:
                self.check_message = 'OK - groups for put tests recreated'
                return True, {}
            self.check_message = 'FAIL - groups not setup correctly for put tests'
            return False, {}

        if self.check == 'setup_users':
            setup_res = self.setup_test_users()
            if setup_res:
                self.check_message = 'OK - users for put tests recreated'
                return True, {}
            self.check_message = 'FAIL - users not setup correctly for put tests'
            return False, {}

        if self.check == 'put_groups_404':
            data = list(self.test_groups.keys())
            r = requests.put('%s/%s' % (full_url, 'FAKE_GROUP'), json=data)
            return self.check_404(r), r.json()

        if self.check == 'put_bad_user_in_list':
            incorrect_users = ['FAKE_USER1_NO', 'FAKE_USER2_NO']
            group = 'ugly'
            ugly_users = [user['userid'] for user in self.test_groups['ugly']]
            data = ugly_users + incorrect_users
            r = requests.put('%s/%s' % (full_url, group), json=data)
            return self.check_bad_users_in_list(r), r.json()

        if self.check == 'put_all_the_groups':
            for group, group_users in self.test_groups.items():
                data = [user['userid'] for user in group_users]
                r = requests.put('%s/%s' % (full_url, group), json=data)
                if r.status_code != 200:
                    self.check_message = 'FAIL - unexpected status code (%s)' % r.status_code
                    return False, r
                r = requests.get('%s/%s' % (full_url, group))
                if sorted(r.json()) != sorted(data):
                    self.check_message = 'FAIL - group check %s does not match expected' % group
                    return False, r
            self.check_message = 'OK - all groups appear to match expected'
            return True, r


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @Time    : 2021/3/30 下午5:53
# @Author  : MaybeShewill-CV
# @Site    : https://github.com/MaybeShewill-CV/image-classification-tensorflow
# @File    : repvgg.py
# @IDE: PyCharm
"""
repvgg model created by Xiaohan Ding "RepVGG: Making VGG-style ConvNets Great Again"
"""
import argparse
import time
import json

import tensorflow as tf
import numpy as np
import tqdm

from cls_model_zoo import cnn_basenet
from cls_model_zoo import loss
from local_utils import config_utils

class RepVgg(cnn_basenet.CNNBaseModel):
    """
    repvgg model for image classification
    """
    def __init__(self, phase, cfg):
        """

        :param phase:
        :param cfg:
        """
        super(RepVgg, self).__init__()

        self._phase = phase
        self._cfg = cfg

        self._is_training = self._is_net_for_training()
        self._loss_type = self._cfg.SOLVER.LOSS_TYPE.lower()
        self._loss_func = getattr(loss, '{:s}_loss'.format(self._loss_type))
        self._class_nums = self._cfg.DATASET.NUM_CLASSES
        self._weights_decay = self._cfg.SOLVER.WEIGHT_DECAY
        self._repvgg_net_flag = self._cfg.MODEL.REPVGG.NET_FLAG
        self._repvgg_net_block_params = self._get_net_block_params()

    def _is_net_for_training(self):
        """
        if the net is used for training or not
        :return:
        """
        if isinstance(self._phase, tf.Tensor):
            phase = self._phase
        else:
            phase = tf.constant(self._phase, dtype=tf.string)

        return tf.equal(phase, tf.constant('train', dtype=tf.string))

    def _get_net_block_params(self):
        """

        :return:
        """
        repvggnet_name = 'repvgg-{:s}'.format(self._repvgg_net_flag)
        block_params = {
            'repvgg-a0': {
                'block_nums': [1, 2, 4, 14, 1],
                'depth_multiplier': [1.0, 0.75, 0.75, 0.75, 2.5],
                'output_channels': [64, 64, 128, 256, 512],
            },
            'repvgg-a1': {
                'block_nums': [1, 2, 4, 14, 1],
                'depth_multiplier': [1.0, 1.0, 1.0, 1.0, 2.5],
                'output_channels': [64, 64, 128, 256, 512],
            },
            'repvgg-a2': {
                'block_nums': [1, 2, 4, 14, 1],
                'depth_multiplier': [1.0, 1.5, 1.5, 1.5, 2.75],
                'output_channels': [64, 64, 128, 256, 512],
            },
            'repvgg-b0': {
                'block_nums': [1, 4, 6, 16, 1],
                'depth_multiplier': [1.0, 1.0, 1.0, 1.0, 2.5],
                'output_channels': [64, 64, 128, 256, 512],
            },
            'repvgg-b1': {
                'block_nums': [1, 4, 6, 16, 1],
                'depth_multiplier': [1.0, 2.0, 2.0, 2.0, 4.0],
                'output_channels': [64, 64, 128, 256, 512],
            },
            'repvgg-b2': {
                'block_nums': [1, 4, 6, 16, 1],
                'depth_multiplier': [1.0, 2.5, 2.5, 2.5, 5.0],
                'output_channels': [64, 64, 128, 256, 512],
            },
            'repvgg-b3': {
                'block_nums': [1, 4, 6, 16, 1],
                'depth_multiplier': [1.0, 3.0, 3.0, 3.0, 5.0],
                'output_channels': [64, 64, 128, 256, 512],
            },
        }
        try:
            return block_params[repvggnet_name]
        except KeyError:
            raise RuntimeError(
                'Wrong repvgg name, only [repvgg-a0, repvgg-a1, repvgg-a2, repvgg-b0, '
                'repvgg-b1, repvgg-b2, repvgg-b3] supported'
            )

    def _conv_block(self, input_tensor, output_channels, stride, name, padding='VALID',
                    use_bias=False, apply_reparam=False):
        """

        :param input_tensor:
        :param output_channels:
        :param stride:
        :param name:
        :param padding:
        :param use_bias:
        :param apply_reparam:
        :return:
        """
        with tf.variable_scope(name_or_scope=name):
            if apply_reparam:
                if padding == 'VALID':
                    input_tensor = tf.keras.layers.ZeroPadding2D().call(inputs=input_tensor)
                output = self.conv2d(
                    inputdata=input_tensor,
                    out_channel=output_channels,
                    kernel_size=3,
                    padding=padding,
                    stride=stride,
                    use_bias=True,
                    name='conv'
                )
            else:
                input_channles = input_tensor.get_shape().as_list()[-1]
                conv_3x3_input_tensor = tf.keras.layers.ZeroPadding2D().call(inputs=input_tensor) if \
                    padding == 'VALID' else input_tensor
                conv_3x3 = self.conv2d(
                    inputdata=conv_3x3_input_tensor,
                    out_channel=output_channels,
                    kernel_size=3,
                    padding=padding,
                    stride=stride,
                    use_bias=use_bias,
                    name='conv_3x3'
                )
                bn_3x3 = self.layerbn(inputdata=conv_3x3, is_training=self._is_training, name='bn_3x3', scale=True)
                conv_1x1_input_tensor = tf.keras.layers.ZeroPadding2D(padding=0).call(inputs=input_tensor) if \
                    padding == 'VALID' else input_tensor
                conv_1x1 = self.conv2d(
                    inputdata=conv_1x1_input_tensor,
                    out_channel=output_channels,
                    kernel_size=1,
                    padding=padding,
                    stride=stride,
                    use_bias=use_bias,
                    name='conv_1x1'
                )
                bn_1x1 = self.layerbn(inputdata=conv_1x1, is_training=self._is_training, name='bn_1x1', scale=True)

                if input_channles == output_channels and stride == 1:
                    output = tf.add_n(
                        inputs=[
                            bn_3x3,
                            bn_1x1,
                            self.layerbn(
                                inputdata=input_tensor,
                                is_training=self._is_training,
                                name='bn_input',
                                scale=True
                            ),
                        ],
                        name='add_n_fused'
                    )
                else:
                    output = tf.add_n(
                        inputs=[bn_3x3, bn_1x1],
                        name='add_n_fused'
                    )
            output = self.relu(inputdata=output, name='conv_block_output')
        return output

    def _conv_stage(self, input_tensor, output_channels, block_nums, name, apply_reparam=False):
        """

        :param input_tensor:
        :param output_channels:
        :param block_nums:
        :param name:
        :param apply_reparam:
        :return:
        """
        with tf.variable_scope(name_or_scope=name):
            strides = [2] + [1] * (block_nums - 1)
            output = input_tensor
            for index, stride in enumerate(strides):
                output = self._conv_block(
                    input_tensor=output,
                    output_channels=output_channels,
                    stride=stride,
                    name='conv_block_{:d}'.format(index + 1),
                    apply_reparam=apply_reparam
                )
        output = tf.identity(output, name='{:s}_output'.format(name))
        return output

    @classmethod
    def _fuse_conv_bn_weights(cls, conv_kernel, bn_moving_avg_mean, bn_moving_avg_var, bn_gamma, bn_beta, eps=1e-7):
        """

        :param conv_kernel:
        :param bn_moving_avg_mean:
        :param bn_moving_avg_var:
        :param bn_gamma:
        :param bn_beta:
        :return:
        """
        std = np.sqrt(bn_moving_avg_var + eps)
        t = (bn_gamma / std).reshape(1, 1, 1, -1)
        output_kernel = conv_kernel * t
        output_bias = bn_beta - bn_moving_avg_mean * bn_gamma / std

        return output_kernel, output_bias

    @classmethod
    def _pad_1x1_kernel_to_3x3_kernel(cls, conv_1x1_kernel):
        """

        :param conv_1x1_kernel:
        :return:
        """
        h, w, in_channels, output_channels = conv_1x1_kernel.shape
        output = []
        for in_index in range(in_channels):
            output_c = []
            for out_index in range(output_channels):
                output_c.append(
                    np.pad(
                        conv_1x1_kernel[:, :, in_index, out_index],
                        [[1, 1], [1, 1]],
                        mode='constant',
                        constant_values=0.0
                    )
                )
            output.append(output_c)

        return np.array(output, dtype=np.float32).transpose((2, 3, 0, 1))

    @classmethod
    def _make_bn_input_layer_bn_equal_conv_kernel(cls, input_channels):
        """

        :param input_channels:
        :return:
        """
        kernel_value = np.zeros((input_channels, input_channels, 3, 3), dtype=np.float32)
        for i in range(input_channels):
            kernel_value[i, i % input_channels, 1, 1] = 1

        return kernel_value.transpose((2, 3, 0, 1))

    def _fuse_conv_block_params(self, conv_block_params):
        """

        :param conv_block_params:
        :return:
        """
        conv_1x1_sub_block = dict()
        conv_3x3_sub_block = dict()
        conv_input_sub_block = dict()

        for param_name, param_value in conv_block_params:
            if '1x1' in param_name:
                if 'moving_variance' in param_name:
                    conv_1x1_sub_block['bn_moving_avg_var'] = param_value
                elif 'moving_mean' in param_name:
                    conv_1x1_sub_block['bn_moving_avg_mean'] = param_value
                elif 'conv_1x1' in param_name:
                    conv_1x1_sub_block['conv_kernel'] = param_value
                elif 'gamma' in param_name:
                    conv_1x1_sub_block['bn_gamma'] = param_value
                elif 'beta' in param_name:
                    conv_1x1_sub_block['bn_beta'] = param_value
                else:
                    raise ValueError('Unrecognized params: {:s}'.format(param_name))
            elif '3x3' in param_name:
                if 'moving_variance' in param_name:
                    conv_3x3_sub_block['bn_moving_avg_var'] = param_value
                elif 'moving_mean' in param_name:
                    conv_3x3_sub_block['bn_moving_avg_mean'] = param_value
                elif 'conv_3x3' in param_name:
                    conv_3x3_sub_block['conv_kernel'] = param_value
                elif 'gamma' in param_name:
                    conv_3x3_sub_block['bn_gamma'] = param_value
                elif 'beta' in param_name:
                    conv_3x3_sub_block['bn_beta'] = param_value
                else:
                    raise ValueError('Unrecognized params: {:s}'.format(param_name))
            elif 'bn_input' in param_name:
                if 'moving_variance' in param_name:
                    conv_input_sub_block['bn_moving_avg_var'] = param_value
                elif 'moving_mean' in param_name:
                    conv_input_sub_block['bn_moving_avg_mean'] = param_value
                elif 'gamma' in param_name:
                    conv_input_sub_block['bn_gamma'] = param_value
                elif 'beta' in param_name:
                    conv_input_sub_block['bn_beta'] = param_value
                else:
                    raise ValueError('Unrecognized params: {:s}'.format(param_name))
            else:
                raise ValueError('Unrecognized param name: {:s}'.format(param_name))

        conv_1x1_fused_kernel, conv_1x1_fused_bias = self._fuse_conv_bn_weights(
            conv_kernel=conv_1x1_sub_block['conv_kernel'],
            bn_moving_avg_mean=conv_1x1_sub_block['bn_moving_avg_mean'],
            bn_moving_avg_var=conv_1x1_sub_block['bn_moving_avg_var'],
            bn_gamma=conv_1x1_sub_block['bn_gamma'],
            bn_beta=conv_1x1_sub_block['bn_beta']
        )
        conv_3x3_fused_kernel, conv_3x3_fused_bias = self._fuse_conv_bn_weights(
            conv_kernel=conv_3x3_sub_block['conv_kernel'],
            bn_moving_avg_mean=conv_3x3_sub_block['bn_moving_avg_mean'],
            bn_moving_avg_var=conv_3x3_sub_block['bn_moving_avg_var'],
            bn_gamma=conv_3x3_sub_block['bn_gamma'],
            bn_beta=conv_3x3_sub_block['bn_beta']
        )

        if conv_input_sub_block:
            conv_input_sub_block['conv_kernel'] = self._make_bn_input_layer_bn_equal_conv_kernel(
                input_channels=conv_input_sub_block['bn_beta'].shape[0]
            )
            conv_input_fused_kernel, conv_input_fused_bias = self._fuse_conv_bn_weights(
                conv_kernel=conv_input_sub_block['conv_kernel'],
                bn_moving_avg_mean=conv_input_sub_block['bn_moving_avg_mean'],
                bn_moving_avg_var=conv_input_sub_block['bn_moving_avg_var'],
                bn_gamma=conv_input_sub_block['bn_gamma'],
                bn_beta=conv_input_sub_block['bn_beta']
            )
            output_kernel = conv_3x3_fused_kernel + self._pad_1x1_kernel_to_3x3_kernel(conv_1x1_fused_kernel) + \
                            conv_input_fused_kernel
            output_bias = conv_3x3_fused_bias + conv_1x1_fused_bias + conv_input_fused_bias
        else:
            output_kernel = conv_3x3_fused_kernel + self._pad_1x1_kernel_to_3x3_kernel(conv_1x1_fused_kernel)
            output_bias = conv_3x3_fused_bias + conv_1x1_fused_bias

        return output_kernel, output_bias

    def _build_net(self, input_tensor, name, reuse=False, apply_reparam=False):
        """

        :param input_tensor:
        :param name:
        :param reuse:
        :param apply_reparam:
        :return:
        """
        with tf.variable_scope(name_or_scope=name, reuse=reuse):
            output = input_tensor
            for index, block_num in enumerate(self._repvgg_net_block_params['block_nums']):
                depth_multiplier = self._repvgg_net_block_params['depth_multiplier'][index]
                output_channels = int(self._repvgg_net_block_params['output_channels'][index] * depth_multiplier)
                if index == 0:
                    output_channels = min(64, output_channels)
                output = self._conv_stage(
                    input_tensor=output,
                    output_channels=output_channels,
                    block_nums=block_num,
                    name='conv_stage_{:d}'.format(index + 1),
                    apply_reparam=apply_reparam
                )
                if index == 0:
                    break
            # output = self.globalavgpooling(
            #     inputdata=output,
            #     name='global_average_pooling'
            # )
            # logits = self.fullyconnect(
            #     inputdata=output,
            #     out_dim=self._class_nums,
            #     name='final_logits'
            # )
        return output

    def inference(self, input_tensor, name, reuse=False):
        """

        :param input_tensor:
        :param name:
        :param reuse:
        :return:
        """
        logits = self._build_net(
            input_tensor=input_tensor,
            name=name,
            reuse=reuse,
            apply_reparam=True,
        )

        return logits

    def compute_loss(self, input_tensor, label, name, reuse=False):
        """

        :param input_tensor:
        :param label:
        :param name:
        :param reuse:
        :return:
        """
        logits = self._build_net(
            input_tensor=input_tensor,
            name=name,
            reuse=reuse,
            apply_reparam=False
        )

        with tf.variable_scope('repvggnet_loss', reuse=reuse):
            if self._loss_type == 'cross_entropy':
                ret = self._loss_func(
                    logits=logits,
                    label_tensor=label,
                    weight_decay=self._weights_decay,
                    l2_vars=tf.trainable_variables(),
                )
            elif self._loss_type == 'dice_bce':
                ret = self._loss_func(
                    logits=logits,
                    label_tensor=label,
                    weight_decay=self._weights_decay,
                    l2_vars=tf.trainable_variables(),
                    class_nums=self._class_nums,
                )
            else:
                raise NotImplementedError('Loss of type: {:s} has not been implemented'.format(self._loss_type))

        return ret

    def save_trained_model(self, weights_path, name, reuse=True, dump_to_json=False):
        """

        :param weights_path:
        :param name:
        :param reuse:
        :param dump_to_json:
        :return:
        """
        input_tensor = tf.placeholder(
            dtype=tf.float32,
            shape=[None, self._cfg.AUG.EVAL_CROP_SIZE[1], self._cfg.AUG.EVAL_CROP_SIZE[0], 3],
            name='input_tensor'
        )

        _ = self._build_net(
            input_tensor=input_tensor,
            name=name,
            reuse=reuse,
            apply_reparam=False
        )

        with tf.variable_scope(name_or_scope='moving_avg'):
            variable_averages = tf.train.ExponentialMovingAverage(self._cfg.SOLVER.MOVING_AVE_DECAY)
            variables_to_restore = variable_averages.variables_to_restore()

        trained_params = dict()
        saver = tf.train.Saver(variables_to_restore)
        with tf.Session() as sess:
            saver.restore(sess=sess, save_path=weights_path)

            for vv_name, vv in variables_to_restore.items():
                trained_params[vv_name] = sess.run(vv).tolist()

        if dump_to_json:
            with open('./repvgg_trainned_params.json', 'w') as file:
                json.dump(obj=trained_params, fp=file)

        return trained_params

    def convert_reparams(self, repvgg_trainned_params: dict, repvgg_converted_param_names):
        """

        :param repvgg_trainned_params:
        :param repvgg_converted_param_names:
        :return:
        """
        tmp_info = dict()
        converted_params = dict()
        for param_name in tqdm.tqdm(repvgg_converted_param_names):
            param_name_tmp = '/'.join(param_name.split('/')[:3]).replace(':0', '')
            output_kernel = None
            output_bias = None
            tmp_info[param_name] = []
            if 'final_logits' in param_name:
                if 'kernel' in param_name:
                    for trainned_param_name, trainned_param_value in repvgg_trainned_params.items():
                        if 'final_logits' in trainned_param_name and 'kernel' in trainned_param_name:
                            output_kernel = trainned_param_value
                elif 'bias' in param_name:
                    for trainned_param_name, trainned_param_value in repvgg_trainned_params.items():
                        if 'final_logits' in trainned_param_name and 'bias' in trainned_param_name:
                            output_bias = trainned_param_value
                else:
                    raise ValueError('Unrecognized params: {:s}'.format(param_name))
            else:
                corresponding_params = []
                for trainned_param_name, trainned_param_value in repvgg_trainned_params.items():
                    if 'moving_avg' in trainned_param_name:
                        trainned_param_name_tmp = '/'.join(trainned_param_name.split('/')[1:4])
                    else:
                        trainned_param_name_tmp = '/'.join(trainned_param_name.split('/')[:3])
                    if param_name_tmp == trainned_param_name_tmp:
                        corresponding_params.append((trainned_param_name, trainned_param_value))
                        tmp_info[param_name].append(trainned_param_name)

                output_kernel, output_bias = self._fuse_conv_block_params(corresponding_params)

            with open('tmp.json', 'w') as file:
                json.dump(tmp_info, file)

            if 'bias' in param_name or 'b' in param_name.split('/')[-1]:
                converted_params[param_name] = output_bias
            elif 'kernel' in param_name or 'W' in param_name.split('/')[-1] :
                converted_params[param_name] = output_kernel
            else:
                raise ValueError('Unrecognized param: {:s}'.format(param_name))

        return converted_params

def _stats_graph(graph):
    """

    :param graph:
    :return:
    """
    flops = tf.profiler.profile(graph, options=tf.profiler.ProfileOptionBuilder.float_operation())
    params = tf.profiler.profile(graph, options=tf.profiler.ProfileOptionBuilder.trainable_variables_parameter())
    print('FLOPs: {};    Trainable params: {}'.format(flops.total_float_ops, params.total_parameters))

    return

def get_model(phase, cfg):
    """

    :param phase:
    :param cfg:
    :return:
    """
    return RepVgg(phase=phase, cfg=cfg)

def _init_args():
    """

    :return:
    """
    parser = argparse.ArgumentParser()
    parser.add_argument('--trained_weights_path', type=str, help='The trainned repvgg model ckpt weights path')
    parser.add_argument('--rep_params_save_path', type=str)

    return parser.parse_args()

def save_train_params():
    """

    :return:
    """
    tf.reset_default_graph()
    cfg = config_utils.get_config(config_file_path='./config/ilsvrc_2012_repvgg.yaml')
    model = get_model(phase='train', cfg=cfg)
    input_tensor = tf.placeholder(
        dtype=tf.float32,
        shape=[None, 224, 224, 3],
        name='input_tensor'
    )

    _ = model._build_net(
        input_tensor=input_tensor,
        name='RepVgg',
        reuse=False,
        apply_reparam=False
    )

    with tf.variable_scope(name_or_scope='moving_avg'):
        variable_averages = tf.train.ExponentialMovingAverage(cfg.SOLVER.MOVING_AVE_DECAY)
        variables_to_restore = variable_averages.variables_to_restore()

    saver = tf.train.Saver(variables_to_restore)
    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        saver.save(sess=sess, save_path='./log/train/repvgg_train.ckpt')

def convert_repvgg_params():
    """

    :return:
    """
    tf.reset_default_graph()
    args = _init_args()

    # init compute graph
    cfg = config_utils.get_config(config_file_path='./config/ilsvrc_2012_repvgg.yaml')
    model = get_model(phase='train', cfg=cfg)

    # save trainned repvgg params into json
    repvgg_trainned_params = dict()
    trained_params = model.save_trained_model(weights_path=args.trained_weights_path, name='RepVgg', reuse=False)
    for param_name, param_value in trained_params.items():
        repvgg_trainned_params[param_name] = np.array(param_value, dtype=np.float32)

    # build repvgg inference compute graph
    tf.reset_default_graph()
    test_input_tensor = tf.placeholder(dtype=tf.float32, shape=[None, 224, 224, 3], name='test_input')
    _ = model.inference(
        input_tensor=test_input_tensor,
        name='RepVgg',
        reuse=False
    )
    repvgg_converted_names = []
    for vv in tf.trainable_variables():
        repvgg_converted_names.append(vv.name)

    converted_params = model.convert_reparams(
        repvgg_trainned_params=repvgg_trainned_params,
        repvgg_converted_param_names=repvgg_converted_names
    )

    saver = tf.train.Saver(tf.global_variables())

    with tf.Session() as sess:

        sess.run(tf.global_variables_initializer())

        for index, vv in enumerate(tf.trainable_variables()):
            if index == 0:
                print(vv.name)
                print(sess.run(vv))
                print('*' * 20)

        for index, vv in enumerate(tf.trainable_variables()):
            kernel = converted_params[vv.name]
            sess.run(tf.assign(vv, kernel))
            if index == 0:
                print(vv.name)
                print(sess.run(vv))
                print('*' * 20)

        saver.save(sess, save_path=args.rep_params_save_path)

    print('Complete')

def check_converted_model():
    """

    :return:
    """
    tf.reset_default_graph()
    args = _init_args()

    cfg = config_utils.get_config(config_file_path='./config/ilsvrc_2012_repvgg.yaml')
    model = get_model(phase='test', cfg=cfg)
    test_input_ndaray = np.random.random((1, 224, 224, 3)).astype(np.float32)

    # init trained compute graph
    test_input_tensor = tf.placeholder(dtype=tf.float32, shape=[1, 224, 224, 3], name='test_input')
    logits = model._build_net(
        input_tensor=test_input_tensor,
        name='RepVgg',
        reuse=False,
        apply_reparam=False
    )

    with tf.variable_scope(name_or_scope='moving_avg'):
        variable_averages = tf.train.ExponentialMovingAverage(cfg.SOLVER.MOVING_AVE_DECAY)
        variables_to_restore = variable_averages.variables_to_restore()

    saver = tf.train.Saver(variables_to_restore)
    with tf.Session() as sess:
        saver.restore(sess=sess, save_path=args.trained_weights_path)

        train_logits = sess.run(logits, feed_dict={test_input_tensor: test_input_ndaray})
        print(train_logits)
        print('*' * 100)

    # reset compute graph
    tf.reset_default_graph()
    test_input_tensor = tf.placeholder(dtype=tf.float32, shape=[1, 224, 224, 3], name='test_input')
    logits = model.inference(
        input_tensor=test_input_tensor,
        name='RepVgg',
        reuse=False,
    )
    saver = tf.train.Saver(tf.global_variables())

    with tf.Session() as sess:
        saver.restore(sess=sess, save_path=args.rep_params_save_path)

        deploy_logits = sess.run(logits, feed_dict={test_input_tensor: test_input_ndaray})
        print(deploy_logits)
        print('*' * 100)

    print('========================== The diff is')
    print(((train_logits - deploy_logits) ** 2).sum())

if __name__ == '__main__':
    """
    test code
    """
    save_train_params()

    convert_repvgg_params()

    check_converted_model()


"""
FILE STRUCTURE:
1.genuines : genuine signatures of 30 people, 5 sample each.
2.forged : forged signature of the same 30 people as in genuine folder, 5 sample each.

Naming of files : NFI-XXXYYZZZ
explaination    XXX - ID number of a person who has done the signature.
		YY - Image smaple number.
		ZZZ - ID number of person whose signature is in photo.

for example: NFI-00602023 is an image of signature of person number 023 done by person 006.
             This is a forged signature.
	         NFI-02103021 is an image of signature of person number 021 done by person 021.
             This is a genuine signature.

"""

import os
from os import listdir
from os.path import isfile, join
from pathlib import Path
from PIL import Image
import progressbar
import gc

import numpy as np
import pandas as pd

def get_ownername(x):
    # NFI-01005010.png
    x = x[9:12]
    return int(x)

def get_signed_owner(x):
    # NFI-01005010.png
    x = x[4:7]
    return int(x)

def get_sample_number(x):
    # NFI-01005010.png
    x = x[7:9]
    return int(x)

def preprocess(dataset, dtype):
    if dtype == "train":
        dataset["owner_id"] = dataset["file"].apply(get_ownername)
        dataset["genuine_sample_number"] = dataset["file"].apply(get_sample_number)
        dataset["signature_owner_id"] = dataset["file"].apply(get_signed_owner)
        dataset["signed_file"] = "genuine/" + dataset["file"]
    else:
        dataset["owner_id"] = dataset["forged_file"].apply(get_ownername)
        dataset["sample_number"] = dataset["forged_file"].apply(get_sample_number)
        dataset["signature_owner_id"] = dataset["forged_file"].apply(get_signed_owner)
        dataset["signed_file"] = "forged/" + dataset["forged_file"]
    return dataset

gc.collect()

genuine_path = Path('sample_Signature/genuine')
forged_path = Path('sample_Signature/forged')

size = 224

genuine_files = [f for f in listdir(genuine_path) if isfile(join(genuine_path, f))]
forged_files = [f for f in listdir(forged_path) if isfile(join(forged_path, f))]

train_csv = pd.DataFrame(genuine_files, columns =['file'])
test_csv = pd.DataFrame(forged_files, columns =['forged_file'])

train_csv = preprocess(train_csv, "train")
test_csv = preprocess(test_csv, "test")

genuine_owner_id = train_csv["owner_id"].unique()

print(genuine_owner_id)

row_count = len(genuine_owner_id)
#row_count = 1
print(row_count)
bar = progressbar.ProgressBar(maxval=row_count, widgets=[progressbar.Bar('=', '[', ']'), ' ', progressbar.Percentage()])
bar.start()

final_data = pd.DataFrame()

for i in range(row_count):
    #id = genuine_owner_id[i]
    genuine_block = train_csv.loc[train_csv['owner_id'] == genuine_owner_id[i]]
    genuine_block_head = genuine_block[["file", "owner_id", "genuine_sample_number"]]
    genuine_block = genuine_block.drop(columns=["file"])

    genuine_block = genuine_block.rename(columns={
    "genuine_sample_number": "sample_number"}, errors="raise")

    forged_block = test_csv.loc[test_csv['owner_id'] == genuine_owner_id[i]]

    #print(forged_block['owner_id'])
    genuine_data = pd.merge(genuine_block_head, genuine_block, on=['owner_id'], how='left').drop_duplicates().reset_index(drop=True)
    genuine_data["status"] = 0
    forged_data = pd.merge(genuine_block_head, forged_block, on=['owner_id'], how='inner').drop_duplicates().reset_index(drop=True)
    forged_data["status"] = 1

    #genuine_data = genuine_data.query('owner_id == signature_owner_id and genuine_sample_number != sample_number')

    #forged_block["signed_file"] = forged_block["forged_file"].values
    #forged_block = pd.merge(genuine_block, forged_block, on=['signed_owner'], how='left').drop_duplicates().reset_index(drop=True)

    #genuine_block["signed_file"] = genuine_block["file"].values
    forged_data = forged_data.drop(columns=["forged_file"])

    final_data = pd.concat([forged_data, genuine_data, final_data], ignore_index=True)
    bar.update(i+1)
    gc.collect()

#train_csv = pd.merge(train_csv, test_csv, on=['signed_owner'], how='inner').drop_duplicates().reset_index(drop=True)
final_data = final_data.sort_values(by=['owner_id', 'genuine_sample_number', 'signature_owner_id','sample_number'])

#final_data = final_data[(final_data.owner_id != final_data.signature_owner_id) and (final_data.genuine_sample_number != final_data.sample_number) ]

#final_data = final_data.query('owner_id == signature_owner_id and genuine_sample_number != sample_number')

bar.finish()
print(final_data.head())
#print(test_csv.head())

final_data.to_csv("sample_Signature/train_final.csv", index=False)

#train_csv.to_csv("sample_Signature/train.csv", index=False)
#test_csv.to_csv("sample_Signature/test.csv", index=False)

gc.collect()


import time
from typing import Optional

import pytorch_lightning as pl
import torch
import torch.nn as nn
from sklearn.metrics import f1_score

from .cae import Conv2dAutoEncoder

class Conv2dAutoEncoderClassifier(pl.LightningModule):

    def __init__(
        self,
        in_channels: int,
        n_latent_features: int,
        cae_backbone: Optional[Conv2dAutoEncoder] = None,
        lr_mlp: float = 1e-3,
        dropout: float = .5
    ):
        super().__init__()
        self.cae = cae_backbone if cae_backbone is not None else Conv2dAutoEncoder(in_channels, n_latent_features)
        self.cae.requires_grad_(False)

        self.mlp = nn.Sequential(
            nn.Linear(25 * 25 * n_latent_features, 2048),
            nn.BatchNorm1d(2048),
            nn.Dropout(dropout),
            nn.ReLU(),
            nn.Linear(2048, 1024),
            nn.BatchNorm1d(1024),
            nn.Dropout(dropout),
            nn.ReLU(),
            nn.Linear(1024, 1024),
            nn.BatchNorm1d(1024),
            nn.Dropout(dropout),
            nn.ReLU(),
            nn.Linear(2048, 128),
            nn.BatchNorm1d(1024),
            nn.Dropout(dropout),
            nn.ReLU(),
            nn.Linear(128, 128),
            nn.Dropout(dropout),
            nn.ReLU(),
            nn.Linear(128, 1),
            nn.Sigmoid()
        )

        self.lr_mlp = lr_mlp

    def forward(self, x, meta):
        cae_embed = self.cae(x)['latent']
        cae_embed = cae_embed.reshape((cae_embed.shape[0], -1))
        return self.mlp(cae_embed)

    def predict_step(self, x, meta):
        prob = self(x, meta)
        return prob > .5

    def training_step(self, batch, batch_idx):
        x, meta, labels = batch
        out = self.predict_step(x, meta)
        loss = nn.BCELoss()(out, labels)
        f1 = f1_score(labels, out.detach().cpu().numpy())
        self.log("train_loss", loss, on_step=False, on_epoch=True, prog_bar=True)
        self.log("train_f1_score", f1, on_step=False, on_epoch=True, prog_bar=True)
        return {"loss": out['loss'], "f1_score": f1}

    def training_epoch_end(self, outputs):
        self.log("train_time", time.time() - self.time_start, prog_bar=False)

    def validation_step(self, batch, batch_idx):
        x, meta, labels = batch
        out = self.predict_step(x, meta)
        loss = nn.BCELoss()(out, labels)
        f1 = f1_score(labels, out.cpu().numpy())
        self.log("val_loss", loss, on_step=False, on_epoch=True, prog_bar=True)
        self.log("val_f1_score", f1, on_step=False, on_epoch=True, prog_bar=True)
        return {"loss": loss, "f1_score": f1}

    def validation_epoch_end(self, outputs):
        pass

    def test_step(self, batch, batch_idx: int):
        x, meta, labels = batch
        out = self.predict_step(x, meta)
        loss = nn.BCELoss()(out, labels)
        f1 = f1_score(labels, out.cpu().numpy())
        self.log("test_loss", loss, on_step=False, on_epoch=True, prog_bar=True)
        self.log("test_f1_score", f1, on_step=False, on_epoch=True, prog_bar=True)
        return {"loss": loss, "f1_score": f1}

    def test_epoch_end(self, outputs):
        pass

    def configure_optimizers(self):
        optimizer = torch.optim.Adam(self.mlp.parameters(), lr=self.lr_mlp)
        scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'max', .5, 2)

        return {
            'optimizer': optimizer,
            'scheduler': scheduler,
            'monitor': 'val_f1_score'
        }


import sys
import os
from node import BFSNode
sys.path.append(os.path.abspath('/Users/ryanbott/Projects/CMPUT204_data_structs_and_algos/DataStructures'))
sys.path.append(os.path.abspath('/Users/ryanbott/Projects/CMPUT204_data_structs_and_algos/ADT'))
from ryan_queue import Queue
from typing import List

class Edge:
    def __init__(self, start, end, weight=None):
        self.start = start
        self.end = end
        self.weight = weight

    # how can I implement eq or something so that I can check if an EDGE is IN a collection?
    # tried this below but got "unhashable type" 'Edge'
    def __eq__(self, __value: object) -> bool:
        return self.start == __value.start and self.end == __value.end

    def __hash__(self) -> int:
        return self.start * self.end

    def __repr__(self) -> str:
        return f"({self.start}, {self.end})"

class Graph:
    def __init__(self, vertices: List[BFSNode]):
        self.vertices = vertices
        vertex: BFSNode
        self.adj_list = {vertex.key: [] for vertex in vertices}
        self.adj_matrix = [[0 for vertex in vertices] for vertex in vertices]

    def add_edge(self, edge: Edge, weight=None) -> None:
        self.adj_list[edge.start].append(edge.end)
        self.adj_matrix[self.vertices.index(BFSNode(edge.start, None))][self.vertices.index(BFSNode(edge.end, None))] = 1 if weight == None else weight

    def get_adj_list(self):
        return self.adj_list

    def get_adj_matrix(self):
        return self.adj_matrix

    def bfs(self, key):
        bfs_vertices = []
        bfs_edges = []
        for v in self.vertices:
            node = BFSNode(v.key, v.data)
            node.color = 'White'
            node.dist = sys.maxsize
            bfs_vertices.append(node)
        queue = Queue()
        bfs_vertices[key].color = 'Grey'
        bfs_vertices[key].dist = 0
        # source.color = 'White'
        # source.dist = 0
        # an issue here is that the source node is living outside of the self.vertices ecosystem.
        # we need to use that particular node from the vertices. maybe we should pass in the key?
        queue.enqueue(bfs_vertices[key])
        while (not queue.is_empty()):
            origin: BFSNode = queue.dequeue()
            neighbour: BFSNode
            for neighbour in self.adj_list[origin.key]:
                neighbour = neighbour - 1
                if (bfs_vertices[neighbour].color == 'White'):
                    bfs_vertices[neighbour].color = 'Grey'
                    bfs_vertices[neighbour].dist = origin.dist + 1
                    bfs_vertices[neighbour].predecessesor = origin.key
                    bfs_edges.append(Edge(origin.key, bfs_vertices[neighbour].key))
                    queue.enqueue(bfs_vertices[neighbour])
            origin.color = 'Black'
        return bfs_edges

    def dfs_visit(self, source: BFSNode, dfs_edges: List[Edge], dfs_vertices: List[BFSNode], time):
        time = time + 1
        source.color = 'Grey'
        source.time = time
        # neighbour: Number
        for neighbour in self.adj_list[source.key]: # these are just ints
            neighbour = neighbour
            if dfs_vertices[neighbour-1].color == 'White':
                dfs_edges.append(Edge(source.key, neighbour))
                dfs_vertices[neighbour-1].predecessesor = source
                self.dfs_visit(dfs_vertices[neighbour-1], dfs_edges, dfs_vertices, time)
            source.color = 'Black'
            time = time + 1
            source.ftime = time

    def dfs(self):
        dfs_vertices = []
        dfs_edges = []
        time = 0
        for v in self.vertices:
            node = BFSNode(v.key, v.data)
            node.color = 'White'
            node.dist = sys.maxsize
            dfs_vertices.append(node)
        v: BFSNode
        for v in dfs_vertices:
            if v.color == 'White':
                self.dfs_visit(v, dfs_edges, dfs_vertices, time)
        return dfs_edges

    def print(self):
        # print adjacency matrix representation
        max_vert = len(str(len(self.vertices)))
        print(" ", end=" " * (max_vert + max_vert - 1))
        for vertex in self.vertices:
            print(vertex.key, end=" " * (max_vert))
        print()

        for i in range(len(self.vertices)):
            print(self.vertices[i].key, end=" " * (max_vert + max_vert - len(str(i+1))))
            # print(self.vertices[i].key, end=" " * (max_vert-len(str(i))))
            for j in range(len(self.vertices)):
                print(int(self.adj_matrix[i][j]), end=" " * max_vert)
            print()
        # print adjacency list representation
        for vertex, adj_list in self.adj_list.items():
            print(vertex, "->", adj_list)
        print()


import json
import time
import uuid

from common.exceptions import ApiRequestError
from common.local import get_request_username
from django.db import transaction
from django.utils.translation import ugettext as _

from dataflow.component.utils.time_util import get_datetime
from dataflow.pizza_settings import JOBNAVI_STREAM_CLUSTER_ID
from dataflow.shared.flow.flow_helper import FlowHelper
from dataflow.shared.handlers import dataflow_job, processing_udf_job
from dataflow.shared.log import stream_logger as logger
from dataflow.shared.stream.utils.resource_util import get_resource_group_id
from dataflow.shared.utils.concurrency import concurrent_call_func
from dataflow.stream.exceptions.comp_execptions import (
    DeleteJobError,
    ProcessingParameterNotExistsError,
    SubmitFlinkJobError,
)
from dataflow.stream.handlers import processing_job_info, processing_stream_info, processing_stream_job
from dataflow.stream.job.code_job import CodeJob
from dataflow.stream.job.flink_job import FlinkJob
from dataflow.stream.job.monitor_job import query_unhealth_job
from dataflow.stream.processing.processing_for_code import Processing
from dataflow.stream.settings import API_ERR_RETRY_TIMES, RECOVER_JOB_MAX_COROUTINE_NUM, ImplementType

class JobHandler(object):
    """
    job 静态表 Handler
    """

    _job_info = None
    _job = None
    _job_id = None
    is_debug = False

    def __init__(self, id, is_debug=False):
        # 初始化普通变量
        self._job_id = id
        self.is_debug = is_debug
        # 初始化 job 实例
        self._job = self._build_inner_job()

    @property
    def real_job_id(self):
        if self.is_debug:
            return self._job_id.split("__")[1]
        else:
            return self._job_id

    def _build_inner_job(self):
        """
        初始化 handler 时调用，当前共有三种 job
        @return:
        """
        if ImplementType.CODE.value == self.implement_type:
            if self.component_type_without_version in [
                "spark_structured_streaming",
                "flink",
            ]:
                return CodeJob(self._job_id, self.is_debug)
        elif self.component_type in ["flink"]:
            return FlinkJob(self._job_id, self.is_debug)
        elif self.component_type in ["storm"]:
            from dataflow.stream.extend.job.storm_job import StormJob

            return StormJob(self._job_id, self.is_debug)
        else:
            raise Exception("component_type(%s) is not defined." % self.component_type)

    @property
    def component_type_without_version(self):
        return self.component_type.split("-")[0]

    def get_job_info(self, related):
        return self._job.get_job_info(related)

    def lock(self):
        self._job.lock()

    def get_code_version(self):
        return self._job.get_code_version()

    def register(self, jar_name, geog_area_code):
        return self._job.register(jar_name, geog_area_code)

    def submit(self, conf):
        return self._job.submit(conf)

    def sync_status(self, operate_info):
        return self._job.sync_status(operate_info)

    @transaction.atomic()
    def unlock(self, entire=False):
        """
        对实时作业相关信息进行解锁
        @param entire: 是否对所有信息进行解锁
        @return:
        """
        self._job.unlock(entire)

    def cancel(self):
        return self._job.cancel()

    def force_kill(self, timeout=180):
        return self._job.force_kill(timeout)

    @property
    def component_type(self):
        return self.job_info.component_type

    @property
    def implement_type(self):
        return self.job_info.implement_type

    @property
    def job_info(self):
        if not self._job_info:
            self._job_info = processing_job_info.get(self.real_job_id)
        return self._job_info

    @property
    def deploy_mode(self):
        return self.job_info.deploy_mode

    @property
    def deploy_config(self):
        return self.job_info.deploy_config

    @property
    def code_version(self):
        return self.job_info.code_version

    @property
    def job_config(self):
        return json.loads(self.job_info.job_config)

    @transaction.atomic()
    def update_job(self, args):
        self._job.update(args)
        return self._job_id

    @transaction.atomic()
    def update_job_conf(self, args):
        self._job.update_job_conf(args)
        return self._job_id

    @transaction.atomic()
    def delete(self, is_force=False):
        """
        删除作业静态配置和动态配置
        @param is_force: 是否需先确保内部的关联关系已删除
        @return:
        """
        if not is_force:
            if processing_udf_job.exists(job_id=self.real_job_id):
                raise DeleteJobError(_("JOB 存在关联信息，不允许删除"))
            stream_job = processing_stream_job.get(self.real_job_id)
            if stream_job and stream_job.status == "running":
                raise DeleteJobError(_("JOB 正在运行，不允许删除"))
        processing_stream_job.delete(self.real_job_id)
        processing_job_info.delete(self.real_job_id)

    @classmethod
    @transaction.atomic()
    def create_job(cls, args):
        if len(args["processings"]) == 0:
            raise ProcessingParameterNotExistsError()

        # 获取1个processing来确定job的 component_type 和 implement_type
        one_processing = Processing(args["processings"][0])
        deploy_mode = one_processing.deploy_mode
        implement_type = one_processing.implement_type
        component_type = one_processing.component_type
        programming_language = one_processing.programming_language
        args["jobserver_config"] = json.dumps(args["jobserver_config"])

        job_id = "{}_{}".format(args["project_id"], str(uuid.uuid4()).replace("-", ""))

        if implement_type == "code" and component_type == "spark_structured_streaming":
            processor_logic = json.loads(one_processing.processing_info.processor_logic)
            args["job_config"]["processor_type"] = "spark_python_code"
            args["job_config"]["processor_logic"] = {}
            args["job_config"]["processor_logic"]["programming_language"] = programming_language
            args["job_config"]["processor_logic"]["user_args"] = processor_logic["user_args"]
            args["job_config"]["processor_logic"]["advanced"] = processor_logic["advanced"]
        # save processing_stream_info
        for processing in args["processings"]:
            processing_stream_info.update(processing, stream_id=job_id)
        # job_config 存储 processing 信息
        args["job_config"]["processings"] = args["processings"]

        # flow传递过来的cluster_group通过查询资源系统立即转为确切的resource_group_id并存储供后续流程使用
        jobserver_config = json.loads(args["jobserver_config"])
        geog_area_code = jobserver_config["geog_area_code"]
        cluster_group = args["cluster_group"]
        # flink code当前component_type为flink-1.10.1这种结构
        component_type_without_version = component_type.split('-')[0]
        resource_group_id = get_resource_group_id(geog_area_code, cluster_group, component_type_without_version)
        # save processing_stream_job
        processing_job_info.save(
            job_id=job_id,
            processing_type="stream",
            deploy_mode=deploy_mode,
            implement_type=implement_type,
            component_type=component_type,
            programming_language=programming_language,
            code_version=args["code_version"],
            cluster_group=resource_group_id,
            jobserver_config=args["jobserver_config"],
            job_config=json.dumps(args["job_config"]),
            deploy_config=json.dumps(args.get("deploy_config") or {}),
            created_at=get_datetime(),
            created_by=get_request_username(),
        )

        return job_id

    def generate_job_config(self):
        """
        提供给flow调用,用于uc启动时获取该job配置信息
        """
        snapshot_config = processing_stream_job.get(self.real_job_id).snapshot_config
        # 一般情况下获取的都是存储在快照中的submit时生成的uc_config配置,而不是重新生成
        if snapshot_config and not self.is_debug:
            snapshot_config = json.loads(snapshot_config)
            uc_config = snapshot_config["uc_config"]
        else:
            uc_config = self._job.generate_job_config()
        return uc_config

    def get_checkpoint(self, related, extra):
        """
        获取该job在redis中的所有checkpoint记录的信息
        """
        return self._job.get_checkpoint(related, extra)

    def reset_checkpoint_on_migrate_kafka(self, head_rt):
        """
        迁移RT所在kafka时重置redis中RT对应的相关checkpoint
        """
        return self._job.reset_checkpoint_on_migrate_kafka(head_rt)

    def transform_checkpoint(self):
        """
        将该job在redis中的所有checkpoint记录的信息从storm格式转换为flink格式
        """
        return self._job.transform_checkpoint()

    def transform_config(self):
        """
        将该job配置从storm转换为flink配置
        """
        return self._job.transform_config()

    def revert_storm(self):
        """
        转换配置失败时候回滚为storm类型
        """
        self._job.revert_storm()

    def migrate(self, step_id):
        """
        转换配置失败时候回滚为storm类型
        """
        result = {}
        flow_id = list(dataflow_job.filter(job_id=self._job_id).only("flow_id").values()).first()["flow_id"]
        if step_id == 1:
            # 步骤一：停止stream任务
            result["step_1_stop"] = FlowHelper.op_flow_task(flow_id, "stop")
        elif step_id == 2:
            # 步骤二：转换checkpoint
            result["step_2_checkpoint"] = self.transform_checkpoint()
        elif step_id == 3:
            # 步骤三：转换配置
            result["step_3_config"] = self.transform_config()
        elif step_id == 4:
            # 步骤四：启动任务
            result["step_4_start"] = FlowHelper.op_flow_task(flow_id, "start")
        elif step_id == 5:
            # 全部步骤都执行
            result["step_1_stop"] = FlowHelper.op_flow_task(flow_id, "stop")
            result["step_2_checkpoint"] = self.transform_checkpoint()
            result["step_3_config"] = self.transform_config()
            result["step_4_start"] = FlowHelper.op_flow_task(flow_id, "start")
        return result

    @transaction.atomic()
    def update_stream_conf(self, params):
        self._job.update_stream_conf(params)
        return self._job_id

    @classmethod
    @transaction.atomic()
    def create_single_processing_job(cls, args):
        processor_args, job_info_args = cls.build_single_processing_param(args)
        # 1. create processing
        processing_stream_info.save(
            processing_id=args["processings"][0],
            processor_type="code",
            processor_logic=json.dumps(processor_args),
            component_type=args["component_type"],
            created_at=get_datetime(),
            created_by=get_request_username(),
            description="",
        )
        # 2. create job_info
        job_id = JobHandler.create_job(job_info_args)
        return job_id

    def update_single_processing_job(self, args):
        processor_args, job_info_args = self.build_single_processing_param(args)
        # 1. create processing
        processing_stream_info.update(
            processing_id=args["processings"][0],
            processor_type="code",
            processor_logic=json.dumps(processor_args),
            component_type=args["component_type"],
            created_at=get_datetime(),
            created_by=get_request_username(),
            description="",
        )
        # 2. create job_info
        job_id = self.update_job(job_info_args)
        return job_id

    @classmethod
    def build_single_processing_param(cls, args):
        heads = args["job_config"]["heads"].split(",")
        tails = args["job_config"]["tails"].split(",")
        processor_args = {
            "static_data": [],
            "input_result_tables": heads,
            "output_result_tables": tails,
            "code": "",
        }
        processor_args.update(args["job_config"]["processor_logic"])
        job_info_args = {
            "project_id": args["project_id"],
            "code_version": "",
            "deploy_config": "",
            "jobserver_config": {
                "geog_area_code": args["tags"][0],
                "cluster_id": JOBNAVI_STREAM_CLUSTER_ID,
            },
            "cluster_group": args["cluster_group"],
            "processings": args["processings"],
            "job_config": {
                "tails": args["job_config"]["tails"],
                "heads": args["job_config"]["heads"],
                "concurrency": args["deploy_config"]["resource"]["concurrency"],
                "offset": args["job_config"]["offset"],
            },
        }
        return processor_args, job_info_args

    def start_stream(self):
        """
        支持单个spark structured streaming processing任务提交
        兼容从flow迁移过来的AIops任务提交接口
        """
        jar_name = self.get_code_version()
        jobserver_config = json.loads(self.job_info.jobserver_config)
        geog_area_code = jobserver_config["geog_area_code"]
        # 1. 注册stream_job
        submit_params = self._job.register(jar_name, geog_area_code)
        # 2. 提交任务
        try:
            operate_info = self._job.submit(submit_params)
        except Exception as e:
            logger.exception(e)
            raise SubmitFlinkJobError("submit job (%s) failed." % self._job_id)
        operate_info["job_id"] = self._job_id
        return operate_info

    def recover(self, timeout=API_ERR_RETRY_TIMES):
        """
        快速恢复任务运行
        提交任务使用snapshot_config中cache的配置(submit_config)
        获取配置使用snapshot_config中cache的配置(uc_config)
        """
        # step-1: kill任务并删除schedule
        self.force_kill(timeout)
        # step-2: 重新提交任务
        snapshot_config = processing_stream_job.get(self._job_id).snapshot_config
        snapshot_config = json.loads(snapshot_config)
        submit_config = snapshot_config["submit_config"]
        # sometimes the jobnavi's gunicorn may appear 502 response, just retry after sleep
        for i in range(timeout):
            result = None
            try:
                result = self._job.submit(submit_config, is_recover=True)
            except ApiRequestError as e1:
                logger.error(
                    "[recover session] force kill flink job %s call ApiRequestError exception, details is : %s"
                    % (self.session_config["cluster_name"], e1.message)
                )
            if result:
                return self._job_id
            time.sleep(1)
        return None

    def fill_snapshot(self):
        """
        生成并填充快照配置
        """
        jar_name = self.get_code_version()
        jobserver_config = json.loads(self.job_info.jobserver_config)
        geog_area_code = jobserver_config["geog_area_code"]
        submit_params = self._job.register(jar_name, geog_area_code, is_fill_snapshot=True)
        snapshot_config = {
            "submit_config": submit_params,
            "uc_config": self.generate_job_config(),
        }
        processing_stream_job.update(self._job_id, snapshot_config=json.dumps(snapshot_config))
        return snapshot_config

    @classmethod
    def recover_jobs(cls, job_id_list, concurrency):
        if not concurrency:
            concurrency = RECOVER_JOB_MAX_COROUTINE_NUM
        if not job_id_list:
            # cluster_job + session_job 一起恢复
            unhealth_jobs = query_unhealth_job()
            job_id_list = []
            job_id_list = job_id_list + unhealth_jobs["yarn_cluster_job"]
            job_id_list = job_id_list + unhealth_jobs["yarn_session_job"]
            job_id_list = job_id_list + unhealth_jobs["flink_streaming_code_job"]
            job_id_list = job_id_list + unhealth_jobs["spark_structured_streaming_code_job"]
        res_list = []
        func_info = []
        for job_id in job_id_list:
            func_info.append([JobHandler(job_id).recover, {}])

        for segment in [func_info[i : i + concurrency] for i in range(0, len(func_info), concurrency)]:
            threads_res = concurrent_call_func(segment)
            for segment_index, res_info in enumerate(threads_res):
                if res_info:
                    res_list.append(res_info)
        return res_list


import os
import hashlib
import re
from collections import OrderedDict

from annotation_manager.importer import AnnotationImporterFactory, AnnotationImporter
from annotation_manager.common_representation import AnnotatedDocument, DocumentLibrary, AnnotationSet
from pdfloc_converter.pdfloc import PDFLocPair

class AnnotationStorage(object):
    _annotations_dir = None

    def __init__(self, annotations_dir):
        super(AnnotationStorage, self).__init__()

        self._annotations_dir = annotations_dir

    def get_annotation_file(self, document_file):
        assert isinstance(document_file, PocketbookAnnotatedDocument)

        return "%s%s%s_A_%s.html" % (self._annotations_dir, os.sep, document_file.filename, document_file.file_hash)

    def get_annotation_dir(self, document_file):
        assert isinstance(document_file, PocketbookAnnotatedDocument)

        return "%s%s%s_A_%s.files" % (self._annotations_dir, os.sep, document_file.filename, document_file.file_hash)

class PocketbookDocumentLibrary(DocumentLibrary):

    _system_drive_path = None
    _external_drive_path = None
    _annotations_dir = None

    _annotation_storage = None
    _document_roots = []

    def __init__(self, system_drive_path, annotations_dir, external_drive_path=None):
        super(PocketbookDocumentLibrary, self).__init__()

        self._system_drive_path = system_drive_path
        self._external_drive_path = external_drive_path
        self._annotations_dir = annotations_dir

        self._annotation_storage = AnnotationStorage(self._annotations_dir)

        self._document_roots.append(self._system_drive_path)
        if self._external_drive_path is not None:
            self._document_roots.append(self._external_drive_path)

        self._seek_for_documents()

    def _seek_for_documents(self):
        documents = OrderedDict()

        for root in self._document_roots:
            for dirpath, _, filenames in os.walk(root):
                for filename in filenames:
                    if filename.endswith(".pdf") or filename.endswith(".PDF"):
                        document = PocketbookAnnotatedDocument(filename, dirpath, self._annotation_storage)
                        documents[document.full_path] = document

        self.add_documents(**documents)

class PocketbookAnnotatedDocument(AnnotatedDocument):

    _dir = None
    _annotation_storage = None

    _file_hash = None

    _pdfloc_regex = r'


import logging
import sys

logger = logging.getLogger(__name__)

def log_exceptions(cls):
    """
    This decorator is meant to decorate management commands.
    Any exceptions raised in the command's handle method will be logged and re-raised.
    Can be replaced if https://code.djangoproject.com/ticket/27877 gets implemented.
    """
    class NewClass(cls):
        def handle(self, *args, **options):
            try:
                super().handle(args, options)
            except Exception:
                logger.exception("Management command '{}' failed. Traceback follows: ".format(sys.argv[1]))
                raise
    return NewClass


from django.shortcuts import render
import requests
from django.http import HttpResponse

from .models import Greeting
import instaloader

# Create your views here.
def index(request):
    # return HttpResponse('Hello from Python!')
    # return render(request, "index.html")
    # r = requests.get('https://peaceful-gunnison-77284.herokuapp.com/')
    # print(r.text)
    # return HttpResponse('
' + r.text + '
')
    loader = instaloader.Instaloader()
    loader.load_session_from_file("_sri.vat.san_","mysession")
    profile = instaloader.Profile.from_username(loader.context,"_sri.vat.san_")
    followers = profile.get_followers()
    followees = profile.get_followees()

    namestr = "Profile Name " + "_sri.vat.san_" + " \n"
    followeestr = "Followees " + str(followees.count) + " \n"
    followerstr = "Followers " + str(followers.count) + " \n"

    return HttpResponse('
' + namestr + followerstr + followeestr + '
')

def db(request):
    greeting = Greeting()
    greeting.save()

    greetings = Greeting.objects.all()

    return render(request, "db.html", {"greetings": greetings})


import os
import pathlib
import re

import psutil

class FilesystemGeneric:
    @staticmethod
    def escape_filename(filename):
        replacements = [
            (re.compile(r"[/:]"), r"_"),
            (re.compile(r"[\?]"), r"!"),
        ]

        for regexp_replace, regexp_substit in replacements:
            filename = re.sub(regexp_replace, regexp_substit, filename)

        return filename

class FilesystemExFAT(FilesystemGeneric):
    ...

_fsname_klass = {
    "exfat": FilesystemExFAT,
}

def get_fs_class(path):
    path = pathlib.Path(path).resolve(strict=True)

    partitions = {}

    for part in psutil.disk_partitions():
        partitions[pathlib.Path(part.mountpoint)] = part.fstype

    if path in partitions:
        if partitions[path] in _fsname_klass:
            return _fsname_klass[partitions[path]]
        else:
            return FilesystemGeneric

    while path != pathlib.Path("/"):
        path = path.parent

        if path in partitions:
            if partitions[path] in _fsname_klass:
                return _fsname_klass[partitions[path]]
            else:
                return FilesystemGeneric

    return FilesystemGeneric


from collections import deque
T = int(input())
for tc in range(1, T+1):
    F, N = map(int, input().split())
    arr = list(map(int, input().split()))
    n = len(arr)
    arr = deque(arr)
    cnt = 0
    cnt_turn = 0
    flag = -1
    dic = {}
    for i in range(n):
        if arr[i] in dic:
            dic[arr[i]] += 1
        else:
            dic[arr[i]] = 1
    max_num = sorted(dic.items(), key=lambda x:x[1], reverse=True)[0][0]
    while cnt != F-1:
        if cnt_turn % n == 0:
            flag = 0
            cnt_turn = 0
            n = len(arr)
        if dic[arr[0]] == dic[max_num]:
            if flag == 0:
                flag = 1
                dic[arr[0]] -= 1
                sorted_arr = sorted(dic.items(), key=lambda x:x[1], reverse=True)
                max_num = sorted_arr[0][0]
                for j in range(1, len(sorted_arr)):
                    if sorted_arr[j][1] == 0:
                        continue
                    else:
                        break
                else:
                    arr.popleft()
                    break
                arr.popleft()
            else:
                arr.append(arr.popleft())
        else:
            arr.append(arr.popleft())
        cnt += 1
        cnt_turn += 1
    print(f'#{tc} {arr[0]}')


import socket
import datetime

SERVER_PORT = 9965

PROTOCOL_VER = "001"

SERVER_MESSAGE = b"HeartBeatSenderHere" + PROTOCOL_VER
CLIENT_MESSAGE = b"HeartBeatRecHere" + PROTOCOL_VER

############ find server ip and port ###############
sock = socket.socket(socket.AF_INET, # Internet
                     socket.SOCK_DGRAM) # UDP
sock.bind(("0.0.0.0", SERVER_PORT))
(msg, (server_ip, server_port)) = sock.recvfrom(1024)
sock.close()

if msg != SERVER_MESSAGE:
    print("not a valid server package:", server_ip, server_port)
    print(msg)
    exit(-1)

print("server at ", server_ip, server_port)

############# access data from server ###############
sock = socket.socket(socket.AF_INET, # Internet
                     socket.SOCK_DGRAM) # UDP

dev_infos = dict()
heart_beat_package_sent_time = datetime.datetime.now() - datetime.timedelta(seconds=1000)

def print_infos():
    import os
    os.system("cls")
    print("last heartbeat package: ", heart_beat_package_sent_time.isoformat())
    for (name, addr, heartbeat, time) in dev_infos.values():
        print("%20s" % name, "|", addr, "|", "% 3d" % heartbeat, "|", time.isoformat())

while True:
    ##### tell server where are you, you need repeat it no more than one minutes again #####
    if (datetime.datetime.now() - heart_beat_package_sent_time).total_seconds() > 20:
        sock.sendto(CLIENT_MESSAGE, (server_ip, server_port))
        heart_beat_package_sent_time = datetime.datetime.now()
        # print("send alive package")

    ##### receive and parse data from server #####
    (msg, (server_ip, server_port)) = sock.recvfrom(1024)
    ####################### msg format ##########################
    # name '\0' bluetooth_device_mac_addr '\0' heartbeat(4byte) #
    #############################################################
    i_name = 0
    i_addr = msg.index(b'\0')+1
    i_heartbeat = msg.index(b'\0', i_addr) + 1

    name = msg[:i_addr-1].decode()
    addr = msg[i_addr:i_heartbeat-1].decode()
    heartbeat = int.from_bytes(msg[-4:])

    dev_infos[addr] = (name, addr, heartbeat, datetime.datetime.now())
    print_infos()


# Inspired by < https://www.pyimagesearch.com/2018/07/09/face-clustering-with-python/ >

from sklearn.cluster import DBSCAN
from .path import Path
from .util import force_mkdir
import logging
import numpy as np
import shutil

class Clusterer:
    def cluster_encodings(self, encodings):
        logging.debug(f"Clustering {len(encodings)} encodings")

        clt = DBSCAN(
            metric = "euclidean"
        )

        # Generate a list so we also have an index
        faces = [ { "file" : k, "encoding" : v } for k, v in encodings.items()]
        faces_encodings = [ i["encoding"] for i in faces ]

        clt.fit(np.array(faces_encodings))
        output = []

        for fid in np.unique(clt.labels_):
            if fid < 0:
                # Skip outliers
                continue

            fid_items = np.where(clt.labels_ == fid)

            output.append({
                "count" : len(fid_items[0]),
                "files" : [ faces[index]["file"] for index in fid_items[0]],
                "id" : int(fid)
            })

        return output

    def move_files(self, clusters, directory):
        for cluster in clusters:
            cluster_id = cluster["id"]
            cluster_path = Path(directory) / str(cluster_id)
            force_mkdir(cluster_path)

            for path in cluster["files"]:
                new_path = Path(cluster_path) / Path(path).name
                logging.debug(f"Copying '{path}' to '{new_path}'")
                shutil.copy(path, new_path)


import pandas as pd
import json
import matplotlib.pyplot as plt

df = pd.DataFrame({
    'nome' : ['Vinicius', 'Gustavo', 'Chrisley', 'João'],
    'idade' : [19, 19, 22, 50],
    'genero': ['M', 'M', 'M', 'M'],
    'estado': ['MG', 'MG', 'MG', 'MG'],
    'num_filhos': [1, 2, 3, 0],
    'num_pets': [4, 2, 3, 1]
})

df

df.plot(kind="line", x="nome", y="idade", color="green")
plt.show()


# Aufgabe 6.2
# ===========

from copy import deepcopy
from typing import Dict

def nextranking(ranking: Dict[str, float],
                network: Dict[str, set]) -> Dict[str, float]:
    """Returns next ranking for all websites based on network.

    Args:
        ranking (dict[str, float]): The current Ranking of the websites.
        network (dict[str, set[str]): Websites internal linkage.

    Returns:
        dict[str, float]: the new ranking of the current network

    """

    # 85%: Verlinkte Website.
    # 15%: Beliebige Website in network.

    # j,i: Webseiten
    # n seiten von i verlinkt.
    # m: Anzahl aller Seiten im Netzwerk
    # pij = 0.85/n + 0.15/m
    # xij = ranking[i] * pij(i, network)
    # neues ranking (Seite j): Σxij über alle i

    def p(i, j):
        n = len(network[i])
        m = len(network)
        if n == 0:
            return 1/m
        elif j in network[i]:
            return 0.85/n + 0.15/m
        else:
            return 0.15/m

    def x(i, j):
        return ranking[i] * p(i, j)

    nranking = {}

    for j in network:
        for i in network:
            nranking[j] = nranking.get(j, 0) + x(i, j)

    return nranking

def ranking(network: Dict[str, set]) -> Dict[str, float]:
    """Iteratively returns the ranking for the given interlinked network.

    Args:
        network (Dict[str, set]): The known network on which the user will be

    Returns:
        rating (Dict[str, float]): The probabilities of the user being
                                    on each of the sites.

    """

    def isdiff(r1, r2):
        for k in r1.keys():
            if abs(r1[k] - r2[k]) > 1e-6:
                return True
        return False

    frank = {e: 1/len(network) for e in network.keys()}
    srank = nextranking(frank, network)

    while isdiff(frank, srank):
        frank = srank
        srank = nextranking(frank, network)
    return srank


import torch
import numpy as np
from PIL import Image
import glob
import pandas as pd
import cv2
from config import config as cfg
from Data_handler import Tiny

'''# Here we are isolating only the correct valition images from the dataset.
# The dataset has all 200 classes in one folder so this is one way to
# seperate them out
current_classes = glob.glob(cfg.dataset_dir + '/train/*')
for i in range(len(current_classes)):
    current_classes[i] = current_classes[i].split('/')[-1]
val = cfg.dataset_dir + '/val_annotations.txt'
df = pd.read_csv(val, sep='\t', header=None, names=['image', 'im_class', 'None', 'None', 'None', 'None'])
df2 = df[df.im_class.isin(current_classes)]
'''
# Device configuration
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

# Load datasets and latent features
test_dataset = Tiny(root_dir=cfg.dataset_dir + '/val/images', mode='test')
train_dataset = Tiny(root_dir=cfg.dataset_dir + '/train', mode='train')
train_latents = np.load(cfg.train_save_dir)
test_latents = np.load(cfg.test_save_dir)

len_train = len(train_latents)
for i in range(len_train):
    idx = np.random.randint(len_train)
    idx = i
    train_latent = train_latents[idx]
    print('Query image used is:')
    query,_ = train_dataset._sample(idx)
    img = cv2.imread(query)

    print(train_dataset._sample(idx))

    root_sqr_err = (train_latents - train_latent) ** 2
    root_sqr_err = root_sqr_err.sum(axis=1)
    print((np.sort(root_sqr_err) ** 0.5)[:5])
    idx_closest_neighbor = root_sqr_err.argsort()[:5]
    print('Closest images are:')
    for j in idx_closest_neighbor:
        print(train_dataset._sample(j)[0])
        comparison_img = train_dataset._sample(j)[0]
        comparison = cv2.imread(comparison_img)
        img = cv2.hconcat((img, comparison))

    shape = img.shape
    img = cv2.resize(img, (shape[1]*3, shape[0]*3))

    cv2.imshow('image', img)
    cv2.waitKey(0)
    cv2.destroyAllWindows()


# importing required modules
import PyPDF2
from transformers import TFPegasusForConditionalGeneration, PegasusTokenizer
from transformers import pipeline
model_name = "google/pegasus-xsum"
pegasus_tokenizer = PegasusTokenizer.from_pretrained(model_name)
pegasus_model = TFPegasusForConditionalGeneration.from_pretrained(model_name,from_tf=True)
summarizer = pipeline("summarization",model = model_name,tokenizer = pegasus_tokenizer,framework = "pt")

temp = ""
result = []
cname = "Tesla" #Enter company name as given in the database after scraping
year = "2020" #Enter the year (Need to purchase to get latest report)
option = 1 #0 for responsibility report and 1 for annual report
if option==1:
    rep = "AnnualReport"
else:
    rep = "ResponsibilityReport"
filename = cname+rep+year+".pdf"
# creating a pdf file object
pdfFileObj = open(filename, 'rb') #Put the report name

# creating a pdf reader object
pdfReader = PyPDF2.PdfFileReader(pdfFileObj)

# printing number of pages in pdf file
n = pdfReader.numPages
print(pdfReader.numPages)

for i in range (3,n):
    # creating a page object
    pageObj = pdfReader.getPage(i)
    # extracting text from page
    #print(pageObj.extractText())
    temp = pageObj.extractText()
    result.append(temp.split('\n'))#The resultant list consists of lists of each line of a page
    break
    #Run each line into the zero shot model

# closing the pdf file object
pdfFileObj.close()
#print(result)

Result = []
tempo = []
for i in result:
    for j in i:
        hmm = str(j).replace('\t',' ')
        tempo.append(hmm)
    Result.append(tempo)
    tempo = []

#print (Result)
#print(len(result[0]))
tempstr = ""
pages = []
for i in Result:
    for j in i:
        tempstr = tempstr + " " + str(j)
    pages.append(tempstr)
    tempstr = ""

print (pages)
summaries = []
for p in pages:
    summary = summarizer(p,min_length = 30,max_length = 100)
    summaries.append(summary)

print(summaries)


from typing import List

class Solution:
    """
    Given an array of integers nums and an integer target,
    return indices of the two numbers such that they add up to target.
    You may assume that each input would have exactly one solution, and you may not use the same element twice.
    You can return the answer in any order.
    """
    def twoSum(self, nums: List[int], target: int) -> List[int]:
        hashmap = {}
        # Loop all the array and add to the dictionary the number as key and index as value
        for indx, num in enumerate(nums):
            hashmap[num] = indx

        # Substract the number to the target and check if that complement is already in the hashmap
        for indx, num in enumerate(nums):
            complement = target - num
            if complement in hashmap and hashmap[complement] != indx:
                return [indx, hashmap[complement]]

solution = Solution()
print(solution.twoSum(nums=[2,7,11,15], target=9))

# Time complexity = O(n)
# Space complexity = O(n)

# Example 1:

# Input: nums = [2,7,11,15], target = 9
# Output: [0,1]
# Explanation: Because nums[0] + nums[1] == 9, we return [0, 1].

# Example 2:

# Input: nums = [3,2,4], target = 6
# Output: [1,2]

# Example 3:

# Input: nums = [3,3], target = 6
# Output: [0,1]


import logging

from flask_restx import Namespace, Resource, cors

from app.services.citybike_service import CityBikeService

_log = logging.getLogger(__name__)

citybike_ns = Namespace(
    "citybike", description="Citybike related operations", decorators=[cors.crossdomain(origin="*")],
)

@citybike_ns.route("/")
class CityBikeListRest(Resource):
    def get(self):
        try:
            result = CityBikeService.get()

            return result, 200, {"content-type": "application/json"}

        except Exception as e:
            _log.warning(e)
            return {"message": str(e)}, 500, {"content-type": "application/json"}


from typing import (Any, Dict, List, MutableMapping,
    Optional, Sequence, Set, Text)

from django.conf import settings

from zerver.lib.bugdown import (
    version as bugdown_version,
    url_embed_preview_enabled_for_realm
)

from zerver.lib.message import (
    MessageDict,
)

from zerver.lib.stream_topic import StreamTopicTarget
from zerver.models import  Stream,Recipient, Message, get_system_bot

from django.db import transaction, IntegrityError, connection
from zerver.lib.queue import queue_json_publish
from zerver.lib import bugdown
from zerver.tornado.event_queue import  send_event
from collections import defaultdict

from zerver.lib.actions import get_recipient_info, render_incoming_message, create_user_messages,get_service_bot_events,\
    bulk_insert_ums,send_welcome_bot_response,do_claim_attachments,get_active_presence_idle_user_ids

def do_send_table(messages_maybe_none: Sequence[Optional[MutableMapping[str, Any]]],
                     email_gateway: Optional[bool]=False) -> List[int]:
    messages = [message for message in messages_maybe_none if message is not None]
    print(1)
    print(messages)
    # Filter out zephyr mirror anomalies where the message was already sent
    already_sent_ids = []  # type: List[int]
    new_messages = []  # type: List[MutableMapping[str, Any]]
    for message in messages:
        print(2)
        print(message)
        if isinstance(message['message'], int):
            already_sent_ids.append(message['message'])
        else:
            new_messages.append(message)
    messages = new_messages

    links_for_embed = set()  # type: Set[Text]

    for message in messages:
        print(3)
        print(message)
        message['rendered_content'] = message.get('rendered_content', None)
        message['stream'] = message.get('stream', None)
        message['local_id'] = message.get('local_id', None)
        message['sender_queue_id'] = message.get('sender_queue_id', None)
        message['realm'] = message.get('realm', message['message'].sender.realm)

        mention_data = bugdown.MentionData(
            realm_id=message['realm'].id,
            content=message['message'].content,
        )
        print(4)
        message['mention_data'] = mention_data

        if message['message'].is_stream_message():
            stream_id = message['message'].recipient.type_id
            stream_topic = StreamTopicTarget(
                stream_id=stream_id,
                topic_name=message['message'].topic_name()
            )  # type: Optional[StreamTopicTarget]
        else:
            stream_topic = None

        info = get_recipient_info(
            recipient=message['message'].recipient,
            sender_id=message['message'].sender_id,
            stream_topic=stream_topic,
            possibly_mentioned_user_ids=mention_data.get_user_ids(),
        )

        message['active_user_ids'] = info['active_user_ids']
        message['push_notify_user_ids'] = info['push_notify_user_ids']
        message['stream_push_user_ids'] = info['stream_push_user_ids']
        message['um_eligible_user_ids'] = info['um_eligible_user_ids']
        message['long_term_idle_user_ids'] = info['long_term_idle_user_ids']
        message['default_bot_user_ids'] = info['default_bot_user_ids']
        message['service_bot_tuples'] = info['service_bot_tuples']

        # Render our messages.
        assert message['message'].rendered_content is None

        rendered_content = render_incoming_message(
            message['message'],
            message['message'].content,
            message['active_user_ids'],
            message['realm'],
            mention_data=message['mention_data'],
            email_gateway=email_gateway,
        )
        message['message'].rendered_content = rendered_content
        message['message'].rendered_content_version = bugdown_version
        links_for_embed |= message['message'].links_for_preview

        # Add members of the mentioned user groups into `mentions_user_ids`.
        mention_data = message['mention_data']
        for group_id in message['message'].mentions_user_group_ids:
            print(message['message'].mentions_user_group_ids)
            members = message['mention_data'].get_group_members(group_id)
            message['message'].mentions_user_ids.update(members)

        mentioned_user_ids = message['message'].mentions_user_ids
        default_bot_user_ids = message['default_bot_user_ids']
        mentioned_bot_user_ids = default_bot_user_ids & mentioned_user_ids
        message['um_eligible_user_ids'] |= mentioned_bot_user_ids

        # Update calculated fields of the message
        message['message'].update_calculated_fields()

    # Save the message receipts in the database
    user_message_flags = defaultdict(dict)  # type: Dict[int, Dict[int, List[str]]]
    with transaction.atomic():
        Message.objects.bulk_create([message['message'] for message in messages])
        ums = []  # type: List[UserMessageLite]
        for message in messages:
            print(6)
            print(messages)
            # Service bots (outgoing webhook bots and embedded bots) don't store UserMessage rows;
            # they will be processed later.
            mentioned_user_ids = message['message'].mentions_user_ids
            user_messages = create_user_messages(
                message=message['message'],
                um_eligible_user_ids=message['um_eligible_user_ids'],
                long_term_idle_user_ids=message['long_term_idle_user_ids'],
                mentioned_user_ids=mentioned_user_ids,
            )

            for um in user_messages:
                user_message_flags[message['message'].id][um.user_profile_id] = um.flags_list()

            ums.extend(user_messages)

            message['message'].service_queue_events = get_service_bot_events(
                sender=message['message'].sender,
                service_bot_tuples=message['service_bot_tuples'],
                mentioned_user_ids=mentioned_user_ids,
                active_user_ids=message['active_user_ids'],
                recipient_type=message['message'].recipient.type,
            )

        bulk_insert_ums(ums)

        # Claim attachments in message
        for message in messages:
            print(7)
            if Message.content_has_attachment(message['message'].content):
                do_claim_attachments(message['message'])

    for message in messages:

        print(8)
        wide_message_dict = MessageDict.wide_dict(message['message'])

        user_flags = user_message_flags.get(message['message'].id, {})
        sender = message['message'].sender
        message_type = wide_message_dict['type']

        presence_idle_user_ids = get_active_presence_idle_user_ids(
            realm=sender.realm,
            sender_id=sender.id,
            message_type=message_type,
            active_user_ids=message['active_user_ids'],
            user_flags=user_flags,
        )

        event = dict(
            type='message',
            message=message['message'].id,
            message_dict=wide_message_dict,
            presence_idle_user_ids=presence_idle_user_ids,
        )
        user_ids = message['active_user_ids'] | set(user_flags.keys())

        users = [
            dict(
                id=user_id,
                flags=user_flags.get(user_id, []),
                always_push_notify=(user_id in message['push_notify_user_ids']),
                stream_push_notify=(user_id in message['stream_push_user_ids']),
            )
            for user_id in user_ids
        ]

        if message['message'].is_stream_message():
            if message['stream'] is None:
                stream_id = message['message'].recipient.type_id
                message['stream'] = Stream.objects.select_related("realm").get(id=stream_id)
            assert message['stream'] is not None  # assert needed because stubs for django are missing
            if message['stream'].is_public():
                event['realm_id'] = message['stream'].realm_id
                event['stream_name'] = message['stream'].name
            if message['stream'].invite_only:
                event['invite_only'] = True
        if message['local_id'] is not None:
            event['local_id'] = message['local_id']
        if message['sender_queue_id'] is not None:
            event['sender_queue_id'] = message['sender_queue_id']
        send_event(event, users)

        if url_embed_preview_enabled_for_realm(message['message']) and links_for_embed:
            event_data = {
                'message_id': message['message'].id,
                'message_content': message['message'].content,
                'message_realm_id': message['realm'].id,
                'urls': links_for_embed}
            queue_json_publish('embed_links', event_data)

        if (settings.ENABLE_FEEDBACK and settings.FEEDBACK_BOT and
                message['message'].recipient.type == Recipient.PERSONAL):

            feedback_bot_id = get_system_bot(email=settings.FEEDBACK_BOT).id
            if feedback_bot_id in message['active_user_ids']:
                queue_json_publish(
                    'feedback_messages',
                    wide_message_dict,
                )

        if message['message'].recipient.type == Recipient.PERSONAL:
            welcome_bot_id = get_system_bot(settings.WELCOME_BOT).id
            if (welcome_bot_id in message['active_user_ids'] and
                    welcome_bot_id != message['message'].sender_id):
                send_welcome_bot_response(message)

        for queue_name, events in message['message'].service_queue_events.items():
            for event in events:
                queue_json_publish(
                    queue_name,
                    {
                        "message": wide_message_dict,
                        "trigger": event['trigger'],
                        "user_profile_id": event["user_profile_id"],
                    }
                )
    return already_sent_ids + [message['message'].id for message in messages]


import logging
import boto3
from botocore.exceptions import ClientError

def uploadFileS3():

    client = boto3.client('s3')

    conf = input("YES/NO: ")

    while (conf == "YES"):

        bucketName = input("Ingrese el nombre del bucket en el cual desea subir el archivo:   ")
        path = input("Ruta del archivo para subir:  ")
        outputName = input("Nombre de archivo para guardar en el bucket:  ")
        try:
            client.upload_file(path, bucketName, outputName)
            print("El archivo fue cargado con ")
        except ClientError as e:
           logging.error(e)
           print("El archivo no ha sido subido con exito")

def downloadFileS3():

    client = boto3.client('s3')

    conf = input("YES/NO: ")

    while (conf == "YES"):

        bucketName = input("Ingrese el nombre del bucket del cual desea descargar el archivo:   ")
        path = input("Ruta del archivo para descargar:  ")
        saveName = input("Nombre de archivo para descargar del bucket: ")
        try:
            client.download_file(bucketName, path, saveName)
            print("El archivo ha sido descargado")
        except ClientError as e:
            logging.error(e)
            print("No se pudo descargar el archivo")

if __name__ == "__main__":

    print("¿Cargar un archivo a un bucket?")
    uploadFileS3()
    print("¿Descargar un archivo desde un bucket?")
    downloadFileS3()


import numpy as np
import cv2
from ast import literal_eval
from os import makedirs
from os.path import exists, join
from shutil import rmtree
from CharucoBoardToSVG.arucoUtililties import toDict
import json

# Class for loading JSON values

class ReaderJSON:
    def __init__(self,path):
        file = open(path)
        self.dict = json.load(file)
        file.close()

    def hasKey(self,key):
        return key in self.dict.keys()

    def getNodeVal(self, name, dtype=str):
        return dtype(self.dict[name])

    def getNodeMat(self, name):
        return np.array(self.dict[name])

# Reading charuco board json
def readCAUB(path):
    reader = ReaderJSON(path)
    squaresX = reader.getNodeVal("squaresX")                # "14",
    squaresY = reader.getNodeVal("squaresY")                # "20",
    squareLength = reader.getNodeVal("squareLength")      # "0.02",
    markerLength = reader.getNodeVal("markerLength")      # "0.016",
    dictionary = toDict(reader.getNodeVal("dictionary"))        # "DICT_5X5_1000"

    return [squaresX, squaresY, squareLength, markerLength, dictionary]

# Create charuco board object based on path
def createCAUB(path)->cv2.aruco_CharucoBoard:
    squaresX, squaresY, squareLength, markerLength, dictionary = readCAUB(path)
    return cv2.aruco.CharucoBoard_create( squaresX, squaresY, squareLength, markerLength, dictionary )

# Read camera calibration matrix and distortion coefficients from json file
def readCC(path):
    readerCAM = ReaderJSON(path)
    cameraMatrix = readerCAM.getNodeMat('camera_matrix')
    distCoeffs = readerCAM.getNodeMat('distortion_coefficients')
    return cameraMatrix, distCoeffs

# Make a new directory if force is true it will overwrite without prompting
def make_clean_folder(path_folder,force=True):
    if not exists(path_folder):
        makedirs(path_folder)
    else:
        if not force:
            user_input = input("%s not empty. Overwrite? (y/n) : " % path_folder)
            if user_input.lower() == 'y':
                rmtree(path_folder)
                makedirs(path_folder)
            else:
                exit()
        else:
            rmtree(path_folder)
            makedirs(path_folder)


import mimetypes
import shutil
from zipfile import ZipFile

import os
from django.conf import settings
from django.core.files.base import ContentFile
from django.http import Http404, HttpResponse
from django.shortcuts import render, redirect

from apps.extensions.forms import ExtensionUploadForm
from apps.extensions.models import Category, Extension
from apps.extensions.utils import get_upload_path, EXTENSION_JSON

def get_extension(request, extension_id, run=False):
    # Remove this block on August 1, 2017
    if extension_id in ['about', 'develop', 'faq', 'news', 'participants']:
        return redirect('/' + extension_id, permanent=True)

    categories = Category.get_categories()
    context = {
        'title': 'Unknown: ' + extension_id,
        'language': request.LANGUAGE_CODE,
        'categories': categories,
        'run': run,
    }

    try:
        extension = Extension.objects.prefetch_related('categories').get(pk=extension_id)
        context['extension'] = extension
        context['title'] = extension.name
        try:
            category = extension.get_primary_category()
            context['active_category'] = category
        except IndexError:
            pass
        return render(request, 'extensions/extension.html', context)
    except Extension.DoesNotExist:
        context['extension_id'] = extension_id
        return render(request, 'extensions/unknown.html', context)

def show_category(request, pk=None):
    categories = Category.get_categories()

    if pk is not None:
        category = Category.objects.prefetch_related('extensions').get(pk=pk)
        if category is not None:
            extensions = category.get_extensions()
        else:
            extensions = []
    else:
        category = None
        extensions = Extension.get_extensions()

    if category is not None:
        title = category.name
    else:
        title = 'All extensions'

    context = {
        'title': title,
        'language': request.LANGUAGE_CODE,
        'categories': categories,
        'active_category': category,
        'extensions': extensions,
    }
    return render(request, 'extensions/index.html', context)

def show_all_extensions(request):
    return show_category(request)

def upload(request):
    if request.method == 'GET':
        form = ExtensionUploadForm()
        context = {
            'title': 'Upload Extension',
            'form': form,
            'language': request.LANGUAGE_CODE,
        }
        return render(request, 'extensions/upload.html', context)
    elif request.method == 'POST':
        form = ExtensionUploadForm(request.POST, request.FILES)
        if form.is_valid():
            cleaned_data = form.cleaned_data
            source = cleaned_data['source']
            packages = cleaned_data['packages']
            extensions = []

            for key in packages:
                package = packages[key]
                package_json = package[EXTENSION_JSON]
                data = {
                    'id': package_json['id'],
                    'name': package_json['name'],
                    'description': package_json['description'],
                    'version': package_json['version'],
                    'allow_anonymous_user': package_json['allow_anonymous_user'],
                    'source': source,
                    'extract_path': os.path.join(get_upload_path(), package_json['id']),
                }
                extension = Extension(**data)
                extension.save()
                logo = ContentFile(package['logo'])
                extension.logo.save(content=logo, name=data['id'] + '.logo')
                extensions += [extension]
                # FIXME ATTENTION! Vulnerability: big file after extract
                with ZipFile(source) as zip_package:
                    path = os.path.join(settings.MEDIA_ROOT, extension.extract_path)
                    package_root = package['path']
                    if not package_root:
                        zip_package.extractall(path=path)
                    else:
                        extract_package(zip_package, package_root, path)

            context = {
                'title': 'Upload Extension',
                'extensions': extensions,
                'language': request.LANGUAGE_CODE,
            }
            return render(request, 'extensions/uploaded.html', context)
        else:
            context = {
                'title': 'Upload Extension',
                'form': form,
                'language': request.LANGUAGE_CODE,
            }
            return render(request, 'extensions/upload.html', context)

def extract_package(zip_package, package_root, path):
    package_path = package_root + '/'
    package_path_len = len(package_path)
    for member in zip_package.namelist():
        if member.startswith(package_path):
            target_path = os.path.join(path, member[package_path_len:])
            upperdirs = os.path.dirname(target_path)
            if upperdirs and not os.path.exists(upperdirs):
                os.makedirs(upperdirs)
            with zip_package.open(member, 'r') as source, \
                    open(target_path, "wb") as target:
                shutil.copyfileobj(source, target)

def run_extension(request, extension_id):
    return get_extension(request, extension_id, run=True)

def running_extension(request, extension_id, path):
    try:
        extension = Extension.objects.prefetch_related('categories').get(pk=extension_id)
        imports_path = extension.imports_path
        if path and imports_path and path.startswith(extension.imports_path):
            return redirect(settings.STATIC_URL + 'imports/' + path[len(imports_path):])

        if path is None or path == '':
            path = '/index.html'
        if path[-1] == '/':
            path += 'index.html'

        absolute_path = extension.get_source_file_path(path)
        with open(absolute_path, 'rb') as file:
            content = file.read()
            content_type, _ = mimetypes.guess_type(absolute_path)
            return HttpResponse(content=content, content_type=content_type)
    except (Extension.DoesNotExist, FileNotFoundError):
        raise Http404('File not found\n Extension: {0}\nPath:{1}'.format(extension_id, path))


import json
import sys
import matplotlib.pyplot as plt
import numpy as np

with open(sys.argv[1], 'r') as f:
    data = json.load(f)

    curr_time_samples = []
    # disk_repair_percent_2 = []

    for item in data:
        curr_time = float(item['curr_time'])
        curr_time_samples.append(curr_time)
        # disk_repair_percent_2.append(percents[2])
    # print(disk_repair_percent_2)

    plt.hist(curr_time_samples, bins=np.linspace(0.0, 365, num=11))
    plt.xlabel('trigger time')
    plt.ylabel('Frequency')
    plt.title('frequency of failure trigger time')
    plt.savefig('hist.png')