from __future__ import absolute_import
from collections import OrderedDict
import six
from django.http import HttpResponse

from .constants import *  # noqa
from .route_decorators import *  # noqa
from ..api import ResourceApiCommon
from ..exceptions import ImmediateErrorHttpResponse

class ResourceApiBase(type):
    def __new__(mcs, name, bases, attrs):
        super_new = super(ResourceApiBase, mcs).__new__

        # attrs will never be empty for classes declared in the standard way
        # (ie. with the `class` keyword). This is quite robust.
        if name == 'NewBase' and attrs == {}:
            return super_new(mcs, name, bases, attrs)

        parents = [
            b for b in bases
            if isinstance(b, ResourceApiBase) and not (b.__name__ == 'NewBase' and b.__mro__ == (b, object))
        ]
        if not parents:
            # If this isn't a subclass of don't do anything special.
            return super_new(mcs, name, bases, attrs)

        routes = []

        # Get local routes and sort them by route number
        for view, obj in attrs.items():
            if callable(obj) and hasattr(obj, 'route'):
                route_number, path_type, method, action_name = obj.route
                routes.append((route_number, path_type, method, action_name, view))
                delattr(obj, 'route')
        routes = sorted(routes, key=lambda o: o[0])

        # Get routes from parent objects
        for parent in parents:
            if hasattr(parent, 'routes'):
                routes.extend(parent.routes)

        attrs['routes'] = routes

        return super_new(mcs, name, bases, attrs)

@six.add_metaclass(ResourceApiBase)
class ResourceApi(ResourceApiCommon):
    """
    Base class for version 2 of the Resource API.

    The API now uses a meta class to extract routes from methods that have
    been decorated with route information.

    """
    # Table containing lookup information to simplify dispatch of incoming
    # requests to the appropriate views
    route_table = None

    # Respond to the options method.
    respond_to_options = True

    def base_urls(self):
        url_table = OrderedDict()
        route_table = {}
        for route_ in self.routes:
            route_number, path_type, methods, action_name, view = route_
            route_key = "%s-%s" % (path_type, action_name) if action_name else path_type

            # Populate url_table
            if route_key not in url_table:
                if path_type == PATH_TYPE_COLLECTION:
                    regex = action_name or r''
                else:
                    if action_name:
                        regex = r'(?P<resource_id>%s)/%s' % (self.resource_id_regex, action_name)
                    else:
                        regex = r'(?P<resource_id>%s)' % self.resource_id_regex

                url_table[route_key] = self.url(regex, self.wrap_view(route_key))

            # Populate route table
            method_map = route_table.setdefault(route_key, {})
            for method in methods:
                method_map[method] = view

            # Add options
            if self.respond_to_options:
                method_map.setdefault(constants.OPTIONS, 'options_response')

        # Store the route table at the same time
        self.route_table = dict(route_table)
        return list(url_table.values())

    def dispatch_to_view(self, route_key, request, *args, **kwargs):
        """
        Primary method used to dispatch incoming requests to the appropriate method.
        """
        routes = self.route_table[route_key]
        try:
            request_method = routes[request.method]
        except KeyError:
            allow = ','.join(routes.keys())
            raise ImmediateErrorHttpResponse(405, 40500, "Method not allowed", headers={'Allow': allow},
                                             meta={'allow': allow})
        else:
            method = getattr(self, request_method, None)

        # Authorisation hook
        if hasattr(self, 'handle_authorisation'):
            self.handle_authorisation(request)

        # Allow for a pre_dispatch hook, a response from pre_dispatch would indicate an override of kwargs
        if hasattr(self, 'pre_dispatch'):
            response = self.pre_dispatch(request, **kwargs)
            if response is not None:
                kwargs = response

        # Apply route key to kwargs on OPTIONS requests
        if request.method == constants.OPTIONS:
            kwargs['route_key'] = route_key

        result = method(request, **kwargs)

        # Allow for a post_dispatch hook, the response of which is returned
        if hasattr(self, 'post_dispatch'):
            return self.post_dispatch(request, result)
        else:
            return result

    def options_response(self, request, route_key, **kwargs):
        routes = self.route_table[route_key]
        response = HttpResponse(status=204)
        response['Allow'] = ','.join(method for method, route in routes.items() if route != 'options_response')
        return response


from flask import Flask, request
import random
app = Flask(__name__)

@app.route('/')
def index():
    return 'olá'

@app.route('/parametros')
#?nome=matheus
def parametros():
    argumentos = request.args.to_dict()
    return argumentos['nome'], 202

@app.route('/numero')
@app.route('/numero/<seed>')
def numero(seed=False):
    if seed:
        random.seed(seed)

    return {'numero': random.randint(0, 10)}, 201

if __name__ == "__main__":
    app.run(debug=True)


import pandas as pd
import numpy as np
import matplotlib.pyplot as pl

dt = pd.read_csv("titanic.csv") # czytanie z pliku csv

dt["Sex_cleaned"] = np.where(dt["Sex"] == 'male', 0, 1)
dt.loc[dt["Embarked"] == 'S', "Embarked_cleaned"] = 0
dt.loc[dt["Embarked"] == 'C', "Embarked_cleaned"] = 1
dt.loc[dt["Embarked"] == 'Q', "Embarked_cleaned"] = 2
dt = dt.drop(dt.columns[[4, 11]], axis = 1)

correct_dt = dt[["Survived",
                 "Pclass",
                 "Name",
                 "Age",
                 "SibSp",
                 "Parch",
                 "Fare",
                 "Sex_cleaned",
                 "Embarked_cleaned"]].dropna(axis = 0, how = 'any')

survivors = correct_dt.loc[correct_dt["Survived"] == 1]
dead = correct_dt.loc[correct_dt["Survived"] == 0]

pl.figure(figsize = (8, 8))
pl.hist(correct_dt["Age"], color = 'b', label = "wszyscy")
pl.hist([survivors["Age"], dead["Age"]], rwidth = 0.75, color = ['g', 'r'], label = ["przeżyli", "nie żyją"])
# alpha = 0.75 (ustawia przezroczystosc na 75%)
pl.xlabel("WIEK")
pl.ylabel("LICZBA OSÓB")
pl.legend()

pl.figure(figsize = (8, 8))
pl.hist(correct_dt["Fare"], color = 'k', label = "wszyscy")
pl.hist([survivors["Fare"], dead["Fare"]], rwidth = 0.75, color = ['r', 'm'], label = ["przeżyli", "nie żyją"])
pl.xlabel("CENA BILETU")
pl.ylabel("LICZBA OSÓB")
pl.legend()
pl.show()


import torch
import torch.nn as nn
import torch.nn.functional as F

class RidgeRegression(nn.Module):
    def __init__(self, lam, kernel, device=None):
        super(RidgeRegression, self).__init__()
        self.lam = lam
        self.kernel = kernel
        self.alphas = None
        self.X_tr = None
        if device is None:
            self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        else:
            self.device = device

    def fit(self, X, Y):
        if len(X.size()) == 3:
            b, n, d = X.size()
            b, m, l = Y.size()
        elif len(X.size()) == 2:
            n, d = X.size()
            m, l = Y.size()
        assert (
            n == m
        ), "Tensors need to have same dimension, dimensions are {} and {}".format(n, m)

        self.K = self.kernel(X, X)
        K_nl = self.K + self.lam * n * torch.eye(n).to(self.device)
        # To use solve we need to make sure Y is a float
        # and not an int
        self.alphas, _ = torch.solve(Y.float(), K_nl)
        self.X_tr = X

    def predict(self, X):
        return torch.matmul(self.kernel(X, self.X_tr), self.alphas)

class LearnedBiasRidgeRegression(nn.Module):
    def __init__(self, d, log_lam, device=None):
        super(LearnedBiasRidgeRegression, self).__init__()
        self.log_lam = nn.Parameter(torch.tensor(log_lam))
        self.bias = nn.Parameter(torch.tensor(torch.zeros(d)).reshape(-1, 1))
        self.w = None
        if device is None:
            self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        else:
            self.device = device

    def fit(self, X, Y):
        if len(X.size()) == 3:
            b, n, d = X.size()
            b, m, l = Y.size()
        elif len(X.size()) == 2:
            n, d = X.size()
            m, l = Y.size()
        assert (
            n == m
        ), "Tensors need to have same dimension, dimensions are {} and {}".format(n, m)

        C = X.transpose(-2, -1).matmul(X) + n * torch.exp(self.log_lam) * torch.eye(
            d
        ).to(self.device)
        a = X.transpose(-2, -1).matmul(Y) + self.bias
        self.w, _ = torch.solve(a, C)

    def predict(self, X):
        return torch.matmul(X, self.w)

class FeatureMapRidgeRegression(nn.Module):
    """Like RidgeRegression but with an additional feature map phi: X \to Phi

    feature_map is a torch module which is learned together with the rest of the parameters

    TODO: Log is wrong since it can go negative, fix"""

    def __init__(self, lam, kernel, feature_map, normalize_features=False, device=None):
        super(FeatureMapRidgeRegression, self).__init__()
        self.lam = nn.Parameter(torch.tensor(lam))
        self.kernel = kernel
        self.feature_map = feature_map
        self.normalize_features = normalize_features
        self.alphas = None
        self.Phi_tr = None
        if device is None:
            self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        else:
            self.device = device

    def fit(self, X, Y):
        Y = F.one_hot(Y)
        n = X.size()[0]

        # Normalize features
        Phi = self.feature_map(X)  # B x N x D
        if self.normalize_features:
            Phi = F.normalize(Phi, dim=-1)

        self.K = self.kernel(Phi, Phi)
        K_nl = self.K + self.lam * n * torch.eye(n).to(self.device)
        # To use solve we need to make sure nY is a float
        # and not an int
        self.alphas, _ = torch.solve(Y.float(), K_nl)
        self.Phi_tr = Phi

    def predict(self, X):
        return torch.matmul(self.kernel(self.feature_map(X), self.Phi_tr), self.alphas)


"""
Usage: train.py MODEL_DIR NODE_NAMES ...

Arguments:
    MODEL_DIR   path to tensorflow checkpoint
    NODE_NAMES  list of nodes to export from graph

Based on: https://blog.metaflow.fr/tensorflow-how-to-freeze-a-model-and-serve-it-with-a-python-api-d4f3596b3adc
"""
import os
import logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s [%(levelname)s] %(message)s',)

import tensorflow as tf

from typeopt import Arguments

def freeze_graph(model_dir, node_names):
    '''
    Extracts the sub-graph defined by the nodes and converts
    all its variables into constants
    '''
    logger = logging.getLogger(__name__)

    if not tf.gfile.Exists(model_dir):
        raise AssertionError(
            "Export directory doesn't exists. Please specify an export directory: %s" %
            model_dir)

    if not node_names:
        logger.warning("You need to supply the name of a node")
        return

    checkpoint = tf.train.get_checkpoint_state(model_dir)
    input_checkpoint = checkpoint.model_checkpoint_path

    # We precise the file fullname of our freezed graph
    output_graph = os.path.join(os.path.dirname(model_dir), "frozen_model.pb")

    # We clear devices to allow TensorFlow to control on which device it will
    # load operations
    clear_devices = True

    # We start a session using a temporaty fresh Graph
    with tf.Session(graph=tf.Graph()) as sess:
        # We import the meta graph in the current default Graph
        saver = tf.train.import_meta_graph(
            "%s.meta" % input_checkpoint, clear_devices=True)

        # Restore the weights
        saver.restore(sess, input_checkpoint)

        # use built-in TF helper to export variables to constants
        output_graph_def = tf.graph_util.convert_variables_to_constants(
            sess,                                   # session is used to retrieve the weights
            tf.get_default_graph().as_graph_def(),  # The graph_def is used to retrieve the nodes
            node_names,                             # The nodes are used to select the subgraphs nodes
            variable_names_blacklist=['global_step'])

        # Serialize and dump the output subgraph to the filesystem
        with tf.gfile.GFile(output_graph, 'wb') as f:
            f.write(output_graph_def.SerializeToString())

        logger.info("%d ops in the source graph." %
                    len(tf.get_default_graph().get_operations()))
        logger.info("%d ops in the final graph." % len(output_graph_def.node))

if __name__ == '__main__':
    args = Arguments(__doc__, version='text')
    freeze_graph(args.model_dir, args.node_names)


from operator import index
from PyQt5.QtWidgets import QMainWindow, QApplication, QComboBox, QLabel
from PyQt5 import uic
import sys

class UI(QMainWindow):
    def __init__(self):
        super(UI, self).__init__()

        ##load ui file
        uic.loadUi("Window.ui", self)

        #define our widgets
        self.combo1 = self.findChild(QComboBox ,"comboBox")
        self.combo2 = self.findChild(QComboBox ,"comboBox_2")
        self.label = self.findChild(QLabel ,"label")

        #Add items to combobox
        self.combo1.addItem("male",["john", "wes", "Dan"])
        self.combo1.addItem("female", ["April", "steph", "Beth"])

        #Click the 1st opert
        self.combo1.activated.connect(self.clicker)

        ##show app
        self.show()

    def clicker(self, index):
            #clear second box
            self.combo2.clear()
            #Do the dependence
            self.combo2.addItems(self.combo1.itemData(index))

    def clicker2(self):
        self.label.setText(f'you picked: {self.combo2.currentText()} - {self.combo1.currentText()}')

##initialise app
app = QApplication(sys.argv)
UIWiWindow = UI()
app.exec_()


"""Utility functions for Object Detection Networks"""
from __future__ import division

import cv2
import torch
import numpy as np
from PIL import Image, ImageDraw

def xyxy2xywh(box: torch.Tensor) -> torch.Tensor:
    """
        Returns the xywh format of the input bounding box tensor

        Arguments:
            box (torch.Tensor): Bounding box tensor in xyxy format

        Output:
            output (torch.Tensor): Bounding box tensor in xywh format
    """
    output = torch.zeros(box.size())
    output[..., 0] = (box[..., 2] + box[..., 0])/2
    output[..., 1] = (box[..., 3] + box[..., 1])/2
    output[..., 2] = box[..., 2] - box[..., 0]
    output[..., 3] = box[..., 3] - box[..., 1]
    output[..., 4:] = box[..., 4:]
    return output

def xywh2xyxy(box: torch.Tensor) -> torch.Tensor:
    """
        Returns the xyxy format of the input bounding box tensor

        Arguments:
            box (torch.Tensor): Bounding box tensor in xywh format

        Output:
            output (torch.Tensor): Bounding box tensor in xyxy format
    """
    output = torch.zeros(box.size())
    output[..., 0] = (box[..., 0] - box[..., 2]/2)
    output[..., 1] = (box[..., 1] - box[..., 3]/2)
    output[..., 2] = (box[..., 0] + box[..., 2]/2)
    output[..., 3] = (box[..., 1] + box[..., 3]/2)
    output[..., 4:] = box[..., 4:]
    return output

def xywh2YOLO(box: torch.Tensor, stride: float,
              anchor: tuple):
    """
        Returns the bounding box in a format similar to the last output
        layer of the YOLO

        Arguments:
            box (torch.Tensor): Boudning box tensor in xywh format
            stride (float): stride of the current layer to decrease the size
            anchor (list): it is a list of tuples of anchor boxes

        Outputs:
            x_coor (int): x coordinate of the detection grid for the given box
            y_coor (int): y coordinate of the detection grid for the given box
            x   (float): x output of the YOLO layer for the corresponding box
            y   (float): y output of the YOLO layer for the corresponding box
            w   (float): w output of the YOLO layer for the corresponding box
            h   (float): h output of the YOLO layer for the corresponding box
    """
    x = box[..., 0].item()/stride
    x_coor = int(box[..., 0].item()/stride)
    y = box[..., 1].item()/stride
    y_coor = int(box[..., 1].item()/stride)
    x -= x_coor
    y -= y_coor
    w = torch.log(box[..., 2] / anchor[0] + 1e-16).item()
    h = torch.log(box[..., 3] / anchor[1] + 1e-16).item()
    return y_coor, x_coor, y, x, w, h

def draw_boxes(img: torch.Tensor, bbox: torch.Tensor,
               from_tensor=False):
    """
        Draws the given bounding boxes on the sample image and show it to
        user to check whether the comming data is correct or not

        Arguments:
            img (torch.Tensor, PIL.Image): Image sample
            bbox (torch.Tensor): Bounding box for corresponding image
            from_tensor (bool): Flag for whether the input is tensor or PIL image
    """
    if from_tensor:
        img = img.transpose(0, 1).transpose(1, 2).numpy()*255
        img = Image.fromarray(np.uint8(img))
    draw = ImageDraw.Draw(img)
    for b in bbox:
        if b[5] != 1:
            continue
        box = b[:4].numpy()
        bbox = [0, 0, 0, 0]
        bbox[0] = int(box[0] - box[2]/2)
        bbox[1] = int(box[1] - box[3]/2)
        bbox[2] = int(box[0] + box[2]/2)
        bbox[3] = int(box[1] + box[3]/2)
        draw.rectangle(bbox, outline='red')
    img.show()

def confidence_mask(tensor: torch.Tensor, confidence: float) -> torch.Tensor:
    """
        Returns the masked form of the input tensor with respect to given
        confidence

        Arguments:
            tensor (torch.Tensor): tensor to be masked by confidence
            confidence (float): confidence score to use in masking
    """
    # confidence masking for direct output of the YOLO layer
    conf_mask = (tensor[:, :, 4] > confidence).float().unsqueeze(2)
    return tensor*conf_mask

def bbox_iou(box1: torch.Tensor, box2: torch.Tensor) -> torch.Tensor:
    """
    Returns the IoU of two bounding boxes

    Arguments:
        box1 (torch.Tensor) : coor tensor of the first box to calculate IoU
        box2 (torch.Tensor) : coor tensor of the first box to calculate IoU

    Returns:
        iou  (float): intersection/union ratio for the given bounding boxes
    """
    # get the coordinates of bounding boxes
    b1_x1, b1_y1 = box1[..., 0], box1[..., 1]
    b1_x2, b1_y2 = box1[..., 2], box1[..., 3]
    b2_x1, b2_y1 = box2[..., 0], box2[..., 1]
    b2_x2, b2_y2 = box2[..., 2], box2[..., 3]

    # get the corrdinates of the intersection rectangle
    inter_rect_x1 = torch.max(b1_x1, b2_x1)
    inter_rect_y1 = torch.max(b1_y1, b2_y1)
    inter_rect_x2 = torch.min(b1_x2, b2_x2)
    inter_rect_y2 = torch.min(b1_y2, b2_y2)

    # intersection area
    inter_area = torch.clamp(inter_rect_x2 - inter_rect_x1 + 1, min=0) * \
        torch.clamp(inter_rect_y2 - inter_rect_y1 + 1, min=0)

    # union Area
    b1_area = (b1_x2 - b1_x1 + 1)*(b1_y2 - b1_y1 + 1)
    b2_area = (b2_x2 - b2_x1 + 1)*(b2_y2 - b2_y1 + 1)

    iou = inter_area / (b1_area + b2_area - inter_area)

    return iou

def bbox_iou_wh(wh1: tuple, wh2: tuple) -> float:
    """
        Returns the IoU value between an anchor box and bounding
        box width and height

        Arguments:
            wh1 (tuple, list): width and height pair for first box
            wh2 (tuple, list): width and height pair for second box

        Returns:
            iou (float): intersection/union for given w-h pairs
    """
    w1, h1 = wh1[0], wh1[1]
    w2, h2 = wh2[0], wh2[0]
    intersect_area = min(w1, w2) * min(h1, h2)
    union_area = w1*h1 + w2*h2 - intersect_area
    return intersect_area/union_area

def predict_transform(prediction, inp_dim, anchors, num_class,
                      CUDA, TRAIN=False) -> torch.Tensor:
    """
    Returns the prediction tensor with respect to the output of the YOLO
    Detection Layers outputs

    Arguements:
        prediction (torch.Tensor) : output tensor of the Detection Layer
        inp_dim (torch.Tensor) : input image dimensions
        anchors (torch.Tensor) : anchors of the Darknet
        num_class (int) : number of classes can be detected by Darknet
        CUDA (bool): CUDA
        TRAIN (bool):

    Returns:
        detection (torch.Tensor): masked and reorganized form of the detection
    """

    batch_size = prediction.size(0)
    stride = inp_dim // prediction.size(2)
    grid_size = inp_dim // stride
    bbox_attrs = 5 + num_class
    num_anchors = len(anchors)

    prediction = prediction.view(
        batch_size, bbox_attrs*num_anchors, grid_size*grid_size)
    prediction = prediction.transpose(1, 2).contiguous()
    prediction = prediction.view(
        batch_size, grid_size*grid_size*num_anchors, bbox_attrs)

    # sigmoid the  centre_x, centre_Y. and object confidencce
    prediction[:, :, 0] = torch.sigmoid(prediction[:, :, 0])
    prediction[:, :, 1] = torch.sigmoid(prediction[:, :, 1])
    prediction[:, :, 4:] = torch.sigmoid(prediction[:, :, 4:])

    # for trianing no offset addition
    if not TRAIN:
        # decreasing the size of the anchor boxes to match with prediction
        anchors = [(a[0]/stride, a[1]/stride) for a in anchors]

        # log space transform height and the width
        anchors = torch.FloatTensor(anchors)

        # add the center offsets
        grid = torch.arange(grid_size)
        b, a = torch.meshgrid(grid, grid)

        x_offset = a.reshape(-1, 1)
        y_offset = b.reshape(-1, 1)

        if CUDA:
            anchors = anchors.cuda()
            x_offset = x_offset.cuda()
            y_offset = y_offset.cuda()

        x_y_offset = torch.cat((x_offset, y_offset), 1).repeat(
            1, num_anchors).view(-1, 2).unsqueeze(0)

        prediction[:, :, :2] += x_y_offset

        anchors = anchors.repeat(grid_size*grid_size, 1).unsqueeze(0)
        prediction[:, :, 2:4] = torch.exp(prediction[:, :, 2:4])*anchors
        prediction[:, :, :4] *= stride

    return prediction  # -> shape = [batch, #_of_boxes, 5 + #_of_classes]

def write_results(prediction, num_class, confidence=0.6,
                  nms_conf=0.4) -> torch.Tensor:
    """
    Returns the results of the predictions of the Darknet
    as bounding boxes and class of the object

    Arguments:
        prediction (torch.Tensor) : output of the Darknet network
        num_class (int) : number of classes can be detected by Darknet
        confidence (float) : confidence of the detection
        nms_conf (float) : non-max supression threshold (default=0.4)

    Returns:
        output (torch.Tensor, int): Returns 0 if there is no detection and
            returns the bounding boxes attributes of darknet for each object
    """

    # confidence masking
    prediction = confidence_mask(prediction, confidence)

    # transforming box attributes to corner coordinates
    box_corner = prediction.new(prediction.shape)
    box_corner[:, :, 0] = (prediction[:, :, 0] - prediction[:, :, 2]/2)
    box_corner[:, :, 1] = (prediction[:, :, 1] - prediction[:, :, 3]/2)
    box_corner[:, :, 2] = (prediction[:, :, 0] + prediction[:, :, 2]/2)
    box_corner[:, :, 3] = (prediction[:, :, 1] + prediction[:, :, 3]/2)
    prediction[:, :, :4] = box_corner[:, :, :4]

    # obtaining the batch_size of the input tensor
    batch_size = prediction.size(0)

    write = False

    for ind in range(batch_size):
        image_pred = prediction[ind]

        # generating the max confidence sequence
        max_conf, max_conf_score = torch.max(image_pred[:, 5:5 +
                                                        num_class], 1)
        max_conf = max_conf.float().unsqueeze(1)
        max_conf_score = max_conf_score.float().unsqueeze(1)
        seq = (image_pred[:, :5], max_conf, max_conf_score)
        image_pred = torch.cat(seq, 1)

        non_zero_ind = (torch.nonzero(image_pred[:, 4]))
        try:
            image_pred_ = image_pred[non_zero_ind.squeeze(), :].view(-1, 7)
        except IndexError:
            continue

        # if there is no detection for this batch image
        if image_pred_.shape[0] == 0:
            continue

        # get the various classes detected in the image
        # -1 index holds the class index
        img_classes = torch.unique(image_pred_[:, -1])

        for cls in img_classes:
            # perform NMS
            # get the detections with one particular class
            cls_mask = image_pred_ * \
                (image_pred_[:, -1] == cls).float().unsqueeze(1)
            class_mask_ind = torch.nonzero(cls_mask[:, -2]).squeeze()
            image_pred_class = image_pred_[class_mask_ind].view(-1, 7)

            # sorting the detections
            conf_sort_index = torch.sort(
                image_pred_class[:, 4], descending=True)[1]
            image_pred_class = image_pred_class[conf_sort_index]
            idx = image_pred_class.size(0)  # Number of detections

            for i in range(idx):
                # get the IOUs of all boxes
                # in the loop
                try:
                    ious = bbox_iou(image_pred_class[i].unsqueeze(
                                    0), image_pred_class[i+1:])
                except (IndexError, ValueError):
                    break

                # zero out all the detections that have IoU > treshhold
                iou_mask = (ious < nms_conf).float().unsqueeze(1)
                image_pred_class[i+1:] *= iou_mask

                # remove the non-zero entries
                non_zero_ind = torch.nonzero(image_pred_class[:, 4]).squeeze()
                image_pred_class = image_pred_class[non_zero_ind].view(-1, 7)

            # repeat the batch for as many detections of the class in the img
            batch_ind = image_pred_class.new(
                image_pred_class.size(0), 1).fill_(ind)
            seq = batch_ind, image_pred_class

            if not write:
                output = torch.cat(seq, 1)
                write = True
            else:
                out = torch.cat(seq, 1)
                output = torch.cat((output, out))

    try:
        return output
    except NameError:
        return 0

def letterbox_image(img, inp_dim) -> np.ndarray:
    """
    Resize image with unchanged aspect ratio using padding

    Arguments:
        img (np.ndarray) : image which will be reshaped as numpy array
        inp_dim (list) : Darknet input image dimensions

    Returns:
        canvas (np.ndarray): Resized and padded input image for Darknet
    """
    img_w, img_h = img.shape[1], img.shape[0]
    w, h = inp_dim
    new_w = int(img_w * min(w/img_w, h/img_h))
    new_h = int(img_h * min(w/img_w, h/img_h))
    resized_image = cv2.resize(
        img, (new_w, new_h), interpolation=cv2.INTER_CUBIC)

    canvas = np.full((inp_dim[1], inp_dim[0], 3), 128)

    canvas[(h-new_h)//2:(h-new_h)//2 + new_h, (w-new_w) //
           2:(w-new_w)//2 + new_w, :] = resized_image

    return canvas

def prep_image(img, inp_dim, mode='BGR') -> torch.Tensor:
    """
    Prepare image for inputting to the neural network as torch.Tensor

    Arguments:
        img (np.ndarray) : input image as numpy array form
        inp_dim (list) : input image dimensions

    Returns:
        img (torch.Tensor): Transposed, padded and resized input image
    """

    assert mode == 'BGR' or mode == 'RGB'
    # scaling image by protecting the aspect-ratio
    img = (letterbox_image(img, (inp_dim, inp_dim)))
    # transpose for the pytorch tensor
    if mode == 'RGB':
        img = img.transpose((2, 0, 1)).copy()
    else:
        img = img[:, :, ::-1].transpose((2, 0, 1)).copy()
    # normalization of the image
    img = torch.from_numpy(img).float().div(255.0).unsqueeze(0)
    return img

def load_classes(names_file_path: str) -> list:
    """Load the classes from dataset file

    Arguments:
        names_file_path (str) : path of the names file of the dataset

    Returns:
        names (list): class labels for the detected objects
    """
    fp = open(names_file_path, "r")
    names = fp.read().split("\n")[:-1]
    return names


import json
import logging

from django.db import transaction
from django.conf import settings

from core.import_data.utils import (
    DB_DIR_LIST,
    DB_YEAR_MAPPING,
    delete_old_data,
    get_adm_column,
    get_chemical_by_name_or_components,
    get_country_and_year_dict,
    get_cp_report_for_db_import,
    get_or_create_adm_row,
)
from core.models.adm import AdmRecord, AdmRow
from core.models.country_programme import CPPrices, CPRecord

logger = logging.getLogger(__name__)

CP_COLUMNS_MAPPING = {
    "CumulativeAmount": "Cumulative float",
    "TrainedNumberHCFC": "HCFC float",
    "RecoveryAmountHCFC": "HCFC float",
    "RecoveryAmount": "ALL OTHER ODS float",
    "TrainedNumber": "ALL OTHER ODS float",
}

CP_RECORD_FIELDS_MAPPING = {
    "ExportAmount": "export_quotas",
    "ImportAmount": "import_quotas",
}

SECTION = "C"

ADM_RECORD_FIELDS = ["RecoveryAmount", "TrainedNumber"]
CP_RECORD_TITLE_WORDS = ["import quotas", "export quotas", "price"]

def get_columns_dict():
    """
    Get admC columns dict
    @param file_name = str (file path for import file) used for source_file field

    @return columns_dict = dict
        - struct: {
            field_name: AdmColumn object
        }
    """
    columns_dict = {}
    for field_name, column_name in CP_COLUMNS_MAPPING.items():
        column = get_adm_column(column_name, SECTION)
        if not column:
            return None
        columns_dict[field_name] = column

    return columns_dict

def create_adm_row(text, file_data, sort_order, parent_row=None):
    """
    Add adm row to items dict
    - if parent row set, the type will be QUESTION
    @param text = str (row text)
    @param source_data = dict (file data)
    @param sort_order = int (row sort order)
    @param parent_row = AdmRow object (admC parent row)

    @return AdmRow object
    """
    # if article parent id is set, get article id from articles dict

    row_data = {
        "text": text,
        "source_file": str(file_data["file_name"]),
        "sort_order": sort_order,
        "parent": parent_row,
        "section": SECTION,
        **DB_YEAR_MAPPING[file_data["db_name"]],
    }

    row_data["type"] = (
        AdmRow.AdmRowType.QUESTION
        if row_data.get("parent", None)
        else AdmRow.AdmRowType.TITLE
    )

    return get_or_create_adm_row(row_data)

def create_adm_rows_for_articles(file_data):
    """
    Parse admc_Layout json file and create adm rows for articles
    @param file_data = dict (file data)

    @return article_dict = dict
        - struct: {
            article_id: AdmRow object
        }

    """
    article_dict = {}
    with open(file_data["file_name"], "r", encoding="utf8") as f:
        json_data = json.load(f)

    for item_json in json_data:
        # skip items that are not titles
        if not item_json["IsTitle"]:
            continue

        # skip articles that are substances titles
        chemical_title = False
        for word in CP_RECORD_TITLE_WORDS:
            if word in item_json["Label"].lower():
                chemical_title = True
        if chemical_title:
            continue

        item_json_id = item_json["ArticleId"]
        # if item is title, add it as adm row
        article_dict[item_json_id] = create_adm_row(
            item_json["Label"],
            file_data,
            item_json["SortOrder"],
        )

    return article_dict

def get_itmes_dict(file_name, articles_dict):
    """
    Parse items json file and create a dictionary
    - the items that are text will be inserted as adm rows

    @param file_name = str (file path for import file)
    @param articles_dict = dict (article dict)

    @return items_dict)
        - struct items_dict: {
            item_cp_id: {
                type: "blend" | "substance" | "text",
                value: chemical_id | item_text
                display_name: item_name
                field: field_name
                article: AdmRow object
            }
        }
    """
    items_dict = {}
    with open(file_name, "r", encoding="utf8") as f:
        json_data = json.load(f)

    for item_json in json_data:
        # skip titles
        if item_json["IsTitle"]:
            continue

        item_json_id = item_json["ItemId"]

        if item_json["ArticleId"] not in articles_dict:
            # This Item is Chemical
            chemical_name = item_json["Label"].replace("(Optional)", "").strip()
            chemical, chemical_type = get_chemical_by_name_or_components(chemical_name)
            if not chemical:
                logger.warning(
                    f"Chemical not found: {chemical_name} "
                    f"(ItemId: {item_json['ItemId']})",
                )
                continue

            # add item to items dict
            items_dict[item_json_id] = {
                "display_name": item_json["Label"],
                "type": chemical_type,
                "value": chemical.id,
            }
            continue

        items_dict[item_json_id] = {
            "display_name": item_json["Label"],
            "type": "text",
            "value": item_json["Label"],
            "field_name": item_json["ItemField"],
            "article": articles_dict[item_json["ArticleId"]],
            "sort_order": item_json["SortOrder"],
        }

    return items_dict

def udate_cp_record(cp, admc_entry, items_dict, source_file):
    """
    Update cp record for admC entry
    @param cp = CPReport object
    @param admc_entry = dict (admC entry)
    @param items_dict = dict (admC items dict)
    @param items_file = str (file path for items file) used for source_file field
    """
    item = items_dict[admc_entry["ItemId"]]
    record_data = {
        "country_programme_report": cp,
    }

    # set substance_id or blend_id for cp_record
    record_data["substance_id"] = item["value"] if item["type"] == "substance" else None
    record_data["blend_id"] = item["value"] if item["type"] == "blend" else None

    new_data = {}
    for entry_field, record_field in CP_RECORD_FIELDS_MAPPING.items():
        if admc_entry[entry_field]:
            new_data[record_field] = admc_entry[entry_field]

    if not new_data:
        # skip if there is no new data to be updated/ created
        return

    record_data.update(new_data)

    try:
        cp_record, created = CPRecord.objects.update_or_create(
            country_programme_report_id=cp.id,
            substance_id=record_data["substance_id"],
            blend_id=record_data["blend_id"],
            defaults=record_data,
        )
    except CPRecord.MultipleObjectsReturned:
        logger.warning(
            f"[row {admc_entry['Adm_CId']}]: Too many records for "
            f"{cp.name} substance: {record_data['substance_id']} blend:{record_data['blend_id']}"
        )
        return

    if created:
        cp_record.section = "A"
        cp_record.display_name = item["display_name"]
        cp_record.source_file = source_file
        cp_record.save()

def create_cp_price(cp, admc_entry, items_dict, source_file):
    """
    Create cp price for admC entry
    @param cp = CPReport object
    @param admc_entry = dict (admC entry)
    @param items_dict = dict (admC items dict)
    @param items_file = str (file path for items file) used for source_file field

    @return CPPrices object
    """

    item = items_dict[admc_entry["ItemId"]]
    price_data = {
        "country_programme_report": cp,
        "current_year_price": admc_entry["AvgODSPrice"],
        "display_name": item["display_name"],
        "source_file": source_file,
    }
    price_data["substance_id"] = item["value"] if item["type"] == "substance" else None
    price_data["blend_id"] = item["value"] if item["type"] == "blend" else None

    return CPPrices(**price_data)

def create_adm_record(cp, file_data, admc_entry, items_dict, column_dict):
    """
    Create adm record for admC entry
    @param cp = CPReport object
    @param file_data = dict (file data)
    @param admc_entry = dict (admC entry)
    @param items_dict = dict (admC items dict)
    @param column_dict = dict (admC columns dict)

    @return adm_records = list (adm records)
    """

    item = items_dict[admc_entry["ItemId"]]

    adm_row = create_adm_row(
        item["display_name"],
        file_data,
        item["sort_order"],
        item["article"],
    )
    adm_records = []
    hcfc_field = item["field_name"] + "HCFC"
    for field in [item["field_name"], hcfc_field, "CumulativeAmount"]:
        if not admc_entry.get(field, None):
            continue
        adm_record_data = {
            "country_programme_report": cp,
            "row": adm_row,
            "column": column_dict[field],
            "value_text": admc_entry[field],
            "section": SECTION,
            "source_file": file_data["file_name"],
        }
        adm_records.append(AdmRecord(**adm_record_data))

    return adm_records

def import_admc_entries(file_data, items_dict, column_dict):
    """
    Import admC entries from json file
    @param file_data = dict (file data)
    @param items_dict = dict (admC items dict)
    @param column_dict = dict (admC columns dict)
    """

    country_dict, year_dict = get_country_and_year_dict(file_data["dir_path"])

    with open(file_data["file_name"], "r", encoding="utf8") as f:
        json_data = json.load(f)

    admc_records = []
    prices = []
    for admc_entry in json_data:
        if admc_entry["ItemId"] not in items_dict:
            # chemical not found
            continue

        cp = get_cp_report_for_db_import(
            year_dict, country_dict, admc_entry, admc_entry["Adm_CId"]
        )

        if not cp:
            continue

        # update cp_record
        if items_dict[admc_entry["ItemId"]]["type"] == "text":
            admc_records.extend(
                create_adm_record(cp, file_data, admc_entry, items_dict, column_dict)
            )
            continue

        udate_cp_record(cp, admc_entry, items_dict, file_data["file_name"])
        if admc_entry["AvgODSPrice"]:
            prices.append(
                create_cp_price(cp, admc_entry, items_dict, file_data["file_name"])
            )

    AdmRecord.objects.bulk_create(admc_records)
    CPPrices.objects.bulk_create(prices)

def parse_db_files(dir_path, db_name):
    """
    Parse db files and import data
    @param dir_path = str (directory path for import files)
    """

    admc_layout_file = f"{dir_path}/AdmCLayout.json"
    file_data = {
        "file_name": admc_layout_file,
        "db_name": db_name,
    }
    articles_dict = create_adm_rows_for_articles(file_data)

    items_dict = get_itmes_dict(f"{dir_path}/AdmCLayout.json", articles_dict)
    logger.info("✔ item file parsed")

    admc_file = f"{dir_path}/AdmC.json"
    column_dict = get_columns_dict()
    if not column_dict:
        return
    logger.info("✔ columns file parsed")

    delete_old_data(AdmRecord, admc_file)
    delete_old_data(CPPrices, admc_file)
    file_data = {
        "file_name": admc_file,
        "dir_path": dir_path,
        "db_name": db_name,
    }
    import_admc_entries(
        file_data,
        items_dict,
        column_dict,
    )

@transaction.atomic
def import_admc_items():
    """
    Import records from databases
    """
    # delete all admRows and admRecords
    AdmRow.objects.filter(
        source_file__contains="AdmCLayout.json", section=SECTION
    ).delete()

    db_dir_path = settings.IMPORT_DATA_DIR / "databases"
    for database_name in DB_DIR_LIST:
        logger.info(f"⏳ importing admC records from {database_name}")
        parse_db_files(db_dir_path / database_name, database_name)
        logger.info(f"✔ admC records from {database_name} imported")


import sys
#
import csv
import itertools as it
import numpy as np

import pandas as pd

from keras.datasets import mnist
from keras.layers import Input, Dense, Reshape, Flatten, Dropout, multiply
from keras.layers import BatchNormalization, Activation, Embedding
from keras.layers import concatenate, ReLU, ZeroPadding2D
from keras.layers.advanced_activations import LeakyReLU
from keras.layers.convolutional import UpSampling2D, Conv2D
from keras.models import Sequential, Model
# from keras.optimizers import Adam # the Keras version of Adam causes a error
# from tensorflow.keras.optimizers import Adam

import matplotlib.pyplot as plt

import numpy as np
from keras import utils
from keras.utils import to_categorical

import math

from utilities_v2 import Tokenizer, train_model, eval_model

from keras.preprocessing.sequence import pad_sequences

np.random.seed(0)

def get_topic(number):
    if not math.isnan(number):
        return int(number)

def load_data(file_name):
    data = pd.read_csv(file_name, sep='\t', names=["url", "title", "text", "topic1", "topic2", "topic3", "topic4",
                                                   "topic5", "topic6", "topic7", "topic8", "topic9", "topic10"])
    all_texts1 = list(data['text'])
    all_topics = []
    all_texts = []
    for i in range(len(all_texts1)):
        if len(str(all_texts1[i])) > 10:
            article_topics = []
            for j in topic_numbers:
                try:
                    article_topics.append(int(data[j][i]))
                except:
                    pass
            all_texts.append(str(all_texts1[i]))
            all_topics.append(article_topics)
    return all_texts, all_topics

def create_one_hot(item, num_items):
    one_hot = np.zeros(num_items)
    for thing in item:
        one_hot[int(thing)] += 1
    return one_hot

if __name__ == "__main__":

    # Code to load in TOPIC_TRAINING_DATA corpus
    # -----------------------------------------------------------------------------------------------------
    print("Starting to load in data")
    topic_numbers = ["topic1", "topic2", "topic3", "topic4", "topic5", "topic6", "topic7", "topic8", "topic9",
                     "topic10"]

    years = ['2019']  # , '2018']
    news_sources = ['cnn']  # ['atlantic', 'blaze', 'breitbart', 'businessinsider', 'buzzfeed', 'cbs', 'cnbc', 'cnn',
    # 'dailycaller', 'dailynews', 'foxnews', 'guardian', 'huffpo', 'latimes', 'nbc', 'newmax',
    # 'newyorker', 'politico', 'reuters', 'slate', 'time', 'usatoday', 'vox', 'wapo',
    # 'wsj', 'wsjblogs', 'yahoonews']
    texts = []
    topics = []

    for year in years:
        for source in news_sources:
            new_texts, new_topics = load_data('tagtraining' + year + source + '.tsv')
            # new_texts, new_topics = load_data('~/DATA1/DIANBO/TOPIC_TRAINING_DATA/tagtraining' + year + source + '.tsv')
            texts += new_texts
            topics += new_topics

    texts = np.array(texts)
    topics = np.array(topics)

    # Shuffle the data
    permutation = np.random.permutation(len(texts))
    texts = texts[permutation]
    topics = topics[permutation]

    num_examples = len(texts)
    val_index = (7 * num_examples) // 8

    # val_index = 100  # DELETE AFTER TESTING
    # num_examples = 150  # DELETE AFTER TESTING

    train_texts = texts[0:val_index]
    train_labels = topics[0:val_index]

    test_texts = texts[val_index:num_examples]
    test_labels = topics[val_index:num_examples]

    num_topics = 600
    print("Loaded in data")
    # -----------------------------------------------------------------------------------------------------

    corpus = train_texts

    embed_dim = 500
    context_size = 5
    hidden_size = 64
    learning_rate = 0.001
    n_epochs = 3
    n_batch = 500

    max_length = 25000

    # step1: train word2vec embedding

    print("Tokenizing Data")

    tokenizer = Tokenizer()
    tokenizer.build_tokenizer(corpus)

    vocab_size = len(tokenizer.vocabulary)

    tokenized_corpus = tokenizer.tokenize(corpus)

    word_to_token = tokenizer.get_word_to_token()
    word_to_token.pop(',', None)
    w = csv.writer(open("updated/tokenizer.csv", "w"))
    for key, val in word_to_token.items():
        w.writerow([key, val])

    # step 2 train topic classification model

    Tokenized_X_train = tokenized_corpus
    # one_hot_x_train = []
    # for tokenized_text in Tokenized_X_train:
    #     one_hot_x_train.append(create_one_hot(tokenized_text, vocab_size))
    # Tokenized_X_train = vectorize_aggregate(Tokenized_X_train, reps_word2vec)
    lengths = [len(i) for i in Tokenized_X_train]
    # max_length = max(lengths)
    # print(max_length)
    Tokenized_X_train = pad_sequences(Tokenized_X_train, maxlen=max_length, padding='post')

    Y_train = []
    for point in train_labels:
        Y_train.append(create_one_hot(point, num_topics))
    Y_train = np.array(Y_train)

    Tokenized_X_test = tokenizer.tokenize(test_texts)
    # one_hot_x_test = []
    # for tokenized_text in Tokenized_X_test:
    #     one_hot_x_test.append(create_one_hot(tokenized_text, vocab_size))
    # Tokenized_X_test = vectorize_aggregate(Tokenized_X_test, reps_word2vec)
    Tokenized_X_test = pad_sequences(Tokenized_X_test, maxlen=max_length, padding='post')

    Y_test = []
    for point in test_labels:
        Y_test.append(create_one_hot(point, num_topics))
    Y_test = np.array(Y_test)

    name = "w2v"

    print(f"{name} features, {Tokenized_X_train.shape[0]} examples")

    print("Starting classifier training")
    model = train_model(Tokenized_X_train, Y_train, vocab_size, max_length, n_batch=250, n_epochs=6)

    model.save("updated/topic_classifier.h5")

    ACC = eval_model(model, Tokenized_X_test, Y_test)


import threading
import transaction

from dobbin.exc import InvalidObjectReference
from dobbin.exc import WriteConflictError
from dobbin.exc import ReadConflictError
from dobbin.exc import ConflictError
from dobbin.persistent import checkout
from dobbin.persistent import Broken
from dobbin.persistent import Persistent
from dobbin.persistent import sync

ROOT_OID = 0

setattr = object.__setattr__

class Manager(object):
    """Transactional object manager.

    This class implements a two-phase transactional object manager.

    Subclasses must implement:

    -new_oid(obj)
    -read(jar, timestamp)
    -write(obj)

    Subclasses can implement:

    -tpc_begin
    -tpc_vote
    -tpc_abort
    -tpc_finish

    """

    tx_ref = None
    tx_count = 0
    tx_timestamp = None

    def __init__(self):
        # define reentrant thread-lock
        l = threading.RLock()
        self.lock_acquire = l.acquire
        self.lock_release = l.release

        # persistent-id to object mapping
        self._oid2obj = {}

        # initialize thread-local state
        self._thread = ThreadState()

        # load objects from storage
        self._sync()

    def __deepcopy__(self, memo):
        return self

    def __len__(self):
        return len(self._oid2obj)

    def __repr__(self):
        return '<%s size="%d">' % (type(self).__name__, len(self))

    @property
    def root(self):
        if self._thread.timestamp is None:
            transaction.manager.registerSynch(self)
            tx = transaction.manager.get()
            self.newTransaction(tx)
        return self.get(ROOT_OID)

    def abort(self, transaction):
        """Abort changes."""

        self._revert(self._thread.modified)
        self._revert(self._thread.committed)

    def add(self, obj):
        """Add an object to the database.

        Note that the recommended way to add objects is through the
        root object graph.
        """

        if obj._p_jar is None:
            obj._p_jar = self
        elif obj._p_jar is self:
            raise RuntimeError("Object already added to the database.")
        else:
            raise InvalidObjectReference(obj)

        checkout(obj)

    def afterCompletion(self, transaction):
        pass

    def beforeCompletion(self, transaction):
        pass

    def commit(self, transaction):
        """Commit changes to disk."""

        committed = self._thread.committed
        modified = self._thread.modified
        timestamp = self._thread.timestamp

        # update transaction timestamp
        self._thread.timestamp = sync.timestamp

        while modified:
            for obj in tuple(modified):
                # assert that object belongs to this database
                if obj._p_jar is not self:
                    raise InvalidObjectReference(obj)

                # if the object has been updated since we begun our
                # transaction, it's a write-conflict.
                if obj._p_serial is not None and obj._p_serial > timestamp:
                    try:
                        state = self._resolve(obj)
                    except ConflictError:
                        raise WriteConflictError(obj)
                else:
                    state = obj.__getstate__()

                # make sure the object has an oid
                oid = obj._p_oid
                if oid is None:
                    oid = self.new_oid(obj)

                self.write(oid, obj._p_class, state)
                committed.append((obj, state))
                modified.remove(obj)

        # update database timestamp
        self.tx_timestamp = self._thread.timestamp

    def get(self, oid, cls=None):
        obj = self._oid2obj.get(oid)
        if obj is None and cls is not None:
            obj = Broken(oid, cls)
            self._oid2obj[oid] = obj
        return obj

    def elect(self, obj):
        """Elect object as database root."""

        if not isinstance(obj, Persistent):
            raise TypeError(
                "Can't set non-persistent object as database root.")

        if obj._p_oid is not None:
            raise ValueError("Can't elect already persisted object.")

        if self.root is not None:
            raise RuntimeError("This database already has a root object.")

        obj._p_oid = ROOT_OID
        self.add(obj)

    def newTransaction(self, transaction):
        """New transaction."""

        self._sync()

    def save(self, obj):
        return self._register(obj)

    def snapshot(self, database, timestamp=None):
        """Return database snapshot."""

        # restore snapshot from database
        for record in self._read(database, end=timestamp):
            pass

        # write snapshot to storage
        tx = transaction.Transaction()
        tx.join(database)
        tx.commit()

    def sortKey(self):
        """Sort-key.

        This method is required by the transaction machinery; the key
        returned guarantees that the first thread to check out an
        object wins the transaction.
        """

        return id(self), self._thread.timestamp

    def tpc_abort(self, transaction):
        """Abort transaction.

        Called when an exception occurred during ``tpc_vote`` or
        ``tpc_finish``.
        """

        self._tpc_cleanup()

    def tpc_begin(self, transaction):
        """Begin commit (two-phase) of a transaction."""

        # catch up on transactions; other processes may have committed
        # transactions which may conflict
        self._sync()

    def tpc_vote(self, transaction):
        """Vote on transaction."""

    def tpc_finish(self, transaction):
        """Indicate confirmation that the transaction is done."""

        state = self._thread
        oid2obj = self._oid2obj
        timestamp = self._thread.timestamp
        committed = self._thread.committed

        while committed:
            obj, state = committed.pop()
            oid2obj[obj._p_oid] = obj
            obj.__setstate__(state)
            obj._p_serial = timestamp

        self._tpc_cleanup()

    def _register(self, obj):
        if self._thread.needs_to_join:
            transaction.get().join(self)
            self._thread.needs_to_join = False
            self._thread.timestamp = sync.timestamp

        modified = self._thread.modified
        if obj not in modified:
            modified.add(obj)

    def _read(self, jar, start=None, end=None):
        conflicts = set()

        for record, objects in self.read(jar, start):
            timestamp = record.timestamp
            if end and timestamp > end:
                break

            for oid, cls, state in objects:
                obj = jar.get(oid, cls)

                if obj in self._thread.modified:
                    # if our version of the object is persistent-local
                    # then we have a write conflict; it may be
                    # resolved, if the object provides a conflict
                    # resolution method; it gets called with three
                    # arguments: (old_state, saved_state, new_state).
                    try:
                        state = self._resolve(obj, new_state=state)
                    except ConflictError:
                        conflicts.add(obj)
                else:
                    setattr(obj, "__class__", cls)

                # update timestamp
                setattr(obj, '_p_serial', timestamp)

                # associate with this database
                setattr(obj, '_p_jar', jar)

                # set shared state
                obj.__setstate__(state)

            yield record

        if conflicts:
            raise ReadConflictError(*conflicts)

    def _resolve(self, obj, new_state=None):
        self.lock_acquire()
        try:
            if obj._p_resolve_conflict is None:
                raise ConflictError(obj)
            if new_state is None:
                new_state = obj.__getstate__()
            return obj._p_resolve_conflict(
                obj.__oldstate__(),
                super(type(obj), obj).__getstate__(),
                new_state
                )
        finally:
            self.lock_release()

    def _revert(self, objects):
        while objects:
            obj = objects.pop()
            obj.__setstate__()

    def _sync(self):
        for record in self._read(self, self.tx_timestamp):
            self.tx_count += 1
            self.tx_timestamp = record.timestamp

    def _tpc_cleanup(self):
        """Performs cleanup operations to support ``tpc_finish`` and
        ``tpc_abort``."""

        self._thread.needs_to_join = True
        self.tx_count += 1

class ThreadState(threading.local):
    """Thread-local database state."""

    needs_to_join = True
    timestamp = None

    def __init__(self):
        self.modified = set()
        self.committed = []


"""
Parameter handling from CLI
"""

# Python standard library
import argparse
import yaml
import importlib

class Arbutus:
    def __init__(self, cli_path=None, arguments=None):
        self.main_branch = argparse.ArgumentParser(arguments)
        self.parsed_args = {}
        self.breadcrumbs = ''
        self.parameters = {}

    def parse_args(self) -> dict:
        self.parsed_args = vars(self.main_branch.parse_args())
        self.__process_args__()
        return self.parsed_args

    def get_branch(self, branch_name: str) -> argparse.ArgumentParser:
        return self.__dict__[branch_name + '_branch']

    def add_branch(self, branch_name: str, node_name: str = 'main', help_text: str = None) -> argparse.ArgumentParser:
        if not self.__dict__.get(node_name + '_node'):
            self.__dict__[node_name + '_node'] = self.__dict__[node_name + '_branch'].add_subparsers(dest=node_name)
        self.__dict__[branch_name + '_branch'] = \
            self.__dict__[node_name + '_node'].add_parser(branch_name, help=help_text)
        return self.__dict__[branch_name + '_branch']

    def add_argument(self, branch: argparse.ArgumentParser, argument_name: str, **kwargs):
        if 'type' in kwargs and kwargs['type'] in ['int', 'str', 'float']:
            kwargs['type'] = eval(kwargs['type'])
        if 'action' in kwargs:
            if type(kwargs['action']) is dict:
                try:
                    module = importlib.import_module(f"{kwargs['action']['source']}")
                    kwargs['action'] = eval(f"module.{kwargs['action']['name']}")
                except KeyError:
                    raise
            elif type(kwargs['action']) is str:
                if kwargs['action'] not in ['store', 'store_const', 'store_true', 'append', 'append_const', 'count',
                                            'help', 'version']:
                    raise KeyError
            else:
                raise TypeError
        if 'optional' in kwargs and kwargs['optional'] == True:
            argument_name = '--' + argument_name
            print(argument_name)
            kwargs.pop('optional')
        branch.add_argument(argument_name, **kwargs)

    @staticmethod
    def new_action(method):
        class NewAction(argparse.Action):
            def __call__(self, parser, namespace, values, option_string=None):
                method(parser=parser, namespace=namespace, values=values, option_string=option_string)
                setattr(namespace, self.dest, values)
        return NewAction

    def __process_args__(self) -> None:
        all_args = list(self.parsed_args.keys())
        self.breadcrumbs = '{}.{}'.format(all_args[0], all_args[1])
        for x in all_args:
            if self.parsed_args[x] in all_args:
                self.breadcrumbs += '.' + self.parsed_args[self.parsed_args[x]]
            elif x in self.breadcrumbs:
                pass
            else:
                self.parameters[x] = self.parsed_args[x]

    def from_dict(self, dict_: dict):
        for key, value in dict_.items():
            if 'branches' in value:
                for _key in value['branches']:
                    help_text = value['branches'][_key].get('help')
                    self.add_branch(_key, key, help_text)
                self.from_dict(value['branches'])
            elif 'arguments' in value:
                for _key in value['arguments']:
                    self.add_argument(self.get_branch(key), _key, **value['arguments'][_key])
            else:
                self.from_dict(value)

    def from_yaml(self, filename: str = './config/cli.yaml'):
        with open(filename, 'r') as yaml_file:
            yaml_as_dict = yaml.load(yaml_file, Loader=yaml.SafeLoader)
        self.from_dict(yaml_as_dict)


import pickle

import numpy as np
import pandas as pd

from .... import opcodes as OperandDef
from ....core import recursive_tile
from ....serialization.serializables import KeyField, BytesField, DictField, FieldTypes
from ....dataframe.core import SERIES_CHUNK_TYPE, DATAFRAME_CHUNK_TYPE
from ....dataframe.utils import parse_index
from ....tensor.core import TENSOR_TYPE, TensorOrder
from ....utils import has_unknown_shape, ensure_own_data
from ...operands import LearnOperand, LearnOperandMixin, OutputType
from .dmatrix import ToDMatrix, check_data

class XGBPredict(LearnOperand, LearnOperandMixin):
    _op_type_ = OperandDef.XGBOOST_PREDICT

    data = KeyField("data", default=None)
    model = BytesField(
        "model", on_serialize=pickle.dumps, on_deserialize=pickle.loads, default=None
    )
    kwargs = DictField("kwargs", key_type=FieldTypes.string, default_factory=dict)

    def __init__(self, output_types=None, gpu=None, **kw):
        super().__init__(_output_types=output_types, gpu=gpu, **kw)

    def _set_inputs(self, inputs):
        super()._set_inputs(inputs)
        self.data = self._inputs[0]

    def __call__(self):
        num_class = self.model.attr("num_class")
        if num_class is not None:
            num_class = int(num_class)
        if num_class is not None:
            shape = (self.data.shape[0], num_class)
        else:
            shape = (self.data.shape[0],)
        inputs = [self.data]
        if self.output_types[0] == OutputType.tensor:
            # tensor
            return self.new_tileable(
                inputs,
                shape=shape,
                dtype=np.dtype(np.float32),
                order=TensorOrder.C_ORDER,
            )
        elif self.output_types[0] == OutputType.dataframe:
            # dataframe
            dtypes = pd.DataFrame(np.random.rand(0, num_class), dtype=np.float32).dtypes
            return self.new_tileable(
                inputs,
                shape=shape,
                dtypes=dtypes,
                columns_value=parse_index(dtypes.index),
                index_value=self.data.index_value,
            )
        else:
            # series
            return self.new_tileable(
                inputs,
                shape=shape,
                index_value=self.data.index_value,
                name="predictions",
                dtype=np.dtype(np.float32),
            )

    @classmethod
    def tile(cls, op: "XGBPredict"):
        out = op.outputs[0]
        out_chunks = []
        data = op.data
        if data.chunk_shape[1] > 1:
            if has_unknown_shape(op.data):
                yield
            data = yield from recursive_tile(data.rechunk({1: op.data.shape[1]}))
        for in_chunk in data.chunks:
            chunk_op = op.copy().reset_key()
            chunk_index = (in_chunk.index[0],)
            if op.model.attr("num_class"):
                chunk_shape = (in_chunk.shape[0], int(op.model.attr("num_class")))
                chunk_index += (0,)
            else:
                chunk_shape = (in_chunk.shape[0],)
            if op.output_types[0] == OutputType.tensor:
                out_chunk = chunk_op.new_chunk(
                    [in_chunk],
                    shape=chunk_shape,
                    dtype=out.dtype,
                    order=out.order,
                    index=chunk_index,
                )
            elif op.output_types[0] == OutputType.dataframe:
                # dataframe chunk
                out_chunk = chunk_op.new_chunk(
                    [in_chunk],
                    shape=chunk_shape,
                    dtypes=data.dtypes,
                    columns_value=data.columns_value,
                    index_value=in_chunk.index_value,
                    index=chunk_index,
                )
            else:
                # series chunk
                out_chunk = chunk_op.new_chunk(
                    [in_chunk],
                    shape=chunk_shape,
                    dtype=out.dtype,
                    index_value=in_chunk.index_value,
                    name=out.name,
                    index=chunk_index,
                )
            out_chunks.append(out_chunk)

        new_op = op.copy()
        params = out.params
        params["chunks"] = out_chunks
        nsplits = (data.nsplits[0],)
        if out.ndim > 1:
            nsplits += ((out.shape[1],),)
        params["nsplits"] = nsplits
        return new_op.new_tileables(op.inputs, kws=[params])

    @classmethod
    def execute(cls, ctx, op: "XGBPredict"):
        from xgboost import DMatrix

        raw_data = data = ctx[op.data.key]
        if isinstance(data, tuple):
            data = ToDMatrix.get_xgb_dmatrix(ensure_own_data(data))
        else:
            data = data.spmatrix if hasattr(data, "spmatrix") else data
            data = DMatrix(data)

        # do not pass arguments that are None
        kwargs = dict((k, v) for k, v in op.kwargs.items() if v is not None)
        result = op.model.predict(data, **kwargs)

        if isinstance(op.outputs[0], DATAFRAME_CHUNK_TYPE):
            result = pd.DataFrame(result, index=raw_data.index)
        elif isinstance(op.outputs[0], SERIES_CHUNK_TYPE):
            result = pd.Series(result, index=raw_data.index, name="predictions")

        ctx[op.outputs[0].key] = result

def predict(
    model,
    data,
    output_margin=False,
    ntree_limit=None,
    validate_features=True,
    base_margin=None,
    session=None,
    run_kwargs=None,
    run=True,
):
    import xgboost

    data = check_data(data)
    if not isinstance(model, xgboost.Booster):
        raise TypeError(f"model has to be a xgboost.Booster, got {type(model)} instead")

    num_class = model.attr("num_class")
    if isinstance(data, TENSOR_TYPE):
        output_types = [OutputType.tensor]
    elif num_class is not None:
        output_types = [OutputType.dataframe]
    else:
        output_types = [OutputType.series]

    kwargs = {
        "output_margin": output_margin,
        "ntree_limit": ntree_limit,
        "validate_features": validate_features,
        "base_margin": base_margin,
    }
    op = XGBPredict(
        data=data,
        model=model,
        kwargs=kwargs,
        gpu=data.op.gpu,
        output_types=output_types,
    )
    result = op()
    if run:
        result.execute(session=session, **(run_kwargs or dict()))
    return result


import json
import sys

"""
Script accepts three parameters : input file(json formatted) , output file , role, user that needs to be added the role
If the json input passed has an entry for role:"roles/<ROLE>" and the user exists as its member, no action is taken
If the json input passed has an entry for role:"roles/<ROLE>" and the user doesn't exists as its member, it is added to members list
If the json input passed has no entry for role:"roles/<ROLE>", an entry is added with user as member and role as "roles/<ROLE>
"""

input = sys.argv[1]
output = sys.argv[2]
role = sys.argv[3]
users = sys.argv[4].split(',')
users = [ "serviceAccount:" + user  for user in users ]
flag=0
with open(input) as f:
  input = json.load(f)

for user in users:
  for k,v in input.items():
    if k == "bindings":
      for list_item_dict in v:
        if 'role' in list_item_dict and list_item_dict['role']=="roles/" + role :
          if user in list_item_dict['members']:
            flag = 1 #no change needed as the user already exists as roles/<ROLE>
          else:
            list_item_dict['members'].append(user)
            flag = 1 #added the user as roles/<ROLE>

  if flag == 0: # No entry was found for roles/<ROLE>
    #print(input['bindings'])
    for k,v in input.items():
      if k == "bindings":
        v.append({"members":[user],"role":"roles/" + role })

  with open(output,'w') as f:
    json.dump(input,f,indent=2)


import logging

DEBUGGING = False
LOGGING_LEVEL = logging.WARN

class DebugFilter(object):

    def filter(self, record):
        return DEBUGGING and record.levelno == logging.DEBUG

# set up logging to file - this sets up the default logging for the root handler
# logging.basicConfig(level=logging.DEBUG,
#                     format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
#                     datefmt='%m-%d %H:%M',
#                     filename='example.log',
#                     filemode='w')

# add the handlers to logger
log = logging.getLogger('condorpy')
log.setLevel(logging.DEBUG)
log.addHandler(logging.NullHandler())

def activate_debug_logging():
    # define a Handler which writes DEBUG messages to sys.stderr
    debugger = logging.StreamHandler()
    debugger.setLevel(logging.DEBUG)
    debugger.addFilter(DebugFilter())
    formatter = logging.Formatter('%(levelname)-8s %(message)s -- Module: %(module)s in %(funcName)s, line %(lineno)d')
    debugger.setFormatter(formatter)
    log.addHandler(debugger)

def activate_console_logging():
    # define a Handler which writes INFO messages or higher to the sys.stderr
    console = logging.StreamHandler()
    console.setLevel(LOGGING_LEVEL)
    formatter = logging.Formatter('%(levelname)-8s %(message)s')
    console.setFormatter(formatter)
    log.addHandler(console)

def activate_file_logging(file_name='condorpy.log'):
    # define a Handler which logs all messages to a file
    log_file = logging.FileHandler(file_name, mode='w')
    log_file.setLevel(logging.DEBUG)
    formatter = logging.Formatter('%(asctime)s %(levelname)-8s %(message)s')
    log_file.setFormatter(formatter)
    log.addHandler(log_file)


'''
Helpers for interacting with the geocoding api
'''
import requests
from api.settings import MAPQUEST_API_KEY

def geocode(city_name):
    '''
    Geocode a city name
    '''
    endpoint = 'http://open.mapquestapi.com/geocoding/v1/address'
    payload = {'key': MAPQUEST_API_KEY, 'location': city_name}
    result = requests.get(endpoint, payload).json()
    try:
        return (result['results'][0]['locations'][0]['displayLatLng']['lat'],
                result['results'][0]['locations'][0]['displayLatLng']['lng'])
    except IndexError:
        return None

def reverse_geocode(coords):
    '''
    Reverse geocode coordinates
    '''

    endpoint = 'http://www.mapquestapi.com/geocoding/v1/reverse'

    payload = {'key': MAPQUEST_API_KEY, 'location': f"{coords['lat']},{coords['lng']}"}

    result = requests.get(endpoint, payload).json()
    try:
        locations_details = result['results'][0]['locations'][0]
        street = locations_details['street']
        city = locations_details['adminArea5']
        state = locations_details['adminArea3']
        address = ''
        if street and city and state:
            address = f"{street}, {city}, {state}"
        elif city and state:
            address = f"{city}, {state}"
        else:
            raise Exception('Bad address response')
        return address
    except IndexError:
        return None


from papi.plugin.base_classes.iop_base import iop_base

from papi.data.DPlugin import DBlock
from papi.data.DSignal import DSignal
from papi.data.DParameter import DParameter

import numpy as np

import threading

import os
import sys

import socket
import ast
import struct
import json
import time
import pickle
import base64

from threading import Timer
from socketIO_client import SocketIO, LoggingNamespace

class OptionalObject(object):
    def __init__(self, ORTD_par_id, nvalues):
        self.ORTD_par_id = ORTD_par_id
        self.nvalues = nvalues
        self.sendOnReceivePort = True
        self.UseSocketIO = True

class UDP_Plugin(iop_base):
    def cb_get_plugin_configuration(self):
        config = {
            'address': {
                'value': '127.0.0.1',
                'advanced': 'Connection',
                'tooltip': 'IP address of the source',
                'display_text': 'Target IP address'
            },
            'source_port': {
                'value': '20000',
                'advanced': 'Connection',
                'tooltip': 'Port of incoming data',
                'display_text': 'Source Port'
            },
            'out_port': {
                'value': '20001',
                'advanced': 'Connection',
                'tooltip': 'Port for outgoing data',
                'display_text': 'Send Port'
            },
            'SeparateSignals': {
                'value': '0',
                'advanced': 'General',
                'tooltip': 'Split up signal vectors to separate signals',
                'display_text': 'Separate Signals'
            },
            'SendOnReceivePort': {
                'value': '0',
                'advanced': 'Connection',
                'display_text': 'Same port for send and receive',
                'tooltip': 'Use the source port to send data back to the target'
            },
            "UseSocketIO" : {
                'value' : '0',
                'advanced' : 'SocketIO',
                'tooltip' : 'Use socket.io connection to node.js target-server',
                'display_text': 'Use SocketIO',
                'type' : 'bool'
            },
            'socketio_port': {
                'value': '8091',
                'advanced': 'SocketIO',
                'tooltip': 'Port for the SocketIO Connection',
                'display_text': 'SocketIO Port'
            },
            "OnlyInitialConfig" : {
                'value' :'0',
                'tooltip' : 'Use only first configuration, ignore further configurations.',
                'type' : 'bool',
                'advanced': 'General'
            }
        }

        return config

    def cb_initialize_plugin(self):
        print('ORTD', self.__id__, ':process id', os.getpid())
        self.config = self.pl_get_current_config_ref()
        # open UDP
        self.HOST = self.config['address']['value']
        self.SOURCE_PORT = int(self.config['source_port']['value'])
        self.OUT_PORT = int(self.config['out_port']['value'])

        self.LOCALBIND_HOST = '' # config['source_address']['value']     #CK

        self.sendOnReceivePort = True if self.config['SendOnReceivePort']['value'] == '1' else False
        self.UseSocketIO = True if self.config['UseSocketIO']['value'] == '1' else False

        if self.UseSocketIO:
            self.SocketIOPort = int(self.config['socketio_port']['value'])

        #self.sendOnReceivePort = True  # NOTE: remove this
        #self.UseSocketIO = True  # NOTE: remove this

        print ("SendOnReceivePort = ", self.sendOnReceivePort)
        print ("UseSocketIO = ", self.UseSocketIO)

        self.PAPI_SIMULINK_BLOCK = False

        self.separate = int(self.config['SeparateSignals']['value'])

        self.onlyInitialConfig = self.config['OnlyInitialConfig']['value'] == '1'
        self.hasInitialConfig = False

        if (not self.sendOnReceivePort):
            # SOCK_DGRAM is the socket type to use for UDP sockets
            self.sock_parameter = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
            self.sock_parameter.setblocking(1)

        self.ControlBlock = DBlock('ControllerSignals')
        self.ControlBlock.add_signal(DSignal('ControlSignalReset'))
        self.ControlBlock.add_signal(DSignal('ControlSignalCreate'))
        self.ControlBlock.add_signal(DSignal('ControlSignalSub'))
        self.ControlBlock.add_signal(DSignal('ControllerSignalParameter'))
        self.ControlBlock.add_signal(DSignal('ControllerSignalClose'))
        self.ControlBlock.add_signal(DSignal('ActiveTab'))
        self.pl_send_new_block_list([self.ControlBlock])

        self.t = 0

        self.pl_set_event_trigger_mode(True)

        if not self.UseSocketIO:

            self.sock_recv = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)

            if (not self.sendOnReceivePort):
                try:
                    self.sock_recv.bind((self.LOCALBIND_HOST, self.SOURCE_PORT)) # CK
                    print("UDP_Plugin-plugin listening on: ", self.LOCALBIND_HOST, ":", self.SOURCE_PORT)     #CK
                except socket.error as msg:
                    sys.stderr.write("[ERROR] Can't start UDP_Plugin due to %s\n" % msg)
                    return False
            else:
                print ("---- Using client UDP mode (not binding to a port) ----")

            self.sock_recv.settimeout(1)

            self.thread_goOn = True
            self.lock = threading.Lock()
            self.thread = threading.Thread(target=self.thread_execute)
            self.thread.start()

        if self.UseSocketIO:
            print ("Using socket.io connection on port", self.SocketIOPort)

            self.thread_socket_goOn = True
            self.thread_socketio = threading.Thread(target=self.thread_socketio_execute)

        self.blocks = {}
        self.Sources = {}

        self.parameters = {}

        self.signal_values = {}

        self.block_id = 0

        self.config_complete = False
        self.config_buffer = {}

        self.timer = Timer(3,self.callback_timeout_timer)
        self.timer_active = False

        self.ConsoleBlock = DBlock('ConsoleSignals')
        self.ConsoleBlock.add_signal(DSignal('MainSignal'))
        self.pl_send_new_block_list([self.ConsoleBlock])

        self.consoleIn      = DParameter('consoleIn',default='')
        self.pl_send_new_parameter_list([self.consoleIn])

        if self.UseSocketIO:
            self.thread_socketio.start()

        return True

    def SIO_callback_SCISTOUT(self, data):
        # Got a chunk of data from a console interface in the target server
        self.sio_count += 1

        self.pl_send_new_data(self.ConsoleBlock.name, [self.sio_count], {'MainSignal':data['Data']})

    def SIO_callback_PAPICONFIG(self, data):
        # Got a new PaPI plugin configuration in JSON-format via socket.io connection
        # currently the config is transmitted via ORTD packets that may also be encapsulated into socket.io
        # added 12.8.15, CK

        print ("Got a new config in JSON/socket.io format")
        print (data)

        self.check_and_process_cfg(data)

        # TODO: Test this

    def SIO_callback_ORTDPACKET(self, data):
        # Got an encapsulated packet from ORTD, not transfered via UDP but encapsulated within the socket.io connection
        #print ("Got data packet from ORTD via socket.io")
        #print (data)
        self.process_received_package(base64.b64decode(data))  # data must be a binary blob

    def thread_socketio_execute(self):
        #self.sio = SocketIO('localhost', 8091, LoggingNamespace)
        #self.sio.on('SCISTOUT',self.callback_SCISTOUT)

        while True:

            try:

                with SocketIO(self.HOST, self.SocketIOPort, LoggingNamespace) as self.sio:
                    self.sio.on('SCISTOUT',self.SIO_callback_SCISTOUT)
                    self.sio.on('PAPICONFIG',self.SIO_callback_PAPICONFIG)
                    self.sio.on('ORTDPACKET',self.SIO_callback_ORTDPACKET)

                    self.request_new_config_from_ORTD()

                    self.sio_count = 0
                    while self.thread_socket_goOn:
                        self.sio.wait(seconds=1)

            except:
                print("Something failed within socket.io")

    def cb_pause(self):
        self.lock.acquire()
        self.thread_goOn = False
        self.lock.release()
        self.thread.join()

    def cb_resume(self):
        self.thread_goOn = True
        self.thread = threading.Thread(target=self.thread_execute, args=(self.HOST, self.SOURCE_PORT))
        self.thread.start()

    def thread_execute(self):
        time.sleep(2)                       # TODO: Grrr. do not use sleep for thread sync...
        #try:

        if not self.UseSocketIO:
            self.request_new_config_from_ORTD()

        goOn = True
        newData = False
        signal_values = {}
        while goOn:
            try:
                if not self.sendOnReceivePort:
                    rev = self.sock_recv.recv(65507)  # not feasible for network connection other than loopback
                else:
                    #print ("---- Waiting for data ----")
                    rev, server = self.sock_recv.recvfrom(65507)  # not feasible for network connection other than loopback
                    #print ("---- got ----")
                    #print (rev)

            except socket.timeout:
                # print('timeout')
                newData = False

            except socket.error:
                print('ORTD got socket error')
            else:
               newData = True

            if newData:
               self.process_received_package(rev)

            # check if thread should go on
            self.lock.acquire()
            goOn = self.thread_goOn
            self.lock.release()
        # Thread ended
        self.sock_recv.close()

    def process_received_package(self, rev):
        SenderId, Counter, SourceId = struct.unpack_from('<iii', rev)

        if SourceId == -1 and len(self.blocks) > 0:
            # data stream finished
            self.process_finished_action(SourceId, rev)
            self.signal_values = {}

        if SourceId >= 0 and len(self.blocks) > 0:
            # got data stream
            self.process_data_stream(SourceId, rev)

        if SourceId == -2:
            # new config in ORTD available
            # send trigger to get new config
            self.request_new_config_from_ORTD()
            self.config_complete = False
            self.config_buffer = {}

        if SourceId == -100:
            self.PAPI_SIMULINK_BLOCK = True
            # got data stream from the PaPI-Simulink Block
            self.process_papi_data_stream(rev)

        if SourceId == -4:
            self.PAPI_SIMULINK_BLOCK = False
            # new configItem
            # print("Part of a new configuration");
            # receive new config item and execute cfg in PaPI

            # unpack package
            i = 16 # Offset: 4 ints
            unp = ''
            while i < len(rev):
                unp = unp + str(struct.unpack_from('<s',rev,i)[0])[2]
                i += 1

            if int(Counter) not in self.config_buffer:
                self.config_buffer[int(Counter)] = unp
            else:
                if self.config_buffer[int(Counter)] != unp:
                    self.config_buffer = {}

            # Check counter key series for holes
            counters = list(self.config_buffer.keys())
            counters.sort()
            i = 1
            config_file = ''

            self.config_complete = True
            for c in counters:
                if i == c:
                    config_file += self.config_buffer[c]
                    i += 1
                else:
                    self.config_complete = False
                    break
            if self.config_complete:
                if not self.check_and_process_cfg(config_file):
                    self.start_timeout_timer()
                else:
                    self.stop_timeout_timer()

            else:
                self.start_timeout_timer()

    def start_timeout_timer(self):
        if not self.timer_active:
            self.timer = Timer(3,self.callback_timeout_timer)
            self.timer.start()
            self.timer_active = True
        else:
            self.timer.cancel()
            self.timer = Timer(3,self.callback_timeout_timer)
            self.timer.start()

    def stop_timeout_timer(self):
        if self.timer_active:
            self.timer.cancel()
            self.timer_active = False
            #self.config_buffer = {}

    def callback_timeout_timer(self):
        print('ORTD_PLUGIN: Config timeout, requesting a new config')
        self.timer_active = False

        self.config_buffer = {}
        self.request_new_config_from_ORTD()

    def request_new_config_from_ORTD(self):
        Counter = 1
        data = struct.pack('<iiid', 12, Counter, int(-3), float(0))

        if self.UseSocketIO:
            print ("Requesting config via socket.io")

            self.sio.emit('ORTDPACKET', base64.b64encode(data).decode('ascii') )
            # TODO test this

        else:

            if not self.sendOnReceivePort:
                self.sock_parameter.sendto(data, (self.HOST, self.OUT_PORT))
            else:
                self.sock_recv.sendto(data, (self.HOST, self.SOURCE_PORT))

    def check_and_process_cfg(self, config_file):
        try:
            # config completely received
            # extract new configuration
            cfg = json.loads(config_file)
            ORTDSources, ORTDParameters, plToCreate, \
            plToClose, subscriptions, paraConnections, activeTab = self.extract_config_elements(cfg)

            if self.hasInitialConfig and self.onlyInitialConfig:
                return True

            self.hasInitialConfig = True

            self.update_block_list(ORTDSources)
            self.update_parameter_list(ORTDParameters)

            self.process_papi_configuration(plToCreate, plToClose, subscriptions, paraConnections, activeTab)
            return True
        except ValueError as e:
            return False

    def process_papi_configuration(self, toCreate, toClose, subs, paraConnections, activeTab):

        self.pl_send_new_data('ControllerSignals', [1], {'ControlSignalReset': 1,
                                                              'ControlSignalCreate':None,
                                                              'ControlSignalSub':None,
                                                              'ControllerSignalParameter':None,
                                                              'ControllerSignalClose':None,
                                                              'ActiveTab': None })

        self.pl_send_new_data('ControllerSignals', [1], {'ControlSignalReset':0,
                                                              'ControlSignalCreate':toCreate,
                                                              'ControlSignalSub':subs,
                                                              'ControllerSignalParameter':paraConnections,
                                                              'ControllerSignalClose':toClose,
                                                              'ActiveTab': activeTab})

    def parse_json_stream(self,stream):
        decoder = json.JSONDecoder()
        while stream:
            obj, idx = decoder.raw_decode(stream)
            yield obj
            stream = stream[idx:].lstrip()

    def update_parameter_list(self, ORTDParameter):

        newList ={}

        for para_id in ORTDParameter:
            para_name = ORTDParameter[para_id]['ParameterName']
            if para_name in self.parameters:
                para_object = self.parameters.pop(para_name)
            else:
                val_count = int(ORTDParameter[para_id]['NValues'])
                opt_object = OptionalObject(para_id, val_count)

                if "initial_value" in ORTDParameter[para_id]:
                    val = ORTDParameter[para_id]['initial_value']
                    if val_count > 1:
                        val = val[1:-1]
                        init_value = list(map(float,val.split(',')))
                    else:
                        init_value = float(val)
                else:
                    init_value = 0

                para_object = DParameter(para_name, default=str(init_value), OptionalObject=opt_object)
                self.pl_send_new_parameter_list([para_object])

            newList[para_name] = para_object

        toDeleteDict = self.parameters
        self.parameters = newList

        for par in toDeleteDict:
            self.pl_send_delete_parameter(par)

    def update_block_list(self,ORTDSources):
        #self.block_id = self.block_id +1
        #newBlock = DBlock('SourceGroup'+str(self.block_id))
        #self.blocks['SourceGroup'+str(self.block_id)] = newBlock
        if 'SourceGroup0' in self.blocks:
            self.pl_send_delete_block('SourceGroup0')
        newBlock = DBlock('SourceGroup0')
        self.blocks['SourceGroup0'] = newBlock
        self.Sources = ORTDSources
        keys = list(self.Sources.keys())
        for key in keys:
            Source = self.Sources[key]
            sig_name = Source['SourceName']
            newBlock.add_signal(DSignal(sig_name))

        self.pl_send_new_block_list([newBlock])

        # Remove BLOCKS
        #if 'SourceGroup'+str(self.block_id-1) in self.blocks:
            #self.pl_send_delete_block(self.blocks.pop('SourceGroup'+str(self.block_id-1)).name)

    def process_papi_data_stream(self, rev):

        timestamp = None

        offset = 4*4
        for i in range(len(self.Sources)):

            try:
                val = []
#                offset += i*(4+4+4)

                # Get current signal ID:

#                signal_id,data = struct.unpack_from('<id', rev, offset)
                signal_id, data = struct.unpack_from('<id', rev, offset)

                # print('Offset=' + str(offset))
                #
                # print('SignalID: ' + str(signal_id))
#                print('Data: ' + str(data))
                if str(signal_id) in self.Sources:

                    Source = self.Sources[str(signal_id)]
                    NValues = int(Source['NValues_send'])

                    # print("NValues : " +  str(NValues))

                    #print("Offset:" + str(offset))

                    offset += 4
                    for n in range(NValues):
                        # print('#Value=' + str(n))
                        # print('Offset=' + str(offset))
                        try:
                            data = struct.unpack_from('<d', rev, offset)[0]
                            # print('Data=' + str(data))

                            val.append(data)
                        except struct.error:
                            # print(sys.exc_info()[0])
                            # print('!!!! except !!!!')
                            val.append(0)

                        offset += 8

                    # print('Data: ' + str(val))

                    # if NValues > 1:
                    #     signal_id,data = struct.unpack_from('<id%sd' %NValues, rev, offset)
                    #     offset += (NValues-1)*(4+4)
                    if self.Sources[str(signal_id)]["SourceName"] == "SourceTime":
                            timestamp = val[0]

                    self.signal_values[signal_id] = val

                #print("Signal: " + str(signal_id) + " Data: " + str(data)  );
            except struct.error:
                print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno))
                print(sys.exc_info()[1])
                print("Can't unpack.")
        self.process_finished_action(-1,None, timestamp)

    def process_data_stream(self, SourceId, rev):
        # Received a data packet
        # Lookup the Source behind the given SourceId
        if str(SourceId) in self.Sources:
            Source = self.Sources[str(SourceId)]
            NValues = int(Source['NValues_send'])

            # Read NVales from the received packet
            val = []
            for i in range(NValues):
                try:
                    val.append(struct.unpack_from('<d', rev, 3 * 4 + i * 8)[0])
                except:
                    val.append(0)

            self.signal_values[SourceId] = val

        else:
            dp_info = self.pl_get_dplugin_info()
            print('ORTD_PLUGIN - '+dp_info.uname+': received data with an unknown id ('+str(SourceId)+')')

    def process_finished_action(self, SourceId, rev, timestamp=None):
        if SourceId == -1:
            # unpack group ID
            # GroupId = struct.unpack_from('<i', rev, 3 * 4)[0]
            self.t += 1.0

            keys = list(self.signal_values.keys())
            keys.sort()                    # REMARK: Die liste keys nur einmal sortieren; bei initialisierung

            signals_to_send = {}
            for key in keys:
                if str(key) in self.Sources:
                    sig_name = self.Sources[str(key)]['SourceName']
                    signals_to_send[sig_name] = self.signal_values[key]

            if len( list(self.blocks.keys()) ) >0:
                block = list(self.blocks.keys())[0]

                if len(self.blocks[block].signals) == len(signals_to_send)+1:
                    if timestamp is None:
                        self.pl_send_new_data(block, [self.t], signals_to_send )
                    else:
                        self.pl_send_new_data(block, [timestamp], signals_to_send )

    def cb_execute(self, Data=None, block_name = None, plugin_uname = None):
        raise Exception('Should not be called!')

    def cb_set_parameter(self, name, value):
        if name in self.parameters:
            parameter = self.parameters[name]
            Pid = parameter.OptionalObject.ORTD_par_id
            Counter = 111
            if value is not None:
                data = None

                # get values in float from string

                valueCast = ast.literal_eval(value)
                # check is it is a list, if not, cast to list
                if not isinstance(valueCast,list):
                    valueCast = [valueCast]

                if self.PAPI_SIMULINK_BLOCK:
                    data = struct.pack('<iii%sd' %len(valueCast), 12, Counter, int(Pid),*valueCast)
                else:
                    data = struct.pack('<iii%sd' %len(valueCast), 12, Counter, int(Pid),*valueCast)

                #if isinstance(valueCast, list):
                    #data += struct.pack('%sd' %len(valueCast),*valueCast)
                #    for i in range(0,len(valueCast)):
                #        data += struct.pack('<d',valueCast[i])
                #else:
                #    data +=  struct.pack('d',float(value))

                if self.UseSocketIO:
                    print ("Setting parameter via socket.io")

                    self.sio.emit('ORTDPACKET', base64.b64encode(data).decode('ascii') )
                    # TODO test this

                else:

                    if not self.sendOnReceivePort:
                        self.sock_parameter.sendto(data, (self.HOST, self.OUT_PORT))
                    else:
                        self.sock_recv.sendto(data, (self.HOST, self.SOURCE_PORT))
        else:
            if name == 'consoleIn' and self.UseSocketIO:
                self.sio.emit('ConsoleCommand', { 'ConsoleId' : '1' ,  'Data' : value  })

    def cb_quit(self):
        self.lock.acquire()
        self.thread_goOn = False
        self.lock.release()
        self.thread.join()
        if not self.sendOnReceivePort:
            self.sock_parameter.close()
        print('ORTD-Plugin will quit')

    def cb_plugin_meta_updated(self):
        pass

    def plconf(self):
        cfg   = {}
        subs  = {}
        paras = {}
        close = {}
        if 'PaPIConfig' in self.ProtocolConfig:
            if 'ToCreate' in self.ProtocolConfig['PaPIConfig']:
                cfg = self.ProtocolConfig['PaPIConfig']['ToCreate']
            if 'ToSub' in self.ProtocolConfig['PaPIConfig']:
                subs = self.ProtocolConfig['PaPIConfig']['ToSub']
            if 'ToControl' in self.ProtocolConfig['PaPIConfig']:
                paras = self.ProtocolConfig['PaPIConfig']['ToControl']
            if 'ToClose' in self.ProtocolConfig['PaPIConfig']:
                close = self.ProtocolConfig['PaPIConfig']['ToClose']
        return cfg, subs, paras, close

    def extract_config_elements(self, configuration):
        plToCreate   = {}
        subscriptions  = {}
        paraConnections = {}
        plToClose = {}
        ORTDSources = {}
        ORTDParameters = {}
        activeTab = 'PaPI-Tab'

        if 'PaPIConfig' in configuration:
            if 'ToCreate' in configuration['PaPIConfig']:
                plToCreate = configuration['PaPIConfig']['ToCreate']

            if 'ToSub' in configuration['PaPIConfig']:
                subscriptions = configuration['PaPIConfig']['ToSub']

            if 'ToControl' in configuration['PaPIConfig']:
                paraConnections = configuration['PaPIConfig']['ToControl']

            if 'ToClose' in configuration['PaPIConfig']:
                plToClose = configuration['PaPIConfig']['ToClose']

            if 'ActiveTab' in configuration['PaPIConfig']:
                activeTab = configuration['PaPIConfig']['tab']

        if 'SourcesConfig' in configuration:
            ORTDSources = configuration['SourcesConfig']

        if 'ParametersConfig' in configuration:
            ORTDParameters = configuration['ParametersConfig']
        return ORTDSources, ORTDParameters, plToCreate, plToClose, subscriptions, paraConnections, activeTab


# === Install package ===========
# > pip install pandas, aspose-cells
# > pip install openpyxl
# ===============================
# Import
import sys
import os
import pandas as pd
import xlsxwriter
import time

folder_iexcels = sys.argv[1]
folder_oexcels = sys.argv[2]

# === process ==========
def bar_process(i_process, max_process, title):
    if i_process <= max_process:
        print("%-1s (%d%-1s%d)" % (title, i_process, "/", max_process), end = "\r")
    if i_process == max_process:
        print("%-1s (%d%-1s%d)" % (title, i_process, "/", max_process))

# === move table to 0, 0 position in sheet =========
def fomart_table_ex(ipath_file_ex, opath_file_ex):

    xls = pd.ExcelFile(ipath_file_ex)
    sheets = xls.sheet_names

    workbook_output = xlsxwriter.Workbook(opath_file_ex)

    for sheet in sheets:
        df = pd.read_excel(xls, sheet)
        worksheet_output = workbook_output.add_worksheet(sheet)
        leng_row = len(df)
        leng_col = len(df.columns)

        w_row = 0
        w_col = 0
        selc_col = 0
        selc_row = 0

        for col in range(leng_col):
            if col >= selc_col:
                for row in range(leng_row):
                    if row >= selc_row:
                        value = str(df.iloc[row][col])
                        if value != "nan":
                            print(value)
                            worksheet_output.write(w_row, w_col, value)
                            w_row += 1
                w_col += 1
                w_row = 0

    workbook_output.close()

# === move table to 0, 0 position in sheet ===========
def fomart_table_exs(ifolder_ex, ffolder_ex):
    # === Get all file in folder
    all_excels = os.listdir(ifolder_ex)
    max_file = len(os.listdir(ifolder_ex))
    i_process = 0
    for file in all_excels:
        fomart_table_ex(ifolder_ex + file, ffolder_ex + file)

        i_process += 1
        bar_process(i_process, max_file, "Formarting file ...")

fomart_table_exs(folder_iexcels, folder_oexcels)