text:
import tensorflow as tf
import numpy as np
import collections
from rpn_tools.my_anchor_target_layer_modified import AnchorTargetLayer
from rpn_tools.proposal_layer_modified import ProposalLayer_Chunk
from rpn_tools.proposal_target_layer_modified import ProposalTargetLayer
from tensorflow.python.framework import ops
from tensorflow.python.ops import array_ops
import csv
np.set_printoptions(threshold=np.inf)
import cv2
import glob
from roi_pool_tools import roi_pooling_op
from roi_pool_tools import roi_pooling_op_grad
from cnn_tools.tools import *

# image_files are original images, while gt_files are csv annotations in
image_files = sorted(glob.glob('/home/alfonso/VOCdevkit2012/VOC2012/JPEGImages/*.jpg'))
gt_files = sorted(glob.glob('/home/alfonso/VOCdevkit2012/VOC2012/Annotations_3/*.csv'))

# hyperparameter settings
resnet_length = 101 # resnet lengths can be {50, 101, 152}
_tr_bn = True # train BN parameters
_add_l2 = True # include weight decay of 1e-4: set to False
_rpn_stat = True # train RPN weights
_rcnn_stat = True # train RCNN weights
_fc_stat = True # train "fc" -- which are hconv5 residual layers and up
_layer0_stat = False # train hconv2
_layer1_stat = True # train hconv3
_layer2_stat = True # train hconv4
lr_w = 0.001 # learning rate for weights
lr_b = 0.002 # learning rate for biases
s1 = 3 # smoothL1 loss sigma hyperparameter for RPN anchor bbox_pred
s2 = 1 # smoothL1 loss sigma hyperparameter for classification bbox_pred

# other hyperparameter settings: modify with care...
A = 9
height = 38
width = 63
rpn_size = 256
rpn_batch_size = 128
im_info = [600, 1000]

def anchor(x, g):
  x = np.array(x)
  g = np.array(g)
  anchor = AnchorTargetLayer()
  anchor.setup(x, g)
  labels, bbox_targets, bbox_inside_weights, bbox_outside_weights, n = \
      anchor.forward(x, g)
  return labels.astype(np.int64), bbox_targets, bbox_inside_weights, \
      bbox_outside_weights, np.array(n).astype(np.float32)

def proposal(cls, bbox):
  cls = np.array(cls)
  bbox = np.array(bbox)
  proposer = ProposalLayer_Chunk()
  proposer.setup(cls, bbox)
  blob = proposer.forward(cls, bbox)
  return blob

def proposal_target(rpn_rois, gt):
  rpn_rois = np.array(rpn_rois)
  gt = np.array(gt)
  proposer_target = ProposalTargetLayer()
  proposer_target.setup()
  rois, labels, bbox_targets, bbox_inside_weights, bbox_outside_weights = \
      proposer_target.forward(rpn_rois, gt)
  return rois.astype(np.int32), labels.astype(np.int64), bbox_targets, \
      bbox_inside_weights, bbox_outside_weights

def resnet(inpt, n, loc):
  if n == 50:
    num_conv = []
    num_conv.append(3)
    num_conv.append(4)
    num_conv.append(6)
    num_conv.append(3)
  elif n == 101:
    num_conv = []
    num_conv.append(3)
    num_conv.append(4)
    num_conv.append(23)
    num_conv.append(3)
  elif n == 152:
    num_conv = []
    num_conv.append(3)
    num_conv.append(8)
    num_conv.append(36)
    num_conv.append(3)
  elif n == 34:
    num_conv = []
    num_conv.append(3)
    num_conv.append(4)
    num_conv.append(6)
    num_conv.append(3)

  layers = []

  with tf.variable_scope('conv1'):
    conv1 = conv_layer(inpt, [7, 7, 3, 64], 2, loc, tr_stat = _layer0_stat, \
        bn_tr_stat = _tr_bn, add_l2_stat = _add_l2, state = "split")
    max1 = max_pool_3x3(conv1)
    layers.append(max1)

  for i in range (num_conv[0]):
    with tf.variable_scope('conv2_%d' % (i + 1)):
      conv2 = residual_block(layers[-1], 64, False, loc, tr_stat = _layer0_stat, \
          bn_tr_stat = _tr_bn, add_l2_stat = _add_l2, branch = "near")
      layers.append(conv2)

  for i in range (num_conv[1]):
    down_sample = True if i == 0 else False
    with tf.variable_scope('conv3_%d' % (i + 1)):
      conv3 = residual_block(layers[-1], 128, down_sample, loc, tr_stat = _layer1_stat, \
          bn_tr_stat = _tr_bn, add_l2_stat = _add_l2, branch = "far")
      layers.append(conv3)

  for i in range (num_conv[2]):
    down_sample = True if i == 0 else False
    with tf.variable_scope('conv4_%d' % (i + 1)):
      conv4 = residual_block(layers[-1], 256, down_sample, loc, tr_stat = _layer2_stat, \
          bn_tr_stat = _tr_bn, add_l2_stat = _add_l2, branch = "far")
      layers.append(conv4)

  return layers[-1]

#SETUP
num_labels = 21
batch_size = 1

prep_img = tf.placeholder(tf.float32, [1, 600, 1000, 3])
gt_box = tf.placeholder(tf.int64)
gt_boxbatch = tf.reshape(tf.pack(gt_box), [-1, 5])
post_img = tf.py_func(process_img, [prep_img, gt_boxbatch], [tf.float32])
im_batch = tf.reshape(post_img, [1, 600, 1000, 3])

#VGG_TRUNK
with tf.name_scope("trunk"):
  h_conv13 = resnet(im_batch, resnet_length, "trunk")

#RPN
with tf.name_scope("rpn"):
  gate = tf.placeholder(tf.float32)
  h_rpn_input = (h_conv13 * (1-gate)) # gate is a residue of alternative-optimization
  W_rpn3 = weight_variable([3,3,1024,1024], "rpn", tr_stat = _rpn_stat, add_l2_stat = _add_l2)
  b_rpn3 = bias_variable([1024], "rpn", tr_stat = _rpn_stat, add_l2_stat = _add_l2)
  h_rpn3 = tf.nn.relu(conv2d(h_rpn_input, W_rpn3) + b_rpn3)

  W_cls_score = weight_variable([1,1,1024,18], "rpn", tr_stat = _rpn_stat, add_l2_stat = _add_l2)
  b_cls_score = bias_variable([18], "rpn", tr_stat = _rpn_stat, add_l2_stat = _add_l2)
  rpn_cls_score = (conv2d_nopad(h_rpn3, W_cls_score) + b_cls_score)

  W_bbox_pred = weight_variable_bbox([1,1,1024,36], "rpn", tr_stat = _rpn_stat, add_l2_stat = _add_l2)
  b_bbox_pred = bias_variable([36], "rpn", tr_stat = _rpn_stat, add_l2_stat = _add_l2)
  rpn_bbox_pred = (conv2d_nopad(h_rpn3, W_bbox_pred) + b_bbox_pred)

#RPN loss and accuracy calculation
rpn_bbox_pred = tf.reshape(rpn_bbox_pred, [1, height, width, A * 4])
rpn_cls_score_reshape = tf.reshape(rpn_cls_score, [-1, 2]) + 1e-20

rpn_labels_ind, rpn_bbox_targets, rpn_bbox_inside_weights, rpn_bbox_outside_weights, rpn_size = \
  tf.py_func(anchor, [rpn_cls_score, gt_boxbatch], [tf.int64, tf.float32, tf.float32, tf.float32, tf.float32])

rpn_labels_ind = tf.reshape(tf.pack(rpn_labels_ind), [-1])
rpn_bbox_targets = tf.reshape(tf.pack(rpn_bbox_targets), [1, height, width, A * 4])
rpn_bbox_inside_weights = tf.reshape(tf.pack(rpn_bbox_inside_weights), [1, height, width, A * 4])
rpn_bbox_outside_weights = tf.reshape(tf.pack(rpn_bbox_outside_weights), [1, height, width, A * 4])

rpn_cls_soft = tf.nn.softmax(rpn_cls_score_reshape)
rpn_cls_score_x = tf.reshape(tf.gather(rpn_cls_score_reshape,tf.where(tf.not_equal(rpn_labels_ind,-1))),[-1,2])
rpn_label = tf.reshape(tf.gather(rpn_labels_ind, tf.where(tf.not_equal(rpn_labels_ind,-1))),[-1])
rpn_loss_cls = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(rpn_cls_score_x, rpn_label))

unique_rpn_cls, o_cls, o_cls_ind = tf.py_func(cls_unique, \
    [rpn_cls_soft, rpn_labels_ind], [tf.float32, tf.float32, tf.float32])
unique_rpn_cls = tf.pack(unique_rpn_cls)

rpn_correct_prediction = tf.py_func(rpn_accuracy, [rpn_cls_soft, rpn_labels_ind], [tf.float32])
rpn_correct_prediction = tf.reshape(tf.pack(rpn_correct_prediction), [-1])
rpn_cls_accuracy = tf.reduce_mean(tf.cast(rpn_correct_prediction, tf.float32))

sigma1 = s1 * s1
smoothL1_sign = tf.cast(tf.less(tf.abs(tf.sub(rpn_bbox_pred, rpn_bbox_targets)),1/sigma1),tf.float32)
rpn_loss_bbox = tf.mul(tf.reduce_mean(tf.reduce_sum(tf.mul(rpn_bbox_outside_weights,tf.add( \
    tf.mul(tf.mul(tf.pow(tf.mul(rpn_bbox_inside_weights, \
        tf.sub(rpn_bbox_pred, rpn_bbox_targets)),2),0.5*sigma1), smoothL1_sign), \
    tf.mul(tf.sub(tf.abs(tf.sub(rpn_bbox_pred, rpn_bbox_targets)),0.5/sigma1),\
        tf.abs(smoothL1_sign-1)))), reduction_indices=[1,2])),1)
rpn_loss_bbox_label = rpn_loss_bbox
zero_count, one_count = tf.py_func(bbox_counter, [rpn_labels_ind], [tf.float32, tf.float32])

#ROI PROPOSAL
rpn_cls_prob = rpn_cls_soft
rpn_cls_prob_reshape = tf.reshape(rpn_cls_prob, [1, height, width, 18])

rpn_rois = tf.py_func(proposal, [rpn_cls_prob_reshape, rpn_bbox_pred], [tf.float32])
rpn_rois = tf.reshape(rpn_rois, [-1, 5])

rcnn_rois, rcnn_labels_ind, rcnn_bbox_targets, rcnn_bbox_inside_w, rcnn_bbox_outside_w = \
  tf.py_func(proposal_target, [rpn_rois, gt_boxbatch], [tf.int32, tf.int64, tf.float32, tf.float32, tf.float32])
rcnn_rois = tf.cast(tf.reshape(tf.pack(rcnn_rois), [-1, 5]), tf.float32)
rcnn_labels_ind = tf.reshape(tf.pack(rcnn_labels_ind), [-1])
rcnn_bbox_targets = tf.reshape(tf.pack(rcnn_bbox_targets), [-1, 21 * 4])
rcnn_bbox_inside_w = tf.reshape(tf.pack(rcnn_bbox_inside_w), [-1, 21 * 4])
rcnn_bbox_outside_w = tf.reshape(tf.pack(rcnn_bbox_outside_w), [-1, 21 * 4])

#RCNN
with tf.name_scope("rcnn"):

  output_dim_tensor = tf.constant((7,7))
  h_conv13 = tf.transpose(h_conv13, [0,3,1,2]) #transpose since roi pool takes in NCHW
  [pool5, argmax] = roi_pooling_op.roi_pool(h_conv13, rcnn_rois, 7, 7, 1.0/16)
  pool5 = tf.transpose(pool5, [0,2,3,1]) #revert back to NHWC
  h_fc6 = tf.reshape(pool5, [-1, 7, 7, 1024])

  r7 = residual_block(h_fc6, 512, False, "rcnn", tr_stat = _rcnn_stat, bn_tr_stat = _tr_bn, add_l2_stat = _add_l2, branch = "near")
  r8 = residual_block(r7, 512, False, "rcnn", tr_stat = _rcnn_stat, bn_tr_stat = _tr_bn, add_l2_stat = _add_l2, branch = "far")
  r9 = residual_block(r8, 512, False, "rcnn", tr_stat = _rcnn_stat, bn_tr_stat = _tr_bn, add_l2_stat = _add_l2, branch = "far")
  gp = tf.reduce_mean(r9, [1,2])

with tf.name_scope("fc"):

  W_end_cls = weight_variable([2048, 21], "fc",  tr_stat = _fc_stat, add_l2_stat = _add_l2)
  b_end_cls = bias_variable([21], "fc", tr_stat = _fc_stat, add_l2_stat = _add_l2)
  end_cls = tf.matmul(gp, W_end_cls) + b_end_cls + 1e-20

  W_end_bbox = weight_variable([2048, 84], "fc",  tr_stat = _fc_stat, add_l2_stat = _add_l2)
  b_end_bbox = bias_variable([84], "fc", tr_stat = _fc_stat, add_l2_stat = _add_l2)
  end_bbox = tf.matmul(gp, W_end_bbox) + b_end_bbox + 1e-20

#END_LOSS
end_cls_soft = tf.nn.softmax(end_cls)
loss_cls = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(end_cls, rcnn_labels_ind))
loss_cls_label = loss_cls

pred = tf.argmax(end_cls_soft, 1)
end_correct_prediction = tf.equal(pred, rcnn_labels_ind)
end_cls_accuracy = tf.reduce_mean(tf.cast(end_correct_prediction, tf.float32))

sigma2 = s2 * s2
smoothL1_sign_bbox = tf.cast(tf.less(tf.abs(tf.sub(end_bbox, rcnn_bbox_targets)),1/sigma2),tf.float32)
loss_bbox = tf.mul(tf.reduce_mean(tf.reduce_sum(tf.mul(rcnn_bbox_outside_w,tf.add( \
    tf.mul(tf.mul(tf.pow(tf.mul(rcnn_bbox_inside_w, tf.sub(end_bbox, rcnn_bbox_targets))*1,2),0.5*sigma2), smoothL1_sign_bbox), \
    tf.mul(tf.sub(tf.abs(tf.sub(end_bbox, rcnn_bbox_targets)),0.5/sigma2),tf.abs(smoothL1_sign_bbox-1)))), reduction_indices=[1])),1)

#loss1 = rpn_loss_cls + rpn_loss_bbox #+ (tf.add_n(tf.get_collection('weight_losses_trunkbase')) + tf.add_n(tf.get_collection('weight_losses_rpn')))/256
#loss2 = loss_cls + loss_bbox #+ ((tf.add_n(tf.get_collection('weight_losses_trunk')) + tf.add_n(tf.get_collection('weight_losses_rcnn')))/rpn_batch_size)
#loss3 = rpn_loss_cls + rpn_loss_bbox #+ tf.add_n(tf.get_collection('weight_losses_rpn'))/256
#loss4 = loss_cls + loss_bbox #+ tf.add_n(tf.get_collection('weight_losses_rcnn'))/rpn_batch_size
total_loss = rpn_loss_cls + rpn_loss_bbox + loss_cls + loss_bbox + tf.add_n(tf.get_collection('weight_losses_trunk')) + tf.add_n(tf.get_collection('weight_losses_rpn')) +\
     tf.add_n(tf.get_collection('weight_losses_rcnn')) +  tf.add_n(tf.get_collection('weight_losses_fc'))

#VARIABLES, OPTIMIZERS, AND LOSSES
trunk_weights = [v for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "trunk") if "weights" in v.name]
trunk_biases = [v for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "trunk") if "biases" in v.name]
rpn_weights = [v for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "rpn") if "weights" in v.name]
rpn_biases = [v for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "rpn") if "biases" in v.name]
rcnn_weights = [v for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "rcnn") if "weights" in v.name]
rcnn_biases = [v for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "rcnn") if "biases" in v.name]
fc_weights = [v for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "fc") if "weights" in v.name]
fc_biases = [v for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "fc") if "biases" in v.name]

lr = tf.placeholder(tf.float32)
var_list1_w =  rcnn_weights + fc_weights + trunk_weights + rpn_weights
var_list1_b = rcnn_biases + fc_biases + trunk_biases + rpn_biases
opt1_w = tf.train.MomentumOptimizer(lr, momentum=0.9, use_nesterov=True)
opt1_b = tf.train.MomentumOptimizer(lr*2, momentum=0.9, use_nesterov=True)
grads1 = tf.gradients(total_loss,  var_list1_w + var_list1_b)
grads1_w = grads1[:len(var_list1_w)]
grads1_b = grads1[len(var_list1_w):]
train_op1_w = opt1_w.apply_gradients(zip(grads1_w, var_list1_w))
train_op1_b = opt1_b.apply_gradients(zip(grads1_b, var_list1_b))
batchnorm_updates = tf.get_collection('update_ops')
batchnorm_updates_op = tf.group(*batchnorm_updates)
first_train_op = tf.group(train_op1_w, train_op1_b, batchnorm_updates_op)

anchor_fraction = one_count / (zero_count + one_count)

#TRAINING

trunk_vars = [v for v in tf.all_variables() if np.logical_and("trunk" in v.name, "Momentum" not in v.name)]
rpn_vars = [v for v in tf.all_variables() if np.logical_and("rpn" in v.name, "Momentum" not in v.name)]
rcnn_vars = [v for v in tf.all_variables() if np.logical_and("rcnn" in v.name, "Momentum" not in v.name)]
saver_all_trunkrcnn = tf.train.Saver(trunk_vars + rcnn_vars)
saver_all = tf.train.Saver()

run_size = len(image_files)

accu_rpn_losscls = 0
accu_rpn_lossbbox = 0
accu_rcnn_losscls = 0
accu_rcnn_lossbbox = 0
accu_rpn_accuracy = 0
accu_rcnn_accuracy = 0
accu_anchor_fraction = 0
accu_anchor_count = 0

epoch = 40

init = tf.initialize_all_variables()
with tf.Session() as sess:
  sess.run(init, feed_dict = {gate : [0.0]})
  #saver_all_trunkrcnn.restore(sess, "./imagenet_resnet.ckpt")
  #saver_all_trunkrcnn.restore(sess, "./caltech_160k.ckpt")
  #saver_all.restore(sess,"./z7_300k.ckpt")
  saver_all.restore(sess, "./z7_end_to_end_girschick.ckpt")
  for x in range(epoch):
      rand_id = np.random.permutation(np.arange(run_size))
      for i in range(run_size):
          if i % 10 == 0:
              print "epoch: " + str(x) + "  iter: " + str(x*run_size+i)

          #PROCESS IMAGE AND LABELS
          img_train = cv2.imread(image_files[rand_id[i]])
          img_train = cv2.resize(img_train, (1000, 600))
          lb_train = np.array(list(csv.reader(open(gt_files[rand_id[i]],"rb"),delimiter=','))).astype('int')
          gt_train = np.zeros((lb_train.shape[0], 5),dtype=np.int64)
          gt_train[:,4] = lb_train[:,0]
          gt_train[:,:4] = lb_train[:,1:]
          img_train, gt_train = flip(img_train, gt_train, im_info)
          img_train = np.reshape(img_train, [1, 600, 1000, 3])

          decay = np.floor(x/2)
          learn_rate = 0.001 * np.power(0.1,decay)

          #RUN TRAIN OP
          _, rpnlosscls, rpnlossbbox, losscls, lossbbox, rpnaccuracy, rcnnaccruacy, anchorfraction, anchorcount = \
              sess.run([first_train_op, rpn_loss_cls, rpn_loss_bbox, loss_cls, \
                  loss_bbox, rpn_cls_accuracy, end_cls_accuracy, anchor_fraction, o_cls], \
              feed_dict = {lr : learn_rate, gate: [0.0], prep_img : img_train, gt_box : gt_train})

          accu_rpn_losscls += rpnlosscls
          accu_rpn_lossbbox += rpnlossbbox
          accu_rcnn_losscls += losscls
          accu_rcnn_lossbbox += lossbbox
          accu_rpn_accuracy += rpnaccuracy
          accu_rcnn_accuracy += rcnnaccruacy
          accu_anchor_fraction += anchorfraction
          accu_anchor_count += anchorcount

          if np.logical_and(i % 100 == 0, (x+i) != 0):
              print ""
              print "training update:"
              print "average rpn_cls_loss:     " + str(accu_rpn_losscls/100)
              print "average rpn_bbox_loss:    " + str(accu_rpn_lossbbox/100)
              print "average end_cls_loss:     " + str(accu_rcnn_losscls/100)
              print "average end_bbox_loss:    " + str(accu_rcnn_lossbbox/100)
              print "average rpn accuracy:     " + str(accu_rpn_accuracy/100)
              print "average rcnn accuracy:    " + str(accu_rcnn_accuracy/100)
              u_anchors, indexes, pred_, lb_ = sess.run([unique_rpn_cls, o_cls_ind, pred, rcnn_labels_ind], \
                  feed_dict = {lr : learn_rate, gate: [0.0], prep_img : img_train, gt_box : gt_train})
              print "unique_anchors:           " + str(u_anchors/1)
              print "average correct anchors:  " + str(accu_anchor_count/100)
              print "average anchor_fraction:  " + str(accu_anchor_fraction/100)
              if np.logical_and(i == 100, x == 0):
                  old_indexes = indexes
              if i > 0:
                  if indexes.shape == old_indexes.shape:
                      change = np.equal(indexes, old_indexes)
                      test = np.sum(change == 0)
                      print "anchor change:            " + str(test)
                  else:
                      print "there's change"
                      print ""
                  old_indexes = indexes
              print "predictions:"
              print pred_
              print "gt:"
              print lb_
              print ""

              accu_rpn_losscls = 0
              accu_rpn_lossbbox = 0
              accu_rcnn_losscls = 0
              accu_rcnn_lossbbox = 0
              accu_rpn_accuracy = 0
              accu_rcnn_accuracy = 0
              accu_anchor_fraction = 0
              accu_anchor_count = 0

          if i % 2000 == 0:
              save_path = saver_all.save(sess, "./z7_end_to_end_girschick.ckpt")
              print ""
              print "model saved"
              print ""

  sess.close()

max_stars_repo_name: rancheng/TF-FRCNN

max_issues_repo_name: rancheng/TF-FRCNN

max_forks_repo_name: rancheng/TF-FRCNN


text:
import math
import re
from collections import deque
from typing import Dict, List, cast
import glob

import numpy as np
from nptyping import Float, NDArray, Shape  # type: ignore

from .ModelBase import Justify, TextEffects
from .ModelSchema import (Arc, Circle, GlobalLabel, Pin, PinList, Polyline,
                          Rectangle, Symbol, isUnit)
from .Typing import POS_T, PTS_T

def search_font(name: str):
    paths: List[str] = ['/usr/share/fonts', '/usr/local/share/fonts', '~/.local/share/fonts']
    for path in paths:
        for file in glob.glob(f'{path}/**/{name}.ttf'):
            return file

def f_coord(arr) -> PTS_T:
    arr = np.array(arr)
    return [(np.min(arr[..., 0]), np.min(arr[..., 1])),
            (np.max(arr[..., 0]), np.max(arr[..., 1]))]

MIRROR = {
    '': np.array((1, 0, 0, -1)),
    'x': np.array((1, 0, 0, 1)),
    'y': np.array((-1, 0, 0, -1)),
    # 3: np.array((0, -1)),
}

def totuple(a: NDArray[Shape["2, 2"], Float]):  # type: ignore
    if len(a) == 0:
        return a
    if isinstance(a[0], np.ndarray):
        return tuple(map(tuple, a))
    return (a[0], a[1])

def add(pos: POS_T, summand: POS_T) -> POS_T:
    """
    Add two positions.

    :param pos POS_T: Origin position.
    :param summand POS_T: The position to add.
    :rtype POS_T: Sum of the two positions.
    """
    return cast(POS_T, totuple(np.round(np.array(pos) + np.array(summand), 3)))

def sub(pos: POS_T, subtrahend: POS_T | PTS_T) -> POS_T:
    """
    Subtract two positions.

    :param pos POS_T: Origin position.
    :param subtrahend POS_T: The position to subtract.
    :rtype POS_T: Difference of the two positions.
    """
    return cast(POS_T, totuple(np.round(np.array(pos) - np.array(subtrahend), 3)))  # type: ignore

def mul(pos: POS_T, multiplier: POS_T) -> POS_T:
    """
    Multiply a position by a scalar.

    :param pos POS_T: Origin position.
    :param multiplier POS_T: The scalar to multiply by.
    :rtype POS_T: Multiplied position.
    """
    return cast(POS_T, totuple(np.round(np.array(pos) * np.array(multiplier), 3)))

def transform(symbol: Symbol | Pin | GlobalLabel, path=(0, 0)) -> PTS_T:
    """
    Transform the coordinates of a Symbol or Pin.

    :param symbol Symbol|Pin: Symbol or Pin
    :param path List[float]: points to transform relative to the Symbol.
    :rtype POS_T: Transformed coordinates
    :raises TypeError: When the element is not a Symbol or Pin.
    """
    if isinstance(symbol, Symbol):
        theta = np.deg2rad(-symbol.angle)
        trans = np.reshape(MIRROR[symbol.mirror], (2, 2)).T
        rot = np.array([[math.cos(theta), -math.sin(theta)],
                        [math.sin(theta), math.cos(theta)]])

        verts = np.matmul(path, rot)
        verts = np.matmul(verts, trans)
        verts = (symbol.pos + verts)
        verts = np.round(verts, 3)
        return cast(PTS_T, totuple(verts))

    if isinstance(symbol, Pin):
        theta = np.deg2rad(symbol.angle)
        rot = np.array([math.cos(theta), math.sin(theta)])
        verts = np.array([symbol.pos, symbol.pos + rot * symbol.length])
        verts = np.round(verts, 3)
        return cast(PTS_T, totuple(verts))

#    if isinstance(symbol, Footprint):
#        theta = np.deg2rad(symbol.angle)
#        rot = np.array([[math.cos(theta), -math.sin(theta)],
#                        [math.sin(theta), math.cos(theta)]])
#        verts = np.matmul(path, rot)
#        verts = (symbol.pos + verts)
#        verts = np.round(verts, 3)
#        return cast(PTS_T, totuple(verts))
#
    if isinstance(symbol, GlobalLabel):
        theta = np.deg2rad(symbol.angle)
        rot = np.array([[math.cos(theta), -math.sin(theta)],
                        [math.sin(theta), math.cos(theta)]])
        verts = np.matmul(path, rot)
        verts = (symbol.pos + verts)
        verts = np.round(verts, 3)
        return cast(PTS_T, totuple(verts))

    raise TypeError(f'unknown type {type(symbol)}')

def symbol_size(symbol: Symbol):
    """
    Calculate the Symbol size.

    :param symbol Symbol: [TODO:description]
    """
    sizes = []
    assert symbol.library_symbol, "Library symbol is not set"
    for unit in symbol.library_symbol.units:
        if isUnit(unit, symbol.unit):
            for graph in unit.graphics:
                if isinstance(graph, Polyline):
                    sizes.append(f_coord(np.array(graph.points)))
                elif isinstance(graph, Rectangle):
                    sizes.append(np.array([(graph.start_x, graph.start_y),
                                           (graph.end_x, graph.end_y)]))
                elif isinstance(graph, Circle):
                    sizes.append(np.array([(graph.center[0] - graph.radius,
                                            graph.center[1] + graph.radius),
                                           (graph.center[0] + graph.radius,
                                            graph.center[1] + graph.radius)]))
                else:
                    #raise TypeError(f'unknown type {type(graph)}')
                    print(f"TypeError(f'unknown type {type(graph)}')")
            for pin in unit.pins:
                sizes.append(transform(pin))

    if len(sizes) == 0:
        return np.array([[0, 0], [0, 0]])
    return f_coord(np.array(sizes))

def get_pins(symbol: Symbol) -> PinList:
    """
    Get the pins of a symbol.

    :param symbol Symbol: The symbol.
    :rtype PinList: The pins.
    """
    _pins = PinList()
    assert symbol.library_symbol, 'library symbol is not set'
    for subsym in symbol.library_symbol.units:
        if isUnit(subsym, symbol.unit):
            _pins.extend(symbol, subsym.pins)
    return _pins

def pinPosition(symbol) -> List[int]:
    positions = pinByPositions(symbol)
    return [len(positions['west']), len(positions['south']),
            len(positions['east']), len(positions['north'])]

def pinByPositions(symbol: Symbol) -> Dict[str, List[Pin]]:
    """
    Get a Dictionary with the Pins sorted by position.

    :param
    """
    position: deque[List[Pin]] = deque([[], [], [], []])

    for pin in get_pins(symbol):
        assert pin.angle <= 270, "pin angle greater then 270°"
        position[int(pin.angle / 90)].append(pin)

    if 'x' in symbol.mirror:
        pos1 = position[1]
        pos3 = position[3]
        position[1] = pos3
        position[3] = pos1

    if 'y' in symbol.mirror:
        pos0 = position[0]
        pos2 = position[2]
        position[2] = pos0
        position[0] = pos2

    position.rotate(int(symbol.angle/90))
    return {'west': position[0], 'south': position[1], 'east': position[2], 'north': position[3]}

def placeFields(symbol: Symbol) -> None:
    positions = pinPosition(symbol)
    # TODO print(f'placeFields: {symbol.angle} {symbol.property("Reference").value} {positions}')
    vis_fields = [
        x for x in symbol.properties if not x.text_effects or not x.text_effects.hidden]
    _size = f_coord(transform(symbol, symbol_size(symbol)))
    if len(get_pins(symbol)) == 1:
        if positions[0] == 1:
            # left
            vis_fields[0].pos = (_size[1][0]+1.28, symbol.pos[1])
            assert vis_fields[0].text_effects, "pin has no text_effects"
            vis_fields[0].text_effects.justify = [Justify.LEFT]
            vis_fields[0].angle = 360 - symbol.angle

        elif positions[1] == 1:
            # bottom
            vis_fields[0].pos = (symbol.pos[0], _size[0][1]-1.28)
            assert vis_fields[0].text_effects, "pin has no text_effects"
            vis_fields[0].text_effects.justify = [Justify.CENTER]

        elif positions[2] == 1:
            # right
            vis_fields[0].pos = (_size[0][0]-1.28, symbol.pos[1])
            assert vis_fields[0].text_effects, "pin has no text_effects"
            vis_fields[0].text_effects.justify = [Justify.RIGHT]
            vis_fields[0].angle = 360 - symbol.angle

        elif positions[3] == 1:
            # up
            vis_fields[0].pos = (symbol.pos[0], _size[1][1]+1.28)
            assert vis_fields[0].text_effects, "pin has no text_effects"
            vis_fields[0].text_effects.justify = [Justify.CENTER]

    else:
        if positions[1] == 0:
            # fields top
            top_pos = _size[0][1] - ((len(vis_fields)-1) * 2) - 1.28
            for pin in vis_fields:
                pin.pos = (symbol.pos[0], top_pos)
                if pin.text_effects:
                    pin.text_effects.justify = [Justify.CENTER]
                else:
                    pin.text_effects = TextEffects(justify=[Justify.CENTER])
                pin.angle = 360 - symbol.angle
                top_pos += 2

        elif positions[2] == 0:
            # fields west
            top_pos = _size[0][1] + \
                ((_size[1][1] - _size[0][1]) / 2) - \
                ((len(vis_fields)-1) * 2) / 2
            for pin in vis_fields:
                pin.pos = (_size[1][0]+0.762, top_pos)
                if pin.text_effects:
                    pin.text_effects.justify = [
                        Justify.LEFT] if symbol.angle == 0 else [Justify.RIGHT]
                else:
                    pin.text_effects = TextEffects(
                        justify=[Justify.LEFT] if symbol.angle == 0 else [Justify.RIGHT])

                pin.angle = 0
                top_pos += 2

        elif positions[0] == 0:
            # fields east
            top_pos = _size[0][1] + \
                ((_size[1][1] - _size[0][1]) / 2) - \
                ((len(vis_fields)-1) * 2) / 2
            for pin in vis_fields:
                pin.pos = (_size[0][0]-0.762, top_pos)
                if pin.text_effects:
                    pin.text_effects.justify = [
                        Justify.RIGHT] if symbol.angle == 0 else [Justify.LEFT]
                else:
                    pin.text_effects = TextEffects(
                        justify=[Justify.RIGHT] if symbol.angle == 0 else [Justify.LEFT])
                pin.angle = 0
                top_pos += 2
            #assert False, "implement"
        elif positions[3] == 0:
            assert False, "implement, single pin fields bottom"
        else:
            assert False, "implement, single pin all sides have pins"

ext: py

max_stars_repo_name: spielhuus/nukleus

max_issues_repo_name: spielhuus/nukleus

max_forks_repo_name: spielhuus/nukleus

reason: import numpy


text:
''' Evaluation of agent trajectories '''

import json
import pprint; pp = pprint.PrettyPrinter(indent=4)  # NoQA

from .utils import load_datasets, Tokenizer
import numpy as np
from .bleu import multi_bleu

class SpeakerEvaluation(object):
    ''' Results submission format:
        [{'instr_id': string,
          'trajectory':[(viewpoint_id, heading_rads, elevation_rads),]}] '''

    def __init__(self, splits, instructions_per_path=None):
        self.splits = splits
        self.gt = {}
        self.instr_ids = []
        self.scans = []
        if instructions_per_path is None:
            instructions_per_path = 3
        self.instructions_per_path = instructions_per_path

        for item in load_datasets(splits):
            item['instructions'] = item['instructions'][:instructions_per_path]
            self.gt[item['path_id']] = item
            self.scans.append(item['scan'])
            self.instr_ids += ['%d_%d' % (item['path_id'], i)
                               for i in range(len(item['instructions']))]

        self.scans = set(self.scans)
        self.instr_ids = set(self.instr_ids)

    def score_results(self, results, verbose=False):
        # results should be a dictionary mapping instr_ids to dictionaries,
        # with each dictionary containing (at least) a 'words' field
        instr_ids = set(self.instr_ids)
        instr_count = 0
        results_by_base_id = {}
        mismatches = []
        for instr_id, result in results.items():
            if instr_id in instr_ids:
                instr_ids.remove(instr_id)

                base_id = int(instr_id.split('_')[0])

                if base_id in results_by_base_id:
                    old_predicted = results_by_base_id[base_id]['words']
                    new_predicted = result['words']
                    if old_predicted != new_predicted:
                        mismatches.append((old_predicted, new_predicted))
                else:
                    results_by_base_id[base_id] = result

        if mismatches:
            print("mismatching outputs for sentences:")
            for old_pred, new_pred in mismatches:
                print(old_pred)
                print(new_pred)
                print()

        assert len(instr_ids) == 0, \
            'Missing %d of %d instruction ids from %s' % (
            len(instr_ids), len(self.instr_ids), ",".join(self.splits))

        all_refs = []
        all_hyps = []

        model_scores = []

        instruction_replaced_gt = []

        skip_count = 0
        skipped_refs = set()
        for base_id, result in sorted(results_by_base_id.items()):
            instr_count += 1
            gt = self.gt[base_id]
            tokenized_refs = [
                Tokenizer.split_sentence(ref) for ref in gt['instructions']]
            tokenized_hyp = result['words']

            replaced_gt = gt.copy()
            replaced_gt['instructions'] = [' '.join(tokenized_hyp)]
            instruction_replaced_gt.append(replaced_gt)

            if 'score' in result:
                model_scores.append(result['score'])

            if len(tokenized_refs) != self.instructions_per_path:
                skip_count += 1
                skipped_refs.add(base_id)
                continue
            all_refs.append(tokenized_refs)
            all_hyps.append(tokenized_hyp)

            if verbose and instr_count % 100 == 0:
                for i, ref in enumerate(tokenized_refs):
                    print("ref {}:\t{}".format(i, ' '.join(ref)))
                print("pred  :\t{}".format(' '.join(tokenized_hyp)))
                print()

        if skip_count != 0:
            print("skipped {} instructions without {} refs: {}".format(
                skip_count, self.instructions_per_path, ' '.join(
                    str(i) for i in skipped_refs)))

        model_score = np.mean(model_scores)
        bleu, unpenalized_bleu = multi_bleu(all_refs, all_hyps)

        score_summary = {
            'model_score': model_score,
            'bleu': bleu,
            'unpenalized_bleu': unpenalized_bleu,
        }
        return score_summary, instruction_replaced_gt

    def score_file(self, output_file, verbose=False):
        ''' Evaluate each agent trajectory based on how close it got to the
        goal location '''
        with open(output_file) as f:
            return self.score_results(json.load(f), verbose=verbose)

def eval_seq2seq():
    import train_speaker
    outfiles = [
        train_speaker.RESULT_DIR + 'seq2seq_teacher_imagenet_%s_iter_5000.json',  # NoQA
        train_speaker.RESULT_DIR + 'seq2seq_sample_imagenet_%s_iter_20000.json',  # NoQA
    ]
    for outfile in outfiles:
        for split in ['val_seen', 'val_unseen']:
            ev = SpeakerEvaluation([split])
            score_summary, _ = ev.score_file(outfile % split)
            print('\n%s' % outfile)
            pp.pprint(score_summary)

if __name__ == '__main__':
    # from train import make_arg_parser
    # utils.run(make_arg_parser(), eval_simple_agents)
    # eval_seq2seq()
    pass


text:
###############################################################################
#   GaussianAmplitudeWrapperPotential.py: Wrapper to modulate the amplitude
#                                         of a potential with a Gaussian
###############################################################################
import numpy
from .WrapperPotential import parentWrapperPotential
from .Potential import _APY_LOADED
from galpy.util import bovy_conversion
if _APY_LOADED:
    from astropy import units
class GaussianAmplitudeWrapperPotential(parentWrapperPotential):
    """Potential wrapper class that allows the amplitude of a Potential object to be modulated as a Gaussian. The amplitude A applied to a potential wrapped by an instance of this class is changed as

    .. math::

        A(t) = amp\\,\\exp\\left(-\\frac{[t-t_0]^2}{2\\,\\sigma^2}\\right)
    """
    def __init__(self,amp=1.,pot=None,to=0.,sigma=1.,ro=None,vo=None):
        """
        NAME:

           __init__

        PURPOSE:

           initialize a GaussianAmplitudeWrapper Potential

        INPUT:

           amp - amplitude to be applied to the potential (default: 1.)

           pot - Potential instance or list thereof; this potential is made to rotate around the z axis by the wrapper

           to= (0.) time at which the Gaussian peaks

           sigma= (1.) standard deviation of the Gaussian (can be a Quantity)

        OUTPUT:

           (none)

        HISTORY:

           2018-02-21 - Started - Bovy (UofT)

        """
        if _APY_LOADED and isinstance(to,units.Quantity):
            to= to.to(units.Gyr).value\
                /bovy_conversion.time_in_Gyr(self._vo,self._ro)
        if _APY_LOADED and isinstance(sigma,units.Quantity):
            sigma= sigma.to(units.Gyr).value\
                /bovy_conversion.time_in_Gyr(self._vo,self._ro)
        self._to= to
        self._sigma2= sigma**2.
        self.hasC= True
        self.hasC_dxdv= True

    def _smooth(self,t):
        return numpy.exp(-0.5*(t-self._to)**2./self._sigma2)

    def _wrap(self,attribute,*args,**kwargs):
        return self._smooth(kwargs.get('t',0.))\
                *self._wrap_pot_func(attribute)(self._pot,*args,**kwargs)

max_stars_repo_name: turnergarrow/galpy

max_issues_repo_name: BurcuAkbulut/galpy

max_forks_repo_name: BurcuAkbulut/galpy

reason: import numpy,from astropy


text:
'''
Created on May 22, 2018

@author: rsepulveda3
'''

#import sys
#sys.path.append('c:/Users/rsepulveda3/AppData/Local/Programs/Python/Python36-32/Lib/site-packages')

import numpy as np
import argparse
import time
import cv2

'''
asdf

 Users\rsepulveda3\workspace\pythonTest\model

 --image add path to image
 Users\rsepulveda3\workspace\pythonTest\model

 --prototxt path to Caffe 'deploy' prototxt file
 Users\rsepulveda3\workspace\pythonTest\model\bvlc_googlenet.prototxt

 --model path to Caffe pre-trained model
 Users\rsepulveda3\workspace\pythonTest\model\bvlc_googlenet.caffemodel

 --labels path to ImageNet labels (i.e., syn-sets)
 Users\rsepulveda3\workspace\pythonTest\model

asdf
'''

image = cv2.imread('dogo.jpg')
#blop= cv2.resize(image,(224,224))
blob = cv2.dnn.blobFromImage(image, 1, (224, 224),(104, 117, 123))
'''

image = cv2.imread('spain.jpg')
image2 = cv2.resize(image,(500,600))
blop = image2[50:300,100:400]
blop = cv2.resize(blop,(224, 224))
blob = cv2.dnn.blobFromImage(blop, 1, (224, 224),(104, 117, 123))
'''
rows = open('synset_words.txt').read().strip().split("\n")
classes = [r[r.find(" ") + 1:].split(",")[0] for r in rows]

'''
img=cv2.imread('Piedmont.jpg')
'''
cv2.imshow('image',image)
#cv2.imshow('image',blop)
cv2.waitKey(0)
cv2.destroyAllWindows()

model='C:/Users/rsepulveda3/workspace/pythonTest/model/bvlc_googlenet.caffemodel'
prototxt='C:/Users/rsepulveda3/workspace/pythonTest/model/bvlc_googlenet.prototxt'
net = cv2.dnn.readNetFromCaffe(prototxt, model)

net.setInput(blob)
start = time.time()
preds = net.forward()
end = time.time()
print("[INFO] classification took {:.5} seconds".format(end - start))

idxsTest = np.argsort(preds[0])
idxs = np.argsort(preds[0])[::-1][:5]

#what=preds[0, 0, 1, 2]

for (i, idx) in enumerate(idxs):
    # draw the top prediction on the input image
    if i == 0:
        text = "Label: {}, {:.2f}%".format(classes[idx],
            preds[0][idx] * 100)
        cv2.putText(image, text, (5, 25),  cv2.FONT_HERSHEY_SIMPLEX,
            0.7, (0, 0, 255), 2)

    # display the predicted label + associated probability to the
    # console
    print("[INFO] {}. label: {}, probability: {:.5}".format(i + 1,
        classes[idx], preds[0][idx]))

print(idxs)
print(preds[0,idxs[0]])

max_stars_repo_name: SirLaughalot/pythonTest

max_issues_repo_name: SirLaughalot/pythonTest

max_forks_repo_name: SirLaughalot/pythonTest


text:
# Copyright 2018,2019,2020,2021 Sony Corporation.
# Copyright 2021 Sony Group Corporation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

'''
Parser utilities for Darknet weight files,
used inside functions in darknet19.py or yolov2.py.
'''

def load_weights_raw(filename):
    '''
    Load float32 flattened weights from .weights file.
    '''
    from struct import unpack
    import numpy as np
    with open(filename, 'rb') as fd:
        # Reading header
        # <https://github.com/pjreddie/darknet/blob/b13f67bfdd87434e141af532cdb5dc1b8369aa3b/src/parser.c#L1230-L1242>
        major, minor, revision = unpack('<iii', fd.read(4 * 3))
        if (major * 10 + minor >= 2) and (major < 1000) and (minor < 1000):
            # Reading `size_t seen;`
            fd.read(8)
        else:
            # Reading `int seen;`
            fd.read(4)

        # Reading weights in float32
        dn_weights = np.fromfile(fd, dtype=np.float32)
    return dn_weights

def get_convolutional_params(params, prefix, no_bn=False):
    '''
    Returns conv/W, bn/beta, bn/gamma, bn/mean, bn/var

    '''
    if no_bn:
        return (
            params['/'.join((prefix, 'conv/W'))],
            params['/'.join((prefix, 'conv/b'))],
        )
    return (
        params['/'.join((prefix, 'conv/W'))],
        params['/'.join((prefix, 'bn/beta'))],
        params['/'.join((prefix, 'bn/gamma'))],
        params['/'.join((prefix, 'bn/mean'))],
        params['/'.join((prefix, 'bn/var'))],
    )

def set_param_and_get_next_cursor(dn_params, cursor, param):
    '''
    '''
    param.d = dn_params[cursor:cursor + param.size].reshape(param.shape)
    return cursor + param.size

def load_convolutional_and_get_next_cursor_core(dn_params, cursor, w, b,
                                                g=None, m=None, v=None):
    '''
    '''
    cursor = set_param_and_get_next_cursor(dn_params, cursor, b)
    if g is not None:
        assert (m is not None and v is not None)
        cursor = set_param_and_get_next_cursor(dn_params, cursor, g)
        cursor = set_param_and_get_next_cursor(dn_params, cursor, m)
        cursor = set_param_and_get_next_cursor(dn_params, cursor, v)
    cursor = set_param_and_get_next_cursor(dn_params, cursor, w)
    return cursor

def load_convolutional_and_get_next_cursor(dn_params, cursor,
                                           params, prefix, no_bn=False):
    '''
    '''
    return load_convolutional_and_get_next_cursor_core(
        dn_params, cursor, *get_convolutional_params(params, prefix, no_bn))

max_stars_repo_name: AaratiAkkapeddi/nnabla-examples

max_issues_repo_name: AaratiAkkapeddi/nnabla-examples

max_forks_repo_name: AaratiAkkapeddi/nnabla-examples


text:
# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for rdp_privacy_accountant."""

import math
import sys

from absl.testing import absltest
from absl.testing import parameterized
import mpmath
import numpy as np

from tensorflow_privacy.privacy.analysis import dp_event
from tensorflow_privacy.privacy.analysis import privacy_accountant
from tensorflow_privacy.privacy.analysis import privacy_accountant_test
from tensorflow_privacy.privacy.analysis import rdp_privacy_accountant

def _get_test_rdp(event, count=1):
  accountant = rdp_privacy_accountant.RdpAccountant(orders=[2.71828])
  accountant.compose(event, count)
  return accountant._rdp[0]

def _log_float_mp(x):
  # Convert multi-precision input to float log space.
  if x >= sys.float_info.min:
    return float(mpmath.log(x))
  else:
    return -np.inf

def _compute_a_mp(sigma, q, alpha):
  """Compute A_alpha for arbitrary alpha by numerical integration."""

  def mu0(x):
    return mpmath.npdf(x, mu=0, sigma=sigma)

  def _mu_over_mu0(x, q, sigma):
    return (1 - q) + q * mpmath.exp((2 * x - 1) / (2 * sigma**2))

  def a_alpha_fn(z):
    return mu0(z) * _mu_over_mu0(z, q, sigma)**alpha

  bounds = (-mpmath.inf, mpmath.inf)
  a_alpha, _ = mpmath.quad(a_alpha_fn, bounds, error=True, maxdegree=8)
  return a_alpha

class RdpPrivacyAccountantTest(privacy_accountant_test.PrivacyAccountantTest,
                               parameterized.TestCase):

  def _make_test_accountants(self):
    return [
        rdp_privacy_accountant.RdpAccountant(
            [2.0], privacy_accountant.NeighboringRelation.ADD_OR_REMOVE_ONE),
        rdp_privacy_accountant.RdpAccountant(
            [2.0], privacy_accountant.NeighboringRelation.REPLACE_ONE)
    ]

  def test_supports(self):
    aor_accountant = rdp_privacy_accountant.RdpAccountant(
        [2.0], privacy_accountant.NeighboringRelation.ADD_OR_REMOVE_ONE)
    ro_accountant = rdp_privacy_accountant.RdpAccountant(
        [2.0], privacy_accountant.NeighboringRelation.REPLACE_ONE)

    event = dp_event.GaussianDpEvent(1.0)
    self.assertTrue(aor_accountant.supports(event))
    self.assertTrue(ro_accountant.supports(event))

    event = dp_event.SelfComposedDpEvent(dp_event.GaussianDpEvent(1.0), 6)
    self.assertTrue(aor_accountant.supports(event))
    self.assertTrue(ro_accountant.supports(event))

    event = dp_event.ComposedDpEvent(
        [dp_event.GaussianDpEvent(1.0),
         dp_event.GaussianDpEvent(2.0)])
    self.assertTrue(aor_accountant.supports(event))
    self.assertTrue(ro_accountant.supports(event))

    event = dp_event.PoissonSampledDpEvent(0.1, dp_event.GaussianDpEvent(1.0))
    self.assertTrue(aor_accountant.supports(event))
    self.assertFalse(ro_accountant.supports(event))

    composed_gaussian = dp_event.ComposedDpEvent(
        [dp_event.GaussianDpEvent(1.0),
         dp_event.GaussianDpEvent(2.0)])
    event = dp_event.PoissonSampledDpEvent(0.1, composed_gaussian)
    self.assertTrue(aor_accountant.supports(event))
    self.assertFalse(ro_accountant.supports(event))

    event = dp_event.SampledWithoutReplacementDpEvent(
        1000, 10, dp_event.GaussianDpEvent(1.0))
    self.assertFalse(aor_accountant.supports(event))
    self.assertTrue(ro_accountant.supports(event))

    event = dp_event.SampledWithoutReplacementDpEvent(1000, 10,
                                                      composed_gaussian)
    self.assertFalse(aor_accountant.supports(event))
    self.assertTrue(ro_accountant.supports(event))

    event = dp_event.SampledWithReplacementDpEvent(
        1000, 10, dp_event.GaussianDpEvent(1.0))
    self.assertFalse(aor_accountant.supports(event))
    self.assertFalse(ro_accountant.supports(event))

  def test_rdp_composition(self):
    base_event = dp_event.GaussianDpEvent(3.14159)
    base_rdp = _get_test_rdp(base_event)

    rdp_with_count = _get_test_rdp(base_event, count=6)
    self.assertAlmostEqual(rdp_with_count, base_rdp * 6)

    rdp_with_self_compose = _get_test_rdp(
        dp_event.SelfComposedDpEvent(base_event, 6))
    self.assertAlmostEqual(rdp_with_self_compose, base_rdp * 6)

    rdp_with_self_compose_and_count = _get_test_rdp(
        dp_event.SelfComposedDpEvent(base_event, 2), count=3)
    self.assertAlmostEqual(rdp_with_self_compose_and_count, base_rdp * 6)

    rdp_with_compose = _get_test_rdp(dp_event.ComposedDpEvent([base_event] * 6))
    self.assertAlmostEqual(rdp_with_compose, base_rdp * 6)

    rdp_with_compose_and_self_compose = _get_test_rdp(
        dp_event.ComposedDpEvent([
            dp_event.SelfComposedDpEvent(base_event, 1),
            dp_event.SelfComposedDpEvent(base_event, 2),
            dp_event.SelfComposedDpEvent(base_event, 3)
        ]))
    self.assertAlmostEqual(rdp_with_compose_and_self_compose, base_rdp * 6)

    base_event_2 = dp_event.GaussianDpEvent(1.61803)
    base_rdp_2 = _get_test_rdp(base_event_2)
    rdp_with_heterogeneous_compose = _get_test_rdp(
        dp_event.ComposedDpEvent([base_event, base_event_2]))
    self.assertAlmostEqual(rdp_with_heterogeneous_compose,
                           base_rdp + base_rdp_2)

  def test_zero_poisson_sample(self):
    accountant = rdp_privacy_accountant.RdpAccountant([3.14159])
    accountant.compose(
        dp_event.PoissonSampledDpEvent(0, dp_event.GaussianDpEvent(1.0)))
    self.assertEqual(accountant.get_epsilon(1e-10), 0)
    self.assertEqual(accountant.get_delta(1e-10), 0)

  def test_zero_fixed_batch_sample(self):
    accountant = rdp_privacy_accountant.RdpAccountant(
        [3.14159], privacy_accountant.NeighboringRelation.REPLACE_ONE)
    accountant.compose(
        dp_event.SampledWithoutReplacementDpEvent(
            1000, 0, dp_event.GaussianDpEvent(1.0)))
    self.assertEqual(accountant.get_epsilon(1e-10), 0)
    self.assertEqual(accountant.get_delta(1e-10), 0)

  def test_epsilon_non_private_gaussian(self):
    accountant = rdp_privacy_accountant.RdpAccountant([3.14159])
    accountant.compose(dp_event.GaussianDpEvent(0))
    self.assertEqual(accountant.get_epsilon(1e-1), np.inf)

  def test_compute_rdp_gaussian(self):
    alpha = 3.14159
    sigma = 2.71828
    event = dp_event.GaussianDpEvent(sigma)
    accountant = rdp_privacy_accountant.RdpAccountant(orders=[alpha])
    accountant.compose(event)
    self.assertAlmostEqual(accountant._rdp[0], alpha / (2 * sigma**2))

  def test_compute_rdp_multi_gaussian(self):
    alpha = 3.14159
    sigma1, sigma2 = 2.71828, 6.28319

    rdp1 = alpha / (2 * sigma1**2)
    rdp2 = alpha / (2 * sigma2**2)
    rdp = rdp1 + rdp2

    accountant = rdp_privacy_accountant.RdpAccountant(orders=[alpha])
    accountant.compose(
        dp_event.PoissonSampledDpEvent(
            1.0,
            dp_event.ComposedDpEvent([
                dp_event.GaussianDpEvent(sigma1),
                dp_event.GaussianDpEvent(sigma2)
            ])))
    self.assertAlmostEqual(accountant._rdp[0], rdp)

  def test_effective_gaussian_noise_multiplier(self):
    np.random.seed(0xBAD5EED)
    sigmas = np.random.uniform(size=(4,))

    event = dp_event.ComposedDpEvent([
        dp_event.GaussianDpEvent(sigmas[0]),
        dp_event.SelfComposedDpEvent(dp_event.GaussianDpEvent(sigmas[1]), 3),
        dp_event.ComposedDpEvent([
            dp_event.GaussianDpEvent(sigmas[2]),
            dp_event.GaussianDpEvent(sigmas[3])
        ])
    ])

    sigma = rdp_privacy_accountant._effective_gaussian_noise_multiplier(event)
    multi_sigmas = list(sigmas) + [sigmas[1]] * 2
    expected = sum(s**-2 for s in multi_sigmas)**-0.5
    self.assertAlmostEqual(sigma, expected)

  def test_compute_rdp_poisson_sampled_gaussian(self):
    orders = [1.5, 2.5, 5, 50, 100, np.inf]
    noise_multiplier = 2.5
    sampling_probability = 0.01
    count = 50
    event = dp_event.SelfComposedDpEvent(
        dp_event.PoissonSampledDpEvent(
            sampling_probability, dp_event.GaussianDpEvent(noise_multiplier)),
        count)
    accountant = rdp_privacy_accountant.RdpAccountant(orders=orders)
    accountant.compose(event)
    self.assertTrue(
        np.allclose(
            accountant._rdp, [
                6.5007e-04, 1.0854e-03, 2.1808e-03, 2.3846e-02, 1.6742e+02,
                np.inf
            ],
            rtol=1e-4))

  def test_compute_epsilon_delta_pure_dp(self):
    orders = range(2, 33)
    rdp = [1.1 for o in orders]  # Constant corresponds to pure DP.

    epsilon = rdp_privacy_accountant._compute_epsilon(orders, rdp, delta=1e-5)
    # Compare with epsilon computed by hand.
    self.assertAlmostEqual(epsilon, 1.32783806176)

    delta = rdp_privacy_accountant._compute_delta(
        orders, rdp, epsilon=1.32783806176)
    self.assertAlmostEqual(delta, 1e-5)

  def test_compute_epsilon_delta_gaussian(self):
    orders = [0.001 * i for i in range(1000, 100000)]

    # noise multiplier is chosen to obtain exactly (1,1e-6)-DP.
    rdp = rdp_privacy_accountant._compute_rdp_poisson_subsampled_gaussian(
        1, 4.530877117, orders)

    eps = rdp_privacy_accountant._compute_epsilon(orders, rdp, delta=1e-6)
    self.assertAlmostEqual(eps, 1)

    delta = rdp_privacy_accountant._compute_delta(orders, rdp, epsilon=1)
    self.assertAlmostEqual(delta, 1e-6)

  params = ({
      'q': 1e-7,
      'sigma': .1,
      'order': 1.01
  }, {
      'q': 1e-6,
      'sigma': .1,
      'order': 256
  }, {
      'q': 1e-5,
      'sigma': .1,
      'order': 256.1
  }, {
      'q': 1e-6,
      'sigma': 1,
      'order': 27
  }, {
      'q': 1e-4,
      'sigma': 1.,
      'order': 1.5
  }, {
      'q': 1e-3,
      'sigma': 1.,
      'order': 2
  }, {
      'q': .01,
      'sigma': 10,
      'order': 20
  }, {
      'q': .1,
      'sigma': 100,
      'order': 20.5
  }, {
      'q': .99,
      'sigma': .1,
      'order': 256
  }, {
      'q': .999,
      'sigma': 100,
      'order': 256.1
  })

  # pylint:disable=undefined-variable
  @parameterized.parameters(p for p in params)
  def test_compute_log_a_equals_mp(self, q, sigma, order):
    # Compare the cheap computation of log(A) with an expensive, multi-precision
    # computation.
    log_a = rdp_privacy_accountant._compute_log_a(q, sigma, order)
    log_a_mp = _log_float_mp(_compute_a_mp(sigma, q, order))
    np.testing.assert_allclose(log_a, log_a_mp, rtol=1e-4)

  def test_delta_bounds_gaussian(self):
    # Compare the optimal bound for Gaussian with the one derived from RDP.
    # Also compare the RDP upper bound with the "standard" upper bound.
    orders = [0.1 * x for x in range(10, 505)]
    eps_vec = [0.1 * x for x in range(500)]
    rdp = rdp_privacy_accountant._compute_rdp_poisson_subsampled_gaussian(
        1, 1, orders)
    for eps in eps_vec:
      delta = rdp_privacy_accountant._compute_delta(orders, rdp, epsilon=eps)
      # For comparison, we compute the optimal guarantee for Gaussian
      # using https://arxiv.org/abs/1805.06530 Theorem 8 (in v2).
      delta0 = math.erfc((eps - .5) / math.sqrt(2)) / 2
      delta0 = delta0 - math.exp(eps) * math.erfc((eps + .5) / math.sqrt(2)) / 2
      self.assertLessEqual(delta0, delta + 1e-300)  # need tolerance 10^-300

      # Compute the "standard" upper bound, which should be an upper bound.
      # Note, if orders is too sparse, this will NOT be an upper bound.
      if eps >= 0.5:
        delta1 = math.exp(-0.5 * (eps - 0.5)**2)
      else:
        delta1 = 1
      self.assertLessEqual(delta, delta1 + 1e-300)

  def test_epsilon_delta_consistency(self):
    orders = range(2, 50)  # Large range of orders (helps test for overflows).
    for q in [0, 0.01, 0.1, 0.8, 1.]:
      for multiplier in [0.0, 0.1, 1., 10., 100.]:
        event = dp_event.PoissonSampledDpEvent(
            q, dp_event.GaussianDpEvent(multiplier))
        accountant = rdp_privacy_accountant.RdpAccountant(orders)
        accountant.compose(event)
        for delta in [.99, .9, .1, .01, 1e-3, 1e-5, 1e-9, 1e-12]:
          epsilon = accountant.get_epsilon(delta)
          delta2 = accountant.get_delta(epsilon)
          if np.isposinf(epsilon):
            self.assertEqual(delta2, 1.0)
          elif epsilon == 0:
            self.assertLessEqual(delta2, delta)
          else:
            self.assertAlmostEqual(delta, delta2)

if __name__ == '__main__':
  absltest.main()

max_stars_repo_name: sarvex/privacy

max_issues_repo_name: sarvex/privacy

max_forks_repo_name: sarvex/privacy

reason: import numpy,import mpmath


text:
from automl_infrastructure.classifiers import Classifier
import numpy as np
import pandas as pd
from automl_infrastructure.utils import random_str

class EnsembleClassifier(Classifier):

    """
    The EnsembleClassifier class represents a complex classifier that consists of one or more sub-classifiers.
    The way the complex classifier works is:
        - applying every sub-classifier on the dataset
        - gather from every sub-classifiers the classes probabilities prediction, and apply ensemble model
          on them with extra features (optional)
    """
    def __init__(self, name, input_models, ensemble_model, ensemble_extra_features=None):
        """
        :param name: the name of the classifier.
        :type name: str

        :param input_models: list of sub-classifiers to aggregate their predictions.
        :type input_models: list of :class:`automl_infrastructure.classifiers.base.Classifier`

        :param ensemble_model: the aggregator classifier on top the sub-classifiers and possibly extra features.
        :type ensemble_model: :class:`automl_infrastructure.classifiers.base.Classifier`

        :param ensemble_extra_features: list of extra features that the ensemble model should work on.
        :type ensemble_extra_features: list of str ,optional
        """
        super().__init__(name)
        self._input_models = input_models
        self._ensemble_model = ensemble_model
        self._ensemble_features = ensemble_extra_features

        # used to create dataframe over ensemble features
        self._temporary_ensemble_features = None

    def _prepare_features(self, x, recreate_features_names=False):
        """
        Prepare features for the ensemble model, using the sub-classifier classes probabilities prediction along side
        extra features.

        :param x: the original dataset.
        :type x: pandas.DataFrame

        :param recreate_features_names: weather to regenerate new features cols names
        :type recreate_features_names: bool ,optional
        """
        # make probability prediction for every sub-classifier
        new_features = []
        for input_model in self._input_models:
            new_features.append(input_model.predict_proba(x))
        if self._ensemble_features is not None:
            new_features.append(x[self._ensemble_features])

        features_array = np.concatenate(new_features, axis=1)
        # create random features names if none supplied
        if recreate_features_names:
            new_features_names = ['FEATURE_{}_{}'.format(i, random_str(length=8))
                                  for i in range(features_array.shape[1])]
            self._temporary_ensemble_features = new_features_names
        else:
            if self._temporary_ensemble_features is None:
                raise Exception('You must initialize new features names.')
            new_features_names = self._temporary_ensemble_features

        # transform to pandas DataFrame form
        new_features_df = pd.DataFrame(features_array, columns=new_features_names)
        return new_features_df

    def fit(self, x, y, **kwargs):
        # train input models
        for input_model in self._input_models:
            input_model.fit(x, y, **kwargs)

        # train ensemble model based on input models predictions
        final_x = self._prepare_features(x, recreate_features_names=True)
        self._ensemble_model.fit(final_x, y, **kwargs)

    def predict(self, x):
        final_x = self._prepare_features(x)
        return self._ensemble_model.predict(final_x)

    def predict_proba(self, x):
        final_x = self._prepare_features(x)
        return self._ensemble_model.predict_proba(final_x)

    def set_params(self, params):
        for input_model in self._input_models:
            if input_model.name in params:
                input_model.set_params(params[input_model.name])
        if self._ensemble_model.name in params:
            self._ensemble_model.set_params(params[self._ensemble_model.name])

    def get_params(self, deep=True):
        params = {self._ensemble_model.name: self._ensemble_model.get_params()}
        if deep:
            for input_model in self._input_models:
                params[input_model.name] = input_model.get_params()
        return params


text:
#!/usr/bin/env python
# coding=utf-8

"""
python pred.py Animals model/mobile_Animals_wgt.h5
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from keras.applications.xception import Xception, preprocess_input
from keras.preprocessing import image
from keras.models import Model
from keras.layers import Dense, Activation
from utils import *
import numpy as np
import sys
import os

def main(superclass, model_weight):
    date = '20180321'
    classNum = {'A': 40, 'F': 40, 'V': 40, 'E': 40, 'H': 24}
    classAttrsNums = {'Animals': 123, 'Fruits': 58}
    unseen_labels = {
        'Animals': ['Label_A_02', 'Label_A_05', 'Label_A_08', 'Label_A_14', 'Label_A_20', 'Label_A_29', 'Label_A_31', 'Label_A_35', 'Label_A_39', 'Label_A_41'],
        'Fruits': ['Label_F_03', 'Label_F_09', 'Label_F_10', 'Label_F_17', 'Label_F_25', 'Label_F_29', 'Label_F_31', 'Label_F_34', 'Label_F_43', 'Label_F_49']
    }

    class_attrs_path = '../zsl_a_%s_train_%s/zsl_a_%s_train_annotations_attributes_per_class_%s.txt' % (superclass.lower(), date, superclass.lower(), date)
    attr_acc_path = 'attr_pred_acc_%s.txt' % (superclass.lower())
    low_entropy_attrs_path = 'low_entropy_attrs_%s.txt' % (superclass.lower())
    test_dir= '../zsl_a_%s_test_%s/' % (superclass.lower(), date)
    pred_path = 'pred_%s.txt' % (superclass)

    facc = open(attr_acc_path, 'r', encoding='utf-8')
    attr_acc = facc.readlines()
    facc.close()
    attr_acc = [float(row.strip().split(' ')[1]) for row in attr_acc]

    #fent = open(low_entropy_attrs_path, 'r', encoding='utf-8')
    #low_entropy_attrs = fent.readline()
    #fent.close()
    #low_entropy_attrs = [int(x) for x in low_entropy_attrs.strip().split(' ')]

    #classAttrsNum = classAttrsNums[superclass] - len(low_entropy_attrs)

    fattrs = open(class_attrs_path, 'r', encoding='utf-8')
    attrs = fattrs.readlines()
    fattrs.close()
    label_attrs = {}
    label_attrs_list = []
    for row in attrs:
        pair = row.strip().split(',')
        if pair[0] not in unseen_labels[superclass]:
            continue
        label_attrs[pair[0]] = list(map(lambda x: float(x), pair[1].strip().split(' ')[1:-1]))
        #label_attrs[pair[0]] = list(np.delete(np.array(label_attrs[pair[0]]), low_entropy_attrs))
        label_attrs_list.append(label_attrs[pair[0]])
    label_attrs_weight = np.sum(np.array(label_attrs_list),axis=0)
    label_attrs_weight = 1 - label_attrs_weight/10.0

    base_model = Xception(include_top=True, weights=None,
                      input_tensor=None, input_shape=(72,72,3),
                      pooling=None, classes=classNum[superclass[0]])
    output = Dense(classAttrsNum, activation='sigmoid', name='predictions')(base_model.get_layer('avg_pool').output)
    model = Model(inputs=base_model.input, outputs=output)
    model.load_weights(model_weight)

    Y = {}
    images = os.listdir(test_dir)
    for c,i in enumerate(images):
        if i[0] == '.':
            continue
        img_path = test_dir + '/' + i
        img = image.load_img(img_path, target_size=(72, 72))
        x = image.img_to_array(img)
        x = np.expand_dims(x, axis=0)
        #x = preprocess_input(x)
        x = x/255.0
        y_pred = model.predict(x)
        y_pred = np.array(list(map(lambda x: round(x,2), y_pred[0])))

        nearest = 0.0
        y_label = ''
        y_pred = y_pred*np.array(attr_acc)
        for label, attr in label_attrs.items():
            attr = np.array(attr)
            topK = 90
            y_pred_index = np.argsort(y_pred)
            attrs_sum = np.sum(label_attrs_weight[np.where(attr >= 1)])
            pred_sum = np.sum(label_attrs_weight[y_pred_index])
            union_sum = sum([i*y_pred[index] for index, i in enumerate(label_attrs_weight) if index in y_pred_index and attr[index]>=1])
            dist = -(union_sum)/(pred_sum + attrs_sum - union_sum)
            #dist = -(union_sum)
            if nearest == 0:
                nearest = dist
                y_label = label
            if dist < nearest:
                nearest = dist
                y_label = label
        Y[i] = label

    fpred = open(pred_path, 'w', encoding='utf-8')
    fpred.write('\n'.join([image+' '+label for image,label in Y.items()]))
    fpred.close()

if __name__ == "__main__":
    if len(sys.argv) == 3:
        superclass = sys.argv[1]
        model_weight = sys.argv[2]

        main(superclass, model_weight)

    else:
        print('Param error')
        exit()

max_stars_repo_name: Carbyn/xman

max_issues_repo_name: Carbyn/xman

max_forks_repo_name: Carbyn/xman


text:
from chainer import cuda
from chainer import function
from chainer.utils import type_check
from ctypes import *
import numpy as np
import cupy
import time
import os
dllpath = '/home/monica/Documents/chainer-1.17.0-RAP/chainer/functions/cnet/libcnet.so'
lib = CDLL(dllpath, RTLD_GLOBAL)

class data_t(Structure):
    _fields_ = [("size", c_int),
                ("val", POINTER(c_float)),
                ("grad", POINTER(c_float))]

class layer(Structure):
    _fields_ = [("batch", c_int),
                ("input", data_t),
                ("output", data_t),
                ("weight", data_t),
                ("bias", data_t),
                ("extra", data_t)]

forward_connected_layer = lib.fc_layer_forward
forward_connected_layer.argtypes = [POINTER(layer)]
backward_connected_layer = lib.fc_layer_backward
backward_connected_layer.argtypes = [POINTER(layer)]

def _as_mat(x):
    if x.ndim == 2:
        return x
    return x.reshape(len(x), -1)

class CnetLinearFunction(function.Function):

    def check_type_forward(self, in_types):
        n_in = in_types.size()
        type_check.expect(2 <= n_in, n_in <= 3)
        x_type, w_type = in_types[:2]

        type_check.expect(
            x_type.dtype == np.float32,
            w_type.dtype == np.float32,
            x_type.ndim >= 2,
            w_type.ndim == 2,
            type_check.prod(x_type.shape[1:]) == w_type.shape[1],
        )
        if n_in.eval() == 3:
            b_type = in_types[2]
            type_check.expect(
                b_type.dtype == np.float32,
                b_type.ndim == 1,
                b_type.shape[0] == w_type.shape[0],
            )

    def forward_cpu(self, inputs):
        x = _as_mat(inputs[0])
        # print(inputs[0].shape)
        # x = inputs[0]
        W = inputs[1].flatten()
        b = inputs[2]
        T_output = np.size(b)
        T_input = np.size(W) / T_output
        T_batch = np.size(x) / T_input
        output_size = T_batch * T_output
        # o = np.zeros(T_output, dtype=np.float)
        o = (c_float * output_size)()

        # l = layer(T_batch, T_input, T_output, 0, 0, cast(b.ctypes.data, POINTER(c_float)), None,
        #           cast(W.ctypes.data, POINTER(c_float)), None, None, o)
        l = layer()
        l.batch = T_batch
        l.input = data_t(T_input, cast(x.ctypes.data, POINTER(c_float)))
        # print(np.size(x))
        l.output = data_t(T_output, o)
        l.bias = data_t(np.size(b), cast(b.ctypes.data, POINTER(c_float)))
        l.weight = data_t(np.size(W), cast(W.ctypes.data, POINTER(c_float)))
        # print("forward connected layer(c)")
        forward_connected_layer(byref(l))
        # print("end connected layer")

        y = np.ctypeslib.as_array((c_float * (T_batch * T_output)).from_address(addressof(o)))

        # print(y.shape)
        y = np.reshape(y, (T_batch, T_output))
        rey = np.copy(y)
        # time.sleep(0.1)
        # sec = time.time()
        # name = 'cnet_linear_for' + str(sec) + ".npy"
        # np.save(name, rey.flatten())
        return rey,

    def forward_gpu(self, inputs):
        x = _as_mat(inputs[0])
        nx = cupy.asnumpy(x)
        # x = inputs[0]
        W = inputs[1].flatten()
        nW = cupy.asnumpy(W)
        b = inputs[2]
        nb = cupy.asnumpy(b)
        T_output = np.size(b)
        T_input = np.size(W) / T_output
        T_batch = np.size(x) / T_input
        output_size = T_batch * T_output
        # o = np.zeros(T_output, dtype=np.float)
        o = (c_float * output_size)()

        # l = layer(T_batch, T_input, T_output, 0, 0, cast(nb.ctypes.data, POINTER(c_float)), None,
        #           cast(nW.ctypes.data, POINTER(c_float)), None, None, o)
        l = layer()
        l.batch = T_batch
        l.input = data_t(T_input, cast(nx.ctypes.data, POINTER(c_float)))
        # print(np.size(x))
        l.output = data_t(T_output, o)
        l.bias = data_t(np.size(b), cast(nb.ctypes.data, POINTER(c_float)))
        l.weight = data_t(np.size(W), cast(nW.ctypes.data, POINTER(c_float)))
        # print("forward connected layer(g)")
        forward_connected_layer(byref(l))
        # print("end connected layer_g")

        y = np.ctypeslib.as_array((c_float * (T_batch * T_output)).from_address(addressof(o)))
        y = np.reshape(y, (T_batch, T_output))
        cy = cupy.array(y)
        # print(y.shape)
        return cy,

    def backward_cpu(self, inputs, grad_outputs):
        x = _as_mat(inputs[0])
        # x = _inputs[0]
        W = inputs[1].flatten()
        b = inputs[2]
        # Wb = numpy.where(W>=0, 1, -1).astype(numpy.float32, copy=False)
        T_output = np.size(b)
        T_input = np.size(W) / T_output
        T_batch = np.size(x) / T_input

        bias_grad = (c_float * T_output)()
        w = np.size(W)
        weight_grad = (c_float * w)()
        d_size = T_input * T_batch
        input_grad = (c_float * d_size)()

        l = layer()
        l.batch = T_batch
        l.input = data_t(T_input, cast(x.ctypes.data, POINTER(c_float)), input_grad)
        l.output = data_t(T_output, grad=cast(grad_outputs[0].ctypes.data, POINTER(c_float)))
        l.bias = data_t(np.size(b), cast(b.ctypes.data, POINTER(c_float)), bias_grad)
        l.weight = data_t(np.size(W), cast(W.ctypes.data, POINTER(c_float)), weight_grad)
        # print("backward connected layer(c)")
        backward_connected_layer(byref(l))
        # print("end backward connected layer_c")
        gx = np.ctypeslib.as_array((c_float * np.size(x)).from_address(addressof(input_grad)))
        gW = np.ctypeslib.as_array((c_float * np.size(W)).from_address(addressof(weight_grad)))
        gb = np.ctypeslib.as_array((c_float * np.size(b)).from_address(addressof(bias_grad)))
        gx = gx.reshape(inputs[0].shape)
        gW = gW.reshape(inputs[1].shape)
        rgx = np.copy(gx)
        rgW = np.copy(gW)
        rgb = np.copy(gb)
        # time.sleep(0.1)
        # sec = time.time()
        # namex = 'cnet_linear_back_x' + str(sec) + ".npy"
        # namew = 'cnet_linear_back_w' + str(sec) + ".npy"
        # nameb = 'cnet_linear_back_b' + str(sec) + ".npy"
        # np.save(namex, rgx.flatten())
        # np.save(namew, rgW.flatten())
        # np.save(nameb, rgb.flatten())
        # print("linear back size")
        # print(rgx.shape)
        return rgx, rgW, rgb

    def backward_gpu(self, inputs, grad_outputs):
        x = _as_mat(inputs[0])
        nx = cupy.asnumpy(x)
        # print(nx.shape)
        # x = _inputs[0]
        W = inputs[1].flatten()
        nW = cupy.asnumpy(W)
        # print(nW.shape)
        b = inputs[2]
        nb = cupy.asnumpy(b)
        out_g = cupy.asnumpy(grad_outputs[0])
        # print(nb.shape)
        # Wb = numpy.where(W>=0, 1, -1).astype(numpy.float32, copy=False)
        T_output = np.size(b)
        T_input = np.size(W) / T_output
        T_batch = np.size(x) / T_input
        bias_grad = (c_float * T_output)()
        w = np.size(W)
        weight_grad = (c_float * w)()
        d_size = T_input * T_batch
        input_grad = (c_float * d_size)()

        l = layer()
        l.batch = T_batch
        l.input = data_t(T_input, cast(nx.ctypes.data, POINTER(c_float)), input_grad)
        l.output = data_t(T_output, grad=cast(out_g.ctypes.data, POINTER(c_float)))
        l.bias = data_t(np.size(b), cast(nb.ctypes.data, POINTER(c_float)), bias_grad)
        l.weight = data_t(np.size(W), cast(nW.ctypes.data, POINTER(c_float)), weight_grad)
        # print("backward connected layer(g)")
        backward_connected_layer(byref(l))
        # print("end backward connected layer_g")
        gx = np.ctypeslib.as_array((c_float * np.size(x)).from_address(addressof(input_grad)))
        gW = np.ctypeslib.as_array((c_float * np.size(W)).from_address(addressof(weight_grad)))
        gb = np.ctypeslib.as_array((c_float * np.size(b)).from_address(addressof(bias_grad)))
        ygb = cupy.array(gb)
        gx = gx.reshape(inputs[0].shape)
        ygx = cupy.array(gx)
        gW2 = gW.reshape(inputs[1].shape)
        gw_return = np.copy(gW2)
        ygw_return = cupy.array(gw_return)
        # print(ygx)
        # print(ygw_return)
        # print(ygb)
        return ygx, ygw_return, ygb

def cnet_linear(x, W, b=None):
    """Binary Linear function, or affine transformation.

    It accepts two or three arguments: an input minibatch ``x``, a weight
    matrix ``W``, and optionally a bias vector ``b``. It computes
    :math:`Y = xW^\\top + b`.

    Args:
        x (~chainer.Variable): Input variable. Its first dimension is assumed
            to be the *minibatch dimension*. The other dimensions are treated
            as concatenated one dimension whose size must be ``N``.
        W (~chainer.Variable): Weight variable of shape ``(M, N)``.
        b (~chainer.Variable): Bias variable (optional) of shape ``(M,)``..

    Returns:
        ~chainer.Variable: Output variable.

    .. seealso:: :class:`~chainer.links.Linear`

    """
    if b is None:
        return CnetLinearFunction()(x, W)
    else:
        return CnetLinearFunction()(x, W, b)

max_stars_repo_name: asrlabncku/RAP

max_issues_repo_name: asrlabncku/RAP

max_forks_repo_name: asrlabncku/RAP

reason: import numpy,import cupy


text:
import os
import numpy as np

from qcodes import Measurement

def test_device(station, chip_config, dac, lockin):
    assert os.path.exists(chip_config)
    station.load_config_file(chip_config)
    chip = station.load_MockChip_123(station=station)
    assert station.dac == dac
    assert chip.device1.gate.source_parameters == (station.dac.ch01.voltage,)
    assert chip.device1.source.source_parameters == (
        lockin.frequency,
        lockin.amplitude,
        lockin.phase,
        lockin.time_constant
    )
    assert chip.device1.drain.source_parameters == (
        lockin.X,
        lockin.Y
    )

def test_device_meas(station, chip):
    meas = Measurement(station=station)
    device = chip.device1
    meas.register_parameter(device.gate)
    meas.register_parameter(device.drain, setpoints=(device.gate,))

    with meas.run() as datasaver:
        for set_v in np.linspace(0, 1.5, 10):
            device.gate.set(set_v)
            get_a = device.drain_X.get()
            datasaver.add_result((device.gate, set_v), (device.drain, get_a))
            datasaver.flush_data_to_database()
        assert len(
            datasaver.dataset.to_pandas_dataframe_dict()["device1_drain"]) == 10

max_stars_repo_name: RosenblumLabUser/Qcodes

max_issues_repo_name: RosenblumLabUser/Qcodes

max_forks_repo_name: nikhartman/Qcodes


text:
import script_tools as st
import registration as reg
import numpy as np
import landmarks
import dist_table as dt
#import matplotlib
import matplotlib.pyplot as plt
from matplotlib import mlab

def main(USE_PYRAMIDS, RANDOMIZE_TRANSFORMS, DO_AFFINE_REGISTRATION, DO_TRANSFORM_LANDMARKS, EVAL_LANDMARKS, metric_name, transformation_size, noise_level):

  # Windows
  #dataset_path = "C:/cygwin64/home/johan/itkAlphaCut/assets/01.png"
  #bin_path = "C:/cygwin64/home/johan/itkAlphaCut-release/Release/"

  # Linux
  dataset_path = "/home/johof680/work/itkAlphaCut-4j/itkAlphaCut/assets/01.png"
  bin_path = "/home/johof680/work/itkAlphaCut-4j/build-release/"

  #weight_path = "/home/johof680/work/VironovaRegistrationImages/"
  out_path = dataset_path#"/home/johof680/work/VironovaRegistrationImages/"

  # image extension
  im_ext = "tif"

  # number of registrations to perform
  count = 1000

  # create registration object
  parallel_count = 6
  image_dimensions = 2
  r = reg.Registration(parallel_count, bin_path, dim = image_dimensions, enable_logging = True)

  out_path1 = "/home/johof680/work/itkAlphaCut-4j/cilia_param_search7/" + transformation_size + "/" + noise_level + "/"
  #out_path1 = "C:/cygwin64/home/johan/cilia_random/large4/"

  # Generate random transforms

  if RANDOMIZE_TRANSFORMS:

    rnd_param = r.get_random_transforms_param_defaults()
    if noise_level == "none":
      rnd_param["noise"] = "0.0"
    elif noise_level == "large":
      rnd_param["noise"] = "0.1"
    else:
      raise "Illegal noise level."

    if transformation_size == "small":
      rnd_param["rotation"] = "10"
      rnd_param["translation"] = "10"
      rnd_param["min_rotation"] = "0"
      rnd_param["min_translation"] = "0"
    elif transformation_size == "medium":
      rnd_param["rotation"] = "20"
      rnd_param["translation"] = "20"
      rnd_param["min_rotation"] = "10"
      rnd_param["min_translation"] = "10"
    elif transformation_size == "large":
      rnd_param["rotation"] = "30"
      rnd_param["translation"] = "30"
      rnd_param["min_rotation"] = "20"
      rnd_param["min_translation"] = "20"
    elif transformation_size == "all":
      rnd_param["rotation"] = "30"
      rnd_param["translation"] = "30"
      rnd_param["min_rotation"] = "0"
      rnd_param["min_translation"] = "0"
    else:
      raise "Illegal transformation size."

    r.random_transforms(dataset_path, out_path1, count, rnd_param)

    st.create_directory(out_path1)

    r.run("Random transformations")

  in_path = out_path1
  out_path1 = out_path1 + metric_name + "/"
  st.create_directory(out_path1)

  #learning_rates_list = [0.001, 0.01, 0.1, 0.5, 1.0]
  #sampling_fractions_list = [0.01, 0.05, 0.1, 0.2, 0.5, 1.0]
  #normalizations_list = [0.0, 0.01, 0.025, 0.05]
  #sampling_fractions_list = [0.01, 0.05, 0.1, 0.2, 0.5, 1.0]

  exponents = [-3.75, -3.5, -3.25, -3, -2.75, -2.5, -2.25, -2.0, -1.75, -1.5, -1.25, -1.0, -0.75, -0.5, -0.25, 0.0]
  #sampling_fractions_list = [np.power(10.0, i) for i in exponents]
  #sampling_fractions_list = [0.1]
  #learning_rates_list = [0.5]
  #normalizations_list = [0.0]
  #alpha_levels_list = [7]

  sampling_fractions_list = [0.1]
  learning_rates_list = [0.5]
  normalizations_list = [0.0]
  alpha_levels_list = range(1, 256)

  param_comb = [(a, b, c, d) for a in learning_rates_list for b in sampling_fractions_list for c in normalizations_list for d in alpha_levels_list]

  print("Parameter format (learning_rate, sampling_fraction, normalization percentile)")
  print(param_comb)

  # Do the registrations

  def register(rp, input1_path, input2_path, output_path, metric, cnt):
      for i in xrange(cnt):
          for j in xrange(len(param_comb)):
            (learning_rate, sampling_fraction, normalization, alpha_levels) = param_comb[j]

            pth = output_path + ("registration_%d_%d/" % (j+1, i+1))
            rpar = rp.get_register_param_defaults()
            rpar.pop("weights1", None)
            rpar.pop("weights2", None)
            in1 = input1_path + "ref_image_%d.%s" %(i+1, im_ext)
            in2 = input2_path + "transformed_image_%d.%s" % (i+1, im_ext)
            msk2 = input2_path + "transformed_mask_%d.%s" % (i+1, im_ext)
            if USE_PYRAMIDS:
              rpar["multiscale_sampling_factors"] = "4x2x1"
              rpar["multiscale_smoothing_sigmas"] = "5x3x0"
            else:
              rpar["multiscale_sampling_factors"] = "1"
              rpar["multiscale_smoothing_sigmas"] = "0"
            rpar["metric"] = metric
            rpar["learning_rate"] = str(learning_rate)
            rpar["alpha_outlier_rejection"] = "0.0"
            rpar["sampling_fraction"] = str(sampling_fraction)
            rpar["normalization"] = str(normalization)
            rpar["alpha_levels"] = str(alpha_levels)

            rpar["mask1"] = "circle"
            rpar["mask2"] = msk2
            rp.register_affine(in1, in2, pth, rpar)

    #def landmark_transform(self, landmarks_in_path, out_path, out_name, transform_path):
  def transform_landmarks(rp, landmark_path, transform_path_base, cnt):
      for i in xrange(cnt):
          for j in xrange(len(param_comb)):
            transform_path = transform_path_base + ("registration_%d_%d/transform_complete.txt" % (j+1, i+1))
            input_path = landmark_path + "transformed_landmarks_%d.csv" % (i + 1)
            out_name = "registered_landmarks_%d_%d.csv" % (j + 1, i + 1)

            rp.landmark_transform(input_path, transform_path_base, out_name, transform_path)

  if DO_AFFINE_REGISTRATION:
      register(r, in_path, in_path, out_path1, metric_name, count)
      r.run("Affine Registration")

  if DO_TRANSFORM_LANDMARKS:
      transform_landmarks(r, in_path, out_path1, count)
      r.run("Transform Landmarks")

  def eval_landmarks(ref_landmark_path, output_path, cnt):
      ref_lm = landmarks.read_csv(ref_landmark_path + "ref_landmarks.csv", False) #csv_2_np.read_csv(out1_path + "ref_landmarks.csv", False)
      out_succ_freq = []
      out_succ_means = []
      out_means = []
      out_stddevs = []
      out_dists = []
      out_dist_summary = []

      for j in xrange(len(param_comb)):
        dists = np.zeros(cnt)
        for i in xrange(cnt):
            tra_lm = landmarks.read_csv(output_path + "registered_landmarks_%d_%d.csv" % (j + 1, i+1))
            dists[i] = landmarks.mean_euclidean(ref_lm, tra_lm)
        succ_freq = np.count_nonzero(np.where(dists <= 1.0))/np.float(cnt)
        succ_means = np.mean(dists[np.where(dists <= 1.0)])

        out_succ_freq.append(succ_freq)
        out_succ_means.append(succ_means)
        out_means.append(np.mean(dists))
        out_stddevs.append(np.std(dists))
        out_dists.append(dists)
        out_dist_summary.append(dt.make_distribution(dists))
          #np.sort(dists)
      return (out_succ_freq, out_succ_means, out_means, out_stddevs, out_dists, out_dist_summary)
          #print("%.4d: %f" % (i+1, dist))

          #print(eval_landmarks(out_path1, out_path1, count))

  def filter_set(values, tup, tup_index, tup_value):
    vals = []
    tups = []
    for i in xrange(len(tup)):
      if tup[i][tup_index] == tup_value:
        tups.append(tup[i])
        vals.append(values[i])
    return (np.array(vals), tups)

  if EVAL_LANDMARKS:
    (succ_freq, succ_means, mn, stddev, full_distri, distri) = eval_landmarks(in_path, out_path1, count)
    np.set_printoptions(precision=5)
    np.set_printoptions(suppress=True)
    print("")
    #print(full_distri)
    #print(distri)
    print("Succ freq: ")
    print(succ_freq)
    print("Succ means: ")
    print(succ_means)
    print("Means: ")
    print(mn)
    print(stddev)

    M = 9
    if M > len(param_comb):
      M = len(param_comb)

    argsort = np.argsort(mn)

    best_indices = argsort[0:M]
    best_params = [param_comb[ind] for ind in best_indices]
    best_vals = [mn[ind] for ind in best_indices]

    worst_indices = argsort[len(argsort)-M:]
    worst_params = [param_comb[ind] for ind in worst_indices]
    worst_vals = [mn[ind] for ind in worst_indices]

    print("Best param: " + str(best_params))
    print("Best vals: " + str(best_vals))

    print("Worst param: " + str(worst_params))
    print("Worst vals: " + str(worst_vals))

    for i in xrange(len(full_distri)):
      np.savetxt(out_path1 + metric_name + "%d.csv" % (i+1), full_distri[i], delimiter=",")

    print("For learning rate 0.5")
    (mn_lr_0_5, tup_lr_0_5) = filter_set(mn, param_comb, 0, 0.5)
    print(mn_lr_0_5)
    print(tup_lr_0_5)

    #print(str(param_comb))
    #make_plot(distri)

  # Eval

  # param = r.get_register_param_defaults()

  # param["weights1"] = "hann"
  # param["weights2"] = "hann"
  # param["learning_rate"] = "0.05"
  # param["relaxation_factor"] = "0.99"
  # param["alpha_levels"] = "7"
  # param["alpha_max_distance"] = "128"
  # param["seed"] = "1000"
  # param["metric"] = "alpha_smd"
  # param["iterations"] = "1000"
  # param["sampling_fraction"] = "0.001"
  # param["spacing_mode"] = "default"
  # param["normalization"] = "0.005"
  # param["multiscale_sampling_factors"] = "4x2x1"
  # param["multiscale_smoothing_sigmas"] = "4.0x2.0x1.0"

  #def make_dir(path, index):
  #  return st.makedir_string(path + "Out%.3d" % (index))

  #def make_register(index, in1, in2, out, params):
  #  return exe_path + " -in1 " + in1 + " -in2 " + in2 + " -out " + out + st.param_dict_to_string(params)

  def ind_str(prefix, index, postfix):
    return prefix + ("%.3d" % index) + postfix

  #def make_cmd(index, in1, in2, out, params):
  #  print("# Image %.3d" % (index))
  #  print(make_dir(out_path, index))
  #  print(make_register(index, in1, in2, out, params))
  #  print("")

  #registration
  # N = 18
  # DO_REGISTRATION = 1

  # if DO_REGISTRATION == 1:
  #   for i in xrange(9, N):
  #     r.register_rigid(in_path + "im008.tif", in_path + ind_str("im", i, ".tif"), dataset_path + ind_str("Out", i, "/"), param)

  #   r.run("Rigid registration")

  #transformation

  # par = r.get_transform_param_defaults()
  # par["16bit"] = "1"

  # for i in xrange(9, N):
  #   r.transform(in_path + "im008.tif", in_path + ind_str("im", i, ".tif"), dataset_path + ind_str("Out", i, "/"), "transformed.png", dataset_path + ind_str("Out", i, "/transform_complete.txt"), par)

  # r.run("Transformations")

    #make_cmd(i, in_path + "im008.tif", in_path + ind_str("im", i, ".tif"), out_path + ind_str("Out", i, "/"), param)

  #mkdir ../../../VironovaRegistrationImages/Out009/
  #../../build6/ACRegister -in1 ../../../VironovaRegistrationImages/1Original/im008.tif -in2 ../../../VironovaRegistrationImages/1Original/im009.tif -weights1 ../../../VironovaRegistrationImages/hann_mask.tif -weights2 ../../../VironovaRegistrationImages/hann_mask.tif -out ../../../VironovaRegistrationImages/Out009/ -learning_rate 0.5 -relaxation_factor 0.99 -alpha_levels 7 -alpha_max_distance 128 -seed 1000 -metric alpha_smd -iterations 1000 -sampling_fraction 0.01 -spacing_mode remove -normalization 0.005 -rigid 1 -smoothing 2.0

if __name__ == "__main__":
  RUN_MODE = True
  PYRAMID_MODE = True
  #main(PYRAMID_MODE, RUN_MODE, RUN_MODE, RUN_MODE, True, "alpha_smd", "small", "large")
#  main(PYRAMID_MODE, False, RUN_MODE, RUN_MODE, True, "ssd", "small", "large")
#  main(PYRAMID_MODE, False, RUN_MODE, RUN_MODE, True, "ncc", "small", "large")
#  main(PYRAMID_MODE, False, RUN_MODE, RUN_MODE, True, "mi", "small", "large")

  main(PYRAMID_MODE, RUN_MODE, RUN_MODE, RUN_MODE, True, "alpha_smd", "all", "large")
#  main(PYRAMID_MODE, False, RUN_MODE, RUN_MODE, True, "ssd", "medium", "large")
#  main(PYRAMID_MODE, False, RUN_MODE, RUN_MODE, True, "ncc", "medium", "large")
#  main(PYRAMID_MODE, False, RUN_MODE, RUN_MODE, True, "mi", "medium", "large")

  #main(PYRAMID_MODE, RUN_MODE, RUN_MODE, RUN_MODE, True, "alpha_smd", "large", "large")
#  main(PYRAMID_MODE, False, RUN_MODE, RUN_MODE, True, "ssd", "large", "large")
#  main(PYRAMID_MODE, False, RUN_MODE, RUN_MODE, True, "ncc", "large", "large")
#  main(PYRAMID_MODE, False, RUN_MODE, RUN_MODE, True, "mi", "large", "large")

  #main(PYRAMID_MODE, RUN_MODE, RUN_MODE, RUN_MODE, True, "alpha_smd", "small", "none")
#  main(PYRAMID_MODE, False, RUN_MODE, RUN_MODE, True, "ssd", "small", "none")
#  main(PYRAMID_MODE, False, RUN_MODE, RUN_MODE, True, "ncc", "small", "none")
#  main(PYRAMID_MODE, False, RUN_MODE, RUN_MODE, True, "mi", "small", "none")

  #main(PYRAMID_MODE, RUN_MODE, RUN_MODE, RUN_MODE, True, "alpha_smd", "medium", "none")
#  main(PYRAMID_MODE, False, RUN_MODE, RUN_MODE, True, "ssd", "medium", "none")
#  main(PYRAMID_MODE, False, RUN_MODE, RUN_MODE, True, "ncc", "medium", "none")
#  main(PYRAMID_MODE, False, RUN_MODE, RUN_MODE, True, "mi", "medium", "none")

  #main(PYRAMID_MODE, RUN_MODE, RUN_MODE, RUN_MODE, True, "alpha_smd", "large", "none")
  #main(PYRAMID_MODE, False, RUN_MODE, RUN_MODE, True, "ssd", "large", "none")
  #main(PYRAMID_MODE, False, RUN_MODE, RUN_MODE, True, "ncc", "large", "none")
  #main(PYRAMID_MODE, False, RUN_MODE, RUN_MODE, True, "mi", "large", "none")

max_stars_repo_name: MIDA-group/itkAlphaAMD

max_issues_repo_name: MIDA-group/itkAlphaAMD

max_forks_repo_name: MIDA-group/itkAlphaAMD