text:
import os, sys
import numpy as np
import json
from collections import OrderedDict
from tqdm import tqdm
from .. import pipeline as pipe
from .. import tf_tools

def run(data_type,
        checkpoint_dir,
        batch_sizes,
        image_shapes,
        image_shape_max_ratio,
        view_planes,
        view_angles,
        all_patients):
    """
    Parameters
    ----------
    data_type : {unit8, int16, float32}
        Data type of prob maps.
    """
    # check if enought batch_sizes given for image_shapes
    if len(image_shapes) != len(batch_sizes):
        raise ValueError('Need same number of batch_sizes and image_shapes for nodule_seg.')
    if len(view_planes) == 0 or len([c for c in view_planes if c not in ['x', 'y', 'z']]) > 0:
        raise ValueError('view_planes ' + str(view_planes) + 'must only contain x, y, z chars.')
    resample_lungs_json = pipe.load_json('out.json', 'resample_lungs')
    if isinstance(resample_lungs_json, FileNotFoundError):
        raise FileNotFoundError(str(resample_lungs_json) + '\n--> Run step "resample_lungs" first.')
    HU_tissue_range = pipe.load_json('params.json', 'resample_lungs')['HU_tissue_range']
    # sort nets in ascending size y * x
    image_shapes = sorted(image_shapes, key=lambda shape: shape[0]*shape[1])
    # split patients by scan_shape for the nodule_segmentation_nets with distinct image_shapes -> save computing time
    patients_per_net = [[] for n in range(len(image_shapes))]
    registered_patients = set()
    considered_patients = pipe.patients if all_patients else pipe.patients_by_split['va']
    for net_num, net_shape in enumerate(image_shapes):
        net_shape = [net_shape[0] for i in range(3)] # need three dimensions
        ratio = 1 if net_num == len(image_shapes) - 1 else image_shape_max_ratio # for avoiding border effects net due to padding, ...
        for patient in considered_patients:
            if patient in registered_patients:
                continue
            scan_shape_z = resample_lungs_json[patient]['resampled_scan_shape_zyx_px'][0]
            bound_box_coords_yx_px = resample_lungs_json[patient]['bound_box_coords_yx_px']
            scan_shape_yx = [bound_box_coords_yx_px[1] + 1 - bound_box_coords_yx_px[0],
                             bound_box_coords_yx_px[3] + 1 - bound_box_coords_yx_px[2]]
            scan_shape = [scan_shape_z] + scan_shape_yx
            # does the scan fit into the shrinked net? it needs to fit in all three directions if we use all planes
            if np.all(np.array(scan_shape) < ratio * np.array(net_shape)):
                patients_per_net[net_num].append(patient)
                registered_patients.add(patient)
    if len(registered_patients) != len(considered_patients):
        raise ValueError('Registering patients for different nets is inconsistent.')
    pipe.log.info('patients distribution on nets: ' + str([len(l) for l in patients_per_net]))
    reuse_init = True
    # loop over nodule_segmentation_nets with distinct image_shapes
    for net_num, net_shape in enumerate(image_shapes):
        if len(patients_per_net[net_num]) == 0:
            continue
        reuse = None if reuse_init else True
        reuse_init = False
        batch_size = batch_sizes[net_num]
        config = json.load(open(checkpoint_dir + '/config.json'))
        image_shape = net_shape + [config['image_shape'][2]]
        label_shape = net_shape + [config['label_shape'][2]]
        prediction = np.zeros([batch_size] + list(label_shape), dtype=np.float32)
        prediction_rot = np.zeros_like(prediction)
        sess, pred_ops, data = tf_tools.load_network(checkpoint_dir, image_shape=image_shape, reuse=reuse)
        pipe.log.info('predicting ' + str(len(patients_per_net[net_num]))
                      + ' patients with net {} with x-y image shape {}'.format(net_num, net_shape))
        # loop over patients in nets list
        patients_json = OrderedDict()
        for patient in tqdm(patients_per_net[net_num]):
            org_img_array = pipe.load_array(resample_lungs_json[patient]['basename'], step_name='resample_lungs') # z, y, x
            prob_map = np.zeros_like(org_img_array, dtype=np.float32)
            for view_plane_cnt, view_plane in enumerate(view_planes):
                if view_plane_cnt > 0: # reload
                    org_img_array = pipe.load_array(resample_lungs_json[patient]['basename'], step_name='resample_lungs') # z, y, x
                if org_img_array.dtype == np.int16: # [0, 1400] -> [-0.25, 0.75] normalized and zero_centered
                    org_img_array = (org_img_array/(HU_tissue_range[1] - HU_tissue_range[0]) - 0.25).astype(np.float32)
                if view_plane == 'x': # move z axis to position 1
                    org_img_array = np.rollaxis(org_img_array, 0, 2) # z, y, x -> x, z, y
                elif view_plane == 'y':
                    org_img_array = np.swapaxes(org_img_array, 1, 2) # -> z, x, y
                elif view_plane == 'z':
                    org_img_array = np.swapaxes(org_img_array, 2, 0) # -> x, y, z
                # for example, if view_plane == 'z', then the 'z' dimension becomes the third dimension in img_array
                #              and the convolution is performed in the first two dimensions, here the y-x dimensions
                img_array = (-0.25) * np.ones((image_shape[0], image_shape[1], org_img_array.shape[2]), dtype=np.float32) # 'black' array
                # embed img in xy center in image_shape
                offset_y = int((img_array.shape[0] - org_img_array.shape[0])/2)
                offset_x = int((img_array.shape[1] - org_img_array.shape[1])/2)
                img_array[offset_y : offset_y + org_img_array.shape[0],
                          offset_x : offset_x + org_img_array.shape[1], :] = org_img_array
                num_batches = int(np.ceil(img_array.shape[2] / batch_size))
                batch = (-0.25) * np.ones(([batch_size] + image_shape), dtype=np.float32)
                batch_rot = batch.copy()
                for batch_cnt in range(num_batches):
                    batch[:] = batch_rot[:] = -0.25 # reset to black
                    prediction[:] = prediction_rot[:] = 0 # reset to 0
                    for cnt in range(batch_size): # fill batch
                        layer_cnt = batch_cnt * batch_size + cnt
                        if layer_cnt >= img_array.shape[2]:
                            break
                        # leave some channels above empty at top and below at bottom
                        min_z = max(0, int(layer_cnt - (image_shape[2] - 1) / 2))
                        max_z = min(int(layer_cnt + (image_shape[2] - 1) / 2) + 1, img_array.shape[2])
                        batch_idx_z = [image_shape[2] - max(0, max_z - min_z), image_shape[2] - min(0, max_z - min_z)]
                        batch[cnt, :, :, batch_idx_z[0]:batch_idx_z[1]] = img_array[:, :, min_z:max_z]
                    for view_angle in view_angles:
                        if view_angle != 0:
                            M = cv2.getRotationMatrix2D((batch.shape[2]//2, batch.shape[1]//2), view_angle, 1)
                            for img_cnt in range(batch.shape[0]):
                                batch_rot[img_cnt] = rotate_3d(((batch[img_cnt].copy() + 0.25) * 255).astype(np.uint8), M, 2)/255 - 0.25
                        else:
                            batch_rot = batch.copy()
                        # get probability for nodules and reshape flat prediction to batchsize, z, x, 1
                        prediction_rot = np.reshape(sess.run(pred_ops, feed_dict = {data['images']: batch_rot})['probs'], prediction.shape)
                        # below, np.clip is called, shouldn't the prediction stay above zero and below one?
                        if np.max(prediction_rot) > 1 or np.min(prediction_rot) < 0:
                            pipe.log.warning('prediction not within [0, 1] for patient ' + patient)
                        # rotate back prediction
                        if view_angle != 0:
                            M_back = cv2.getRotationMatrix2D((prediction_rot.shape[2]//2, prediction_rot.shape[1]//2), -view_angle, 1)
                            for img_cnt in range(batch.shape[0]):
                                prediction_rot[img_cnt] = np.clip(rotate_3d((prediction_rot[img_cnt] * 255).astype(np.uint8), M_back, 2) / 255, 0, 1)
                        # mean over view_angles
                        prediction += prediction_rot / len(view_angles)
                    # crop from embedded layers
                    prediction_embed = prediction[:, offset_y : offset_y + org_img_array.shape[0],
                                                     offset_x : offset_x + org_img_array.shape[1]] / len(view_planes)
                    layer_start = batch_size * batch_cnt
                    layer_end = min(img_array.shape[2], batch_size + layer_start)
                    # TODO: is the following correct?
                    if view_plane == 'x':
                        prob_map[:, :, layer_start:layer_end] += np.swapaxes(prediction_embed[:layer_end-layer_start, :, :, 0], 0, 2)
                    elif view_plane == 'y':
                        prob_map[:, layer_start:layer_end, :] += np.swapaxes(prediction_embed[:layer_end-layer_start, :, :, 0], 0, 1)
                    elif view_plane == 'z':
                        prob_map[layer_start:layer_end, :, :] +=  np.swapaxes(prediction_embed[:layer_end-layer_start, :, :, 0], 1, 2)
            if np.min(prob_map) < 0 or np.max(prob_map) > 1:
                 pipe.log.warning('nodule seg prob_map not in value range [0, 1] for patient ' + patient)
            if data_type == 'uint8':
                prob_map = (prob_map * 255).astype(np.uint8)
            elif data_type == 'uint16':
                prob_map = (prob_map * 65535).astype(np.uint16)
            patients_json[patient] = OrderedDict()
            patients_json[patient]['basename'] = basename = patient + '_prob_map.npy'
            patients_json[patient]['pathname'] = pipe.save_array(basename, prob_map)
        pipe.save_json('out.json', patients_json, mode='w' if reuse is None else 'a') # open in 'w' mode when something is written for the first time
        sess.close()

def rotate(in_tensor, M):
    dst = cv2.warpAffine(in_tensor, M, (in_tensor.shape[1], in_tensor.shape[0]),
                         flags=cv2.INTER_CUBIC)
    if len(dst) == 2:
        dst = np.expand_dims(dst, 2)
    return dst

def rotate_3d(tensor, M, axis):
    if axis == 0:
        for i in range(tensor.shape[0]):
            tensor[i, :, :] = rotate(tensor[i, :, :], M)
    elif axis == 1:
        for i in range(tensor.shape[1]):
            tensor[:, i, :] = rotate(tensor[:, i, :], M)
    elif axis == 2:
        for i in range(tensor.shape[2]):
            tensor[:, :, i] = rotate(tensor[:, :, i], M)
    return tensor


text:
import time
from random import randrange
from socketIO_client import SocketIO, LoggingNamespace

import sys
import json
import requests

import numpy as np

if __name__ == '__main__':
    with SocketIO('localhost', 5000, LoggingNamespace) as socketIO:
        while True:

            y_pred = np.random.random(7)
            print(y_pred)
            # Create dict
            send_data = {}
            labels = ["Chat", "Email", "File_Transfer",
                      "P2P", "Streaming", "VoIP", "Web_Browsing"]
            for y, label in zip(y_pred, labels):
                send_data[label] = y

            # Send
            socketIO.emit('my event', send_data)
            time.sleep(0.5)

ext: py

max_stars_repo_name: fmeslet/SPPNet

max_issues_repo_name: fmeslet/SPPNet

max_forks_repo_name: fmeslet/SPPNet

reason: import numpy


text:
from keras import backend as K
from keras.utils import np_utils
from keras.optimizers import Adamax
from model import DFNet
import numpy as np
import time
import random
import os
import argparse
import logging
import const
import keras
import tensorflow as tf
config = tf.ConfigProto( device_count = {'GPU': 1 , 'CPU': 20} )
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
keras.backend.set_session(sess)

def init_logger():
    logger = logging.getLogger('kf')
    logger.setLevel(logging.DEBUG)
    # create console handler and set level to debug
    ch = logging.StreamHandler()
    # create formatter
    formatter = logging.Formatter(const.LOG_FORMAT)
    # add formatter to ch
    ch.setFormatter(formatter)
    # add ch to logger
    logger.addHandler(ch)
    return logger

def loadData(fpath):
	train = np.load(fpath).item()
	train_X ,train_y = train['feature'], train['label']
	return train_X, train_y

if __name__ == "__main__":
    random.seed(0)
    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

    EXP_Type = 'OpenWorld_NoDef'
    print ("Experimental Type: ", EXP_Type)
    # network and training
    NB_EPOCH = 30
    print ("Number of Epoch: ", NB_EPOCH)
    BATCH_SIZE = 128
    VERBOSE = 1
    LENGTH = 5000
    OPTIMIZER = Adamax(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0)

    NB_CLASSES = 100+1 # number of outputs: 95 Monitored websites + 1 Unmonitored websites
    INPUT_SHAPE = (LENGTH,1)

    '''initialize logger'''
    logger = init_logger()
    '''read config'''
    parser = argparse.ArgumentParser(description='k-FP attack')
    parser.add_argument('feature_path',
                        metavar='<feature path>',
                        help='Path to the directory of the extracted features')
    args = parser.parse_args()

    X_train, y_train = loadData(args.feature_path)
    K.set_image_dim_ordering("tf") # tf is tensorflow
    # consider them as float and normalize
    X_train = X_train.astype('float32')
    y_train = y_train.astype('float32')
    # print(y_train)

    # we need a [Length x 1] x n shape as input to the DFNet (Tensorflow)
    X_train = X_train[:, :,np.newaxis]

    print(X_train.shape[0], 'training samples')

    # # convert class vectors to binary class matrices
    # y_train = np_utils.to_categorical(y_train, NB_CLASSES)
    # y_valid = np_utils.to_categorical(y_valid, NB_CLASSES)

    print ("Preparing Data for training")
    # initialize the optimizer and model
    print (time.sleep(2))
    model = DFNet.build(input_shape=INPUT_SHAPE, classes=NB_CLASSES)

    model.compile(loss="categorical_crossentropy", optimizer=OPTIMIZER,
    	metrics=["accuracy"])
    print ("Model compiled")

    # Start training
    history = model.fit(X_train, y_train,
    		batch_size=BATCH_SIZE, epochs=NB_EPOCH, verbose=VERBOSE,validation_split=0.1)

    # Save model
    print ("Saving Model")
    model_id = args.feature_path.split('/')[-1].split('.')[0].split('_train')[0]
    savedpath = os.path.join(const.modeldir,'%s.h5'%str(model_id))
    model.save(savedpath)
    print ("Saving Model Done!", savedpath)

max_stars_repo_name: WFDetector/WFDetection

max_issues_repo_name: WFDetector/WFDetection

max_forks_repo_name: WFDetector/WFDetection


text:
# --------------------------------------------------------
# Faster R-CNN
# Copyright (c) 2015 Microsoft
# Licensed under The MIT License [see LICENSE for details]
# Written by Ross Girshick and Sean Bell
# --------------------------------------------------------

import os
import caffe
import yaml
from fast_rcnn.config import cfg
import numpy as np
import numpy.random as npr
from generate_anchors import generate_anchors
from utils.cython_bbox import bbox_overlaps
from fast_rcnn.bbox_transform import bbox_transform

DEBUG = False

class AnchorTargetLayer(caffe.Layer):
    """
    Assign anchors to ground-truth targets. Produces anchor classification
    labels and bounding-box regression targets.
    """

    def setup(self, bottom, top):
        layer_params = yaml.load(self.param_str_)
        anchor_scales = layer_params.get('scales', (8, 16, 32))
        self._anchors = generate_anchors(scales=np.array(anchor_scales))
        self._num_anchors = self._anchors.shape[0]
        self._feat_stride = layer_params['feat_stride']

        if DEBUG:
            print('anchors:')
            print(self._anchors)
            print('anchor shapes:')
            print(np.hstack((
                self._anchors[:, 2::4] - self._anchors[:, 0::4],
                self._anchors[:, 3::4] - self._anchors[:, 1::4],
            )))
            self._counts = cfg.EPS
            self._sums = np.zeros((1, 4))
            self._squared_sums = np.zeros((1, 4))
            self._fg_sum = 0
            self._bg_sum = 0
            self._count = 0

        # allow boxes to sit over the edge by a small amount
        self._allowed_border = layer_params.get('allowed_border', 0)

        height, width = bottom[0].data.shape[-2:]
        if DEBUG:
            print('AnchorTargetLayer: height', height, 'width', width)

        A = self._num_anchors
        # labels
        top[0].reshape(1, 1, A * height, width)
        # bbox_targets
        top[1].reshape(1, A * 4, height, width)
        # bbox_inside_weights
        top[2].reshape(1, A * 4, height, width)
        # bbox_outside_weights
        top[3].reshape(1, A * 4, height, width)

    def forward(self, bottom, top):
        # Algorithm:
        #
        # for each (H, W) location i
        #   generate 9 anchor boxes centered on cell i
        #   apply predicted bbox deltas at cell i to each of the 9 anchors
        # filter out-of-image anchors
        # measure GT overlap

        assert bottom[0].data.shape[0] == 1, \
            'Only single item batches are supported'

        # map of shape (..., H, W)
        height, width = bottom[0].data.shape[-2:]
        # GT boxes (x1, y1, x2, y2, label)
        gt_boxes = bottom[1].data
        # im_info
        im_info = bottom[2].data[0, :]

        if DEBUG:
            print('')
            print('im_size: ({}, {})'.format(im_info[0], im_info[1]))
            print('scale: {}'.format(im_info[2]))
            print('height, width: ({}, {})'.format(height, width))
            print('rpn: gt_boxes.shape', gt_boxes.shape)
            print('rpn: gt_boxes', gt_boxes)

        # 1. Generate proposals from bbox deltas and shifted anchors
        shift_x = np.arange(0, width) * self._feat_stride
        shift_y = np.arange(0, height) * self._feat_stride
        shift_x, shift_y = np.meshgrid(shift_x, shift_y)
        shifts = np.vstack((shift_x.ravel(), shift_y.ravel(),
                            shift_x.ravel(), shift_y.ravel())).transpose()
        # add A anchors (1, A, 4) to
        # cell K shifts (K, 1, 4) to get
        # shift anchors (K, A, 4)
        # reshape to (K*A, 4) shifted anchors
        A = self._num_anchors
        K = shifts.shape[0]
        all_anchors = (self._anchors.reshape((1, A, 4)) +
                       shifts.reshape((1, K, 4)).transpose((1, 0, 2)))
        all_anchors = all_anchors.reshape((K * A, 4))
        total_anchors = int(K * A)

        # only keep anchors inside the image
        inds_inside = np.where(
            (all_anchors[:, 0] >= -self._allowed_border) &
            (all_anchors[:, 1] >= -self._allowed_border) &
            (all_anchors[:, 2] < im_info[1] + self._allowed_border) &  # width
            (all_anchors[:, 3] < im_info[0] + self._allowed_border)    # height
        )[0]

        if DEBUG:
            print('total_anchors', total_anchors)
            print('inds_inside', len(inds_inside))

        # keep only inside anchors
        anchors = all_anchors[inds_inside, :]
        if DEBUG:
            print('anchors.shape', anchors.shape)

        # label: 1 is positive, 0 is negative, -1 is dont care
        labels = np.empty((len(inds_inside), ), dtype=np.float32)
        labels.fill(-1)

        # overlaps between the anchors and the gt boxes
        # overlaps (ex, gt)
        overlaps = bbox_overlaps(
            np.ascontiguousarray(anchors, dtype=np.float),
            np.ascontiguousarray(gt_boxes, dtype=np.float))
        argmax_overlaps = overlaps.argmax(axis=1)
        max_overlaps = overlaps[np.arange(len(inds_inside)), argmax_overlaps]
        gt_argmax_overlaps = overlaps.argmax(axis=0)
        gt_max_overlaps = overlaps[gt_argmax_overlaps,
                                   np.arange(overlaps.shape[1])]
        gt_argmax_overlaps = np.where(overlaps == gt_max_overlaps)[0]

        if not cfg.TRAIN.RPN_CLOBBER_POSITIVES:
            # assign bg labels first so that positive labels can clobber them
            labels[max_overlaps < cfg.TRAIN.RPN_NEGATIVE_OVERLAP] = 0

        # fg label: for each gt, anchor with highest overlap
        labels[gt_argmax_overlaps] = 1

        # fg label: above threshold IOU
        labels[max_overlaps >= cfg.TRAIN.RPN_POSITIVE_OVERLAP] = 1

        if cfg.TRAIN.RPN_CLOBBER_POSITIVES:
            # assign bg labels last so that negative labels can clobber positives
            labels[max_overlaps < cfg.TRAIN.RPN_NEGATIVE_OVERLAP] = 0

        # subsample positive labels if we have too many
        num_fg = int(cfg.TRAIN.RPN_FG_FRACTION * cfg.TRAIN.RPN_BATCHSIZE)
        fg_inds = np.where(labels == 1)[0]
        if len(fg_inds) > num_fg:
            disable_inds = npr.choice(
                fg_inds, size=(len(fg_inds) - num_fg), replace=False)
            labels[disable_inds] = -1

        # subsample negative labels if we have too many
        num_bg = cfg.TRAIN.RPN_BATCHSIZE - np.sum(labels == 1)
        bg_inds = np.where(labels == 0)[0]
        if len(bg_inds) > num_bg:
            disable_inds = npr.choice(
                bg_inds, size=(len(bg_inds) - num_bg), replace=False)
            labels[disable_inds] = -1
            #print "was %s inds, disabling %s, now %s inds" % (
                #len(bg_inds), len(disable_inds), np.sum(labels == 0))

        bbox_targets = np.zeros((len(inds_inside), 4), dtype=np.float32)
        bbox_targets = _compute_targets(anchors, gt_boxes[argmax_overlaps, :])

        bbox_inside_weights = np.zeros((len(inds_inside), 4), dtype=np.float32)
        bbox_inside_weights[labels == 1, :] = np.array(cfg.TRAIN.RPN_BBOX_INSIDE_WEIGHTS)

        bbox_outside_weights = np.zeros((len(inds_inside), 4), dtype=np.float32)
        if cfg.TRAIN.RPN_POSITIVE_WEIGHT < 0:
            # uniform weighting of examples (given non-uniform sampling)
            num_examples = np.sum(labels >= 0)
            positive_weights = np.ones((1, 4)) * 1.0 / num_examples
            negative_weights = np.ones((1, 4)) * 1.0 / num_examples
        else:
            assert ((cfg.TRAIN.RPN_POSITIVE_WEIGHT > 0) &
                    (cfg.TRAIN.RPN_POSITIVE_WEIGHT < 1))
            positive_weights = (cfg.TRAIN.RPN_POSITIVE_WEIGHT /
                                np.sum(labels == 1))
            negative_weights = ((1.0 - cfg.TRAIN.RPN_POSITIVE_WEIGHT) /
                                np.sum(labels == 0))
        bbox_outside_weights[labels == 1, :] = positive_weights
        bbox_outside_weights[labels == 0, :] = negative_weights

        if DEBUG:
            self._sums += bbox_targets[labels == 1, :].sum(axis=0)
            self._squared_sums += (bbox_targets[labels == 1, :] ** 2).sum(axis=0)
            self._counts += np.sum(labels == 1)
            means = self._sums / self._counts
            stds = np.sqrt(self._squared_sums / self._counts - means ** 2)
            print('means:')
            print(means)
            print('stdevs:')
            print(stds)

        # map up to original set of anchors
        labels = _unmap(labels, total_anchors, inds_inside, fill=-1)
        bbox_targets = _unmap(bbox_targets, total_anchors, inds_inside, fill=0)
        bbox_inside_weights = _unmap(bbox_inside_weights, total_anchors, inds_inside, fill=0)
        bbox_outside_weights = _unmap(bbox_outside_weights, total_anchors, inds_inside, fill=0)

        if DEBUG:
            print('rpn: max max_overlap', np.max(max_overlaps))
            print('rpn: num_positive', np.sum(labels == 1))
            print('rpn: num_negative', np.sum(labels == 0))
            self._fg_sum += np.sum(labels == 1)
            self._bg_sum += np.sum(labels == 0)
            self._count += 1
            print('rpn: num_positive avg', self._fg_sum / self._count)
            print('rpn: num_negative avg', self._bg_sum / self._count)

        # labels
        labels = labels.reshape((1, height, width, A)).transpose(0, 3, 1, 2)
        labels = labels.reshape((1, 1, A * height, width))
        top[0].reshape(*labels.shape)
        top[0].data[...] = labels

        # bbox_targets
        bbox_targets = bbox_targets \
            .reshape((1, height, width, A * 4)).transpose(0, 3, 1, 2)
        top[1].reshape(*bbox_targets.shape)
        top[1].data[...] = bbox_targets

        # bbox_inside_weights
        bbox_inside_weights = bbox_inside_weights \
            .reshape((1, height, width, A * 4)).transpose(0, 3, 1, 2)
        assert bbox_inside_weights.shape[2] == height
        assert bbox_inside_weights.shape[3] == width
        top[2].reshape(*bbox_inside_weights.shape)
        top[2].data[...] = bbox_inside_weights

        # bbox_outside_weights
        bbox_outside_weights = bbox_outside_weights \
            .reshape((1, height, width, A * 4)).transpose(0, 3, 1, 2)
        assert bbox_outside_weights.shape[2] == height
        assert bbox_outside_weights.shape[3] == width
        top[3].reshape(*bbox_outside_weights.shape)
        top[3].data[...] = bbox_outside_weights

    def backward(self, top, propagate_down, bottom):
        """This layer does not propagate gradients."""
        pass

    def reshape(self, bottom, top):
        """Reshaping happens during the call to forward."""
        pass

def _unmap(data, count, inds, fill=0):
    """ Unmap a subset of item (data) back to the original set of items (of
    size count) """
    if len(data.shape) == 1:
        ret = np.empty((count, ), dtype=np.float32)
        ret.fill(fill)
        ret[inds] = data
    else:
        ret = np.empty((count, ) + data.shape[1:], dtype=np.float32)
        ret.fill(fill)
        ret[inds, :] = data
    return ret

def _compute_targets(ex_rois, gt_rois):
    """Compute bounding-box regression targets for an image."""

    assert ex_rois.shape[0] == gt_rois.shape[0]
    assert ex_rois.shape[1] == 4
    assert gt_rois.shape[1] == 5

    return bbox_transform(ex_rois, gt_rois[:, :4]).astype(np.float32, copy=False)

max_stars_repo_name: songhappy/ai-matrix

max_issues_repo_name: songhappy/ai-matrix

max_forks_repo_name: songhappy/ai-matrix


text:
import numpy as np
import pandas as pd
import csv
import re
from string import ascii_uppercase
import logging
from config import conf

logger = logging.getLogger('parser_matriz_gauss')
logger.setLevel(conf.get_logging_level('parser_matriz_gauss'))
logger.disabled = conf.data['debugging']['parser_matriz_gauss']['disabled']

def cargar_matriz_coeficientes(n_dimension):
    n_dimension_matriz_gauss = conf.data['vars']['n_dimension_matriz_gauss']
    filename = 'csv/' + conf.data['env']['path'] + '/matriz_coeficientes.csv'
    matriz_reader = list(csv.reader(open(filename)))
    # Se crean los pratones y reemplazos para cada variable
    # Patrones y reemplazos de los materiales (Eps)
    pattern_e0 = re.compile(r'E0')
    replace_e0 = '1'
    pattern_er = re.compile(r'(Er)([\* | \s])')
    replace_er = '2\g<2>'
    pattern_ern = re.compile(r'(Er)([1-9])')
    replace_ern = str('(\g<2>+1)')
    # Matriz diagonal de unos
    pattern_uno = re.compile(r'Uno')
    replace_uno = f"np.eye({n_dimension})"
    # Matriz cuadrada de ceros
    pattern_cero = re.compile(r'Cero')
    replace_cero = f"np.diag(np.zeros({n_dimension}))"
    # Constanstes k
    pattern_k = re.compile(r'k\d+')
    replace_k = "constantes_k.loc['\g<0>', 'calcular']"
    # Vectores eigen
    pattern_eig = re.compile(r'(?<!\d)([A-Z])([\*|\s])')
    replace_eig = "matrices_diagonales_valores_eig.loc['\g<1>'].to_numpy()\g<2>"
    # Matrices diagonales de funciones hiperbolicas
    pattern_hyp = re.compile(r'D\d+')
    replace_hyp = "matrices_diagonales_hiperbolicas.loc['\g<0>'].to_numpy()"
    # Matrices acopladoras con un digito
    pattern_acop_1_dig = re.compile(r'(?<!\')(M\d)(?!(T|\'))')
    replace_acop_1_dig = "matrices_acoplamiento.loc['\g<1>'].to_numpy()"
    # Matrices acopladoras con dos digitos
    pattern_acop_2_dig = re.compile(r'(M\d\d)(?!(T|\'))')
    replace_acop_2_dig = "matrices_acoplamiento.loc['\g<1>'].to_numpy()"
    # Matrices acopladoras transpuestas
    pattern_acop_t = re.compile(r'(?<!\')(M\d+)(T|\')')
    replace_acop_t = "matrices_acoplamiento_trans.loc['\g<1>'].to_numpy()"

    # Lista que contiene cada patron y reemplazo en forma de tupla
    repl_patter_list = [
        (pattern_e0, replace_e0), (pattern_er, replace_er), (pattern_e0, replace_e0),
        (pattern_ern, replace_ern), (pattern_uno, replace_uno), (pattern_cero, replace_cero),
        (pattern_k, replace_k), (pattern_eig, replace_eig), (pattern_hyp, replace_hyp),
        (pattern_acop_2_dig, replace_acop_2_dig), (pattern_acop_t, replace_acop_t),
        (pattern_acop_1_dig, replace_acop_1_dig),
        ]

    matriz_coeficientes_list = []
    for row in matriz_reader:
        row_list = []
        for expression in row:
            expr_mod = expression.split('=')[1]
            for pattern, replace in repl_patter_list:
                expr_mod = re.sub(pattern, replace, expr_mod)
            row_list.append(expr_mod)
        matriz_coeficientes_list.append(row_list)
    index = [ascii_uppercase[i] for i in range(0, n_dimension_matriz_gauss)]
    columns = ['C' + str(i) for i in range(1, n_dimension_matriz_gauss + 1)]
    matriz_coeficientes_calcular_str = pd.DataFrame(matriz_coeficientes_list, index=index, columns=columns)
    return matriz_coeficientes_calcular_str

def cargar_variable_valores_dependientes(n_dimension):
    n_dimension_matriz_gauss = conf.data['vars']['n_dimension_matriz_gauss']
    filename = 'csv/' + conf.data['env']['path'] + '/vectores_terminos_dependientes.csv'
    vectores_reader = list(csv.reader(open(filename)))
    vectores_valores_dependientes_list = []
    # patron y reemplazo de los vectores S1,S2,...,Sn
    pattern_s = re.compile(r'(S[0-9]+)')
    replace_s = "vectores_distorsionadores.loc['\g<0>'].to_numpy()"
    # reemplazo para CeroV
    replace_v = f"np.zeros({n_dimension})"
    for expression in vectores_reader:
        expr_mod = expression[0].split('=')[1].replace(' ', '')
        expr_mod = re.sub(pattern_s, replace_s, expr_mod)
        if expr_mod == 'CeroV':
            expr_mod = replace_v
        vectores_valores_dependientes_list.append(expr_mod)
    index = [ascii_uppercase[i] for i in range(0, n_dimension_matriz_gauss)]
    vectores_valores_dependientes_calcular_str = pd.Series(vectores_valores_dependientes_list, index=index, dtype='object')
    return vectores_valores_dependientes_calcular_str


text:
# -*- coding: utf-8 -*-
"""
Created on Mon Mar 26 16:55:55 2018

@author: kncv078

Input refined data from Noé script and annotation data (cmpd counts per assay). to make htsfp. also define threshold for minimum nuber of compounds per assay.
"""

import pandas as pd
import time
import argparse
import numpy as np
#import csv
import os
from scipy.sparse import coo_matrix, save_npz
#import pubchempy as pcp

print('---'*30)
start = time.time()

# In[1] define inputs and output paths
ap = argparse.ArgumentParser(description='Takes refined hts data and generates a flag based fingerprint')
ap.add_argument('-a', action="store", dest='annotation_data', required=True, help='Path to annotation file')    # file with counts of compounds per assay post refinement with Noés script
ap.add_argument('-i', action="store", dest='hts_data', required=True, help='Path to refined hts data')          # folder with refined data produced by Noés script.
ap.add_argument('-o', action="store", dest='outpath', default='htsfp_gen_out/' , help='Optional: set output path for fingerprint file')
ap.add_argument('-t', action="store", dest='threshold', default=20000, type=int, help='Optional: threshold for minimum number of compounds per assay')

args = ap.parse_args()

if args.hts_data[-1] != '/':
    args.hts_data = args.hts_data+'/'

# additional inputs: file created by script: pubchem website or pubchempy
cid2smi_file    = 'C:/CESFP_project/CID2smi_pubchem.txt'

# set path for output
if not os.path.exists(args.outpath):
    os.makedirs(args.outpath)
outpath = args.outpath+'htsfp_t'+str(args.threshold)

# In[2] Load files
# load Annotation file and select assays with more than N cmpds using info from annotation file
print('loading annotation file...')
statsdf = pd.read_csv(args.annotation_data, sep='\t')
#statsdf.columns = ['assay','count']
assay_selection = statsdf.loc[statsdf['count']>=args.threshold]['assay'].values
print('Number of assays containing at least', args.threshold, 'Assays:', len(assay_selection))
print(time.time() - start)

# load compoundID 2 canonical smiles file
print('loading CID 2 smiles file...')
cid2smi = {}
with open(cid2smi_file, 'r') as f:
    for line in f:
        linels = line.strip().split('\t')
        cid2smi[linels[0]] = linels[1]
print(time.time() - start)

# In[4] MAIN PART: Make dictionary containing cmpd_ID : list(assay activity flags)
#   PART 1/2: make empty dict with all values x's of correct dimensions
print('Creating empty dict...')
main_cmpd_dict = {}
for cmpd in cid2smi:
#    main_cmpd_dict[cmpd] = ['x']*len(assay_selection)
    main_cmpd_dict[cmpd] = [0]*len(assay_selection)
print('done.')
print('dimensions of htsfp:\t{} x {}'.format(len(main_cmpd_dict),len(assay_selection)))
print(time.time() - start)

# In[5] PART 2/2: fill dictionary with 'flags' from refined HTS data
print('Filling dict with flag values...')
idx = -1
Alength = len(assay_selection)
miss_cid = 0
for assay in assay_selection:
    idx += 1
    if idx % 10 == 0: print('... {}%'.format(round((idx+1)/Alength*100),0), end='\r')
    assay_file = args.hts_data+assay
    with open(assay_file, 'r') as f:
        j = 0
        for line in f:
            j += 1
            if j == 1: continue
            line_list = line.strip().split('\t')
            Cid = line_list[0]
            flag = line_list[1]
            if flag not in ['A','N']: print('{} unknown flag: {}'.format(assay, flag), end='\n'); continue
            if Cid == '': miss_cid += 1; continue
            elif Cid in main_cmpd_dict:
#                main_cmpd_dict[Cid][idx] = flag
                if flag == 'A': main_cmpd_dict[Cid][idx] = 1
            else:
                print('{}, CID not in main cmpd dict: {}'.format(assay, Cid))
                continue

print('number of entries without CID: {}'.format(miss_cid))
print('\ndone')
print(time.time() - start)
# In[7] make sparse
print('converting results to sparse matrix (coo)...')
row=[]
col=[]
data=[]
cmpdlist=[]
for i,(key,value) in enumerate(sorted(main_cmpd_dict.items())):
    cmpdlist.append(key)
    for j, bit in enumerate(value):
        if int(bit)==1:
            row.append(i)
            col.append(j)
            data.append(np.int8(int(bit)))

cooM = coo_matrix((data, (row, col)), shape=(len(main_cmpd_dict), len(assay_selection)))
save_npz('htsfp_sparse.npz',cooM)

#save corresponding cmpd list
with open('htsfp_sparse_cmpdlist.txt','w') as outf:
    for cmpd in cmpdlist:
        outf.write('{}\n'.format(cmpd))
with open('htsfp_sparse_assaylist.txt','w') as outf:
    for assay in assay_selection:
        assay_id = assay.split('.')[0]
        outf.write("{}\n".format(assay_id))

# In[6] save output file containing: cmpd ID: HTSFP

#print('saving output files')
#
#with open(outpath+'.txt', 'w') as csv_file:
#    for key, value in main_cmpd_dict.items():
#       csv_file.write('{} {}\n'.format(key,' '.join(value)))
#print('HTSFP matrix file saved to:', outpath+'.txt')
#
## save a file containing an ordered list of assay IDs used to generate this HTSFP
#print('saving assay list to file')
#with open(outpath+'-Assay_list.txt','w') as assay_list:
#    for item in assay_selection:
#        assay_id = item.split('.')[0]
#        assay_list.write("{}\n".format(assay_id))
#print('Assay list saved to:', outpath+'-Assay_list.txt')
#
#print('Total time taken:',int(time.time() - start),'seconds')

reason: import numpy,from scipy


text:
from copy import deepcopy
from functools import partial
from itertools import groupby
from autoconstructive.utils import helpers
from joblib import Parallel, delayed
import numpy as np
import math
from typing import Any, Counter, List
from torch import nn
from torch._C import Value
from torch.functional import Tensor
from torch.nn import init
import torch
from torch.nn.modules.linear import Linear
from torch.nn.parameter import Parameter
from torch.multiprocessing import Pool, set_start_method, freeze_support

def build_model_ids(
    repetitions: int,
    activation_functions: list,
    min_neurons: int,
    max_neurons: int,
    step: int,
):
    """Creates a list with model ids to relate to hidden representations.
    1. Creates a list containing the number of hidden neurons for each architecture (independent of activation functions and/or repetitions)
    using the following formula neurons_structures=range(min_neurons, max_neurons+1, step)
    2. Calculates the number of independent models (parallel mlps) = len(neurons_structures) * len(activations) * repetitions

    Raises:
        ValueError: [description]
        ValueError: [description]
        RuntimeError: [description]
        ValueError: [description]

    Returns:
        hidden_neurons__model_id: List indicating for each global neuron the model_id that it belongs to
        output__model_id: List containing the id of the model for each output
        output__architecture_id: List containing the id of the architecture (neuron structure AND activation function) that the output belongs.
            Architectures with the same id means that it only differs the repetition number, but have equal neuron structure and activation function.
    """

    if len(activation_functions) == 0:
        raise ValueError(
            "At least one activation function must be passed. Try `nn.Identity()` if you want no activation."
        )

    activation_names = [a.__class__.__name__ for a in activation_functions]
    if len(set(activation_names)) != len(activation_names):
        raise ValueError("activation_functions must have only unique values.")

    num_activations = len(activation_functions)

    neurons_structure = torch.arange(min_neurons, max_neurons + 1, step).tolist()
    num_different_neurons_structures = len(neurons_structure)
    num_parallel_mlps = num_different_neurons_structures * num_activations * repetitions

    i = 0
    hidden_neuron__model_id = []
    while i < num_parallel_mlps:
        for structure in neurons_structure:
            hidden_neuron__model_id += [i] * structure
            i += 1

    output__model_id = [i[0] for i in groupby(hidden_neuron__model_id)]
    output__architecture_id = output__model_id[:num_activations * num_different_neurons_structures] * repetitions

    return hidden_neuron__model_id, output__model_id, output__architecture_id

class ParallelMLPs(nn.Module):
    def __init__(
        self,
        in_features: int,
        out_features: int,
        hidden_neuron__model_id: List[int],
        output__model_id: List[int],
        output__architecture_id: List[int],
        activations: List[nn.Module],
        bias: bool = True,
        device: str = "cuda",
        logger: Any= None
    ):
        super().__init__()
        self.device = device
        self.in_features = in_features
        self.out_features = out_features
        self.activations = activations
        self.logger = logger

        # Mappings: index -> id
        self.hidden_neuron__model_id = (
            torch.Tensor(hidden_neuron__model_id).long().to(self.device)
        )
        self.output__model_id = torch.Tensor(output__model_id).long().to(self.device)
        self.output__architecture_id = torch.Tensor(output__architecture_id).long().to(self.device)

        self.total_hidden_neurons = len(self.hidden_neuron__model_id)
        self.unique_model_ids = sorted(list(set(hidden_neuron__model_id)))
        self.model_id__num_hidden_neurons = torch.from_numpy(
            np.bincount(self.hidden_neuron__model_id.cpu().numpy())
        ).to(self.device)
        # self.model_id__start_neuron = torch.zeros_like(self.model_id__num_hidden_neurons)
        self.model_id__start_idx = torch.cat([torch.tensor([0]).to(self.device), self.model_id__num_hidden_neurons.cumsum(0)[:-1]])
        self.model_id__end_idx = self.model_id__start_idx + self.model_id__num_hidden_neurons
        # self.model_id__start_neuron =

        # self.model_id__num_hidden_neurons = torch.bincount(
        #     self.hidden_neuron__model_id
        # ).to(self.device)

        self.num_unique_models = len(self.unique_model_ids)
        self.num_activations = len(activations)

        self.activations_split = self.total_hidden_neurons // self.num_activations

        self.hidden_layer = nn.Linear(self.in_features, self.total_hidden_neurons)
        self.weight = Parameter(
            torch.Tensor(self.out_features, self.total_hidden_neurons)
        )
        if bias:
            self.bias = Parameter(
                torch.Tensor(self.num_unique_models, self.out_features)
            )
        else:
            self.bias = None
            self.register_parameter("bias", None)

        self.reset_parameters()
        self.to(device)
        self.logger.info(f"Model sent to {device}!")

    def _build_outputs_ids(self):
        return [i[0] for i in groupby(self.hidden_neuron__model_id)]

    def reset_parameters(self, layer_ids=None):
        if layer_ids == None:
            layer_ids = self.unique_model_ids

        with torch.no_grad():
            for layer_id in layer_ids:
                start = self.model_id__start_idx[layer_id]
                end = self.model_id__end_idx[layer_id]
                hidden_w = self.hidden_layer.weight[start:end, :]
                hidden_b = self.hidden_layer.bias[start:end]

                out_w = self.weight[:, start:end]
                out_b = self.bias[layer_id, :]

                for w, b in [(hidden_w, hidden_b), (out_w, out_b)]:
                    init.kaiming_uniform_(w, a=math.sqrt(5))
                    fan_in, _ = init._calculate_fan_in_and_fan_out(w)
                    bound = 1 / math.sqrt(fan_in)
                    init.uniform_(b, -bound, bound)

    def apply_activations(self, x: Tensor) -> Tensor:
        tensors = x.split(self.activations_split, dim=1)
        output = []
        sub_tensor_out_features = tensors[0].shape[1]
        for (act, sub_tensor) in zip(self.activations, tensors):
            if sub_tensor.shape[1] != sub_tensor_out_features:
                raise RuntimeError(
                    f"sub_tensors with different number of parameters per activation {[t.shape for t in tensors]}"
                )
            output.append(act(sub_tensor))
        output = torch.cat(output, dim=1)
        return output

    def forward(self, x: Tensor) -> Tensor:
        batch_size = x.shape[0]
        x = self.hidden_layer(x)  # [batch_size, total_hidden_neurons]
        x = self.apply_activations(x)  # [batch_size, total_hidden_neurons]

        x = (
            x[:, :, None] * self.weight.T[None, :, :]
        )  # [batch_size, total_hidden_neurons, out_features]

        # [batch_size, total_repetitions, num_architectures, out_features]
        adjusted_out = (
            torch.zeros(
                batch_size, self.num_unique_models, self.out_features, device=x.device
            ).scatter_add_(
                1,
                # self.hidden_neuron__layer_id,
                self.hidden_neuron__model_id[None, :, None].expand(
                    batch_size, -1, self.out_features
                ),
                x,
            )
        ) + self.bias[None, :, :]

        # [batch_size, num_unique_models, out_features]
        return adjusted_out

    def calculate_loss(self, loss_func, preds, target):
        if hasattr(loss_func, "reduction"):
            assert loss_func.reduction == "none"

        if preds.ndim == 3:
            batch_size, num_models, neurons = preds.shape
            loss = loss_func(
                preds.permute(0, 2, 1), target[:, None].expand(-1, num_models)
            )
        else:
            loss = loss_func(preds, target)

        return loss

    def extract_mlp(self, model_id: int) -> nn.Sequential:
        """Extracts a completely independent MLP.
        """
        if model_id >= self.num_unique_models:
            raise ValueError(
                f"model_id {model_id} > num_uniqe_models {self.num_unique_models}"
            )

        with torch.no_grad():
            model_neurons = self.hidden_neuron__model_id == model_id
            hidden_weight = self.hidden_layer.weight[model_neurons, :]
            hidden_bias = self.hidden_layer.bias[model_neurons]

            out_weight = self.weight[:, model_neurons]
            out_bias = self.bias[model_id, :]

            hidden_layer = nn.Linear(
                in_features=hidden_weight.shape[1], out_features=hidden_weight.shape[0]
            )
            activation_index = (
                torch.nonzero(self.hidden_neuron__model_id == model_id)[0]
                // self.activations_split
            )
            activation = deepcopy(self.activations[activation_index])
            out_layer = nn.Linear(
                in_features=hidden_layer.out_features, out_features=self.out_features
            )

            hidden_layer.weight[:, :] = hidden_weight.clone()
            hidden_layer.bias[:] = hidden_bias.clone()

            out_layer.weight[:, :] = out_weight.clone()
            out_layer.bias[:] = out_bias.clone()

        return nn.Sequential(hidden_layer, activation, out_layer).to(self.device)

    def extra_repr(self) -> str:
        return "in_features={}, out_features={}, bias={}".format(
            self.in_features, self.out_features, self.bias is not None
        )


text:
import copy

import numpy as np
import pytest
from mmcv.utils import assert_dict_has_keys
from numpy.testing import assert_array_equal

from mmaction.datasets.pipelines import (AudioFeatureSelector,
                                         DenseSampleFrames, SampleAVAFrames,
                                         SampleFrames, SampleProposalFrames,
                                         UntrimmedSampleFrames)
from .base import BaseTestLoading

class TestSampling(BaseTestLoading):

    def test_sample_frames(self):
        target_keys = [
            'frame_inds', 'clip_len', 'frame_interval', 'num_clips',
            'total_frames'
        ]

        with pytest.warns(UserWarning):
            # start_index has been deprecated
            config = dict(
                clip_len=3, frame_interval=1, num_clips=5, start_index=1)
            SampleFrames(**config)

        # Sample Frame with tail Frames
        video_result = copy.deepcopy(self.video_results)
        frame_result = copy.deepcopy(self.frame_results)
        config = dict(
            clip_len=3, frame_interval=1, num_clips=5, keep_tail_frames=True)
        sample_frames = SampleFrames(**config)
        sample_frames(video_result)
        sample_frames(frame_result)

        # Sample Frame with no temporal_jitter
        # clip_len=3, frame_interval=1, num_clips=5
        video_result = copy.deepcopy(self.video_results)
        frame_result = copy.deepcopy(self.frame_results)
        config = dict(
            clip_len=3, frame_interval=1, num_clips=5, temporal_jitter=False)
        sample_frames = SampleFrames(**config)
        sample_frames_results = sample_frames(video_result)
        assert assert_dict_has_keys(sample_frames_results, target_keys)
        assert len(sample_frames_results['frame_inds']) == 15
        sample_frames_results = sample_frames(frame_result)
        assert len(sample_frames_results['frame_inds']) == 15
        assert np.max(sample_frames_results['frame_inds']) <= 5
        assert np.min(sample_frames_results['frame_inds']) >= 1
        assert repr(sample_frames) == (f'{sample_frames.__class__.__name__}('
                                       f'clip_len={3}, '
                                       f'frame_interval={1}, '
                                       f'num_clips={5}, '
                                       f'temporal_jitter={False}, '
                                       f'twice_sample={False}, '
                                       f'out_of_bound_opt=loop, '
                                       f'test_mode={False})')

        # Sample Frame with no temporal_jitter
        # clip_len=5, frame_interval=1, num_clips=5,
        # out_of_bound_opt='repeat_last'
        video_result = copy.deepcopy(self.video_results)
        frame_result = copy.deepcopy(self.frame_results)
        config = dict(
            clip_len=5,
            frame_interval=1,
            num_clips=5,
            temporal_jitter=False,
            out_of_bound_opt='repeat_last')
        sample_frames = SampleFrames(**config)
        sample_frames_results = sample_frames(video_result)
        assert repr(sample_frames) == (f'{sample_frames.__class__.__name__}('
                                       f'clip_len={5}, '
                                       f'frame_interval={1}, '
                                       f'num_clips={5}, '
                                       f'temporal_jitter={False}, '
                                       f'twice_sample={False}, '
                                       f'out_of_bound_opt=repeat_last, '
                                       f'test_mode={False})')

        def check_monotonous(arr):
            length = arr.shape[0]
            for i in range(length - 1):
                if arr[i] > arr[i + 1]:
                    return False
            return True

        assert assert_dict_has_keys(sample_frames_results, target_keys)
        assert len(sample_frames_results['frame_inds']) == 25
        frame_inds = sample_frames_results['frame_inds'].reshape([5, 5])
        for i in range(5):
            assert check_monotonous(frame_inds[i])

        sample_frames_results = sample_frames(frame_result)
        assert len(sample_frames_results['frame_inds']) == 25
        frame_inds = sample_frames_results['frame_inds'].reshape([5, 5])
        for i in range(5):
            assert check_monotonous(frame_inds[i])
        assert np.max(sample_frames_results['frame_inds']) <= 5
        assert np.min(sample_frames_results['frame_inds']) >= 1

        # Sample Frame with temporal_jitter
        # clip_len=4, frame_interval=2, num_clips=5
        video_result = copy.deepcopy(self.video_results)
        frame_result = copy.deepcopy(self.frame_results)
        config = dict(
            clip_len=4, frame_interval=2, num_clips=5, temporal_jitter=True)
        sample_frames = SampleFrames(**config)
        sample_frames_results = sample_frames(video_result)
        assert assert_dict_has_keys(sample_frames_results, target_keys)
        assert len(sample_frames_results['frame_inds']) == 20
        sample_frames_results = sample_frames(frame_result)
        assert len(sample_frames_results['frame_inds']) == 20
        assert np.max(sample_frames_results['frame_inds']) <= 5
        assert np.min(sample_frames_results['frame_inds']) >= 1
        assert repr(sample_frames) == (f'{sample_frames.__class__.__name__}('
                                       f'clip_len={4}, '
                                       f'frame_interval={2}, '
                                       f'num_clips={5}, '
                                       f'temporal_jitter={True}, '
                                       f'twice_sample={False}, '
                                       f'out_of_bound_opt=loop, '
                                       f'test_mode={False})')

        # Sample Frame with no temporal_jitter in test mode
        # clip_len=4, frame_interval=1, num_clips=6
        video_result = copy.deepcopy(self.video_results)
        frame_result = copy.deepcopy(self.frame_results)
        config = dict(
            clip_len=4,
            frame_interval=1,
            num_clips=6,
            temporal_jitter=False,
            test_mode=True)
        sample_frames = SampleFrames(**config)
        sample_frames_results = sample_frames(video_result)
        assert assert_dict_has_keys(sample_frames_results, target_keys)
        assert len(sample_frames_results['frame_inds']) == 24
        sample_frames_results = sample_frames(frame_result)
        assert len(sample_frames_results['frame_inds']) == 24
        assert np.max(sample_frames_results['frame_inds']) <= 5
        assert np.min(sample_frames_results['frame_inds']) >= 1
        assert repr(sample_frames) == (f'{sample_frames.__class__.__name__}('
                                       f'clip_len={4}, '
                                       f'frame_interval={1}, '
                                       f'num_clips={6}, '
                                       f'temporal_jitter={False}, '
                                       f'twice_sample={False}, '
                                       f'out_of_bound_opt=loop, '
                                       f'test_mode={True})')

        # Sample Frame with no temporal_jitter in test mode
        # clip_len=3, frame_interval=1, num_clips=6
        video_result = copy.deepcopy(self.video_results)
        frame_result = copy.deepcopy(self.frame_results)
        config = dict(
            clip_len=3,
            frame_interval=1,
            num_clips=6,
            temporal_jitter=False,
            test_mode=True)
        sample_frames = SampleFrames(**config)
        sample_frames_results = sample_frames(video_result)
        assert assert_dict_has_keys(sample_frames_results, target_keys)
        assert len(sample_frames_results['frame_inds']) == 18
        sample_frames_results = sample_frames(frame_result)
        assert len(sample_frames_results['frame_inds']) == 18
        assert np.max(sample_frames_results['frame_inds']) <= 5
        assert np.min(sample_frames_results['frame_inds']) >= 1

        # Sample Frame with no temporal_jitter to get clip_offsets
        # clip_len=1, frame_interval=1, num_clips=8
        video_result = copy.deepcopy(self.video_results)
        frame_result = copy.deepcopy(self.frame_results)
        frame_result['total_frames'] = 6
        config = dict(
            clip_len=1,
            frame_interval=1,
            num_clips=8,
            temporal_jitter=False,
            test_mode=True)
        sample_frames = SampleFrames(**config)
        sample_frames_results = sample_frames(video_result)
        assert assert_dict_has_keys(sample_frames_results, target_keys)
        assert len(sample_frames_results['frame_inds']) == 8
        sample_frames_results = sample_frames(frame_result)
        assert len(sample_frames_results['frame_inds']) == 8
        assert_array_equal(sample_frames_results['frame_inds'],
                           np.array([1, 2, 2, 3, 4, 5, 5, 6]))

        # Sample Frame with no temporal_jitter to get clip_offsets
        # clip_len=1, frame_interval=1, num_clips=8
        video_result = copy.deepcopy(self.video_results)
        frame_result = copy.deepcopy(self.frame_results)
        frame_result['total_frames'] = 6
        config = dict(
            clip_len=1,
            frame_interval=1,
            num_clips=8,
            temporal_jitter=False,
            test_mode=True)
        sample_frames = SampleFrames(**config)
        sample_frames_results = sample_frames(video_result)
        assert sample_frames_results['start_index'] == 0
        assert assert_dict_has_keys(sample_frames_results, target_keys)
        assert len(sample_frames_results['frame_inds']) == 8
        sample_frames_results = sample_frames(frame_result)
        assert len(sample_frames_results['frame_inds']) == 8
        assert_array_equal(sample_frames_results['frame_inds'],
                           np.array([1, 2, 2, 3, 4, 5, 5, 6]))

        # Sample Frame with no temporal_jitter to get clip_offsets zero
        # clip_len=6, frame_interval=1, num_clips=1
        video_result = copy.deepcopy(self.video_results)
        frame_result = copy.deepcopy(self.frame_results)
        frame_result['total_frames'] = 5
        config = dict(
            clip_len=6,
            frame_interval=1,
            num_clips=1,
            temporal_jitter=False,
            test_mode=True)
        sample_frames = SampleFrames(**config)
        sample_frames_results = sample_frames(video_result)
        assert sample_frames_results['start_index'] == 0
        assert assert_dict_has_keys(sample_frames_results, target_keys)
        assert len(sample_frames_results['frame_inds']) == 6
        sample_frames_results = sample_frames(frame_result)
        assert len(sample_frames_results['frame_inds']) == 6
        assert_array_equal(sample_frames_results['frame_inds'],
                           [1, 2, 3, 4, 5, 1])

        # Sample Frame with no temporal_jitter to get avg_interval <= 0
        # clip_len=12, frame_interval=1, num_clips=20
        video_result = copy.deepcopy(self.video_results)
        frame_result = copy.deepcopy(self.frame_results)
        frame_result['total_frames'] = 30
        config = dict(
            clip_len=12,
            frame_interval=1,
            num_clips=20,
            temporal_jitter=False,
            test_mode=False)
        sample_frames = SampleFrames(**config)
        sample_frames_results = sample_frames(video_result)
        assert sample_frames_results['start_index'] == 0
        assert assert_dict_has_keys(sample_frames_results, target_keys)
        assert len(sample_frames_results['frame_inds']) == 240
        sample_frames_results = sample_frames(frame_result)
        assert len(sample_frames_results['frame_inds']) == 240
        assert np.max(sample_frames_results['frame_inds']) <= 30
        assert np.min(sample_frames_results['frame_inds']) >= 1

        # Sample Frame with no temporal_jitter to get clip_offsets
        # clip_len=1, frame_interval=1, num_clips=8
        video_result = copy.deepcopy(self.video_results)
        frame_result = copy.deepcopy(self.frame_results)
        frame_result['total_frames'] = 6
        config = dict(
            clip_len=1,
            frame_interval=1,
            num_clips=8,
            temporal_jitter=False,
            test_mode=False)
        sample_frames = SampleFrames(**config)
        sample_frames_results = sample_frames(video_result)
        assert assert_dict_has_keys(sample_frames_results, target_keys)
        assert sample_frames_results['start_index'] == 0
        assert len(sample_frames_results['frame_inds']) == 8
        sample_frames_results = sample_frames(frame_result)
        assert len(sample_frames_results['frame_inds']) == 8
        assert_array_equal(sample_frames_results['frame_inds'],
                           np.array([1, 2, 3, 3, 4, 5, 5, 6]))

        # Sample Frame with no temporal_jitter to get clip_offsets zero
        # clip_len=12, frame_interval=1, num_clips=2
        video_result = copy.deepcopy(self.video_results)
        frame_result = copy.deepcopy(self.frame_results)
        frame_result['total_frames'] = 10
        config = dict(
            clip_len=12,
            frame_interval=1,
            num_clips=2,
            temporal_jitter=False,
            test_mode=False)
        sample_frames = SampleFrames(**config)
        sample_frames_results = sample_frames(video_result)
        assert sample_frames_results['start_index'] == 0
        assert assert_dict_has_keys(sample_frames_results, target_keys)
        assert len(sample_frames_results['frame_inds']) == 24
        sample_frames_results = sample_frames(frame_result)
        assert len(sample_frames_results['frame_inds']) == 24
        assert np.max(sample_frames_results['frame_inds']) <= 10
        assert np.min(sample_frames_results['frame_inds']) >= 1

        # Sample Frame using twice sample
        # clip_len=12, frame_interval=1, num_clips=2
        video_result = copy.deepcopy(self.video_results)
        frame_result = copy.deepcopy(self.frame_results)
        frame_result['total_frames'] = 40
        config = dict(
            clip_len=12,
            frame_interval=1,
            num_clips=2,
            temporal_jitter=False,
            twice_sample=True,
            test_mode=True)
        sample_frames = SampleFrames(**config)
        sample_frames_results = sample_frames(video_result)
        assert sample_frames_results['start_index'] == 0
        assert assert_dict_has_keys(sample_frames_results, target_keys)
        assert len(sample_frames_results['frame_inds']) == 48
        sample_frames_results = sample_frames(frame_result)
        assert len(sample_frames_results['frame_inds']) == 48
        assert np.max(sample_frames_results['frame_inds']) <= 40
        assert np.min(sample_frames_results['frame_inds']) >= 1

    def test_dense_sample_frames(self):
        target_keys = [
            'frame_inds', 'clip_len', 'frame_interval', 'num_clips',
            'total_frames'
        ]

        # Dense sample with no temporal_jitter in test mode
        # clip_len=4, frame_interval=1, num_clips=6
        video_result = copy.deepcopy(self.video_results)
        frame_result = copy.deepcopy(self.frame_results)
        config = dict(
            clip_len=4,
            frame_interval=1,
            num_clips=6,
            temporal_jitter=False,
            test_mode=True)
        dense_sample_frames = DenseSampleFrames(**config)
        dense_sample_frames_results = dense_sample_frames(video_result)
        assert dense_sample_frames_results['start_index'] == 0
        assert assert_dict_has_keys(dense_sample_frames_results, target_keys)
        assert len(dense_sample_frames_results['frame_inds']) == 240
        dense_sample_frames_results = dense_sample_frames(frame_result)
        assert len(dense_sample_frames_results['frame_inds']) == 240
        assert repr(dense_sample_frames) == (
            f'{dense_sample_frames.__class__.__name__}('
            f'clip_len={4}, '
            f'frame_interval={1}, '
            f'num_clips={6}, '
            f'sample_range={64}, '
            f'num_sample_positions={10}, '
            f'temporal_jitter={False}, '
            f'out_of_bound_opt=loop, '
            f'test_mode={True})')

        # Dense sample with no temporal_jitter
        # clip_len=4, frame_interval=1, num_clips=6
        video_result = copy.deepcopy(self.video_results)
        frame_result = copy.deepcopy(self.frame_results)
        config = dict(
            clip_len=4, frame_interval=1, num_clips=6, temporal_jitter=False)
        dense_sample_frames = DenseSampleFrames(**config)
        dense_sample_frames_results = dense_sample_frames(video_result)
        assert dense_sample_frames_results['start_index'] == 0
        assert assert_dict_has_keys(dense_sample_frames_results, target_keys)
        assert len(dense_sample_frames_results['frame_inds']) == 24
        dense_sample_frames_results = dense_sample_frames(frame_result)
        assert len(dense_sample_frames_results['frame_inds']) == 24

        # Dense sample with no temporal_jitter, sample_range=32 in test mode
        # clip_len=4, frame_interval=1, num_clips=6
        video_result = copy.deepcopy(self.video_results)
        frame_result = copy.deepcopy(self.frame_results)
        config = dict(
            clip_len=4,
            frame_interval=1,
            num_clips=6,
            sample_range=32,
            temporal_jitter=False,
            test_mode=True)
        dense_sample_frames = DenseSampleFrames(**config)
        dense_sample_frames_results = dense_sample_frames(video_result)
        assert dense_sample_frames_results['start_index'] == 0
        assert assert_dict_has_keys(dense_sample_frames_results, target_keys)
        assert len(dense_sample_frames_results['frame_inds']) == 240
        dense_sample_frames_results = dense_sample_frames(frame_result)
        assert len(dense_sample_frames_results['frame_inds']) == 240

        # Dense sample with no temporal_jitter, sample_range=32
        # clip_len=4, frame_interval=1, num_clips=6
        video_result = copy.deepcopy(self.video_results)
        frame_result = copy.deepcopy(self.frame_results)
        config = dict(
            clip_len=4,
            frame_interval=1,
            num_clips=6,
            sample_range=32,
            temporal_jitter=False)
        dense_sample_frames = DenseSampleFrames(**config)
        dense_sample_frames_results = dense_sample_frames(video_result)
        assert dense_sample_frames_results['start_index'] == 0
        assert assert_dict_has_keys(dense_sample_frames_results, target_keys)
        assert len(dense_sample_frames_results['frame_inds']) == 24
        dense_sample_frames_results = dense_sample_frames(frame_result)
        assert len(dense_sample_frames_results['frame_inds']) == 24
        assert repr(dense_sample_frames) == (
            f'{dense_sample_frames.__class__.__name__}('
            f'clip_len={4}, '
            f'frame_interval={1}, '
            f'num_clips={6}, '
            f'sample_range={32}, '
            f'num_sample_positions={10}, '
            f'temporal_jitter={False}, '
            f'out_of_bound_opt=loop, '
            f'test_mode={False})')

        # Dense sample with no temporal_jitter, sample_range=1000 to check mod
        # clip_len=4, frame_interval=1, num_clips=6
        video_result = copy.deepcopy(self.video_results)
        frame_result = copy.deepcopy(self.frame_results)
        config = dict(
            clip_len=4,
            frame_interval=1,
            num_clips=6,
            sample_range=1000,
            temporal_jitter=False)
        dense_sample_frames = DenseSampleFrames(**config)
        dense_sample_frames_results = dense_sample_frames(video_result)
        assert dense_sample_frames_results['start_index'] == 0
        assert assert_dict_has_keys(dense_sample_frames_results, target_keys)
        assert len(dense_sample_frames_results['frame_inds']) == 24
        dense_sample_frames_results = dense_sample_frames(frame_result)
        assert len(dense_sample_frames_results['frame_inds']) == 24

        # Dense sample with no temporal_jitter in test mode
        # sample_range=32, num_sample_positions=5
        # clip_len=4, frame_interval=1, num_clips=6
        video_result = copy.deepcopy(self.video_results)
        frame_result = copy.deepcopy(self.frame_results)
        config = dict(
            clip_len=4,
            frame_interval=1,
            num_clips=6,
            num_sample_positions=5,
            sample_range=32,
            temporal_jitter=False,
            test_mode=True)
        dense_sample_frames = DenseSampleFrames(**config)
        dense_sample_frames_results = dense_sample_frames(video_result)
        assert dense_sample_frames_results['start_index'] == 0
        assert assert_dict_has_keys(dense_sample_frames_results, target_keys)
        assert len(dense_sample_frames_results['frame_inds']) == 120
        dense_sample_frames_results = dense_sample_frames(frame_result)
        assert len(dense_sample_frames_results['frame_inds']) == 120
        assert repr(dense_sample_frames) == (
            f'{dense_sample_frames.__class__.__name__}('
            f'clip_len={4}, '
            f'frame_interval={1}, '
            f'num_clips={6}, '
            f'sample_range={32}, '
            f'num_sample_positions={5}, '
            f'temporal_jitter={False}, '
            f'out_of_bound_opt=loop, '
            f'test_mode={True})')

    def test_untrim_sample_frames(self):

        target_keys = [
            'frame_inds', 'clip_len', 'frame_interval', 'num_clips',
            'total_frames'
        ]

        frame_result = dict(
            frame_dir=None,
            total_frames=100,
            filename_tmpl=None,
            modality='RGB',
            start_index=0,
            label=1)
        video_result = copy.deepcopy(self.video_results)

        config = dict(clip_len=1, frame_interval=16, start_index=0)
        sample_frames = UntrimmedSampleFrames(**config)
        sample_frames_results = sample_frames(frame_result)
        assert assert_dict_has_keys(sample_frames_results, target_keys)
        assert len(sample_frames_results['frame_inds']) == 6
        assert_array_equal(sample_frames_results['frame_inds'],
                           np.array([8, 24, 40, 56, 72, 88]))
        assert repr(sample_frames) == (f'{sample_frames.__class__.__name__}('
                                       f'clip_len={1}, '
                                       f'frame_interval={16})')

        config = dict(clip_len=1, frame_interval=16, start_index=0)
        sample_frames = UntrimmedSampleFrames(**config)
        sample_frames_results = sample_frames(video_result)
        assert assert_dict_has_keys(sample_frames_results, target_keys)
        frame_inds = np.array(list(range(8, 300, 16)))
        assert len(sample_frames_results['frame_inds']) == frame_inds.shape[0]
        assert_array_equal(sample_frames_results['frame_inds'], frame_inds)
        assert repr(sample_frames) == (f'{sample_frames.__class__.__name__}('
                                       f'clip_len={1}, '
                                       f'frame_interval={16})')

        config = dict(clip_len=1, frame_interval=16)
        sample_frames = UntrimmedSampleFrames(**config)
        frame_result_ = copy.deepcopy(frame_result)
        frame_result_['start_index'] = 1
        sample_frames_results = sample_frames(frame_result_)
        assert assert_dict_has_keys(sample_frames_results, target_keys)
        assert len(sample_frames_results['frame_inds']) == 6
        assert_array_equal(sample_frames_results['frame_inds'],
                           np.array([8, 24, 40, 56, 72, 88]) + 1)
        assert repr(sample_frames) == (f'{sample_frames.__class__.__name__}('
                                       f'clip_len={1}, '
                                       f'frame_interval={16})')

        config = dict(clip_len=3, frame_interval=16, start_index=0)
        sample_frames = UntrimmedSampleFrames(**config)
        sample_frames_results = sample_frames(frame_result)
        assert assert_dict_has_keys(sample_frames_results, target_keys)
        assert len(sample_frames_results['frame_inds']) == 18
        assert_array_equal(
            sample_frames_results['frame_inds'],
            np.array([
                7, 8, 9, 23, 24, 25, 39, 40, 41, 55, 56, 57, 71, 72, 73, 87,
                88, 89
            ]))
        assert repr(sample_frames) == (f'{sample_frames.__class__.__name__}('
                                       f'clip_len={3}, '
                                       f'frame_interval={16})')

    def test_sample_ava_frames(self):
        target_keys = [
            'fps', 'timestamp', 'timestamp_start', 'shot_info', 'frame_inds',
            'clip_len', 'frame_interval'
        ]
        config = dict(clip_len=32, frame_interval=2)
        sample_ava_dataset = SampleAVAFrames(**config)
        ava_result = sample_ava_dataset(results=self.ava_results)
        assert assert_dict_has_keys(ava_result, target_keys)
        assert ava_result['clip_len'] == 32
        assert ava_result['frame_interval'] == 2
        assert len(ava_result['frame_inds']) == 32
        assert repr(sample_ava_dataset) == (
            f'{sample_ava_dataset.__class__.__name__}('
            f'clip_len={32}, '
            f'frame_interval={2}, '
            f'test_mode={False})')

        # add test case in Issue #306
        config = dict(clip_len=8, frame_interval=8)
        sample_ava_dataset = SampleAVAFrames(**config)
        ava_result = sample_ava_dataset(results=self.ava_results)
        assert assert_dict_has_keys(ava_result, target_keys)
        assert ava_result['clip_len'] == 8
        assert ava_result['frame_interval'] == 8
        assert len(ava_result['frame_inds']) == 8
        assert repr(sample_ava_dataset) == (
            f'{sample_ava_dataset.__class__.__name__}('
            f'clip_len={8}, '
            f'frame_interval={8}, '
            f'test_mode={False})')

    def test_sample_proposal_frames(self):
        target_keys = [
            'frame_inds', 'clip_len', 'frame_interval', 'num_clips',
            'total_frames', 'start_index'
        ]

        # test error cases
        with pytest.raises(TypeError):
            proposal_result = copy.deepcopy(self.proposal_results)
            config = dict(
                clip_len=1,
                frame_interval=1,
                body_segments=2,
                aug_segments=('error', 'error'),
                aug_ratio=0.5,
                temporal_jitter=False)
            sample_frames = SampleProposalFrames(**config)
            sample_frames(proposal_result)

        # test normal cases
        # Sample Frame with no temporal_jitter
        # clip_len=1, frame_interval=1
        # body_segments=2, aug_segments=(1, 1)
        proposal_result = copy.deepcopy(self.proposal_results)
        proposal_result['total_frames'] = 9
        config = dict(
            clip_len=1,
            frame_interval=1,
            body_segments=2,
            aug_segments=(1, 1),
            aug_ratio=0.5,
            temporal_jitter=False)
        sample_frames = SampleProposalFrames(**config)
        sample_frames_results = sample_frames(proposal_result)
        assert assert_dict_has_keys(sample_frames_results, target_keys)
        assert len(sample_frames_results['frame_inds']) == 8
        assert repr(sample_frames) == (f'{sample_frames.__class__.__name__}('
                                       f'clip_len={1}, '
                                       f'body_segments={2}, '
                                       f'aug_segments={(1, 1)}, '
                                       f'aug_ratio={(0.5, 0.5)}, '
                                       f'frame_interval={1}, '
                                       f'test_interval={6}, '
                                       f'temporal_jitter={False}, '
                                       f'mode=train)')

        # Sample Frame with temporal_jitter
        # clip_len=1, frame_interval=1
        # body_segments=2, aug_segments=(1, 1)
        proposal_result = copy.deepcopy(self.proposal_results)
        proposal_result['total_frames'] = 9
        config = dict(
            clip_len=1,
            frame_interval=1,
            body_segments=2,
            aug_segments=(1, 1),
            aug_ratio=0.5,
            temporal_jitter=True)
        sample_frames = SampleProposalFrames(**config)
        sample_frames_results = sample_frames(proposal_result)
        assert assert_dict_has_keys(sample_frames_results, target_keys)
        assert len(sample_frames_results['frame_inds']) == 8
        assert repr(sample_frames) == (f'{sample_frames.__class__.__name__}('
                                       f'clip_len={1}, '
                                       f'body_segments={2}, '
                                       f'aug_segments={(1, 1)}, '
                                       f'aug_ratio={(0.5, 0.5)}, '
                                       f'frame_interval={1}, '
                                       f'test_interval={6}, '
                                       f'temporal_jitter={True}, '
                                       f'mode=train)')

        # Sample Frame with no temporal_jitter in val mode
        # clip_len=1, frame_interval=1
        # body_segments=2, aug_segments=(1, 1)
        proposal_result = copy.deepcopy(self.proposal_results)
        proposal_result['total_frames'] = 9
        config = dict(
            clip_len=1,
            frame_interval=1,
            body_segments=2,
            aug_segments=(1, 1),
            aug_ratio=0.5,
            temporal_jitter=False,
            mode='val')
        sample_frames = SampleProposalFrames(**config)
        sample_frames_results = sample_frames(proposal_result)
        assert assert_dict_has_keys(sample_frames_results, target_keys)
        assert len(sample_frames_results['frame_inds']) == 8
        assert repr(sample_frames) == (f'{sample_frames.__class__.__name__}('
                                       f'clip_len={1}, '
                                       f'body_segments={2}, '
                                       f'aug_segments={(1, 1)}, '
                                       f'aug_ratio={(0.5, 0.5)}, '
                                       f'frame_interval={1}, '
                                       f'test_interval={6}, '
                                       f'temporal_jitter={False}, '
                                       f'mode=val)')

        # Sample Frame with no temporal_jitter in test mode
        # test_interval=2
        proposal_result = copy.deepcopy(self.proposal_results)
        proposal_result['out_proposals'] = None
        proposal_result['total_frames'] = 10
        config = dict(
            clip_len=1,
            frame_interval=1,
            body_segments=2,
            aug_segments=(1, 1),
            aug_ratio=0.5,
            test_interval=2,
            temporal_jitter=False,
            mode='test')
        sample_frames = SampleProposalFrames(**config)
        sample_frames_results = sample_frames(proposal_result)
        assert assert_dict_has_keys(sample_frames_results, target_keys)
        assert len(sample_frames_results['frame_inds']) == 5
        assert repr(sample_frames) == (f'{sample_frames.__class__.__name__}('
                                       f'clip_len={1}, '
                                       f'body_segments={2}, '
                                       f'aug_segments={(1, 1)}, '
                                       f'aug_ratio={(0.5, 0.5)}, '
                                       f'frame_interval={1}, '
                                       f'test_interval={2}, '
                                       f'temporal_jitter={False}, '
                                       f'mode=test)')

        # Sample Frame with no temporal_jitter to get clip_offsets zero
        # clip_len=1, frame_interval=1
        # body_segments=2, aug_segments=(1, 1)
        proposal_result = copy.deepcopy(self.proposal_results)
        proposal_result['total_frames'] = 3
        config = dict(
            clip_len=1,
            frame_interval=1,
            body_segments=2,
            aug_segments=(1, 1),
            aug_ratio=0.5,
            temporal_jitter=False)
        sample_frames = SampleProposalFrames(**config)
        sample_frames_results = sample_frames(proposal_result)
        assert assert_dict_has_keys(sample_frames_results, target_keys)
        assert len(sample_frames_results['frame_inds']) == 8
        assert repr(sample_frames) == (f'{sample_frames.__class__.__name__}('
                                       f'clip_len={1}, '
                                       f'body_segments={2}, '
                                       f'aug_segments={(1, 1)}, '
                                       f'aug_ratio={(0.5, 0.5)}, '
                                       f'frame_interval={1}, '
                                       f'test_interval={6}, '
                                       f'temporal_jitter={False}, '
                                       f'mode=train)')

        # Sample Frame with no temporal_jitter to
        # get clip_offsets zero in val mode
        # clip_len=1, frame_interval=1
        # body_segments=4, aug_segments=(2, 2)
        proposal_result = copy.deepcopy(self.proposal_results)
        proposal_result['total_frames'] = 3
        config = dict(
            clip_len=1,
            frame_interval=1,
            body_segments=4,
            aug_segments=(2, 2),
            aug_ratio=0.5,
            temporal_jitter=False,
            mode='val')
        sample_frames = SampleProposalFrames(**config)
        sample_frames_results = sample_frames(proposal_result)
        assert assert_dict_has_keys(sample_frames_results, target_keys)
        assert len(sample_frames_results['frame_inds']) == 16
        assert repr(sample_frames) == (f'{sample_frames.__class__.__name__}('
                                       f'clip_len={1}, '
                                       f'body_segments={4}, '
                                       f'aug_segments={(2, 2)}, '
                                       f'aug_ratio={(0.5, 0.5)}, '
                                       f'frame_interval={1}, '
                                       f'test_interval={6}, '
                                       f'temporal_jitter={False}, '
                                       f'mode=val)')

    def test_audio_feature_selector(self):
        target_keys = ['audios']
        # test frame selector with 2 dim input
        inputs = copy.deepcopy(self.audio_feature_results)
        inputs['frame_inds'] = np.arange(0, self.audio_total_frames,
                                         2)[:, np.newaxis]
        inputs['num_clips'] = 1
        inputs['length'] = 1280
        audio_feature_selector = AudioFeatureSelector()
        results = audio_feature_selector(inputs)
        assert assert_dict_has_keys(results, target_keys)
        assert repr(audio_feature_selector) == (
            f'{audio_feature_selector.__class__.__name__}('
            f'fix_length={128})')

reason: import numpy,from numpy


text:
# @Author: dileep
# @Last Modified by:   dileep

from halo import Halo
import numpy as np
from sklearn.metrics import confusion_matrix, accuracy_score
import torch
from torch.autograd import Variable
import torch.nn.functional as F
from torch import nn
from torch.utils.data import DataLoader, TensorDataset
from ...loaders import Churn

#TODO: Make this more flexible wrt hidden layers
class Classifier(nn.Module):
    """
        Pytorch classifier to classify data samples in the Churn Modelling dataset
        Parameters:
        ----------
        n_inputs : int
            Number of inputs/features used for classification
        n_outputs : int
            Number of classes that are part of the classification
        Attributes:
        ----------
        input_size : int
            Number of features the classifier takes as input
        output_size : int
            Number of classes the classifier needs to classify into
        hidden_size : Tuple
            Tuple containing integers that represent the size of each hidden layer
        n_layers : int
            Number of layer the classifier has
    """
    def __init__(self, n_inputs: int, n_outputs: int) -> None:
        super().__init__()
        self.input_size = n_inputs
        self.output_size = n_outputs
        self.n_layers = 3
        self.hidden_size = ((n_inputs + n_outputs) // 2, )
        self.layer1 = nn.Linear(n_inputs, self.hidden_size[0])
        self.layer2 = nn.Linear(self.hidden_size[0], self.hidden_size[0])
        self.layer3 = nn.Linear(self.hidden_size[0], n_outputs)

    def forward(self, x):
        """ Forward pass of the neural network """
        y = F.relu(self.layer1(x))
        z = F.relu(self.layer2(y))
        out = self.layer3(z)
        return out

class ChurnTrainer:
    """
        Trains the neural network on the churn dataset
        Parameters:
        ----------
        train_data : np.ndarray
            Training data for the model
        test_data : np.ndarray
            Testing data for the model
        #TODO: Define these later
        loss : str, optional
            Loss function to be used
        optimizer : str, optional
            Optimizer to be used
        learning_rate : float, optional
            Learning rate to be used in the optimizer
        Attributes:
        ----------
        model : nn.Module
            Pytorch neural network
    """
    def __init__(self, train_features: np.ndarray, train_labels: np.ndarray,
                 test_features: np.ndarray, test_labels: np.ndarray) -> None:
        self.train_data = TensorDataset(self._tofeat(train_features), self._tolabel(train_labels))
        self.test_data = TensorDataset(self._tofeat(test_features), self._tolabel(test_labels))
        self.input_size = train_features.shape[1]
        self.output_size = train_labels.shape[1]
        self.model = Classifier(self.input_size, 2)
        self.criterion = torch.nn.CrossEntropyLoss()
        self.optimizer = torch.optim.Adam(self.model.parameters())

    @staticmethod
    def _tofeat(x):
        return torch.from_numpy(x).float()

    @staticmethod
    def _tolabel(x):
        return torch.from_numpy(x).long().squeeze()

    def train(self, epochs: int) -> float:
        """
            Trains the model and returns the accuracy on the test_data
            Parameters:
            ----------
            epochs : int
                Number of epochs for the training
            Returns:
            -------
            float
                Accuracy of the model on the test_data
        """
        trainloader = DataLoader(self.train_data, batch_size=100, shuffle=True, num_workers=2)
        testloader = DataLoader(self.test_data, batch_size=100, shuffle=True, num_workers=2)
        halo = Halo(text='Loading', spinner='dots')
        halo.start()
        for epoch in range(epochs):
            for i, data in enumerate(trainloader, 0):
                features, targets = data
                features = Variable(features, requires_grad=False)
                targets = Variable(targets, requires_grad=False)
                self.optimizer.zero_grad()
                outputs = self.model(features)
                loss = self.criterion(outputs, targets)
                loss.backward()
                self.optimizer.step()
                halo.text = f"Epoch:{epoch}, Step:{(i+1)/40*100}, Loss:{loss.data[0]}"
        halo.stop()
        features = self.test_data.data_tensor
        targets = self.test_data.target_tensor
        features = Variable(features, requires_grad=False)
        _, output = self.model(features).max(dim=1)
        print(confusion_matrix(targets.numpy(), output.data.numpy()))
        print("accuracy", accuracy_score(targets.numpy(), output.data.numpy()))

max_stars_repo_name: dileep-kishore/deeplearning-examples

max_issues_repo_name: dileep-kishore/deeplearning-examples

max_forks_repo_name: dileep-kishore/deeplearning-examples


text:
import lms_code.lib.rep2 as rep2
import numpy as np
from codim1.core import *
from codim1.core.quad_strategy import AdaptiveInteriorQuad2
import matplotlib.pyplot as plt
from codim1.core.tools import plot_mesh
from lms_code.plots import plot_all as lms_plot
from mpi4py import MPI
import itertools
import sys

def plot_points(points):
    fig = plt.figure()
    ax = fig.add_subplot(1, 1, 1)
    x, y = zip(*points)
    ax.scatter(x, y, marker = 'o', c = 'b', s = 5)
    plt.show()

def get_field_type(type):
    if type == 'u':
        field_normal = [0.0, 0.0]
        field_type = 'displacement'
    elif type == 'sx':
        field_normal = [1.0, 0.0]
        field_type = 'traction'
    elif type == 'sy':
        field_normal = [0.0, 1.0]
        field_type = 'traction'
    else:
        raise Exception('Unknown field type')
    return field_normal, field_type

def pick_field_and_region(int_eval):
    print('Number of points to evaluate: ' + str(int_eval.meshpy_pts.shape[0]))
    int_eval.viz_mesh()
    region_num = raw_input('Type the region #: ')
    name = raw_input('Type the region name: ')
    type = raw_input('[u, sx, sy]: ')
    plt.close('all')
    return region_num, name, type

def create_quad(q_min):
    return AdaptiveInteriorQuad2(q_min, 1e50)

def one_chunk(model, region_num, subset_num, total_subsets, name, type, which_side):
    bem_soln = rep2.load('bem_' + model)
    int_eval = rep2.load('interior_mesh_' + model)

    field_normal, field_type = get_field_type(type)

    q_min = 4

    ek = ElasticKernelSet(bem_soln['shear_modulus'], bem_soln['poisson_ratio'])
    qs = create_quad(q_min)

    int_data = dict()
    int_data['region_num'] = region_num
    int_data['subset_num'] = subset_num
    int_data['total_subsets'] = total_subsets
    int_data['name'] = name
    int_data['type'] = type

    int_eval.choose_subregion(int(region_num), subset_num, total_subsets)
    int_data['tris'] = int_eval.selected_tris
    print 'Number of triangles: ' + str(int_data['tris'].shape)

    soln = int_eval.eval_soln(qs, ek, bem_soln['soln_coeffs'],
                              which_side, field_normal, field_type)
    int_data['soln'] = soln
    rep2.save(interior_fname(model, name, subset_num, type), int_data)

def interior_fname(model, name, subset_num, type):
    return 'interior_' + model + '_' + name + '/' + str(subset_num) + '_' + type

def chunkify(lst, n):
    return [lst[i::n] for i in xrange(n)]

def main(args):
    model = 'all_details'
    int_eval = rep2.load('interior_mesh_' + model)

    if len(args) == 2 and args[1] == 'choose':
        pick_field_and_region(int_eval)
        sys.exit()

    info = lms_plot.interior_params()
    options = [info['regions'], range(info['subsets']), info['types']]
    jobs = list(itertools.product(*options))

    comm = MPI.COMM_WORLD
    total_procs = comm.size
    chunks = chunkify(jobs, total_procs)
    my_chunk = chunks[comm.rank]
    if comm.rank == 0:
        print('There are ' + str(len(jobs)) + ' jobs to run.')
        print('Each of ' + str(comm.size) + ' processor is running ' + str(len(my_chunk)) + ' of the jobs.')
    for (region_num, name), subset_num, type in my_chunk:
        which_side = 'positive'
        if name == 'sichuan':
            which_side = 'negative'
        one_chunk(model, region_num, subset_num,
                  info['subsets'], name, type, which_side)

if __name__ == '__main__':
    main(sys.argv)


text:
# Importing modules from the Python Standard Library.
import sys
import math

# Importing third-party modules.
import numpy as np

# Importing Matplot's Python Library.
import matplotlib
from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg
from matplotlib.figure import Figure

# Importing PyQt5.
from PyQt5 import QtCore, QtWidgets, QtGui

class MainWindow(QtWidgets.QMainWindow):
    """"The MainWindow class inherits from the QMainWindow. It acts as a container for the MainWindow class
        (QWidget) while also setting the window title and creating the statusbar."""

    def __init__(self):
        super().__init__()

        # Setting the window title, icon, and assigning the MainWindow class as the central widget.
        self.setWindowTitle("Quanta")
        self.setWindowIcon(QtGui.QIcon("Icon.png"))

        # Creating instances of the tabs within the QMainWindow class.
        self.ViewTab = ViewTab()
        self.ConsoleTab = ConsoleTab()
        self.ParametersTab = ParametersTab()

        # Creating the QTabWidget and adding the instances of the tabs to it.
        Tabs = QtWidgets.QTabWidget()
        Tabs.addTab(self.ViewTab, "View")
        Tabs.addTab(self.ConsoleTab, "Console")
        Tabs.addTab(self.ParametersTab, "Parameters")
        self.setCentralWidget(Tabs)

        # Creating the Statusbar.
        StatusBar = self.statusBar()
        StatusBar.showMessage('Ready')

        # Creating the toolbar.
        self.Toolbar = self.addToolBar("Toolbar")
        self.Toolbar.setMovable(False)

        # Creating the progressbar.
        self.ProgressBar = QtWidgets.QProgressBar(self)

        # Creating buttons.
        self.UpdateView = QtWidgets.QAction("Update View", self)

        # Adding widgets and buttons(actions) to the toolbar.
        self.Toolbar.addAction(self.UpdateView)
        self.Toolbar.addWidget(self.ProgressBar)

        # Buttons are connected to their respective handlers.
        self.UpdateView.triggered.connect(MainWindow.UpdateView_handler)

        # Displaying the Graphical User Interface.
        self.show()

    def UpdateView_handler():

        # Console output.
        print("Event: UpdateView_handler")

class MplCanvas(FigureCanvasQTAgg):

    def __init__(self, parent=None, width=5, height=4, dpi=100):
        figure = Figure(figsize=(width, height), dpi=dpi)
        self.axes = figure.gca(projection='3d')
        super(MplCanvas, self).__init__(figure)

class ViewTab(QtWidgets.QWidget):

    def __init__(self):
        super().__init__()

        self.length_x = 1
        self.length_y = 1
        self.quantum_number_x = 3
        self.quantum_number_y = 2

        """Generating x-coordinates."""
        x = np.linspace(0, self.length_x, 100)

        """Generating y-coordinates."""
        y = np.linspace(0, self.length_y, 100)

        """Generating all possible xy-coordinates."""
        X,Y = np.meshgrid(x,y)

        """Generating z-coordinates from xy-coordinates."""
        Z = np.sqrt((2)/(self.length_x*self.length_y))*np.sin((math.pi*self.quantum_number_x*X)/(self.length_x))*np.sin((math.pi*self.quantum_number_y*Y)/(self.length_y))

        plt = MplCanvas(self, width=5, height=4, dpi=100)
        contour = plt.axes.plot_surface(X,Y,Z,cmap='hot')
        plt.figure.colorbar(contour, shrink = 1)
        plt.axes.view_init(elev = 20, azim = -135)
        plt.axes.set_xlabel(r'$x$')
        plt.axes.set_ylabel(r'$y$')
        plt.axes.set_zlabel(r'$\psi(x, y)$')
        Layout = QtWidgets.QVBoxLayout()
        Layout.addWidget(plt)
        self.setLayout(Layout)

class ConsoleTab(QtWidgets.QWidget):

    def __init__(self):
        super().__init__()

        # Creating the Console(QTextBrowser class) within the ConsoleTab(QTabWidget class).
        self.Textbox = QtWidgets.QTextBrowser(self)
        self.Textbox.setReadOnly(True)
        self.Textbox.setOpenExternalLinks(True)

        # Adding the Console(QTextBrowser class) to the layout ConsoleTab(QTabWidget class)..
        Layout = QtWidgets.QVBoxLayout()
        Layout.addWidget(self.Textbox)
        self.setLayout(Layout)

        # Printing starting text.
        self.Textbox.insertPlainText("===========================")
        self.Textbox.insertPlainText("\n")
        self.Textbox.insertHtml("<b> Quanta Version: 0.0.1 </b>")
        self.Textbox.insertPlainText("\n")
        self.Textbox.insertHtml("<b> Created by Prithvi R. </b>")
        self.Textbox.insertPlainText("\n")
        self.Textbox.insertPlainText("======================")

class ParametersTab(QtWidgets.QWidget):

    def __init__(self):
        super().__init__()

        GridLayout = QtWidgets.QGridLayout()
        self.setLayout(GridLayout)

        Group = QtWidgets.QGroupBox("Quantum System")
        self.QuantumSystem = QtWidgets.QComboBox(self)
        self.QuantumSystem.resize(200, 25)
        self.QuantumSystem.addItem("Particle in a Box (2D)")
        GridLayout.addWidget(Group)

        self.QuantumFunction = QtWidgets.QComboBox(self)
        self.QuantumFunction.resize(200, 25)
        self.QuantumFunction.addItem("Wavefunction")
        self.QuantumFunction.addItem("Probability Density Function")

        GridLayout.addWidget(Group)
        GroupLayout = QtWidgets.QVBoxLayout()
        GroupLayout.addWidget(self.QuantumSystem)
        GroupLayout.addWidget(self.QuantumFunction)
        Group.setLayout(GroupLayout)

if __name__ == "__main__":
    application = QtWidgets.QApplication(sys.argv)
    GUI = MainWindow()
    application.exec_()