text:
"""
author:yunnaidan
@time: 2018/11/23
@file: ConfidenceLevel.py
"""
import os
import re
import time
import numpy as np
import pandas as pd
from scipy import stats
import multiprocessing

from dyntripy.plot_utils import distribution

def cl41event(row, threshold, out_file, figure_out_file):
    try:
        tele_pir = row['remote_earthquake_ratio']
        if tele_pir != 'None':
            tele_pir = float(tele_pir)
            background_pirs = re.split(' +', row['background_day_ratio'])
            background_pir_list = np.array(background_pirs, dtype='float64')
            if len(background_pir_list) < 30:
                confidence_level_value = 'Insufficient RBs'
                tri_flag = None
            else:
                mean, std = stats.norm.fit(background_pir_list)
                background_norm = stats.norm(loc=mean, scale=std)
                confidence_level_value = background_norm.cdf(tele_pir)
                if confidence_level_value >= threshold:
                    tri_flag = int(1)
                else:
                    tri_flag = int(0)

                if figure_out_file is not None:
                    distribution(
                        background_pir_list,
                        tele_pir,
                        figure_out_file,
                        ax=None)

        else:
            confidence_level_value = 'No RE'
            tri_flag = None

        with open(out_file, 'a') as f:
            f.write(','.join([str(row['time']), str(confidence_level_value), str(tri_flag)]))
            f.write('\n')
    except Exception as err_msg:
        print(err_msg)

    return None

def run_cl_parallel(
        background_pir_associated_file,
        threshold,
        out_file,
        figure_out_folder,
        cores):
    if os.path.exists(out_file):
        os.remove(out_file)
    with open(out_file, 'a') as f:
        f.write(','.join(['time', 'cl', 'triggering']))
        f.write('\n')

    tele_background_pir_df = pd.read_csv(background_pir_associated_file)

    pool = multiprocessing.Pool(processes=cores)
    tasks = []
    for row_index, row in tele_background_pir_df.iterrows():
        print ('Prepare cl task %s' % row['time'], end='\r')

        if figure_out_folder is not None:
            figure_out_file = os.path.join(figure_out_folder, row['time'] + '.png')
        else:
            figure_out_file = None

        tasks.append((row, threshold, out_file, figure_out_file))
        # cl41event(row, threshold, out_file, figure_out_file)
    print ('\n')

    # chunksize is how many tasks will be processed by one processor
    rs = pool.starmap_async(cl41event, tasks, chunksize=1)
    # close() & join() is necessary
    pool.close()
    # simple progress bar
    while (True):
        remaining = rs._number_left
        print("finished:{0}/{1}".format(len(tasks) - remaining, len(tasks)),
              end='\r')  # '\r' means remove the last line
        if (rs.ready()):
            break
        time.sleep(0.5)

    pool.join()
    print ('\n')

max_stars_repo_name: yigexiaoshuiping/trigger

max_issues_repo_name: yunndlalala/dyntrigger

max_forks_repo_name: yunndlalala/dyntrigger

reason: import numpy,from scipy


text:
import numpy as np

def CalculDistanceMots(mot1, mot2):
    '''
    L'objectif de cette fonction est de retourner la
    distance qu'il existe entre deux mots rentrés en
    argument de la fonction qui sont ici appelés mot1 et mot2.
    '''
    # Nous utilisons le package NumPy pour faciliter l'écriture
    # de la matrice d.
    d = np.zeros((len(mot1)+1,len(mot2)+1))
    d[:,0] = np.arange(0,len(mot1)+1)
    d[0] = np.arange(0, len(mot2)+1)

    cout = 0
    for i in range(1,len(mot1)+1):
        for j in range(1,len(mot2)+1):
            if mot1[i-1] == mot2[j-1]:
                cout = 0
            else:
                cout = 1

            d[i][j] = min(d[i-1][j]+1, d[i][j-1]+1, d[i-1][j-1]+cout)

            if i > 1 and j > 1 and mot1[i-1] == mot2[j-2] and mot1[i-2] == mot2[j-1]:
                d[i][j] = min(d[i][j], d[i-2][j-2]+1)

    return int(d[len(mot1)][len(mot2)])

ext: py

reason: import numpy


text:
# -*- coding: utf-8 -*-
"""
Created on Wed Jan 23 15:07:43 2019

@author: GKX
"""

from nilearn.image import reorder_img, new_img_like
from sitk_utils import resample_to_spacing, calculate_origin_offset
import SimpleITK as sitk
import os
import numpy as np
import nibabel as nib
import matplotlib.pyplot as plt
from skimage import measure

def save_max_objects(img):
    labels = measure.label(img)  # 返回打上标签的img数组
    jj = measure.regionprops(labels)  # 找出连通域的各种属性。  注意，这里jj找出的连通域不包括背景连通域
    # is_del = False
#    print(len(jj))
    if len(jj) == 1:
        out = img
        # is_del = False
    else:
        # 通过与质心之间的距离进行判断
        num = labels.max()  #连通域的个数
        del_array = np.array([0] * (num + 1))#生成一个与连通域个数相同的空数组来记录需要删除的区域（从0开始，所以个数要加1）
        save_index =None

        for k in range(num):#TODO：这里如果遇到全黑的图像的话会报错
            if k == 0:
                initial_area = jj[0].area
                save_index = 1  # 初始保留第一个连通域
            else:
                k_area = jj[k].area  # 将元组转换成array

                if initial_area < k_area:
                    initial_area = k_area
                    save_index = k + 1

        del_array[save_index] = 1
        del_mask = del_array[labels]
        out = img * del_mask
        # is_del = True
    return out

def resize(image, new_shape, interpolation="linear"):
    image = reorder_img(image, resample=interpolation)
    zoom_level = np.divide(new_shape, image.shape)
    new_spacing = np.divide(image.header.get_zooms(), zoom_level)
    new_data = resample_to_spacing(image.get_data(), image.header.get_zooms(), new_spacing,
                                   interpolation=interpolation)
    new_affine = np.copy(image.affine)
    np.fill_diagonal(new_affine, new_spacing.tolist() + [1])
    new_affine[:3, 3] += calculate_origin_offset(new_spacing, image.header.get_zooms())
    return new_img_like(image, new_data, affine=new_affine)

def binaryzation(filefullname):
    ###   二值化
    image = sitk.ReadImage(filefullname)
    image_array = sitk.GetArrayFromImage(image)
    threshold_value = 0
    image_array[image_array<threshold_value] = 0
    image_array[image_array>threshold_value] = 1
    d,h,w= image_array.shape
    ###   去除小的连通区域
    for i in range(d):
        image_array[i, :, :] = save_max_objects(image_array[i, :, :])
    image_array[image_array==0] = 20000
    image_array[image_array==1] = 0
    return image_array

def get_bound(Arrays):

    H,W,L = Arrays.shape

    for i in range(0,H):
        if np.max(Arrays[i,:,:])>0:
#            print(np.max(Arrays[i,:,:]))
            H_min = i
            break

    for i in range(H-1,0,-1):
        if np.max(Arrays[i,:,:])>0:
#            print(np.max(Arrays[i,:,:]))
            H_max = i
            break

    for i in range(0,W):
        if np.max(Arrays[:,i,:])>0:
#            print(np.max(Arrays[:,i,:]))
            W_min = i
            break

    for i in range(W-1,0,-1):
        if np.max(Arrays[:,i,:])>0:
#            print(np.max(Arrays[:,i,:]))
            W_max = i
            break

    for i in range(0,L):
        if np.max(Arrays[:,:,i])>0:
#            print(np.max(Arrays[:,:,i]))
            L_min = i
            break

    for i in range(L-1,0,-1):
        if np.max(Arrays[:,:,i])>0:
#            print(np.max(Arrays[:,:,i]))
            L_max = i
            break
#
    return Arrays[H_min:H_max,W_min:W_max,L_min:L_max]

def _transform(dcm_dir,save_dir):
###   from dcm to nii
    reader = sitk.ImageSeriesReader()
    dicom_names = reader.GetGDCMSeriesFileNames(dcm_dir)
    reader.SetFileNames(dicom_names)
    image2 = reader.Execute()

###   transform 3D image to array
    image_array = sitk.GetArrayFromImage(image2)  #  z,y,x

###   crop the dark voxel
#    new_array,range_list = get_bound(image_array)

###   transform array to 3D image
    image3 = sitk.GetImageFromArray(image_array)

###   save 3D image
    name = dcm_dir.split('/')[-1] + '.nii'
    save_path = os.path.join(save_dir,name)   #   get the save path
    sitk.WriteImage(image3,save_path)

def nib_resize1(save_dir1,name1,image_shape):

    save_path = os.path.join(save_dir1,name1)
    # load
    image1 = nib.load(save_path)
    image_data1 = image1.get_data()
    # print('before resize :',image_data1.shape)
    # resize
    image1 = resize(image1, image_shape)

    ###   transform to array
    image_data1 = image1.get_data()
    # image_data1 = image_data1[:,:,::-1]

    # print('after resize : ',image_data1.shape)
    image1 = sitk.GetImageFromArray(image_data1)
    sitk.WriteImage(image1,save_path)

import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--d", type=int, default=128)
parser.add_argument("--W", type=int, default=128)
args = parser.parse_args()

image_shape = (args.W,args.W,args.d)

ori_dir = './train_dataset/'
ori_nii_dir = './train_binary_'+ str(args.W) + '_'+ str(args.W)+ '_' + str(args.d) + '/'

isExists = os.path.exists(ori_nii_dir)

if not isExists:
    os.makedirs(ori_nii_dir)
    print(ori_nii_dir + ' build successfully !')
else:
    print(ori_nii_dir + ' already exist !')

if __name__ == '__main__':

    dirnum_names = os.listdir(ori_dir)
    count = 0
    for name in dirnum_names:
        dcm_dir = os.path.join(ori_dir,name)
        count+=1
        _transform(dcm_dir,ori_nii_dir) #   dicom转化为nii
        ori_nii_path = os.path.join(ori_nii_dir,name+'.nii')
        #   二值化和去除连通阈
        new_arrary = binaryzation(ori_nii_path)
        ori_image = sitk.ReadImage(ori_nii_path)
        ori_arrary = sitk.GetArrayFromImage(ori_image)
        #   映像到原图
        new_arrary1 = ori_arrary+new_arrary
        new_arrary1[new_arrary1 >10000] = 0
        crop_arrary = get_bound(new_arrary1)
        crop_image = sitk.GetImageFromArray(crop_arrary)
        sitk.WriteImage(crop_image,ori_nii_path)
        print(image_shape,'--------->>',count)
        nib_resize1(ori_nii_dir,name+'.nii',image_shape)


text:
"""
    Modelling of time series with different lengths example.

    Supplementary code for:
    D. Hafner and C. Igel. "Signal Processing with Recurrent Neural Networks in TensorFlow"
    """

import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf

def generate_toy_time_series(sequences = 10, max_length = 100, gap = 5, noise = 0.1, cut_sequence = True):
    """
        Generate toy time series data set, where the time series can have different lengths.
        :param sequences: Number of time series to generate
        :param max_length: Maximum sequence length
        :param gap: Prediction horizon (how many steps the labels are in the future)
        :param noise: Standard deviation of additive Gaussian noise
        :param cut_sequence: If true, the sequences have variable lengths.
        :return: Input values, target values, and lenths of time series
        """
    X = np.empty((sequences, max_length))
    Y = np.empty((sequences, max_length))
    sequence_lengths = np.empty(sequences, dtype=int)
    for i in range(sequences):
        t = np.arange(0, max_length + gap)
        t0 = np.random.rand() * max_length
        f = np.random.rand() * 2
        x = np.sin(t + t0)
        X[i,:] = x[0:max_length]
        Y[i,:] = x[gap:max_length+gap] + np.random.normal(0, noise, max_length)
        if cut_sequence:
            length = np.random.randint(max_length/2,max_length) + 1;
            sequence_lengths[i] = length
            X[i,length:] = np.zeros(max_length-length)
            Y[i,length:] = np.zeros(max_length-length)
        else:
            sequence_lengths[i] = max_length
    return X.reshape((sequences, max_length, 1)), Y.reshape((sequences, max_length, 1)), sequence_lengths

def generate_plot(time_series, filename, plot_input=False):
    """
        Plots time series
        :param time_series: List of three arrays of time series
        :param filename: Name of the image
        :param plot_input: Plot input, default False
        """
    fig, ax = plt.subplots(time_series[0].shape[0], sharex=True)
    for i in range(0, time_series[0].shape[0]):
        if plot_input:
            ax[i].plot(time_series[0][i], label='input', color='lightgray', linestyle='--', linewidth=3)
        ax[i].plot(time_series[1][i], label='target', linestyle='-', linewidth=3)
        ax[i].plot(time_series[2][i], label='prediction', linestyle='-', linewidth=3)
        ax[i].set_yticks([0])
        ax[i].set_yticks([0])
    plt.rc('font', size=14)
    plt.xlabel('time [t]')
    plt.ylabel('signal')
    lgd = plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
    plt.savefig(filename, bbox_extra_artists=(lgd,), bbox_inches='tight')
    plt.close()

# Basic definitions
N = 32  # Size of recurrent neural network
T = 100  # Maximum length of training time series
n = 10  # Number of training sequences
n_test = 2 # Number of test sequences
m = 1  # Output dimension
d = 1  # Input dimension
epochs = 200  # Maximum mnumber of training epochs
learning_rate = 0.1  # Learning rate

# Generate data
train_X, train_Y, train_lengths = generate_toy_time_series(
    sequences = n, max_length = T, gap = 5, noise = 0.5, cut_sequence=True)
test_X, test_Y, test_lengths = generate_toy_time_series(
    sequences = n_test, max_length = T, gap = 5, noise = 0.5, cut_sequence=True)

# Placeholders
inputs  = tf.placeholder(tf.float32, [None, None, d])
target = tf.placeholder(tf.float32, [None, None, m])
lengths = tf.placeholder(tf.int64)

# Network architecture
cell = tf.nn.rnn_cell.GRUCell(N)
rnn_output, _ = tf.nn.dynamic_rnn(
    cell, inputs, sequence_length=lengths, dtype=tf.float32)

# Note the following reshaping:
#   We want a prediction for every time step.
#   Weights of fully connected layer should be the same (shared) for every time step.
#   This is achieved by flattening the first two dimensions.
#   Now all time steps look the same as individual inputs in a batch fed into a feed-forward network.
rnn_output_flat = tf.reshape(rnn_output, [-1, N])
target_flat = tf.reshape(target, [-1, m])
prediction_flat = tf.layers.dense(rnn_output_flat, m, activation=None)
prediction  = tf.reshape(prediction_flat, [-1, tf.shape(inputs)[1], m])

# Error function
loss = tf.reduce_sum(tf.square(target_flat - prediction_flat))
loss /= tf.cast(tf.reduce_sum(lengths), tf.float32)

# Optimizer
train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)

# Create graph and initialize variables
with tf.Session() as sess:
    init = tf.global_variables_initializer()
    sess.run(init)
    sess.graph.finalize()  # Graph is read-only after this statement.

    # Do the learning
    for i in range(epochs):
        sess.run(train_step, feed_dict={
            inputs: train_X, target: train_Y, lengths: train_lengths})
        if i==0 or (i+1)%100==0:
            temp_loss = sess.run(loss,
                feed_dict={
                    inputs: train_X, target: train_Y, lengths: train_lengths})
            print(i+1, 'loss =', temp_loss)

    # Visualize modelling of training data
    model = sess.run(prediction, feed_dict={inputs: train_X, lengths: train_lengths})
    generate_plot([train_X, train_Y, model], 'sineTrain.pdf')

    # Visualize modelling of test data
    model = sess.run(prediction, feed_dict={inputs: test_X, lengths: test_lengths})
    generate_plot([test_X, test_Y, model], 'sineTest.pdf')

max_stars_repo_name: glahr/rnn-tutorial

max_issues_repo_name: glahr/rnn-tutorial

max_forks_repo_name: glahr/rnn-tutorial


text:
# coding=utf-8
# Copyright 2020 The TensorFlow Datasets Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Kitti dataset."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import collections
import csv
import os
import numpy as np
import tensorflow.compat.v2 as tf
import tensorflow_datasets.public_api as tfds

_CITATION = """\
@inproceedings{Geiger2012CVPR,
  author = {Andreas Geiger and Philip Lenz and Raquel Urtasun},
  title = {Are we ready for Autonomous Driving? The KITTI Vision Benchmark Suite},
  booktitle = {Conference on Computer Vision and Pattern Recognition (CVPR)},
  year = {2012}
}
"""
_DESCRIPTION = """\
Kitti contains a suite of vision tasks built using an autonomous driving
platform. The full benchmark contains many tasks such as stereo, optical flow,
visual odometry, etc. This dataset contains the object detection dataset,
including the monocular images and bounding boxes. The dataset contains 7481
training images annotated with 3D bounding boxes. A full description of the
annotations can be found in the readme of the object development kit readme on
the Kitti homepage.
"""
_HOMEPAGE_URL = "http://www.cvlibs.net/datasets/kitti/"
_DATA_URL = "https://s3.eu-central-1.amazonaws.com/avg-kitti"
_IMAGES_FNAME = "data_object_image_2.zip"
_LABELS_FNAME = "data_object_label_2.zip"
_DEVKIT_FNAME = "devkit_object.zip"
_OBJECT_LABELS = [
    "Car",
    "Van",
    "Truck",
    "Pedestrian",
    "Person_sitting",
    "Cyclist",
    "Tram",
    "Misc",
]
# The percentage of trainset videos to put into validation and test sets.
# The released test images do not have labels.
_VALIDATION_SPLIT_PERCENT_VIDEOS = 10
_TEST_SPLIT_PERCENT_VIDEOS = 10

# Raw Kitti representation of a bounding box. Coordinates are in pixels,
# measured from the top-left hand corner.
RawBoundingBox = collections.namedtuple("RawBoundingBox",
                                        ["top", "bottom", "left", "right"])

class Kitti(tfds.core.GeneratorBasedBuilder):
  """Kitti dataset."""

  VERSION = tfds.core.Version("3.1.0")
  SUPPORTED_VERSIONS = [
      tfds.core.Version(
          "2.0.0", "New split API (https://tensorflow.org/datasets/splits)"),
  ]

  def _info(self):
    # Annotation descriptions are in the object development kit.
    annotations = {
        "type": tfds.features.ClassLabel(names=_OBJECT_LABELS),
        "truncated": tfds.features.Tensor(shape=(), dtype=tf.float32),
        "occluded": tfds.features.ClassLabel(num_classes=4),
        "alpha": tfds.features.Tensor(shape=(), dtype=tf.float32),
        "bbox": tfds.features.BBoxFeature(),
        "dimensions": tfds.features.Tensor(shape=(3,), dtype=tf.float32),
        "location": tfds.features.Tensor(shape=(3,), dtype=tf.float32),
        "rotation_y": tfds.features.Tensor(shape=(), dtype=tf.float32),
    }
    return tfds.core.DatasetInfo(
        builder=self,
        description=_DESCRIPTION,
        features=tfds.features.FeaturesDict({
            "image": tfds.features.Image(),
            "image/file_name": tfds.features.Text(),  # E.g. "000001.png".
            "objects": tfds.features.Sequence(annotations),
        }),
        homepage=_HOMEPAGE_URL,
        citation=_CITATION,
    )

  def _split_generators(self, dl_manager):
    filenames = {
        "images": os.path.join(_DATA_URL, _IMAGES_FNAME),
        "annotations": os.path.join(_DATA_URL, _LABELS_FNAME),
        "devkit": os.path.join(_DATA_URL, _DEVKIT_FNAME),
    }
    files = dl_manager.download(filenames)
    train_images, validation_images, test_images = _build_splits(
        dl_manager.iter_archive(files["devkit"]))

    return [
        tfds.core.SplitGenerator(
            name=tfds.Split.TRAIN,
            gen_kwargs={
                "images": dl_manager.iter_archive(files["images"]),
                "annotations": dl_manager.iter_archive(files["annotations"]),
                "subdir": "training",
                "image_ids": train_images,
            }),
        tfds.core.SplitGenerator(
            name=tfds.Split.VALIDATION,
            gen_kwargs={
                "images": dl_manager.iter_archive(files["images"]),
                "annotations": dl_manager.iter_archive(files["annotations"]),
                "subdir": "training",
                "image_ids": validation_images,
            }),
        tfds.core.SplitGenerator(
            name=tfds.Split.TEST,
            gen_kwargs={
                "images": dl_manager.iter_archive(files["images"]),
                "annotations": dl_manager.iter_archive(files["annotations"]),
                "subdir": "training",
                "image_ids": test_images,
            }),
    ]

  def _generate_examples(self, images, annotations, subdir, image_ids):
    """Yields images and annotations.

    Args:
      images: object that iterates over the archive of images.
      annotations: object that iterates over the archive of annotations.
      subdir: subdirectory from which to extract images and annotations, e.g.
        training or testing.
      image_ids: file ids for images in this split.

    Yields:
      A tuple containing the example's key, and the example.
    """
    cv2 = tfds.core.lazy_imports.cv2

    all_annotations = dict()
    for fpath, fobj in annotations:
      prefix, ext = os.path.splitext(fpath)
      if ext != ".txt":
        continue
      if prefix.split("/")[0] != subdir:
        continue

      # Key is the datapoint id. E.g. training/label_2/label_000016 -> 16.
      all_annotations[int(prefix[-6:])] = _parse_kitti_annotations(fobj)

    for fpath, fobj in images:
      prefix, ext = os.path.splitext(fpath)
      if ext != ".png":
        continue
      if prefix.split("/")[0] != subdir:
        continue
      image_id = int(prefix[-6:])
      if image_id not in image_ids:
        continue
      annotations = all_annotations[image_id]
      img = cv2.imdecode(np.fromstring(fobj.read(), dtype=np.uint8),
                         cv2.IMREAD_COLOR)
      img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
      height, width, _ = img.shape
      for obj in annotations:
        obj["bbox"] = _build_bounding_box(obj["bbox_raw"], height, width)
        del obj["bbox_raw"]
      _, fname = os.path.split(fpath)
      record = {"image": img, "image/file_name": fname, "objects": annotations}
      yield fname, record

def _build_bounding_box(bbox, height, width):
  """Builds and returns TFDS bounding box.

  Args:
    bbox: RawBoundingBox, bounding box in Kitti coordinates (origin top left).
    height: Image height in pixels.
    width: Image width in pixels.

  Returns:
    A TFDS BBox (origin bottom left).
  """
  return tfds.features.BBox(
      ymin=(height - bbox.bottom) / height,
      ymax=(height - bbox.top) / height,
      xmin=bbox.left / width,
      xmax=bbox.right / width,
  )

def _parse_kitti_annotations(annotations_csv):
  """Loads and parses the Kitti object annotations.

  Args:
    annotations_csv: csv file containing annotations for a single image.

  Returns:
    A list of labelled bounding boxes. Each bounding box is stored as a
    dictionary of features.
  """
  annotations = []
  for line in annotations_csv:
    (obj_type, truncated, occluded, alpha, left, top, right, bottom, height,
     width, length, x, y, z,
     rotation_y) = list(csv.reader([line.decode()], delimiter=" "))[0]
    # DontCare objects lack annotations, so skip them.
    if obj_type == "DontCare":
      continue
    annotations.append({
        "type": obj_type,
        "truncated": float(truncated),
        "occluded": int(occluded),
        "alpha": float(alpha),
        "bbox_raw": RawBoundingBox(
            top=float(top),
            bottom=float(bottom),
            left=float(left),
            right=float(right)),
        "dimensions": [float(v) for v in [height, width, length]],
        "location": [float(v) for v in [x, y, z]],
        "rotation_y": float(rotation_y),
    })
  return annotations

def _build_splits(devkit):
  """Splits the train data into train/val/test by video.

  Ensures that images from the same video do not traverse the splits.

  Args:
    devkit: object that iterates over the devkit archive.

  Returns:
    train_images: File ids for the training set images.
    validation_images: File ids for the validation set images.
    test_images: File ids for the test set images.
  """
  mapping_line_ids = None
  mapping_lines = None
  for fpath, fobj in devkit:
    if fpath == "mapping/train_rand.txt":
      # Converts 1-based line index to 0-based line index.
      mapping_line_ids = [
          int(x.strip()) - 1 for x in fobj.read().decode("utf-8").split(",")
      ]
    if fpath == "mapping/train_mapping.txt":
      mapping_lines = fobj.readlines()
      mapping_lines = [x.decode("utf-8") for x in mapping_lines]

  assert mapping_line_ids
  assert mapping_lines

  video_to_image = collections.defaultdict(list)
  for image_id, mapping_lineid in enumerate(mapping_line_ids):
    line = mapping_lines[mapping_lineid]
    video_id = line.split(" ")[1]
    video_to_image[video_id].append(image_id)

  # Sets numpy random state.
  numpy_original_state = np.random.get_state()
  np.random.seed(seed=123)

  # Max 1 for testing.
  num_test_videos = max(1,
                        _TEST_SPLIT_PERCENT_VIDEOS * len(video_to_image) // 100)
  num_validation_videos = max(
      1,
      _VALIDATION_SPLIT_PERCENT_VIDEOS * len(video_to_image) // 100)
  test_videos = set(
      np.random.choice(
          sorted(list(video_to_image.keys())), num_test_videos, replace=False))
  validation_videos = set(
      np.random.choice(
          sorted(list(set(video_to_image.keys()) - set(test_videos))),
          num_validation_videos,
          replace=False))
  test_images = []
  validation_images = []
  train_images = []
  for k, v in video_to_image.items():
    if k in test_videos:
      test_images.extend(v)
    elif k in validation_videos:
      validation_images.extend(v)
    else:
      train_images.extend(v)

  # Resets numpy random state.
  np.random.set_state(numpy_original_state)
  return train_images, validation_images, test_images

max_stars_repo_name: bosecodes/datasets

max_issues_repo_name: bosecodes/datasets

max_forks_repo_name: bosecodes/datasets


text:
# -*- coding: utf-8 -*-
"""
Created on Fri Dec  7 20:19:51 2018

@author: Emanuele

Test backpropagation for a random net.
"""

import copy as cp
import numpy as np
import sys

sys.path.append('../cnn')

import activations as act
import convolution as conv
import nn as nn

# mimic activation of a 2 layer neural net, a convolutional layer followed by a
#  dense layer
def activation(x, w_conv, stride, act_conv, w_dense, b_dense, act_dense):

    # convolution
    tmp = conv.conv_1d(x, w_conv, stride)
    tmp = act.dict_activations[act_conv](tmp)
    # dense layer
    tmp = np.dot(tmp, w_dense) + bias_dense
    tmp = act.dict_activations[act_dense](tmp)

    return np.sum(tmp)

if __name__ == '__main__':

    net_blocks = {'n_inputs': 100,
                  'layers': [
                          {'type': 'conv', 'activation': 'relu', 'shape': (1, 25), 'stride': 5},
                          {'type': 'dense', 'activation': 'relu', 'shape': (None, 32)}
                          ]
                      }

    net = nn.NN(net_blocks)

    # initialize the parameters
    net.init_parameters(['uniform', -1., 1.])

    # calculate partial derivative for each layer
    input_ = np.random.rand(1, net.n_inputs)
    # activate the net for a random input
    net.activation(input_, accumulate=True)
    # calculate partial derivative for each layer
    net.derivative(None)
    target = cp.copy(net.output)
    # use as loss the 'none' function which returns the prediction itself
    net.backpropagation(target, 'none', 'ada', 1e-3)

    derivative_by_calc = np.concatenate([net.layers[0].delta_weights.flatten(),
                               net.layers[1].delta_weights.flatten(),
                               net.layers[1].delta_bias.flatten()
                               ]).flatten()

    # calculate derivative with the definition
    weights_conv = cp.copy(net.layers[0].weights)
    stride_conv = net.layers[0].stride
    act_conv = net.layers[0].act
    weights_dense = cp.copy(net.layers[1].weights)
    bias_dense = cp.copy(net.layers[1].bias)
    act_dense = net.layers[1].act

    derivative_by_def = list()
    epsilon = 1e-5

    # convolutional layer: weights
    for i in range(weights_conv.shape[1]):

        weights_conv[:,i] += 2*epsilon

        f_plus = activation(input_,
                            weights_conv, stride_conv, act_conv,
                            weights_dense, bias_dense, act_dense)

        weights_conv[:,i] -= 2*epsilon

        f_minus = activation(input_,
                            weights_conv, stride_conv, act_conv,
                            weights_dense, bias_dense, act_dense)

        weights_conv[:,i] += epsilon

        derivative_by_def.append((f_plus-f_minus)/(2*epsilon))

    # dense layer: weights
    for i in range(weights_dense.shape[0]):

        for j in range(weights_dense.shape[1]):

            weights_dense[i,j] += 2*epsilon

            f_plus = activation(input_,
                                weights_conv, stride_conv, act_conv,
                                weights_dense, bias_dense, act_dense)

            weights_dense[i,j] -= 2*epsilon

            f_minus = activation(input_,
                                weights_conv, stride_conv, act_conv,
                                weights_dense, bias_dense, act_dense)

            weights_dense[i,j] += epsilon

            derivative_by_def.append((f_plus-f_minus)/(2*epsilon))

    # dense layer: bias
    for i in range(bias_dense.shape[1]):

        bias_dense[:,i] += 2*epsilon

        f_plus = activation(input_,
                            weights_conv, stride_conv, act_conv,
                            weights_dense, bias_dense, act_dense)

        bias_dense[:,i]-= 2*epsilon

        f_minus = activation(input_,
                            weights_conv, stride_conv, act_conv,
                            weights_dense, bias_dense, act_dense)

        bias_dense[:,i] += epsilon

        derivative_by_def.append((f_plus-f_minus)/(2*epsilon))

    # evaluate error
    derivative_by_def = np.array(derivative_by_def).flatten()

    error = np.linalg.norm(derivative_by_calc-derivative_by_def)

    print("Error: ", error)

max_stars_repo_name: EmanueleLM/CNN

max_issues_repo_name: EmanueleLM/CNN

max_forks_repo_name: EmanueleLM/CNN


text:
import numpy as np
from chainer import Variable, FunctionSet
import chainer.functions as F

class CharRNN(FunctionSet):

    def __init__(self, n_vocab, n_units):
        super(CharRNN, self).__init__(
            embed = F.EmbedID(n_vocab, n_units),
            l1_x = F.Linear(n_units, 4*n_units),
            l1_h = F.Linear(n_units, 4*n_units),
            l2_h = F.Linear(n_units, 4*n_units),
            l2_x = F.Linear(n_units, 4*n_units),
            l3   = F.Linear(n_units, n_vocab),
        )
        for param in self.parameters:
            param[:] = np.random.uniform(-0.08, 0.08, param.shape)

    def forward_one_step(self, x_data, y_data, state, train=True, dropout_ratio=0.5):
        x = Variable(x_data, volatile=not train)
        t = Variable(y_data, volatile=not train)

        h0      = self.embed(x)
        h1_in   = self.l1_x(F.dropout(h0, ratio=dropout_ratio, train=train)) + self.l1_h(state['h1'])
        c1, h1  = F.lstm(state['c1'], h1_in)
        h2_in   = self.l2_x(F.dropout(h1, ratio=dropout_ratio, train=train)) + self.l2_h(state['h2'])
        c2, h2  = F.lstm(state['c2'], h2_in)
        y       = self.l3(F.dropout(h2, ratio=dropout_ratio, train=train))
        state   = {'c1': c1, 'h1': h1, 'c2': c2, 'h2': h2}

        return state, F.softmax_cross_entropy(y, t)

    def predict(self, x_data, state):
        x = Variable(x_data, volatile=True)

        h0      = self.embed(x)
        h1_in   = self.l1_x(h0) + self.l1_h(state['h1'])
        c1, h1  = F.lstm(state['c1'], h1_in)
        h2_in   = self.l2_x(h1) + self.l2_h(state['h2'])
        c2, h2  = F.lstm(state['c2'], h2_in)
        y       = self.l3(h2)
        state   = {'c1': c1, 'h1': h1, 'c2': c2, 'h2': h2}

        return state, F.softmax(y)

def make_initial_state(n_units, batchsize=50, train=True):
    return {name: Variable(np.zeros((batchsize, n_units), dtype=np.float32),
            volatile=not train)
            for name in ('c1', 'h1', 'c2', 'h2')}


text:
#------------------------------
"""
Class :py:class:`FWView` is a QGraphicsView QWidget with interactive scalable scene
=====================================================================================

FW stands for Full Window - no margins for rullers etc.

Usage ::

    # Test
    #-----
    import sys
    from psana.graphqt.FWView import *
    app = QApplication(sys.argv)
    w = FWView(None, rscene=QRectF(0, 0, 100, 100), origin='UL', scale_ctl='HV')
    w.show()
    app.exec_()

    # Constructor and destructor
    #---------------------------
    from psana.graphqt.FWView import FWView
    w = FWView(parent=None, rscene=QRectF(0, 0, 10, 10), origin='UL', scale_ctl='HV', show_mode=0)
    w.__del__() # on w.close()

    # Main methods
    #-------------
    w.set_rect_scene(rs, set_def=True)
    w.reset_original_size()
    w.connect_cursor_on_scene_pos_to(recip)
    w.connect_scene_rect_changed_to(recip)
    w.connect_mouse_press_event_to(recip)

    # Methods
    #---------
    w.set_origin(origin='UL')           # sets attributes for origin
    w.set_scale_control(scale_ctl='HV') # sets attribute _scale_ctl = 0/+1/+2
    sctrl = w.scale_control()
    w.set_style()
    w._set_signs_of_transform()
    w.set_background_style(bkgd="background-color:black; border: 0px solid green")
    w._set_scene_item_rect_zvalue(sc_zvalue=20)

    # Re-calls
    #---------
    w.update_my_scene()
    w.set_cursor_type_on_scene_rect(curs_hover=Qt.CrossCursor, curs_grab=Qt.SizeAllCursor)
    w.set_view(rs=None)

    # Methods for tests
    #------------------
    w.add_rect_to_scene_v1(rect, brush=QBrush(), pen=QPen(Qt.yellow, 4, Qt.DashLine))
    w.add_rect_to_scene(rect, brush=QBrush(), pen=QPen(Qt.yellow, 4, Qt.DashLine))
    w.add_test_items_to_scene(show_mode=0)

    # Call-back slots
    #----------------
    w.mouseReleaseEvent(e)
    # w.mouseDoubleCkickEvent(e)
    w.mousePressEvent(e)
    w.mouseMoveEvent(e)
    w.wheelEvent(e)
    w.enterEvent(e)
    w.leaveEvent(e)
    w.closeEvent(e)
    w.resizeEvent(e)
    # w.paintEvent(e)
    w.key_usage(self)
    w.keyPressEvent(e)

    w.connect_mouse_move_event_to(recip)
    w.disconnect_mouse_move_event_from(recip)
    w.test_mouse_move_event_reception(e)     # resieves signal mouse_move_event(QMouseEvent)

    w.on_timeout()
    w.emit_signal_if_scene_rect_changed()        # emits signal scene_rect_changed(QRectF)
    w.connect_scene_rect_changed_to(recip)
    w.disconnect_scene_rect_changed_from(recip)
    w.test_scene_rect_changed_reception(rs)

    w.connect_mouse_press_event_to(recip)
    w.disconnect_mouse_press_event_from(recip)
    w.test_mouse_press_event_reception(e)

    # w.connect_wheel_is_stopped_to(recip)
    # w.disconnect_wheel_is_stopped_from(recip)
    # w.test_wheel_is_stopped_reception()

See:
    - :class:`FWView`
    - :class:`FWViewImage`
    - :class:`FWViewAxis`
    - :class:`FWViewColorBar`
    - `lcls2 on github <https://github.com/slac-lcls/lcls2>`_.

This software was developed for the LCLS2 project.
If you use all or part of it, please give an appropriate acknowledgment.

Created on 2017-01-03 by Mikhail Dubrovin
Adopted for LCLS2 on 2018-02-16
"""
#------------------------------

import logging
logger = logging.getLogger(__name__)

from PyQt5.QtWidgets import QGraphicsView, QGraphicsScene, QApplication
from PyQt5.QtGui import QBrush, QPen, QCursor, QColor
from PyQt5.QtCore import Qt, pyqtSignal, QRectF, QPointF, QTimer

#from time import time, sleep
#t0_sec = time()
#logger.debug('time = %.6f' % (time() - t0_sec))

#------------------------------

class FWView(QGraphicsView) :
    #wheel_is_stopped = pyqtSignal()
    #view_is_closed = pyqtSignal()
    mouse_move_event   = pyqtSignal('QMouseEvent')
    mouse_press_event  = pyqtSignal('QMouseEvent')
    scene_rect_changed = pyqtSignal('QRectF')

    def __init__(self, parent=None, rscene=QRectF(0, 0, 10, 10),\
                 origin='UL', scale_ctl='HV',\
                 show_mode=0) :
        """
        Parameters

        - parent (QWidget) - parent object
        - rscene (QRectF) - default rect on scaene
        - origin (str) - location of origin U(T)=Upper(Top), L=Left, other letters are not used but mean Down(Bottom), Right
        - scale_ctl (str) - scale control at mouse move, scroll, H=Horizontal, V=Vertical
        - show_mode (int) - 0(def) - draw nothing, +1 - field of scene rect, +2 - origin
        """

        sc = QGraphicsScene()
        QGraphicsView.__init__(self, sc, parent)

        self._name = self.__class__.__name__

        self.rs = rscene         # default rect on scene, restored at reset_original_size
        self.rs_old = self.rs    # rs_old - is used to check if scene rect changed
        self.rs_item = None      # item on scene to change cursor style at howering

        self.pos_click = None    # QPoint at mousePress
        self._set_scene_item_rect_zvalue()
        self.set_origin(origin)           # sets self._origin_...
        self.set_scale_control(scale_ctl) # sets self._scale_ctl
        self._set_signs_of_transform()

        self.set_style()
        self.set_view()
        self.add_test_items_to_scene(show_mode)

        self.twheel_msec = 500 # timeout for wheelEvent
        self.timer = QTimer()
        self.timer.timeout.connect(self.on_timeout)

    def __del__(self) :
        pass

    def set_origin(self, origin='UL') :
        self._origin = origin
        key = origin.upper()

        self._origin_u = 'U' in key or 'T' in key
        self._origin_d = not self._origin_u

        self._origin_l = 'L' in key
        self._origin_r = not self._origin_l

        self._origin_ul = self._origin_u and self._origin_l
        self._origin_ur = self._origin_u and self._origin_r
        self._origin_dl = self._origin_d and self._origin_l
        self._origin_dr = self._origin_d and self._origin_r

    def set_scale_control(self, scale_ctl='HV') :
        """Sets scale control bit-word
           = 0 - x, y frozen scales
           + 1 - x is interactive
           + 2 - y is interactive
           bit value 0/1 frozen/interactive
        """
        self.str_scale_ctl = scale_ctl
        self._scale_ctl = 0
        if 'H' in scale_ctl : self._scale_ctl += 1
        if 'V' in scale_ctl : self._scale_ctl += 2

    def scale_control(self) :
        return self._scale_ctl

    def str_scale_control(self) :
        return self.str_scale_ctl

    def origin(self) :
        return self._origin

    def set_style(self) :
        self.setGeometry(20, 20, 600, 600)
        self.setWindowTitle("FWView")
        #self.setContentsMargins(0,0,0,0)
        #self.setWindowFlags(self.windowFlags() | Qt.FramelessWindowHint)
        #self.setAttribute(Qt.WA_TranslucentBackground)
        #self.setInteractive(True)
        #self.setFixedSize(600, 22)

        self.brudf = QBrush()
        self.brubx = QBrush(Qt.black, Qt.SolidPattern)
        self.pendf = QPen()
        self.pendf.setStyle(Qt.NoPen)
        self.penbx = QPen(Qt.black, 6, Qt.SolidLine)

        self.set_background_style()

        #self.setViewportUpdateMode(QGraphicsView.SmartViewportUpdate)

    def set_background_style(self, bkgd_color = QColor(50,5,50)) :
        """Sets self.bkgd (str) - background style"""
        # it seems like in pyqt5 setStyleSheet is not enough to re-paint bkgd at drug, zoom etc.
        #bkgd_style="background-color:black; border: 0px solid green",
        #self.setStyleSheet(bkgd_style)
        self.setBackgroundBrush(QBrush(bkgd_color, Qt.SolidPattern))

    def _set_signs_of_transform(self) :
        if not self._origin_ul :
            sx = 1 if self._origin_l else -1
            sy = 1 if self._origin_u else -1
            ts = self.transform().scale(sx, sy)
            self.setTransform(ts)

    def _set_scene_item_rect_zvalue(self, sc_zvalue=20) :
        """Sets self.sc_zvalue (float) - scene rect item z value for cursor behavir"""
        self.sc_zvalue = sc_zvalue

    def update_my_scene(self) :
        """Should be re-implemented, if necessary
        """
        self.set_cursor_type_on_scene_rect()

    def set_cursor_type_on_scene_rect(self, curs_hover=Qt.CrossCursor, curs_grab=Qt.SizeAllCursor) :
        """Optional method: sets current scene rect item self.rs_item for cursor type

        Parameters

        - curs_hover (QCursor) - cursor type at mouse hovering on scene rect
        - curs_grab  (QCursor) - cursor type at mouse click on scene rect
        """
        sc = self.scene()
        rs = sc.sceneRect()
        if self.rs_item is not None : sc.removeItem(self.rs_item)
        self.rs_item = self.add_rect_to_scene(rs, self.brudf, self.pendf)
        self.rs_item.setCursorHover(curs_hover)
        self.rs_item.setCursorGrab(curs_grab)
        self.rs_item.setZValue(self.sc_zvalue)

    def set_view(self, rs=None) :
        """Sets rs - rect on scene to view in viewport, by default self.rs is used
        """
        r = self.rs if rs is None else rs
        self.scene().setSceneRect(r)
        self.fitInView(r, Qt.IgnoreAspectRatio) # KeepAspectRatioByExpanding KeepAspectRatio
        self.update_my_scene()

    def set_rect_scene(self, rs, set_def=True) :
        """Sets new rect to view, set_def=True - updates default for reset.
        """
        if set_def :
            self.rs = rs
        self.set_view(rs)
        self.emit_signal_if_scene_rect_changed() # sends signal

    def reset_original_size(self) :
        """Alias with default parameters"""
        self.set_view()
        self.emit_signal_if_scene_rect_changed() # sends signal

    def mouseReleaseEvent(self, e):
        QApplication.restoreOverrideCursor()
        QGraphicsView.mouseReleaseEvent(self, e)
        #logger.debug('FWView.mouseReleaseEvent, at point: '+str(e.pos()))
        #self.pos_click = e.pos()
        self.pos_click = None
        ### self.set_cursor_type_on_scene_rect()

        self.emit_signal_if_scene_rect_changed()
        #self.update_my_scene()

#    def mouseDoubleCkickEvent(self, e):
#        QGraphicsView.mouseDoubleCkickEvent(self, e)
#        logger.debug('mouseDoubleCkickEvent')

    def mousePressEvent(self, e):
        #logger.debug('FWView.mousePressEvent, at point: ', e.pos() #e.globalX(), e.globalY())
        #QApplication.setOverrideCursor(QCursor(Qt.ClosedHandCursor)) #Qt.SizeAllCursor))# ClosedHandCursor
        #self.emit(QtCore.SIGNAL('mouse_press_event(QMouseEvent)'), e)

        logger.debug('FWView.mousePressEvent but=%d %s scene x=%.1f y=%.1f'%\
                     (e.button(), str(e.pos()), self.x(), self.y())) # self.__class__.__name__

        self.mouse_press_event.emit(e)

        QGraphicsView.mousePressEvent(self, e)
        self.pos_click = e.pos()
        self.rs_center = self.scene().sceneRect().center()
        self.invscalex = 1./self.transform().m11()
        self.invscaley = 1./self.transform().m22()
        #self.pos_click_sc = self.mapToScene(self.pos_click)

    def mouseMoveEvent(self, e):
        QGraphicsView.mouseMoveEvent(self, e)
        #logger.debug('FWView.mouseMoveEvent, at point: %s' % str(e.pos()))
        #self.emit(QtCore.SIGNAL('mouse_move_event(QMouseEvent)'), e)
        self.mouse_move_event.emit(e)

        if self._scale_ctl==0 : return
        if self.pos_click is None : return

        dp = e.pos() - self.pos_click
        dx = dp.x()*self.invscalex if self._scale_ctl & 1 else 0
        dy = dp.y()*self.invscaley if self._scale_ctl & 2 else 0
        dpsc = QPointF(dx, dy)

        sc = self.scene()
        rs = sc.sceneRect()
        rs.moveCenter(self.rs_center - dpsc)
        sc.setSceneRect(rs)

        #self.update_my_scene()

    def wheelEvent(self, e) :
        QGraphicsView.wheelEvent(self, e)

        if self._scale_ctl==0 : return

        #logger.debug('wheelEvent: ', e.angleDelta())
        f = 1 + 0.4 * (1 if e.angleDelta().y()>0 else -1)
        #logger.debug('Scale factor =', f)

        p = self.mapToScene(e.pos())
        px, py = p.x(), p.y()
        #logger.debug('wheel x,y = ', e.x(), e.y(), ' on scene x,y =', p.x(), p.y())

        sc = self.scene()
        rs = sc.sceneRect()
        x,y,w,h = rs.x(), rs.y(), rs.width(), rs.height()
        #logger.debug('Scene x,y,w,h:', x,y,w,h)

        # zoom relative to mouse position
        dxc = (f-1)*(px-x)
        dyc = (f-1)*(py-y)
        dx, sx = (dxc, f*w) if self._scale_ctl & 1 else (0, w)
        dy, sy = (dyc, f*h) if self._scale_ctl & 2 else (0, h)

        rs.setRect(x-dx, y-dy, sx, sy)

        self.set_view(rs)

        self.timer.start(self.twheel_msec)

    def on_timeout(self) :
        """Is activated by timer when wheel is stopped and interval is expired.
        """
        #logger.debug('on_timeout')
        self.timer.stop()
        self.emit_signal_if_scene_rect_changed()
        #self.emit(QtCore.SIGNAL('wheel_is_stopped()'))

#------------------------------

#    def connect_wheel_is_stopped_to(self, recip) :
#        self.connect(self, QtCore.SIGNAL('wheel_is_stopped()'), recip)

#    def disconnect_wheel_is_stopped_from(self, recip) :
#        self.disconnect(self, QtCore.SIGNAL('wheel_is_stopped()'), recip)

#    def test_wheel_is_stopped_reception(self) :
#        logger.debug('GUView.test_wheel_is_stopped_reception')

#------------------------------

    def emit_signal_if_scene_rect_changed(self):
        """Checks if scene rect have changed and submits signal with new rect.
        """
        rs = self.scene().sceneRect()
        #logger.debug('XXX FWView.emit_signal_if_scene_rect_changed old, new:', self.rs_old, rs)

        #if not equal_rects(rs, self.rs_old) :
        if rs != self.rs_old :
            self.rs_old = rs
            #self.emit(QtCore.SIGNAL('scene_rect_changed(QRectF)'), rs)
            self.scene_rect_changed.emit(rs)

#            self.scale_drag_points()

#    def scale_drag_points(self):
#        sx, sy = self.transform().m11(), self.transform().m22()
#        print('scalex=%.3f, scaley=%.3f' %(sx, sy))
#        from psana.graphqt.DragPoint import DragPoint
#        for item in self.scene().items() :
#            if isinstance(item, DragPoint) :
#                 item.setScale(item.sx0/sx)

#------------------------------

    def connect_scene_rect_changed_to(self, recip) :
        #self.connect(self, QtCore.SIGNAL('scene_rect_changed(QRectF)'), recip)
        #self.scene_rect_changed('QRectF').connect(recip)
        self.scene_rect_changed.connect(recip)

    def disconnect_scene_rect_changed_from(self, recip) :
        self.scene_rect_changed.disconnect(recip)

    def test_scene_rect_changed_reception(self, rs) :
        from psana.graphqt.QWUtils import print_rect
        #logger.debug('GUView.test_scene_rect_changed_reception:', rs)
        print_rect(rs, cmt='FWView.test_scene_rect_changed_reception')

#------------------------------

    def enterEvent(self, e) :
    #    logger.debug('enterEvent')
        QGraphicsView.enterEvent(self, e)
        #QApplication.setOverrideCursor(QCursor(Qt.CrossCursor))

    def leaveEvent(self, e) :
    #    logger.debug('leaveEvent')
        QGraphicsView.leaveEvent(self, e)
        #QApplication.restoreOverrideCursor()

    def closeEvent(self, e) :
        #logger.debug('XXX FWView.closeEvent')
        sc = self.scene()
        for item in sc.items() :
            #logger.debug('XXX removeItem:', item)
            sc.removeItem(item)
        QGraphicsView.closeEvent(self, e)
        #logger.debug('XXX closeEvent is over')

    #def moveEvent(self, e) :
    #    logger.debug('moveEvent')
    #    logger.debug('Geometry rect:', self.geometry())

    def resizeEvent(self, e) :
        QGraphicsView.resizeEvent(self, e)
        rs = self.scene().sceneRect()
        self.set_view(rs)

    #def paintEvent(self, e):
    #    logger.debug('paintEvent')
    #    QGraphicsView.paintEvent(self, e)

    def key_usage(self) :
        return 'Keys:'\
               '\n  ESC - exit'\
               '\n  R - reset original size'\
               '\n  W - change scene rect, do not change default'\
               '\n  D - change scene rect and its default'\
               '\n'

    def keyPressEvent(self, e) :
        #logger.debug('keyPressEvent, key=', e.key())
        if   e.key() == Qt.Key_Escape :
            self.close()

        elif e.key() == Qt.Key_R :
            print('Reset original size')
            self.reset_original_size()

        elif e.key() in (Qt.Key_W, Qt.Key_D)  :
            change_def = e.key()==Qt.Key_D
            print('%s: change scene rect %s' % (self._name, 'set new default' if change_def else ''))
            v = ag.random_standard((4,), mu=0, sigma=20, dtype=np.int)
            rs = QRectF(v[0], v[1], v[2]+100, v[3]+100)
            print('Set scene rect: %s' % str(rs))
            self.set_rect_scene(rs, set_def=change_def)

        else :
            print(self.key_usage())

#-----------------------------

    def add_rect_to_scene_v1(self, rect, brush=QBrush(), pen=QPen(Qt.yellow, 4, Qt.DashLine)) :
        """Adds rect to scene, returns QGraphicsRectItem"""
        pen.setCosmetic(True)
        return self.scene().addRect(rect, pen, brush)

    def add_rect_to_scene(self, rect, brush=QBrush(), pen=QPen(Qt.yellow, 4, Qt.DashLine)) :
        """Adds rect to scene, returns QWGraphicsRectItem - for interactive stuff"""

        #logger.debug('XXX:QWGraphicsRectItem TBD')

        from psana.graphqt.QWGraphicsRectItem import QWGraphicsRectItem
        pen.setCosmetic(True)
        item = QWGraphicsRectItem(rect, parent=None, scene=self.scene())
        item.setPen(pen)
        item.setBrush(brush)
        return item
        return None

    def add_test_items_to_scene(self, show_mode=0) :
        colfld = Qt.magenta
        colori = Qt.red
        if show_mode & 1 :
            self.rsi = self.add_rect_to_scene_v1(self.rs, pen=QPen(Qt.NoPen), brush=QBrush(colfld))
        if show_mode & 2 :
            ror=QRectF(-1, -1, 2, 2)
            self.rori = self.add_rect_to_scene_v1(ror, pen=QPen(colori, 0, Qt.SolidLine), brush=QBrush(colori))

#------------------------------

    def connect_mouse_press_event_to(self, recip) :
        self.mouse_press_event.connect(recip)

    def disconnect_mouse_press_event_from(self, recip) :
        self.mouse_press_event.disconnect(recip)

    def test_mouse_press_event_reception(self, e) :
        print('FWViewImage.mouse_press_event, QMouseEvent point: x=%d y=%d' % (e.x(), e.y()))

#-----------------------------

    def test_mouse_move_event_reception(self, e) :
        #logger.debug('mouseMoveEvent, current point: ', e.x(), e.y(), ' on scene: %.1f  %.1f' % (p.x(), p.y()))
        p = self.mapToScene(e.pos())
        self.setWindowTitle('FWView: x=%.1f y=%.1f %s' % (p.x(), p.y(), 25*' '))

    def connect_mouse_move_event_to(self, recip) :
        #self.connect(self, QtCore.SIGNAL('mouse_move_event(QMouseEvent)'), recip)
        self.mouse_move_event['QMouseEvent'].connect(recip)

    def disconnect_mouse_move_event_from(self, recip) :
        #self.disconnect(self, QtCore.SIGNAL('mouse_move_event(QMouseEvent)'), recip)
        self.mouse_move_event['QMouseEvent'].disconnect(recip)

#-----------------------------

    #def drawBackground(self, painter, rect):
        #painter.fillRect(rect, QBrush(QColor(0,0,0), Qt.SolidPattern))

#-----------------------------
#-----------------------------
#-----------------------------
if __name__ == "__main__" :
  def test_fwview(tname) :
    print('%s:' % sys._getframe().f_code.co_name)
    b="background-color:yellow; border: 0px solid green"
    app = QApplication(sys.argv)
    w = None
    if   tname == '0': w=FWView(None, rscene=QRectF(0, 0, 100, 100), origin='DL', show_mode=3, scale_ctl='HV')
    elif tname == '1': w=FWView(None, rscene=QRectF(0, 0, 100, 100), origin='UL', show_mode=3, scale_ctl='HV')
    elif tname == '2': w=FWView(None, rscene=QRectF(0, 0, 100, 100), origin='UR', show_mode=3, scale_ctl='HV')
    elif tname == '3': w=FWView(None, rscene=QRectF(0, 0, 100, 100), origin='DR', show_mode=3, scale_ctl='HV')
    elif tname == '4': w=FWView(None, rscene=QRectF(0, 0, 100, 100), origin='DL', show_mode=3, scale_ctl='')
    elif tname == '5': w=FWView(None, rscene=QRectF(0, 0, 100, 100), origin='DL', show_mode=3, scale_ctl='H')
    elif tname == '6': w=FWView(None, rscene=QRectF(0, 0, 100, 100), origin='DL', show_mode=3, scale_ctl='V')
    elif tname == '7': w=FWView(None, rscene=QRectF(0, 0, 100, 100), origin='DL', show_mode=1, scale_ctl='HV')
    elif tname == '8': w=FWView(None, rscene=QRectF(0, 0, 100, 100), origin='DL', show_mode=3, scale_ctl='HV')
    else :
        print('test %s is not implemented' % tname)
        return

    w.connect_mouse_move_event_to(w.test_mouse_move_event_reception)
    w.connect_scene_rect_changed_to(w.test_scene_rect_changed_reception)
    w.connect_mouse_press_event_to(w.test_mouse_press_event_reception)

    w.show()

    #w.disconnect_mouse_move_event_from(w.test_mouse_move_event_reception)
    #w.disconnect_scene_rect_changed_from(w.test_scene_rect_changed_reception)
    #w.disconnect_mouse_press_event_from(w.test_mouse_press_event_reception)
    #w.close()

    app.exec_()
    app.quit()

    del w
    del app

#------------------------------

if __name__ == "__main__" :
    import sys; global sys
    import numpy as np; global np
    import psana.pyalgos.generic.NDArrGenerators as ag; global ag

    tname = sys.argv[1] if len(sys.argv) > 1 else '0'
    print(50*'_', '\nTest %s' % tname)
    test_fwview(tname)
    sys.exit('End of Test %s' % tname)

#------------------------------


text:
# -*- coding: utf-8 -*-
"""Demo156_Outliers_Detection_LocalOutlierFactor.ipynb

## Outliers

An outlier is an observation that lies outside the overall pattern of a distribution __[Moore and McCabe, 1999]__.

- Outliers can either be treated special completely ignored

- E.g., Fraudulant transactions are outliers, but since we want to avoid them, they must be paid special attention

- If we think that the outliers are errors, we should remove them

## Which of the ML models care about Outliers?

Affected models:

- AdaBoost
- Linear models
- Linear regression
- Neural Networks (if the number is high)
- Logistic regression
- KMeans
- Heirarchical Clustering
- PCA

Unaffected models:

- Decision trees
- Naive bayes
- SVMs
- Random forest
- Gradient boosted trees
- K-Nearest Neighbors

### Identification

- Extreme Value Analysis
  - IQR = 75th quantile - 25th quantile
  - Upper boundary = 75th quantile + (IQR * 1.5)
  - Lower boundary = 25th quantile - (IQR * 1.5)
  - Upper Extreme boundary = 75th quantile + (IQR * 3)
  - Lower Extreme boundary = 25th quantile - (IQR * 3)
"""

import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np

from google.colab import drive
drive.mount('/content/gdrive')
data = pd.read_csv("gdrive/My Drive/Colab Notebooks/FeatureEngineering/train_date.csv")

data.head()

# Outliers according to the quantiles + 1.5 IQR
sns.catplot(x="Survived", y="Fare", kind="box", data=data)
sns.despine(left=False, right=False, top=False)

data['Fare'].describe()

# Get outliers
IQR = data['Fare'].quantile(0.75) - data['Fare'].quantile(0.25)
ub = data['Fare'].quantile(0.75) + (IQR * 3)
lb = data['Fare'].quantile(0.25) - (IQR * 3)
data[(data['Fare']>ub) | (data['Fare']<lb)].groupby('Survived')['Fare'].count()

# First let's plot the histogram to get an idea of the distribution
fig = data.Age.hist(bins=50, color='green')
fig.set_title('Distribution')
fig.set_xlabel('X')
fig.set_ylabel('#')

# Get outliers
IQR = data['Age'].quantile(0.75) - data['Age'].quantile(0.25)
ub = data['Age'].mean() + data['Age'].std()
lb = data['Age'].mean() - data['Age'].std()
data[(data['Age']>ub) | (data['Age']<lb)].groupby('Survived')['Age'].count()

from sklearn.metrics import roc_auc_score
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score

data = data.drop(['Date'], axis=1)

data.columns

data[['Fare','Age']].isnull().mean()

X_train, X_test, y_train, y_test = train_test_split(data[['Age', 'Fare']].fillna(0),
                                                    data['Survived'],
                                                    test_size=0.2)
X_train.shape, X_test.shape

# We will cap the values of outliers
data_processed = data.copy()

_temp = np.ceil(data['Age'].mean() + data['Age'].std())
data_processed.loc[data_processed.Age >= _temp, 'Age'] = _temp

IQR = data['Fare'].quantile(0.75) - data['Fare'].quantile(0.25)
_temp = np.ceil(data['Fare'].quantile(0.75) + (IQR * 3))
data_processed.loc[data_processed.Fare > _temp, 'Fare'] = _temp

X_train_processed, X_test_processed, y_train_processed, y_test_processed = train_test_split(
    data_processed[['Age', 'Fare']].fillna(0),
    data_processed['Survived'],
    test_size=0.2)

from sklearn.neighbors import LocalOutlierFactor

df_outliers = data.copy()
df_outliers = df_outliers.fillna(0)

column_name = 'Fare'
obj = LocalOutlierFactor()
_temp = obj.fit_predict(df_outliers[[column_name]])
print(np.unique(_temp, return_counts=True))
central = df_outliers[_temp==1][column_name].mean()
max_val = df_outliers[_temp==1][column_name].max()
min_val = df_outliers[_temp==1][column_name].min()
df_outliers.loc[_temp==-1,[column_name]] = df_outliers.loc[_temp==-1,[column_name]].apply(lambda x: [max_val if y > central else y for y in x])
df_outliers.loc[_temp==-1,[column_name]] = df_outliers.loc[_temp==-1,[column_name]].apply(lambda x: [min_val if y < central else y for y in x])
print(data.shape)
print(df_outliers.shape)

column_name = 'Age'
obj = LocalOutlierFactor()
_temp = obj.fit_predict(df_outliers[[column_name]])
print(np.unique(_temp, return_counts=True))
central = df_outliers[_temp==1][column_name].mean()
max_val = df_outliers[_temp==1][column_name].max()
min_val = df_outliers[_temp==1][column_name].min()
df_outliers.loc[_temp==-1,[column_name]] = df_outliers.loc[_temp==-1,[column_name]].apply(lambda x: [max_val if y > central else y for y in x])
df_outliers.loc[_temp==-1,[column_name]] = df_outliers.loc[_temp==-1,[column_name]].apply(lambda x: [min_val if y < central else y for y in x])
print(data.shape)
print(df_outliers.shape)

X_train_outliers, X_test_outliers, y_train_outliers, y_test_outliers = train_test_split(
    df_outliers[['Age', 'Fare']],
    df_outliers['Survived'],
    test_size=0.2)

print(X_train.shape)
print(X_train_processed.shape)
print(X_train_outliers.shape)
print(X_test.shape)
print(X_test_processed.shape)
print(X_test_outliers.shape)

from sklearn.linear_model import LogisticRegression
classifier = LogisticRegression()
classifier.fit(X_train,y_train)
y_pred = classifier.predict(X_test)
y_pred = np.round(y_pred).flatten()
print(accuracy_score(y_test, y_pred))

classifier.fit(X_train_processed,y_train_processed)
y_pred_processed = classifier.predict(X_test_processed)
y_pred_processed = np.round(y_pred_processed).flatten()
print(accuracy_score(y_test_processed, y_pred_processed))

classifier.fit(X_train_outliers,y_train_outliers)
y_pred_outliers = classifier.predict(X_test_outliers)
y_pred_outliers = np.round(y_pred_outliers).flatten()
print(accuracy_score(y_test_outliers, y_pred_outliers))

from sklearn.linear_model import RidgeClassifierCV
classifier = RidgeClassifierCV()
classifier.fit(X_train,y_train)
y_pred = classifier.predict(X_test)
y_pred = np.round(y_pred).flatten()
print(accuracy_score(y_test, y_pred))

classifier.fit(X_train_processed,y_train_processed)
y_pred_processed = classifier.predict(X_test_processed)
y_pred_processed = np.round(y_pred_processed).flatten()
print(accuracy_score(y_test_processed, y_pred_processed))

classifier.fit(X_train_outliers,y_train_outliers)
y_pred_outliers = classifier.predict(X_test_outliers)
y_pred_outliers = np.round(y_pred_outliers).flatten()
print(accuracy_score(y_test_outliers, y_pred_outliers))

from sklearn.linear_model import RidgeClassifierCV
classifier = RidgeClassifierCV()
classifier.fit(X_train,y_train)
y_pred = classifier.predict(X_test)
y_pred = np.round(y_pred).flatten()
print(accuracy_score(y_test, y_pred))

classifier.fit(X_train_processed,y_train_processed)
y_pred_processed = classifier.predict(X_test_processed)
y_pred_processed = np.round(y_pred_processed).flatten()
print(accuracy_score(y_test_processed, y_pred_processed))

classifier.fit(X_train_outliers,y_train_outliers)
y_pred_outliers = classifier.predict(X_test_outliers)
y_pred_outliers = np.round(y_pred_outliers).flatten()
print(accuracy_score(y_test_outliers, y_pred_outliers))

from sklearn.svm import SVC
classifier = SVC()
classifier.fit(X_train,y_train)
y_pred = classifier.predict(X_test)
y_pred = np.round(y_pred).flatten()
print(accuracy_score(y_test, y_pred))

classifier.fit(X_train_processed,y_train_processed)
y_pred_processed = classifier.predict(X_test_processed)
y_pred_processed = np.round(y_pred_processed).flatten()
print(accuracy_score(y_test_processed, y_pred_processed))

classifier.fit(X_train_outliers,y_train_outliers)
y_pred_outliers = classifier.predict(X_test_outliers)
y_pred_outliers = np.round(y_pred_outliers).flatten()
print(accuracy_score(y_test_outliers, y_pred_outliers))

from sklearn.neural_network import MLPClassifier
classifier = MLPClassifier()
classifier.fit(X_train,y_train)
y_pred = classifier.predict(X_test)
y_pred = np.round(y_pred).flatten()
print(accuracy_score(y_test, y_pred))

classifier.fit(X_train_processed,y_train_processed)
y_pred_processed = classifier.predict(X_test_processed)
y_pred_processed = np.round(y_pred_processed).flatten()
print(accuracy_score(y_test_processed, y_pred_processed))

classifier.fit(X_train_outliers,y_train_outliers)
y_pred_outliers = classifier.predict(X_test_outliers)
y_pred_outliers = np.round(y_pred_outliers).flatten()
print(accuracy_score(y_test_outliers, y_pred_outliers))

from sklearn.svm import LinearSVC
classifier = LinearSVC()
classifier.fit(X_train,y_train)
y_pred = classifier.predict(X_test)
y_pred = np.round(y_pred).flatten()
print(accuracy_score(y_test, y_pred))

classifier.fit(X_train_processed,y_train_processed)
y_pred_processed = classifier.predict(X_test_processed)
y_pred_processed = np.round(y_pred_processed).flatten()
print(accuracy_score(y_test_processed, y_pred_processed))

classifier.fit(X_train_outliers,y_train_outliers)
y_pred_outliers = classifier.predict(X_test_outliers)
y_pred_outliers = np.round(y_pred_outliers).flatten()
print(accuracy_score(y_test_outliers, y_pred_outliers))

from sklearn.ensemble import RandomForestClassifier
classifier = RandomForestClassifier()
classifier.fit(X_train,y_train)
y_pred = classifier.predict(X_test)
y_pred = np.round(y_pred).flatten()
print(accuracy_score(y_test, y_pred))

classifier.fit(X_train_processed,y_train_processed)
y_pred_processed = classifier.predict(X_test_processed)
y_pred_processed = np.round(y_pred_processed).flatten()
print(accuracy_score(y_test_processed, y_pred_processed))

classifier.fit(X_train_outliers,y_train_outliers)
y_pred_outliers = classifier.predict(X_test_outliers)
y_pred_outliers = np.round(y_pred_outliers).flatten()
print(accuracy_score(y_test_outliers, y_pred_outliers))

from sklearn.tree import DecisionTreeClassifier
classifier = DecisionTreeClassifier()
classifier.fit(X_train,y_train)
y_pred = classifier.predict(X_test)
y_pred = np.round(y_pred).flatten()
print(accuracy_score(y_test, y_pred))

classifier.fit(X_train_processed,y_train_processed)
y_pred_processed = classifier.predict(X_test_processed)
y_pred_processed = np.round(y_pred_processed).flatten()
print(accuracy_score(y_test_processed, y_pred_processed))

classifier.fit(X_train_outliers,y_train_outliers)
y_pred_outliers = classifier.predict(X_test_outliers)
y_pred_outliers = np.round(y_pred_outliers).flatten()
print(accuracy_score(y_test_outliers, y_pred_outliers))

from sklearn.ensemble import GradientBoostingClassifier
classifier = GradientBoostingClassifier()
classifier.fit(X_train,y_train)
y_pred = classifier.predict(X_test)
y_pred = np.round(y_pred).flatten()
print(accuracy_score(y_test, y_pred))

classifier.fit(X_train_processed,y_train_processed)
y_pred_processed = classifier.predict(X_test_processed)
y_pred_processed = np.round(y_pred_processed).flatten()
print(accuracy_score(y_test_processed, y_pred_processed))

classifier.fit(X_train_outliers,y_train_outliers)
y_pred_outliers = classifier.predict(X_test_outliers)
y_pred_outliers = np.round(y_pred_outliers).flatten()
print(accuracy_score(y_test_outliers, y_pred_outliers))

from sklearn.linear_model import SGDClassifier
classifier = SGDClassifier()
classifier.fit(X_train,y_train)
y_pred = classifier.predict(X_test)
y_pred = np.round(y_pred).flatten()
print(accuracy_score(y_test, y_pred))

classifier.fit(X_train_processed,y_train_processed)
y_pred_processed = classifier.predict(X_test_processed)
y_pred_processed = np.round(y_pred_processed).flatten()
print(accuracy_score(y_test_processed, y_pred_processed))

classifier.fit(X_train_outliers,y_train_outliers)
y_pred_outliers = classifier.predict(X_test_outliers)
y_pred_outliers = np.round(y_pred_outliers).flatten()
print(accuracy_score(y_test_outliers, y_pred_outliers))

from sklearn.linear_model import Perceptron
classifier = Perceptron()
classifier.fit(X_train,y_train)
y_pred = classifier.predict(X_test)
y_pred = np.round(y_pred).flatten()
print(accuracy_score(y_test, y_pred))

classifier.fit(X_train_processed,y_train_processed)
y_pred_processed = classifier.predict(X_test_processed)
y_pred_processed = np.round(y_pred_processed).flatten()
print(accuracy_score(y_test_processed, y_pred_processed))

classifier.fit(X_train_outliers,y_train_outliers)
y_pred_outliers = classifier.predict(X_test_outliers)
y_pred_outliers = np.round(y_pred_outliers).flatten()
print(accuracy_score(y_test_outliers, y_pred_outliers))

from sklearn.naive_bayes import GaussianNB
classifier = GaussianNB()
classifier.fit(X_train,y_train)
y_pred = classifier.predict(X_test)
y_pred = np.round(y_pred).flatten()
print(accuracy_score(y_test, y_pred))

classifier.fit(X_train_processed,y_train_processed)
y_pred_processed = classifier.predict(X_test_processed)
y_pred_processed = np.round(y_pred_processed).flatten()
print(accuracy_score(y_test_processed, y_pred_processed))

classifier.fit(X_train_outliers,y_train_outliers)
y_pred_outliers = classifier.predict(X_test_outliers)
y_pred_outliers = np.round(y_pred_outliers).flatten()
print(accuracy_score(y_test_outliers, y_pred_outliers))

from sklearn.neighbors import KNeighborsClassifier
classifier = KNeighborsClassifier()
classifier.fit(X_train,y_train)
y_pred = classifier.predict(X_test)
y_pred = np.round(y_pred).flatten()
print(accuracy_score(y_test, y_pred))

classifier.fit(X_train_processed,y_train_processed)
y_pred_processed = classifier.predict(X_test_processed)
y_pred_processed = np.round(y_pred_processed).flatten()
print(accuracy_score(y_test_processed, y_pred_processed))

classifier.fit(X_train_outliers,y_train_outliers)
y_pred_outliers = classifier.predict(X_test_outliers)
y_pred_outliers = np.round(y_pred_outliers).flatten()
print(accuracy_score(y_test_outliers, y_pred_outliers))


text:
# Copyright (c) 2020 NVIDIA CORPORATION.
# Copyright (c) 2018-2020 Chris Choy (chrischoy@ai.stanford.edu).
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
# Please cite "4D Spatio-Temporal ConvNets: Minkowski Convolutional Neural
# Networks", CVPR'19 (https://arxiv.org/abs/1904.08755) if you use any part
# of the code.
import unittest
import numpy as np
import torch

from MinkowskiEngine import (
    SparseTensor,
    SparseTensorOperationMode,
    SparseTensorQuantizationMode,
    set_sparse_tensor_operation_mode,
    clear_global_coordinate_manager,
    is_cuda_available,
)

from MinkowskiEngine.utils import batched_coordinates, sparse_quantize, sparse_collate
from tests.python.common import data_loader, load_file

class SparseTensorTestCase(unittest.TestCase):
    def test(self):
        print(f"{self.__class__.__name__}: test SparseTensor")
        coords, feats, labels = data_loader(nchannel=2)
        input = SparseTensor(feats, coordinates=coords)
        print(input)

    def test_empty(self):
        print(f"{self.__class__.__name__}: test_empty SparseTensor")
        feats = torch.FloatTensor(0, 16)
        coords = torch.IntTensor(0, 4)
        input = SparseTensor(feats, coordinates=coords)
        print(input)

    def test_tensor_stride(self):
        print(f"{self.__class__.__name__}: test_tensor_stride SparseTensor")
        feats = torch.FloatTensor(4, 16)
        coords = torch.IntTensor(
            [[0, 4, 2, 1], [0, 4, 0, 0], [0, 4, 4, 4], [0, 4, 4, 7]]
        )
        print(coords)
        input = SparseTensor(feats, coordinates=coords, tensor_stride=4)
        self.assertEqual(input.tensor_stride, [4, 4, 4])
        print(input)

    def test_force_creation(self):
        print(f"{self.__class__.__name__}: test_force_creation")
        coords, feats, labels = data_loader(nchannel=2)
        input1 = SparseTensor(feats, coordinates=coords)
        input2 = SparseTensor(
            feats, coordinates=coords, coordinate_manager=input1.coordinate_manager
        )
        print(input1.coordinate_map_key, input2.coordinate_map_key)

    def test_device(self):
        print(f"{self.__class__.__name__}: test_device SparseTensor")
        if not is_cuda_available():
            return

        coords = torch.IntTensor(
            [[0, 1], [0, 1], [0, 2], [0, 2], [1, 0], [1, 0], [1, 1]]
        )
        feats = torch.FloatTensor([[0, 1, 2, 3, 5, 6, 7]]).T

        SparseTensor(feats.to(0), coords.to(0))
        feats = torch.FloatTensor([[0, 1, 2, 3, 5, 6, 7]]).T.to(0)
        st = SparseTensor(feats, coords, device=feats.device)
        print(st)

    def test_device_unique(self):
        print(f"{self.__class__.__name__}: test_device_unique SparseTensor")
        if not is_cuda_available():
            return

        coords = torch.IntTensor(
            [[0, 1], [0, 2], [0, 3], [0, 4], [1, 0], [1, 1], [1, 2]]
        )
        feats = torch.FloatTensor([[0, 1, 2, 3, 5, 6, 7]]).T
        SparseTensor(feats.to(0), coords.to(0))
        feats = torch.FloatTensor([[0, 1, 2, 3, 5, 6, 7]]).T.to(0)
        st = SparseTensor(feats, coords, device=feats.device)
        print(st)

    def test_device2(self):
        print(f"{self.__class__.__name__}: test_device2 SparseTensor")
        if not is_cuda_available():
            return

        coordinates = np.random.rand(8192,3) * 200
        quant_coordinates, quant_features = sparse_quantize(coordinates, coordinates)
        bcoords, bfeats = sparse_collate([quant_coordinates], [quant_features])
        bcoords, bfeats = bcoords.cuda(), bfeats.cuda()
        print(bcoords, bfeats)
        SparseTensor(bfeats, bcoords)

    def test_quantization(self):
        print(f"{self.__class__.__name__}: test_quantization")
        coords, feats, labels = data_loader(nchannel=2)
        # create duplicate coords
        coords[0] = coords[1]
        coords[2] = coords[3]
        input = SparseTensor(feats, coordinates=coords)
        self.assertTrue(len(input) == len(coords) - 2)
        input = SparseTensor(
            feats,
            coordinates=coords,
            quantization_mode=SparseTensorQuantizationMode.UNWEIGHTED_AVERAGE,
        )
        self.assertTrue(len(coords) == 16)
        self.assertTrue(len(input) == 14)

        # 1D
        coords = torch.IntTensor(
            [[0, 1], [0, 1], [0, 2], [0, 2], [1, 0], [1, 0], [1, 1]]
        )
        feats = torch.FloatTensor([[0, 1, 2, 3, 5, 6, 7]]).T
        # 0.5, 2.5, 5.5, 7
        sinput = SparseTensor(
            coordinates=coords,
            features=feats,
            quantization_mode=SparseTensorQuantizationMode.UNWEIGHTED_AVERAGE,
        )
        self.assertTrue(len(sinput) == 4)
        self.assertTrue(0.5 in sinput.features)
        self.assertTrue(2.5 in sinput.features)
        self.assertTrue(5.5 in sinput.features)
        self.assertTrue(7 in sinput.features)
        self.assertTrue(len(sinput.slice(sinput)) == len(coords))

    def test_quantization_gpu(self):
        print(f"{self.__class__.__name__}: test_quantization_gpu")
        coords, feats, labels = data_loader(nchannel=2)
        # create duplicate coords
        coords[0] = coords[1]
        coords[2] = coords[3]
        input = SparseTensor(feats, coordinates=coords)
        self.assertTrue(len(input) == len(coords) - 2)
        input = SparseTensor(
            feats,
            coordinates=coords,
            quantization_mode=SparseTensorQuantizationMode.UNWEIGHTED_AVERAGE,
            device="cuda",
        )
        self.assertTrue(len(coords) == 16)
        self.assertTrue(len(input) == 14)
        print(input)

        # 1D
        coords = torch.IntTensor(
            [[0, 1], [0, 1], [0, 2], [0, 2], [1, 0], [1, 0], [1, 1]]
        )
        feats = torch.FloatTensor([[0, 1, 2, 3, 5, 6, 7]]).T
        # 0.5, 2.5, 5.5, 7
        sinput = SparseTensor(
            coordinates=coords,
            features=feats,
            quantization_mode=SparseTensorQuantizationMode.UNWEIGHTED_AVERAGE,
            device="cuda",
        )
        print(sinput)
        self.assertTrue(len(sinput) == 4)
        self.assertTrue(0.5 in sinput.features)
        self.assertTrue(2.5 in sinput.features)
        self.assertTrue(5.5 in sinput.features)
        self.assertTrue(7 in sinput.features)
        self.assertTrue(len(sinput.slice(sinput)) == len(coords))

    def test_extraction(self):
        print(f"{self.__class__.__name__}: test_extraction")
        coords = torch.IntTensor([[0, 0], [0, 1], [0, 2], [2, 0], [2, 2]])
        feats = torch.FloatTensor([[1.1, 2.1, 3.1, 4.1, 5.1]]).t()
        X = SparseTensor(feats, coords)
        C0 = X.coordinates_at(0)
        F0 = X.features_at(0)
        self.assertTrue(0 in C0)
        self.assertTrue(1 in C0)
        self.assertTrue(2 in C0)

        self.assertTrue(1.1 in F0)
        self.assertTrue(2.1 in F0)
        self.assertTrue(3.1 in F0)

        CC0, FC0 = X.coordinates_and_features_at(0)
        self.assertTrue((C0 == CC0).all())
        self.assertTrue((F0 == FC0).all())

        coords, feats = X.decomposed_coordinates_and_features
        for c, f in zip(coords, feats):
            self.assertEqual(c.numel(), f.numel())
            print(c, f)
        self.assertEqual(len(coords[0]), 3)
        self.assertEqual(len(coords[1]), 0)
        self.assertEqual(len(coords[2]), 2)

        if not is_cuda_available():
            return

        coords = torch.IntTensor([[0, 0], [0, 1], [0, 2], [2, 0], [2, 2]])
        feats = torch.FloatTensor([[1.1, 2.1, 3.1, 4.1, 5.1]]).t()

        X = SparseTensor(feats, coords, device=0)
        coords, feats = X.decomposed_coordinates_and_features
        for c, f in zip(coords, feats):
            self.assertEqual(c.numel(), f.numel())
            print(c, f)

        self.assertEqual(len(coords[0]), 3)
        self.assertEqual(len(coords[1]), 0)
        self.assertEqual(len(coords[2]), 2)

    def test_features_at_coordinates(self):
        print(f"{self.__class__.__name__}: test_features_at_coordinates")
        coords = torch.IntTensor([[0, 0], [0, 1], [0, 2], [2, 0], [2, 2]])
        feats = torch.FloatTensor([[1.1, 2.1, 3.1, 4.1, 5.1]]).t()

        X = SparseTensor(features=feats, coordinates=coords)
        feats = X.features_at_coordinates(
            torch.FloatTensor([[0, 0], [0, 1], [0, 2], [2, 2], [0, 0], [0, 0.5]])
        ).flatten()

        self.assertTrue(feats[0] == 1.1)
        self.assertTrue(feats[3] == 5.1)
        self.assertTrue(feats[4] == 1.1)

    def test_decomposition(self):
        print(f"{self.__class__.__name__}: test_decomposition")
        coords, colors, pcd = load_file("1.ply")
        colors = torch.from_numpy(colors)
        for batch_size in [1, 5, 10, 20, 40]:
            for voxel_size in [0.02]:
                dcoords = torch.from_numpy(np.floor(coords / voxel_size)).int()
                bcoords = batched_coordinates([dcoords for i in range(batch_size)])
                feats = torch.cat([colors for b in range(batch_size)], 0)
                sinput = SparseTensor(feats, bcoords)
                (
                    decomposed_coords,
                    decomposed_feats,
                ) = sinput.decomposed_coordinates_and_features
                print([len(c) for c in decomposed_coords])
                print([len(f) for f in decomposed_feats])
                self.assertEqual(len(decomposed_coords), batch_size)
                self.assertEqual(len(decomposed_feats), batch_size)

    def test_decomposition_gpu(self):
        print(f"{self.__class__.__name__}: test_decomposition_gpu")
        if not torch.cuda.is_available():
            return

        coords, colors, pcd = load_file("1.ply")
        colors = torch.from_numpy(colors)

        for batch_size in [5, 10, 20, 40]:
            for voxel_size in [0.02]:
                dcoords = torch.from_numpy(np.floor(coords / voxel_size)).int()
                bcoords = batched_coordinates([dcoords for i in range(batch_size)])
                feats = torch.cat([colors for b in range(batch_size)], 0)
                sinput = SparseTensor(feats.to(0), bcoords.to(0))
                (
                    decomposed_coords,
                    decomposed_feats,
                ) = sinput.decomposed_coordinates_and_features
                print([len(c) for c in decomposed_coords])
                print([len(f) for f in decomposed_feats])
                self.assertEqual(len(decomposed_coords), batch_size)
                self.assertEqual(len(decomposed_feats), batch_size)

    def test_operation_mode(self):
        print(f"{self.__class__.__name__}: test_operation_mode")
        # Set to use the global sparse tensor coords manager by default
        set_sparse_tensor_operation_mode(
            SparseTensorOperationMode.SHARE_COORDINATE_MANAGER
        )

        coords, feats, labels = data_loader(nchannel=2)

        # Create a sparse tensor on two different coordinates.
        A = SparseTensor(torch.rand(feats.shape), coordinates=coords)
        B = SparseTensor(
            torch.rand(4, 2),
            coordinates=torch.IntTensor([[0, 0, 0], [1, 1, 1], [0, 1, 0], [1, 0, 1]]),
        )

        self.assertTrue(A.coordinate_manager == B.coordinate_manager)

        A.requires_grad_(True)
        B.requires_grad_(True)

        C = A + B

        C.F.sum().backward()

        self.assertTrue(torch.all(A.F.grad == 1).item())
        self.assertTrue(torch.all(B.F.grad == 1).item())

        C = A - B
        C = A * B
        C = A / B

        # Inplace
        A.requires_grad_(False)
        D = SparseTensor(
            torch.rand(feats.shape),
            coordinate_map_key=A.coordinate_map_key,
            coordinate_manager=A.coordinate_manager,
        )
        A -= D
        A *= D
        A /= D
        clear_global_coordinate_manager()
        set_sparse_tensor_operation_mode(
            SparseTensorOperationMode.SEPARATE_COORDINATE_MANAGER
        )

max_stars_repo_name: NNstorm/MinkowskiEngine

max_issues_repo_name: NNstorm/MinkowskiEngine

max_forks_repo_name: NNstorm/MinkowskiEngine


text:
# pylint: disable=missing-docstring
import numpy as np
import torch

import cleverhans.future.torch.utils as utils
from cleverhans.devtools.checks import CleverHansTest

class TestOptimizeLinear(CleverHansTest):
  """
  Identical to the TestOptimizeLinear in tests_tf/test_attacks.
  """
  def setUp(self):
    super(TestOptimizeLinear, self).setUp()
    self.clip_eta = utils.clip_eta
    self.rand_grad = torch.randn(100, 3, 2)
    self.rand_eta = torch.randn(100, 3, 2)
    self.red_ind = list(range(1, len(self.rand_grad.size())))
    # eps needs to be nonnegative
    self.eps_list = [0, .1, 1., 3]

  def test_optimize_linear_linf(self):
    grad = torch.tensor([[1., -2.]])
    eta = utils.optimize_linear(grad, eps=1., norm=np.inf)
    objective = torch.sum(grad * eta)

    self.assertEqual(grad.size(), eta.size())
    self.assertClose(objective, grad.abs().sum())
    self.assertClose(eta.abs(), 1.)

  def test_optimize_linear_l2(self):
    grad = torch.tensor([[.5 ** .5, -.5 ** .5]])
    eta = utils.optimize_linear(grad, eps=1., norm=2)
    objective = torch.sum(grad * eta)

    self.assertEqual(grad.size(), eta.size())
    self.assertClose(objective, 1.)
    self.assertClose(eta.pow(2).sum().sqrt(), 1.)

  def test_optimize_linear_l1(self):
    grad = torch.tensor([[1., -2.]])
    eta = utils.optimize_linear(grad, eps=1., norm=1)
    objective = torch.sum(grad * eta)

    self.assertEqual(grad.size(), eta.size())
    self.assertClose(objective, 2.)
    self.assertClose(eta.abs().sum(), 1.)

  def test_optimize_linear_l1_ties(self):
    grad = torch.tensor([[2., -2.]])
    eta = utils.optimize_linear(grad, eps=1., norm=1)
    objective = torch.sum(grad * eta)

    self.assertEqual(grad.size(), eta.size())
    self.assertClose(objective, 2.)
    self.assertClose(eta.abs().sum(), 1.)

  def test_optimize_linear_linf_satisfies_norm_constraint(self):
    for eps in self.eps_list:
      eta = utils.optimize_linear(self.rand_grad, eps=eps, norm=np.inf)
      self.assertClose(eta.abs(), eps)

  def test_optimize_linear_l1_satisfies_norm_constraint(self):
    for eps in self.eps_list:
      eta = utils.optimize_linear(self.rand_grad, eps=eps, norm=1)
      norm = eta.abs().sum(dim=self.red_ind)
      self.assertTrue(torch.allclose(norm, eps * torch.ones_like(norm)))

  def test_optimize_linear_l2_satisfies_norm_constraint(self):
    for eps in self.eps_list:
      eta = utils.optimize_linear(self.rand_grad, eps=eps, norm=2)
      # optimize_linear uses avoid_zero_div as the divisor for
      # gradients with overly small l2 norms when performing norm
      # normalizations on the gradients so as to safeguard against
      # zero division error. Therefore, the replaced gradient vectors
      # will not be l2-unit vectors after normalization. In this test,
      # these gradients are filtered out by the one_mask
      # below and are not tested.
      # NOTE the value of avoid_zero_div should be the same as the
      # avoid_zero_div used in the optimize_linear function
      avoid_zero_div = torch.tensor(1e-12)
      square = torch.max(
          avoid_zero_div,
          torch.sum(self.rand_grad ** 2, self.red_ind, keepdim=True)
          )
      norm = eta.pow(2).sum(dim=self.red_ind, keepdim=True).sqrt()
      one_mask = (
          (square <= avoid_zero_div).to(torch.float) * norm +
          (square > avoid_zero_div).to(torch.float))
      self.assertTrue(torch.allclose(norm, eps * one_mask))

  def test_clip_eta_linf(self):
    clipped = self.clip_eta(eta=self.rand_eta, norm=np.inf, eps=.5)
    self.assertTrue(torch.all(clipped <= .5))
    self.assertTrue(torch.all(clipped >= -.5))

  def test_clip_eta_l1(self):
    self.assertRaises(
        NotImplementedError, self.clip_eta, eta=self.rand_eta, norm=1, eps=.5)

    # TODO uncomment the actual test below after we have implemented the L1 attack
    # clipped = self.clip_eta(eta=self.rand_eta, norm=1, eps=.5)
    # norm = clipped.abs().sum(dim=self.red_ind)
    # self.assertTrue(torch.all(norm <= .5001))

  def test_clip_eta_l2(self):
    clipped = self.clip_eta(eta=self.rand_eta, norm=2, eps=.5)
    norm = clipped.pow(2).sum(dim=self.red_ind).pow(.5)
    self.assertTrue(torch.all(norm <= .5001))


text:
import numpy as np
from sklearn import utils as skutils
from PIL import Image
from rng import np_rng, py_rng

def center_crop(x, ph, pw=None):
    if pw is None:
        pw = ph
    h, w = x.shape[:2]
    j = int(round((h - ph) / 2.))
    i = int(round((w - pw) / 2.))
    return x[j:j + ph, i:i + pw]

def patch(x, ph, pw=None):
    if pw is None:
        pw = ph
    h, w = x.shape[:2]
    j = py_rng.randint(0, h - ph)
    i = py_rng.randint(0, w - pw)
    x = x[j:j + ph, i:i + pw]
    return x

def list_shuffle(*data):
    idxs = np_rng.permutation(np.arange(len(data[0])))
    if len(data) == 1:
        return [data[0][idx] for idx in idxs]
    else:
        return [[d[idx] for idx in idxs] for d in data]

def shuffle(*arrays, **options):
    if isinstance(arrays[0][0], basestring):
        return list_shuffle(*arrays)
    else:
        return skutils.shuffle(*arrays, random_state=np_rng)

def OneHot(X, n=None, negative_class=0.):
    X = np.asarray(X).flatten()
    if n is None:
        n = np.max(X) + 1
    Xoh = np.ones((len(X), n)) * negative_class
    Xoh[np.arange(len(X)), X] = 1.
    return Xoh

def iter_data(*data, **kwargs):
    size = kwargs.get('size', 512)
    try:
        n = len(data[0])
    except:
        n = data[0].shape[0]
    batches = n / size
    if n % size != 0:
        batches += 1

    for b in range(batches):
        start = b * size
        end = (b + 1) * size
        if end > n:
            end = n
        if len(data) == 1:
            yield data[0][start:end]
        else:
            yield tuple([d[start:end] for d in data])

def convert_img(img):
    H, W, C = img.shape
    con_img = np.zeros((C, H, W), dtype=img.dtype)
    for i in xrange(C):
        con_img[i, :, :] = img[:, :, i]
    return con_img

def convert_img_back(img):
    C, H, W = img.shape
    con_img = np.zeros((H, W, C), dtype=img.dtype)
    for i in xrange(C):
        con_img[:, :, i] = img[i, :, :]
    return con_img

def convert_video(video):
    n, f, c, w, h = video.shape
    con_video = np.zeros((n, c, f, w, h), dtype='float32')
    for i in xrange(f):
        con_video[:, :, i, :, :] = video[:, i, :, :, :]
    return con_video

def convert_video_back(video):
    n, c, f, w, h = video.shape
    con_video = np.zeros((n, f, c, w, h), dtype='float32')
    for i in xrange(f):
        con_video[:, i, :, :, :] = video[:, :, i, :, :]
    return con_video

def coloring(xmb, ymb, p=0.2):
    xmb_color = xmb
    B, C, H, W = xmb.shape
    # print H
    for i in range(B):
        img_input = np.cast['float32'](xmb[i, :, :, :])
        img_real = np.cast['float32'](ymb[i, :, :, :])

        img_input = convert_img_back(img_input)
        img_real = convert_img_back(img_real)

        img_gray = img_real[:, :, 0] + img_real[:, :, 1] + img_real[:, :, 2]
        img_gray = (765 - img_gray)
        img_gray[img_gray > 10] = 255
        img_gray[img_gray < 255] = 0
        img_gray = img_gray / 255
        img_gray = [img_gray] * 3
        img_gray = np.array(img_gray)
        img_gray = img_gray.reshape([3, H, W])
        img_mask = convert_img_back(img_gray)
        rand = np.random.random((H, W))
        rand[rand > p] = 1
        rand[rand < 1] = 0
        rand = [rand] * 3
        rand = np.array(rand)
        rand = rand.reshape([3, H, W])
        rand = convert_img_back(rand)

        img_color = (img_input) * (1 - img_mask) + img_real * img_mask + rand * 1000
        img_color[img_color > 255] = 255
        img_input = img_color * (img_input / 255)
        img_input = np.cast['uint8'](img_input)
        # img_real = np.cast['uint8'](img_real)
        # img_mask = np.cast['uint8'](img_mask*255)
        xmb_color[i, :, :, :] = convert_img(img_input)
    return xmb_color

def processing_img(img, center=True, scale=True, convert=True):
    img = np.array(img)
    img = np.cast['float32'](img)
    if convert is True:
        img = convert_img(img)
    if center and scale:
        img[:] -= 127.5
        img[:] /= 127.5
    elif center:
        img[:] -= 127.5
    elif scale:
        img[:] /= 255.

    return img

def ImgRescale(img, center=True, scale=True, convert_back=False):
    img = np.array(img)
    img = np.cast['float32'](img)
    if convert_back is True:
        img = convert_img_back(img)
    if center and scale:
        img = ((img + 1) / 2 * 255).astype(np.uint8)
    elif center:
        img = (img + 127.5).astype(np.uint8)
    elif scale:
        img = (img * 255).astype(np.uint8)
    return img

def ImgBatchRescale(img, center=True, scale=True, convert_back=False):
    img = np.array(img)
    img = np.cast['float32'](img)
    if convert_back is True:
        b, C, H, W = img.shape
        print img.dtype
        imgh = np.zeros((b, H, W, C), dtype=img.dtype)
        for i in range(b):
            imgh[i, :, :, :] = convert_img_back(img[i, :, :, :])
        img = imgh
    if center and scale:
        img = ((img + 1) / 2 * 255).astype(np.uint8)
    elif center:
        img = (img + 127.5).astype(np.uint8)
    elif scale:
        img = (img * 255).astype(np.uint8)
    return img

def test_color(img, p=0.2):
    img = np.array(img)
    img_color = img
    H, W, C = img.shape
    # print C
    img = np.cast['float32'](img)
    img_color = np.cast['float32'](img_color)

    img = img[:, :, 0] + img[:, :, 1] + img[:, :, 2]
    img[img > 100] = 255
    img[img < 255] = 0
    img = img / 255
    img = [img] * 3
    img = np.array(img)
    img = img.reshape([C, H, W])
    img = convert_img_back(img)
    rand = np.random.random((H, W))
    rand[rand > p] = 1
    rand[rand < 1] = 0
    rand = [rand] * 3
    rand = np.array(rand)
    rand = rand.reshape([C, H, W])
    rand = convert_img_back(rand)

    img_color = img_color + rand * 1000
    img_color[img_color > 255] = 255
    img_color = img_color * img
    img_color = np.cast['uint8'](img_color)

    return img_color

def CropImgs(xmb, ymb, fineSize, nbatch, h1, w1):
    if nbatch == 1:
        xmb_out = xmb[:, :, h1[0]:h1[0] + fineSize, w1[0]:w1[0] + fineSize]
        ymb_out = ymb[:, :, h1[0]:h1[0] + fineSize, w1[0]:w1[0] + fineSize]
    else:
        xmb_out = np.zeros((nbatch, 3, fineSize, fineSize), dtype='uint8')
        ymb_out = np.zeros((nbatch, 3, fineSize, fineSize), dtype='uint8')
        for i in xrange(nbatch):
            x_img = xmb[i, :, :, :]
            y_img = ymb[i, :, :, :]
            h = h1[i]
            w = w1[i]
            x_img_out = x_img[:, h:h + fineSize, w:w + fineSize]
            y_img_out = y_img[:, h:h + fineSize, w:w + fineSize]
            xmb_out[i, :, :, :] = x_img_out
            ymb_out[i, :, :, :] = y_img_out
    xmb_out = xmb_out.reshape([nbatch, 3, fineSize, fineSize])
    ymb_out = ymb_out.reshape([nbatch, 3, fineSize, fineSize])
    return xmb_out, ymb_out

def FlipImgs(xmb, ymb):
    bs, ci, hi, wi = xmb.shape
    xmb_out = np.zeros((bs, ci, hi, wi), dtype='uint8')
    ymb_out = np.zeros((bs, ci, hi, wi), dtype='uint8')
    for i in xrange(bs):
        x_img = xmb[i, :, :, :]
        y_img = ymb[i, :, :, :]
        if np.random.rand(1) > 0.5:
            x_img = convert_img_back(x_img)
            y_img = convert_img_back(y_img)

            x_img = Image.fromarray(x_img)
            y_img = Image.fromarray(y_img)

            x_img = x_img.transpose(Image.FLIP_LEFT_RIGHT)
            y_img = y_img.transpose(Image.FLIP_LEFT_RIGHT)

            x_img = convert_img(np.array(x_img))
            y_img = convert_img(np.array(y_img))
        xmb_out[i, :, :, :] = x_img
        ymb_out[i, :, :, :] = y_img
    xmb_out = xmb_out.reshape([bs, ci, hi, wi])
    ymb_out = ymb_out.reshape([bs, ci, hi, wi])
    return xmb_out, ymb_out

def pix2pixBatch(xmb, ymb, fineSize, input_nc, flip=True):
    bs, ci, hi, wi = xmb.shape
    if hi > fineSize or wi > fineSize:
        h1 = np.random.randint(0, hi - fineSize, size=bs)
        w1 = np.random.randint(0, wi - fineSize, size=bs)
        xmb, ymb = CropImgs(xmb, ymb, fineSize, bs, h1, w1)
    else:
        xmb = xmb.reshape([bs, 3, fineSize, fineSize])
        ymb = ymb.reshape([bs, 3, fineSize, fineSize])
    if flip is True:
        xmb, ymb = FlipImgs(xmb, ymb)

    return xmb, ymb

def Batch(xmb, fineSize, input_nc, flip=True):
    bs, ci, hi, wi = xmb.shape
    if hi > fineSize or wi > fineSize:
        h1 = np.random.randint(0, hi - fineSize, size=bs)
        w1 = np.random.randint(0, wi - fineSize, size=bs)
        xmb, ymb = CropImgs(xmb, xmb, fineSize, bs, h1, w1)
    else:
        xmb = np.array(xmb, dtype='uint8')
        xmb = xmb.reshape([bs, 3, fineSize, fineSize])
        ymb = xmb
    if flip is True:
        xmb, ymb = FlipImgs(xmb, ymb)

    return xmb

max_stars_repo_name: YouKnowT/model

max_issues_repo_name: YouKnowT/model

max_forks_repo_name: YouKnowT/model