text:
import numpy as np
import collections

Map_boundaries_tuple=collections.namedtuple("Map_boundaries_tuple",["lon_0",
    "lat_0","llcrnrlat","llcrnrlon","urcrnrlat","urcrnrlon","proj","resolution"]);

def geo2xyz(lons, lats, lon0, lat0):
  # returns units of meters
  if len(np.atleast_1d(lons))==1:
  	x_array = (lons-lon0)*111000.0*(np.cos(np.deg2rad(lat0)));
  	y_array = (lats-lat0)*111000.0;
  else:
    x_array = np.zeros(np.shape(lons));
    y_array = np.zeros(np.shape(lats));

    for i in range(len(lons)):
      x_array[i]=(lons[i]-lon0)*111000.0*(np.cos(np.deg2rad(lat0)));
      y_array[i]=(lats[i]-lat0)*111000.0;

  return x_array,y_array;

def xyz2geo(x, y, lon0, lat0):
  # takes units of meters
  if len(np.atleast_1d(x))==1:
    lon_array = lon0 + (x/(111000.0*np.cos(np.deg2rad(lat0))));
    lat_array = lat0 + (y/111000.0);
  else:
    lon_array = np.zeros(np.shape(x));
    lat_array = np.zeros(np.shape(y));

    for i in range(len(x)):
      lon_array[i] = lon0 + (x[i]/(111000.0*np.cos(np.deg2rad(lat0))));
      lat_array[i] = lat0 + (y[i]/111000.0);

  return lon_array, lat_array;

def geodetic_to_cartesian(pos_geo, collection):
  '''
  Parameters
  ----------
    pos_geo : (...,D) array
      array of geodetic positions. The first and second component of
      the last axis are longitude and latitude.  The last axis can
      have additional components (e.g. height) and they will be
      returned unaltered.

  Returns
  -------
    pos_cart : (...,D) array
  '''
  pos_cart = np.array(pos_geo,copy=True)
  pos_geo = np.asarray(pos_geo)
  pos_x,pos_y = geo2xyz(pos_geo[...,0],pos_geo[...,1], collection.lon_0, collection.lat_0)
  pos_x = np.asarray(pos_x)
  pos_y = np.asarray(pos_y)
  pos_cart[...,0] = pos_x
  pos_cart[...,1] = pos_y
  return pos_cart

def cartesian_to_geodetic(pos_cart,collection):
  '''
  Parameters
  ----------
    pos_cart : (...,D) array
      array of cartesian positions

  Returns
  -------
    pos_geo : (...,D) array
  '''
  pos_geo = np.array(pos_cart,copy=True)
  pos_cart = np.asarray(pos_cart)
  pos_lon,pos_lat = xyz2geo(pos_cart[...,0],pos_cart[...,1],collection.lon_0, collection.lat_0)
  pos_lon = np.asarray(pos_lon)
  pos_lat = np.asarray(pos_lat)
  pos_geo[...,0] = pos_lon
  pos_geo[...,1] = pos_lat
  return pos_geo

def create_default_basemap(lon_lst,lat_lst, proj='M5i', resolution='i'):
  '''
  creates a named tuple that bounds lat_lst and lon_lst
  '''
  print("Creating collection of map boundaries, for plotting with GMT")
  if (len(lon_lst) == 0) | (len(lat_lst) == 0):
    return Map_boundaries_tuple(lon_0=-90.0, lat_0=41.0, llcrnrlat = 26.0, llcrnrlon=-128.0, urcrnrlat=48.0, urcrnrlon=-53.0, proj='M5i');

  lon_buff = (max(lon_lst) - min(lon_lst))/20.0
  lat_buff = (max(lat_lst) - min(lat_lst))/20.0
  if lon_buff < 0.15:
    lon_buff = 0.15

  if lat_buff < 0.15:
    lat_buff = 0.15

  llcrnrlon = min(lon_lst) - lon_buff
  llcrnrlat = min(lat_lst) - lat_buff
  urcrnrlon = max(lon_lst) + lon_buff
  urcrnrlat = max(lat_lst) + lat_buff
  lon_0 = (llcrnrlon + urcrnrlon)/2.0
  lat_0 = (llcrnrlat + urcrnrlat)/2.0
  return Map_boundaries_tuple(lon_0=lon_0, lat_0=lat_0, llcrnrlat = llcrnrlat,
    llcrnrlon=llcrnrlon, urcrnrlat=urcrnrlat, urcrnrlon=urcrnrlon, proj=proj, resolution=resolution);

max_stars_repo_name: kmaterna/SlipPy

max_issues_repo_name: kmaterna/SlipPy

max_forks_repo_name: kmaterna/SlipPy


text:
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import numpy as np
import argparse
import ast
import paddle
import paddle.fluid as fluid
from paddle.fluid.layer_helper import LayerHelper
from paddle.fluid.dygraph.nn import Conv2D, Pool2D, BatchNorm, Linear
from paddle.fluid.dygraph.base import to_variable

from paddle.fluid import framework

import math
import sys
import time
import reader

IMAGENET1000 = 1281167
base_lr = 0.1
momentum_rate = 0.9
l2_decay = 1e-4

def parse_args():
    parser = argparse.ArgumentParser("Training for Resnet.")
    parser.add_argument(
        "--use_data_parallel",
        type=ast.literal_eval,
        default=False,
        help="The flag indicating whether to use data parallel mode to train the model."
    )
    parser.add_argument(
        "-e", "--epoch", default=120, type=int, help="set epoch")
    parser.add_argument(
        "-b", "--batch_size", default=32, type=int, help="set epoch")
    parser.add_argument("--ce", action="store_true", help="run ce")

    # NOTE:used in benchmark
    parser.add_argument(
        "--max_iter",
        default=0,
        type=int,
        help="the max iters to train, used in benchmark")
    parser.add_argument(
        "--class_dim",
        default=102,
        type=int,
        help="the class number of flowers dataset")
    parser.add_argument(
        "--use_imagenet_data",
        action="store_true",
        help="Use imagenet dataset instead of the flowers dataset(small dataset)"
    )
    parser.add_argument(
        '--data_dir',
        default="./data/ILSVRC2012",
        type=str,
        help="The ImageNet dataset root directory.")
    parser.add_argument(
        '--lower_scale',
        default=0.08,
        type=float,
        help="The value of lower_scale in ramdom_crop")
    parser.add_argument(
        '--lower_ratio',
        default=3. / 4.,
        type=float,
        help="The value of lower_ratio in ramdom_crop")
    parser.add_argument(
        '--upper_ratio',
        default=4. / 3.,
        type=float,
        help="The value of upper_ratio in ramdom_crop")
    parser.add_argument(
        '--resize_short_size',
        default=256,
        type=int,
        help="The value of resize_short_size")
    parser.add_argument(
        '--crop_size', default=224, type=int, help="The value of crop size")
    parser.add_argument(
        '--use_mixup', default=False, type=bool, help="Whether to use mixup")
    parser.add_argument(
        '--mixup_alpha',
        default=0.2,
        type=float,
        help="The value of mixup_alpha")
    parser.add_argument(
        '--reader_thread',
        default=8,
        type=int,
        help="The number of multi thread reader")
    parser.add_argument(
        '--reader_buf_size',
        default=16,
        type=int,
        help="The buf size of multi thread reader")
    parser.add_argument(
        '--interpolation',
        default=None,
        type=int,
        help="The interpolation mode")
    parser.add_argument(
        '--use_aa',
        default=False,
        type=bool,
        help="Whether to use auto augment")
    parser.add_argument(
        '--image_mean',
        nargs='+',
        type=float,
        default=[0.485, 0.456, 0.406],
        help="The mean of input image data")
    parser.add_argument(
        '--image_std',
        nargs='+',
        type=float,
        default=[0.229, 0.224, 0.225],
        help="The std of input image data")
    parser.add_argument(
        '--use_gpu',
        type=ast.literal_eval,
        default=True,
        help='default use gpu.')

    args = parser.parse_args()
    return args

args = parse_args()
batch_size = args.batch_size

class TimeCostAverage(object):
    def __init__(self):
        self.reset()

    def reset(self):
        self.cnt = 0
        self.total_time = 0

    def record(self, usetime):
        self.cnt += 1
        self.total_time += usetime

    def get_average(self):
        if self.cnt == 0:
            return 0
        return self.total_time / self.cnt

def optimizer_setting(parameter_list=None):

    total_images = IMAGENET1000

    step = int(math.ceil(float(total_images) / batch_size))

    epochs = [30, 60, 90]
    bd = [step * e for e in epochs]

    lr = []
    lr = [base_lr * (0.1**i) for i in range(len(bd) + 1)]
    if fluid.in_dygraph_mode():
        optimizer = fluid.optimizer.Momentum(
            learning_rate=fluid.layers.piecewise_decay(
                boundaries=bd, values=lr),
            momentum=momentum_rate,
            regularization=fluid.regularizer.L2Decay(l2_decay),
            parameter_list=parameter_list)
    else:
        optimizer = fluid.optimizer.Momentum(
            learning_rate=fluid.layers.piecewise_decay(
                boundaries=bd, values=lr),
            momentum=momentum_rate,
            regularization=fluid.regularizer.L2Decay(l2_decay))

    return optimizer

class ConvBNLayer(fluid.dygraph.Layer):
    def __init__(self,
                 num_channels,
                 num_filters,
                 filter_size,
                 stride=1,
                 groups=1,
                 act=None):
        super(ConvBNLayer, self).__init__()

        self._conv = Conv2D(
            num_channels=num_channels,
            num_filters=num_filters,
            filter_size=filter_size,
            stride=stride,
            padding=(filter_size - 1) // 2,
            groups=groups,
            act=None,
            bias_attr=False)

        self._batch_norm = BatchNorm(num_filters, act=act)

    def forward(self, inputs):
        y = self._conv(inputs)
        y = self._batch_norm(y)

        return y

class BottleneckBlock(fluid.dygraph.Layer):
    def __init__(self, num_channels, num_filters, stride, shortcut=True):
        super(BottleneckBlock, self).__init__()

        self.conv0 = ConvBNLayer(
            num_channels=num_channels,
            num_filters=num_filters,
            filter_size=1,
            act='relu')
        self.conv1 = ConvBNLayer(
            num_channels=num_filters,
            num_filters=num_filters,
            filter_size=3,
            stride=stride,
            act='relu')
        self.conv2 = ConvBNLayer(
            num_channels=num_filters,
            num_filters=num_filters * 4,
            filter_size=1,
            act=None)

        if not shortcut:
            self.short = ConvBNLayer(
                num_channels=num_channels,
                num_filters=num_filters * 4,
                filter_size=1,
                stride=stride)

        self.shortcut = shortcut

        self._num_channels_out = num_filters * 4

    def forward(self, inputs):
        y = self.conv0(inputs)
        conv1 = self.conv1(y)
        conv2 = self.conv2(conv1)

        if self.shortcut:
            short = inputs
        else:
            short = self.short(inputs)

        y = fluid.layers.elementwise_add(x=short, y=conv2)

        layer_helper = LayerHelper(self.full_name(), act='relu')
        return layer_helper.append_activation(y)

class ResNet(fluid.dygraph.Layer):
    def __init__(self, layers=50, class_dim=102):
        super(ResNet, self).__init__()

        self.layers = layers
        supported_layers = [50, 101, 152]
        assert layers in supported_layers, \
            "supported layers are {} but input layer is {}".format(supported_layers, layers)

        if layers == 50:
            depth = [3, 4, 6, 3]
        elif layers == 101:
            depth = [3, 4, 23, 3]
        elif layers == 152:
            depth = [3, 8, 36, 3]
        num_channels = [64, 256, 512, 1024]
        num_filters = [64, 128, 256, 512]

        self.conv = ConvBNLayer(
            num_channels=3, num_filters=64, filter_size=7, stride=2, act='relu')
        self.pool2d_max = Pool2D(
            pool_size=3, pool_stride=2, pool_padding=1, pool_type='max')

        self.bottleneck_block_list = []
        for block in range(len(depth)):
            shortcut = False
            for i in range(depth[block]):
                bottleneck_block = self.add_sublayer(
                    'bb_%d_%d' % (block, i),
                    BottleneckBlock(
                        num_channels=num_channels[block]
                        if i == 0 else num_filters[block] * 4,
                        num_filters=num_filters[block],
                        stride=2 if i == 0 and block != 0 else 1,
                        shortcut=shortcut))
                self.bottleneck_block_list.append(bottleneck_block)
                shortcut = True

        self.pool2d_avg = Pool2D(
            pool_size=7, pool_type='avg', global_pooling=True)

        self.pool2d_avg_output = num_filters[len(num_filters) - 1] * 4 * 1 * 1

        import math
        stdv = 1.0 / math.sqrt(2048 * 1.0)

        self.out = Linear(
            self.pool2d_avg_output,
            class_dim,
            act='softmax',
            param_attr=fluid.param_attr.ParamAttr(
                initializer=fluid.initializer.Uniform(-stdv, stdv)))

    def forward(self, inputs):
        y = self.conv(inputs)
        y = self.pool2d_max(y)
        for bottleneck_block in self.bottleneck_block_list:
            y = bottleneck_block(y)
        y = self.pool2d_avg(y)
        y = fluid.layers.reshape(y, shape=[-1, self.pool2d_avg_output])
        y = self.out(y)
        return y

def reader_decorator(reader):
    def __reader__():
        for item in reader():
            img = np.array(item[0]).astype('float32').reshape(3, 224, 224)
            label = np.array(item[1]).astype('int64').reshape(1)
            yield img, label

    return __reader__

def eval(model, data):

    model.eval()
    total_loss = 0.0
    total_acc1 = 0.0
    total_acc5 = 0.0
    total_sample = 0
    for batch_id, data in enumerate(data()):
        img = data[0]
        label = data[1]
        label.stop_gradient = True

        out = model(img)

        acc_top1 = fluid.layers.accuracy(input=out, label=label, k=1)
        acc_top5 = fluid.layers.accuracy(input=out, label=label, k=5)

        #dy_out = avg_loss.numpy()

        #total_loss += dy_out
        total_acc1 += acc_top1.numpy()
        total_acc5 += acc_top5.numpy()
        total_sample += 1

        if batch_id % 10 == 0:
            print("test | batch step %d, acc1 %0.3f acc5 %0.3f" % \
                  ( batch_id, total_acc1 / total_sample, total_acc5 / total_sample))
    if args.ce:
        print("kpis\ttest_acc1\t%0.3f" % (total_acc1 / total_sample))
        print("kpis\ttest_acc5\t%0.3f" % (total_acc5 / total_sample))
    print("final eval acc1 %0.3f acc5 %0.3f" % \
          (total_acc1 / total_sample, total_acc5 / total_sample))

def train_resnet():
    epoch = args.epoch

    if not args.use_gpu:
        place = fluid.CPUPlace()
    elif not args.use_data_parallel:
        place = fluid.CUDAPlace(0)
    else:
        place = fluid.CUDAPlace(fluid.dygraph.parallel.Env().dev_id)

    with fluid.dygraph.guard(place):
        if args.ce:
            print("ce mode")
            seed = 33
            np.random.seed(seed)
            fluid.default_startup_program().random_seed = seed
            fluid.default_main_program().random_seed = seed

        if args.use_data_parallel:
            strategy = fluid.dygraph.parallel.prepare_context()

        resnet = ResNet(class_dim=args.class_dim)
        optimizer = optimizer_setting(parameter_list=resnet.parameters())

        if args.use_data_parallel:
            resnet = fluid.dygraph.parallel.DataParallel(resnet, strategy)

        if args.use_imagenet_data:
            imagenet_reader = reader.ImageNetReader(0)
            train_reader = imagenet_reader.train(settings=args)
        else:
            train_reader = paddle.batch(
                reader_decorator(paddle.dataset.flowers.train(use_xmap=True)),
                batch_size=batch_size,
                drop_last=True)

        if args.use_data_parallel:
            train_reader = fluid.contrib.reader.distributed_batch_reader(
                train_reader)

        if args.use_imagenet_data:
            test_reader = imagenet_reader.val(settings=args)
        else:
            test_reader = paddle.batch(
                reader_decorator(paddle.dataset.flowers.test(use_xmap=True)),
                batch_size=batch_size,
                drop_last=True)

        train_loader = fluid.io.DataLoader.from_generator(
            capacity=32,
            use_double_buffer=True,
            iterable=True,
            return_list=True,
            use_multiprocess=True)
        train_loader.set_sample_list_generator(train_reader, places=place)

        test_loader = fluid.io.DataLoader.from_generator(
            capacity=64,
            use_double_buffer=True,
            iterable=True,
            return_list=True,
            use_multiprocess=True)
        test_loader.set_sample_list_generator(test_reader, places=place)

        #NOTE: used in benchmark
        total_batch_num = 0

        for eop in range(epoch):
            epoch_start = time.time()

            resnet.train()
            total_loss = 0.0
            total_acc1 = 0.0
            total_acc5 = 0.0
            total_sample = 0

            train_batch_cost_avg = TimeCostAverage()
            train_reader_cost_avg = TimeCostAverage()
            batch_start = time.time()
            for batch_id, data in enumerate(train_loader()):
                #NOTE: used in benchmark
                if args.max_iter and total_batch_num == args.max_iter:
                    return

                train_reader_cost = time.time() - batch_start

                img, label = data
                label.stop_gradient = True

                out = resnet(img)
                loss = fluid.layers.cross_entropy(input=out, label=label)
                avg_loss = fluid.layers.mean(x=loss)

                acc_top1 = fluid.layers.accuracy(input=out, label=label, k=1)
                acc_top5 = fluid.layers.accuracy(input=out, label=label, k=5)

                dy_out = avg_loss.numpy()

                if args.use_data_parallel:
                    avg_loss = resnet.scale_loss(avg_loss)
                    avg_loss.backward()
                    resnet.apply_collective_grads()
                else:
                    avg_loss.backward()

                optimizer.minimize(avg_loss)
                resnet.clear_gradients()

                total_loss += dy_out
                total_acc1 += acc_top1.numpy()
                total_acc5 += acc_top5.numpy()
                total_sample += 1

                train_batch_cost = time.time() - batch_start
                train_batch_cost_avg.record(train_batch_cost)
                train_reader_cost_avg.record(train_reader_cost)

                total_batch_num = total_batch_num + 1  #this is for benchmark
                if batch_id % 10 == 0:
                    ips = float(
                        args.batch_size) / train_batch_cost_avg.get_average()
                    print(
                        "[Epoch %d, batch %d] loss: %.5f, acc1: %.5f, acc5: %.5f, batch_cost: %.5f sec, reader_cost: %.5f sec, ips: %.5f images/sec"
                        % (eop, batch_id, total_loss / total_sample,
                           total_acc1 / total_sample, total_acc5 / total_sample,
                           train_batch_cost_avg.get_average(),
                           train_reader_cost_avg.get_average(), ips))
                    train_batch_cost_avg.reset()
                    train_reader_cost_avg.reset()
                batch_start = time.time()

            if args.ce:
                print("kpis\ttrain_acc1\t%0.3f" % (total_acc1 / total_sample))
                print("kpis\ttrain_acc5\t%0.3f" % (total_acc5 / total_sample))
                print("kpis\ttrain_loss\t%0.3f" % (total_loss / total_sample))

            train_epoch_cost = time.time() - epoch_start
            print(
                "[Epoch %d], loss %.5f, acc1 %.5f, acc5 %.5f, epoch_cost: %.5f s"
                % (eop, total_loss / total_sample, total_acc1 / total_sample,
                   total_acc5 / total_sample, train_epoch_cost))

            resnet.eval()
            eval(resnet, test_loader)

            save_parameters = (not args.use_data_parallel) or (
                args.use_data_parallel and
                fluid.dygraph.parallel.Env().local_rank == 0)
            if save_parameters:
                fluid.save_dygraph(resnet.state_dict(), 'resnet_params')

if __name__ == '__main__':
    train_resnet()

ext: py

reason: import numpy


text:
import numpy as np # pip install numpy
import os
from random import shuffle
from tqdm import tqdm
import cv2
def my_label(image_name):
    name = image_name.split('.')[-3]
    # if you have two person in your dataset
#     if name=="Ishwar":
#         return np.array([1,0])
#     elif name=="Manish":
#         return np.array([0,1])

    # if you have three person in your dataset
    if name=="hoang":
        return np.array([1,0,0])
    elif name=="mytam":
        return np.array([0,1,0])
    elif name=="ronaldo":
        return np.array([0,0,1])

def my_data():
    data = []
    for img in tqdm(os.listdir("./dataset/")):
        path=os.path.join("./dataset/",img)
        img_data = cv2.imread(path, cv2.IMREAD_GRAYSCALE)
        img_data = cv2.resize(img_data, (50,50))
        data.append([np.array(img_data), my_label(img)])
    shuffle(data)
    return data

# data = my_data()


text:
import collections

# import cv2
import numpy as np

def get_bbox_overlap(bbox1, bbox2):
    y11, x11, y12, x12 = bbox1
    y21, x21, y22, x22 = bbox2
    w1, h1 = x12 - x11, y12 - y11
    w2, h2 = x22 - x21, y22 - y21
    intersect = (max(0, min(x12, x22) - max(x11, x21)) *
                 max(0, min(y12, y22) - max(y11, y21)))
    union = w1 * h1 + w2 * h2 - intersect
    return 1.0 * intersect / union

def get_mask_overlap(mask1, mask2, half_if_nounion=False):
    intersect = np.bitwise_and(mask1, mask2).sum()
    union = np.bitwise_or(mask1, mask2).sum()
    if union == 0:
        if half_if_nounion:
            return 0.5
        else:
            return 0.
    else:
        return 1.0 * intersect / union

# def create_proposal_targets(rois, boxes, labels, masks,
#                             loc_normalize_mean, loc_normalize_std,
#                             mask_size):
#     import chainer
#     from chainercv.links.model.faster_rcnn.utils.proposal_target_creator\
#         import ProposalTargetCreator
#     xp = chainer.cuda.get_array_module(rois)
#     rois = chainer.cuda.to_cpu(rois)
#     boxes = chainer.cuda.to_cpu(boxes)
#     labels = chainer.cuda.to_cpu(labels)
#     masks = chainer.cuda.to_cpu(masks)
#
#     proposal_target_creator = ProposalTargetCreator(n_sample=64)
#     sample_rois, gt_roi_locs, gt_roi_labels = \
#         proposal_target_creator(
#             rois, boxes, labels,
#             loc_normalize_mean, loc_normalize_std)
#
#     N, H, W = masks.shape
#     assert boxes.shape == (N, 4)
#     assert labels.shape == (N,)
#
#     n_sample = len(sample_rois)
#     gt_roi_masks = - np.ones(
#        (n_sample, mask_size, mask_size), dtype=np.int32)
#     for i, (id_cls, roi) in enumerate(zip(gt_roi_labels, sample_rois)):
#         y1, x1, y2, x2 = map(int, roi)
#         assert 0 <= y1 and y2 <= H
#         assert 0 <= x1 and x2 <= W
#         if id_cls == 0:
#             continue
#         idx_ins = np.argmax([get_bbox_overlap(b, roi) for b in boxes])
#         mask_ins = masks[idx_ins]
#         assert mask_ins.dtype == np.int32
#         mask_roi = np.zeros_like(mask_ins)
#         mask_roi[y1:y2, x1:x2] = 1
#         mask_ins = mask_ins & mask_roi
#
#         mask_ins = mask_ins[y1:y2, x1:x2]
#         mask_ins = mask_ins.astype(np.float32)
#         mask_ins = cv2.resize(mask_ins, (mask_size, mask_size))
#         mask_ins = np.round(mask_ins).astype(np.int32)
#         gt_roi_masks[i] = mask_ins
#     if xp != np:
#         sample_rois = chainer.cuda.to_gpu(sample_rois)
#         gt_roi_locs = chainer.cuda.to_gpu(gt_roi_locs)
#         gt_roi_labels = chainer.cuda.to_gpu(gt_roi_labels)
#         gt_roi_masks = chainer.cuda.to_gpu(gt_roi_masks)
#     return sample_rois, gt_roi_locs, gt_roi_labels, gt_roi_masks

def label2instance_boxes(label_instance, label_class, return_masks=False):
    """Convert instance label to boxes.

    Parameters
    ----------
    label_instance: numpy.ndarray, (H, W)
        Label image for instance id.
    label_class: numpy.ndarray, (H, W)
        Label image for class.
    return_masks: bool
        Flag to return each instance mask.

    Returns
    -------
    instance_classes: numpy.ndarray, (n_instance,)
        Class id for each instance.
    boxes: (n_instance, 4)
        Bounding boxes for each instance. (x1, y1, x2, y2)
    instance_masks: numpy.ndarray, (n_instance, H, W), bool
        Masks for each instance. Only returns when return_masks=True.
    """
    instances = np.unique(label_instance)
    instances = instances[instances != -1]
    n_instance = len(instances)
    # instance_class is 'Class of the Instance'
    instance_classes = np.zeros((n_instance,), dtype=np.int32)
    boxes = np.zeros((n_instance, 4), dtype=np.int32)
    H, W = label_instance.shape
    instance_masks = np.zeros((n_instance, H, W), dtype=bool)
    for i, inst in enumerate(instances):
        mask_inst = label_instance == inst
        count = collections.Counter(label_class[mask_inst].tolist())
        instance_class = max(count.items(), key=lambda x: x[1])[0]

        assert inst not in [-1]
        assert instance_class not in [-1, 0]

        where = np.argwhere(mask_inst)
        (y1, x1), (y2, x2) = where.min(0), where.max(0) + 1

        instance_classes[i] = instance_class
        boxes[i] = (y1, x1, y2, x2)
        instance_masks[i] = mask_inst
    if return_masks:
        return instance_classes, boxes, instance_masks
    else:
        return instance_classes, boxes

def instance_boxes2label(labels, bboxes, masks, scores=None):
    if scores is not None:
        # sort ascending order of score
        indices = np.argsort(scores)
        labels = labels[indices]
        bboxes = bboxes[indices]
        masks = masks[indices]

    _, H, W = masks.shape
    lbl_ins = - np.ones((H, W), dtype=np.int32)
    lbl_cls = np.zeros((H, W), dtype=np.int32)
    for ins_id, (label, bbox, mask) in enumerate(zip(labels, bboxes, masks)):
        assert label > 0  # instance must be foreground
        assert mask.dtype == bool
        lbl_cls[mask] = label
        lbl_ins[mask] = ins_id

    return lbl_ins, lbl_cls

def mask_to_bbox(mask):
    """Convert mask image to bounding box.

    Parameters
    ----------
    mask: :class:`numpy.ndarray`
        Input mask image.

    Returns
    -------
    box: tuple (y1, x1, y2, x2)
        Bounding box.
    """
    where = np.argwhere(mask)
    (y1, x1), (y2, x2) = where.min(0), where.max(0) + 1
    bbox = y1, x1, y2, x2
    return bbox

def label_to_bboxes(label, ignore_label=-1):
    """Convert label image to bounding boxes."""
    if not isinstance(ignore_label, collections.Iterable):
        ignore_label = (ignore_label,)
    bboxes = []
    for l in np.unique(label):
        if l in ignore_label:
            continue
        mask = label == l
        bbox = mask_to_bbox(mask)
        bboxes.append(bbox)
    return np.array(bboxes)

def label_rois(rois, label_instance, label_class, overlap_thresh=0.5):
    """Label rois for instance classes.

    Parameters
    ----------
    rois: numpy.ndarray, (n_rois, 4)
    label_instance: numpy.ndarray, (H, W)
    label_class: numpy.ndarray, (H, W)
    overlap_thresh: float, [0, 1]
        Threshold to label as fg. (default: 0.5)

    Returns
    -------
    roi_clss: numpy.ndarray, (n_rois,), numpy.int32
    roi_inst_masks: list of numpy.ndarray
    """
    inst_clss, inst_rois, inst_masks = label2instance_boxes(
        label_instance, label_class, return_masks=True)
    roi_clss = []
    roi_inst_masks = []
    for roi in rois:
        overlaps = [get_bbox_overlap(roi, inst_roi) for inst_roi in inst_rois]
        inst_ind = np.argmax(overlaps)
        overlap = overlaps[inst_ind]

        if overlap > overlap_thresh:
            roi_cls = inst_clss[inst_ind]
            y1, x1, y2, x2 = roi
            roi_inst_mask = inst_masks[inst_ind][y1:y2, x1:x2]
        else:
            roi_cls = 0
            roi_inst_mask = None
        roi_clss.append(roi_cls)
        roi_inst_masks.append(roi_inst_mask)
    roi_clss = np.array(roi_clss, dtype=np.int32)
    return roi_clss, roi_inst_masks

# def instance_label_accuracy_score(lbl_ins1, lbl_ins2):
#     best_overlaps = []
#     for l1 in np.unique(lbl_ins1):
#         if l1 == -1:
#             continue
#         mask1 = lbl_ins1 == l1
#         best_overlap = 0
#         for l2 in np.unique(lbl_ins2):
#             if l2 == -1:
#                 continue
#             mask2 = lbl_ins2 == l2
#             overlap = get_mask_overlap(mask1, mask2)
#             best_overlap = max(best_overlap, overlap)
#         best_overlaps.append(best_overlap)
#     return np.mean(best_overlaps)


text:
from __future__ import print_function
from viewer import Viewer
from shaders import rgb
import h5py
import numpy as np
import gc

class Hdf5Viewer(Viewer):

    def __init__(self, filename = None, show_layer_numbers=False):

        super(Hdf5Viewer, self).__init__()
        self.files = []
        self.num_datasets = 0
        self.show_layer_numbers = show_layer_numbers

        if filename is not None:
            self.add_file(filename)

    def add_file(self, filename, datasets=None):

        # make sure this file is closed before opening it again, h5py doesn't
        # handle this well otherwise
        for obj in gc.get_objects():
            if isinstance(obj, h5py.File):
                try:
                    if obj.filename == filename:
                        print("Closing previously open %s"%filename)
                        obj.close()
                except:
                    pass

        f = h5py.File(filename, 'r')
        self.files.append(f)

        if datasets is None:
            self.__traverse_add(f, filename)
        else:
            for dataset in datasets:
                self.add_dataset(f[dataset], filename + dataset)

    def add_dataset(self, dataset, name):

        kwargs = {}
        if 'offset' in dataset.attrs:
            kwargs['offset'] = tuple(dataset.attrs['offset'][::-1])
        if 'resolution' in dataset.attrs:
            kwargs['voxel_size'] = tuple(dataset.attrs['resolution'][::-1])
        elif 'voxel_size' in dataset.attrs:
            kwargs['voxel_size'] = tuple(dataset.attrs['voxel_size'][::-1])

        # strip 1-dimensions
        while len(dataset.shape) > 3 and dataset.shape[0] == 1:
            dataset = dataset[0]
            if 'offset' in kwargs:
                kwargs['offset'] = kwargs['offset'][:-1]
            if 'voxel_size' in kwargs:
                kwargs['voxel_size'] = kwargs['voxel_size'][:-1]

        if len(dataset.shape) not in [3,4]:
            print("Skipping " + name)
            return

        num_channels = 1
        if len(dataset.shape) > 3:
            num_channels = dataset.shape[0]

        for i in range(0, num_channels, 3):

            if num_channels > 1:
                view = dataset[i:i+3]
                if 'offset' in kwargs:
                    kwargs['offset'] = kwargs['offset'][:len(view.shape)-1]
                if 'voxel_size' in kwargs:
                    kwargs['voxel_size'] = kwargs['voxel_size'][:len(view.shape)-1]
            else:
                view = dataset

            self.num_datasets += 1
            print("Adding dataset", name, "as", self.num_datasets)

            if len(view.shape) == 4 and view.shape[0] == 3:
                kwargs['shader'] = rgb()

            if view.dtype == np.bool:
                view = np.array(view, dtype=np.uint8)*255

            if self.show_layer_numbers:
                self.add(view, name=str(self.num_datasets), **kwargs)
            else:
                self.add(view, name=name, **kwargs)

    def __traverse_add(self, item, filename):

        if isinstance(item, h5py.Dataset):
            self.add_dataset(item, filename + item.name)
        elif isinstance(item, h5py.Group):
            for k in item:
                self.__traverse_add(item[k], filename)
        else:
            print("Skipping " + item.name)

max_stars_repo_name: funkey/nyroglancer

max_issues_repo_name: sheridana/nyroglancer

max_forks_repo_name: sheridana/nyroglancer


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
__author__ = 'Will Brennan'

# built-in modules
import logging
# Standard modules
import cv2
import numpy
import skimage
import skimage.measure
import skimage.segmentation
# Custom modules
import main
import scripts

logger = logging.getLogger('main')

def get_masks(img, n_seg=250):
    logger.debug('SLIC segmentation initialised')
    segments = skimage.segmentation.slic(img, n_segments=n_seg, compactness=10, sigma=1)
    logger.debug('SLIC segmentation complete')
    logger.debug('contour extraction...')
    masks = [[numpy.zeros((img.shape[0], img.shape[1]), dtype=numpy.uint8), None]]
    for region in skimage.measure.regionprops(segments):
        masks.append([masks[0][0].copy(), region.bbox])
        x_min, y_min, x_max, y_max = region.bbox
        masks[-1][0][x_min:x_max, y_min:y_max] = skimage.img_as_ubyte(region.convex_image)
    logger.debug('contours extracted')
    return masks[1:]

def blur_mask_old(img):
    assert isinstance(img, numpy.ndarray), 'img_col must be a numpy array'
    assert img.ndim == 3, 'img_col must be a color image ({0} dimensions currently)'.format(img.ndim)
    blur_mask = numpy.zeros(img.shape[:2], dtype=numpy.uint8)
    for mask, loc in get_masks(img):
        logger.debug('Checking Mask: {0}'.format(numpy.unique(mask)))
        logger.debug('SuperPixel Mask Percentage: {0}%'.format(int((100.0/255.0)*(numpy.sum(mask)/mask.size))))
        img_fft, val, blurry = main.blur_detector(img[loc[0]:loc[2], loc[1]:loc[3]])
        logger.debug('Blurry: {0}'.format(blurry))
        if blurry:
            blur_mask = cv2.add(blur_mask, mask)
    result = numpy.sum(blur_mask)/(255.0*blur_mask.size)
    logger.info('{0}% of input image is blurry'.format(int(100*result)))
    return blur_mask, result

def morphology(msk):
    assert isinstance(msk, numpy.ndarray), 'msk must be a numpy array'
    assert msk.ndim == 2, 'msk must be a greyscale image'
    kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5))
    msk = cv2.erode(msk, kernel, iterations=1)
    kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3))
    msk = cv2.morphologyEx(msk, cv2.MORPH_CLOSE, kernel)
    msk[msk < 128] = 0
    msk[msk > 127] = 255
    return msk

def remove_border(msk, width=50):
    assert isinstance(msk, numpy.ndarray), 'msk must be a numpy array'
    assert msk.ndim == 2, 'msk must be a greyscale image'
    dh, dw = map(lambda i: i//width, msk.shape)
    h, w = msk.shape
    msk[:dh, :] = 255
    msk[h-dh:, :] = 255
    msk[:, :dw] = 255
    msk[:, w-dw:] = 255
    return msk

def blur_mask(img):
    assert isinstance(img, numpy.ndarray), 'img_col must be a numpy array'
    assert img.ndim == 3, 'img_col must be a color image ({0} dimensions currently)'.format(img.ndim)
    msk, val, blurry = main.blur_detector(img)
    logger.debug('inverting img_fft')
    msk = cv2.convertScaleAbs(255-(255*msk/numpy.max(msk)))
    msk[msk < 50] = 0
    msk[msk > 127] = 255
    logger.debug('removing border')
    msk = remove_border(msk)
    logger.debug('applying erosion and dilation operators')
    msk = morphology(msk)
    logger.debug('evaluation complete')
    result = numpy.sum(msk)/(255.0*msk.size)
    logger.info('{0}% of input image is blurry'.format(int(100*result)))
    return msk, result, blurry

if __name__ == '__main__':
    img_path = raw_input("Please Enter Image Path: ")
    img = cv2.imread(img_path)
    msk, val = blur_mask(img)
    scripts.display('img', img)
    scripts.display('msk', msk)
    cv2.waitKey(0)

max_stars_repo_name: toreal/BlurDetection

max_issues_repo_name: toreal/BlurDetection

max_forks_repo_name: toreal/BlurDetection


text:
import matplotlib.pyplot as plt
import numpy as np

with open('resistance_values.txt') as f:
    lines = f.readlines()
    for i in range(3):
        lines.pop(0)
    x = [sbstr.strip().split('\t')[0] for sbstr in lines]
    y = [round(float(sbstr.strip().split('\t')[1]), 4) for sbstr in lines]
    plt.title('Coil resistance ')
    plt.xlabel('Temperature[C]')
    plt.ylabel('Resistance[ohms]')
    plt.xticks(rotation=90)
    plt.plot(x, y)
    plt.show()


text:
from esem import rf_model
from esem.utils import get_uniform_params
from tests.mock import *
from numpy.testing import assert_allclose

class RFTest:
    """
    Tests on the RFModel class and its methods. The actual model is setup
     independently in the concrete test classes below. This abstracts the
     different test cases out and allows the model to only be created once
     for each test case.

    N.B. Fixing random_seed=0 when initialising model, else no guarantee
    that the tolerance will always pass. (RFs use bootstrapping)
    """

    def test_simple_predict(self):

        # Get the actual test data
        #  Use the class method `eval_fn` so 'self' doesn't get passed
        expected = type(self).eval_fn(self.test_params[0])

        pred_m, pred_var = self.model._predict(self.test_params[0:1])

        assert_allclose(expected.data, pred_m, rtol=1e-2)

    def test_predict_interface(self):

        # Get the actual test data
        #  Use the class method `eval_fn` so 'self' doesn't get passed
        expected = type(self).eval_fn(self.test_params[0])

        pred_m, pred_var = self.model.predict(self.test_params[0:1])

        assert_allclose(expected.data, pred_m.data, rtol=1e-2)
        assert pred_m.name() == 'Emulated ' + expected.name()
        assert_allclose(np.full_like(expected.data, np.NaN), pred_var.data, equal_nan=True)
        assert pred_var.name() == 'Variance in emulated ' + expected.name()
        assert pred_m.units == expected.units

    def test_predict_interface_multiple_samples(self):
        from iris.cube import CubeList
        # Get the actual test data
        #  Use the class method `eval_fn` so 'self' doesn't get passed
        expected = CubeList([type(self).eval_fn(p, job_n=i) for i, p in enumerate(self.test_params)])
        expected = expected.concatenate_cube()

        pred_m, pred_var = self.model.predict(self.test_params)

        # For some reason the relative tolerance has to be
        # higher here than in the other tests???
        assert_allclose(expected.data, pred_m.data, rtol=1e-1)
        assert pred_m.name() == 'Emulated ' + (expected.name() or 'data')
        assert_allclose(np.full_like(expected.data, np.NaN), pred_var.data, equal_nan=True)
        assert pred_var.name() == 'Variance in emulated ' + expected.name()
        assert pred_m.units == expected.units

class TestSimple1D(RFTest):
    """
    Setup for the simple 1D 2 parameter test case
    """

    @classmethod
    def setup_class(cls) -> None:
        params, test = pop_elements(get_uniform_params(2), 10, 12)
        ensemble = get_1d_two_param_cube(params)
        m = rf_model(training_params=params,
                     training_data=ensemble,
                     random_state=0)
        m.train()

        cls.model = m
        cls.params = params
        cls.test_params = test
        cls.eval_fn = eval_1d_cube

class TestSimple2D(RFTest):
    """
    Setup for the simple 2D 3 parameter test case.
    """

    @classmethod
    def setup_class(cls) -> None:
        params, test = pop_elements(get_uniform_params(3), 50)
        ensemble = get_three_param_cube(params)
        m = rf_model(training_params=params,
                     training_data=ensemble,
                     random_state=0)
        m.train()

        cls.model = m
        cls.params = params
        cls.test_params = test
        cls.eval_fn = eval_cube

class TestSimple32bit(RFTest):
    """
    Setup for the simple 2D 3 parameter test case with 32bit data
    """

    @classmethod
    def setup_class(cls) -> None:
        params, test = pop_elements(get_uniform_params(3), 50)

        ensemble = get_three_param_cube(params)
        # Create a new, ensemble at lower precision
        ensemble = ensemble.copy(data=ensemble.data.astype('float32'))
        m = rf_model(training_params=params,
                     training_data=ensemble,
                     random_state=0)
        m.train()

        cls.model = m
        cls.params = params
        cls.test_params = test
        cls.eval_fn = eval_cube

reason: from numpy


text:
# -*- coding: utf-8 -*-
import os
import sys
import numpy as np

import unittest
from sklearn.utils.testing import assert_equal

# temporary solution for relative imports in case pyod is not installed
# if suod is installed, no need to use the following line
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))

from suod.models.jl_projection import jl_fit_transform, jl_transform

class TestCostPredictor(unittest.TestCase):
    def setUp(self):
        self.objective_dim = 5
        self.X = np.random.random([30, 8])
        self.X2 = np.random.random([10, 8])

    def test_fit_transform(self):
        X_transformed, jl_transformer = jl_fit_transform(self.X,
                                                         self.objective_dim,
                                                         method='basic')
        assert_equal(X_transformed.shape,
                     (self.X.shape[0], self.objective_dim))
        assert_equal(jl_transformer.shape,
                     (self.X.shape[1], self.objective_dim))

        X_transformed, jl_transformer = jl_fit_transform(self.X,
                                                         self.objective_dim,
                                                         method='discrete')
        assert_equal(X_transformed.shape,
                     (self.X.shape[0], self.objective_dim))
        assert_equal(jl_transformer.shape,
                     (self.X.shape[1], self.objective_dim))

        X_transformed, jl_transformer = jl_fit_transform(self.X,
                                                         self.objective_dim,
                                                         method='circulant')
        assert_equal(X_transformed.shape,
                     (self.X.shape[0], self.objective_dim))
        assert_equal(jl_transformer.shape,
                     (self.X.shape[1], self.objective_dim))

        X_transformed, jl_transformer = jl_fit_transform(self.X,
                                                         self.objective_dim,
                                                         method='toeplitz')
        assert_equal(X_transformed.shape,
                     (self.X.shape[0], self.objective_dim))
        assert_equal(jl_transformer.shape,
                     (self.X.shape[1], self.objective_dim))

    def test_transform(self):
        _, jl_transformer = jl_fit_transform(self.X,
                                             self.objective_dim,
                                             method='basic')
        X_transformed = jl_transform(self.X2, jl_transformer)
        assert_equal(X_transformed.shape,
                     (self.X2.shape[0], self.objective_dim))

max_issues_repo_name: DreamtaleCore/SUOD

max_forks_repo_name: DreamtaleCore/SUOD


text:
#!/usr/bin/env python3
# -*- coding: utf8 -*-
#   Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""grammar defination
"""

import sys
import os
import traceback
import logging
from collections import defaultdict

import numpy as np

CONST_KEYWORD = set(['intersect', 'union', 'except',
                 'count', 'sum', 'min', 'max', 'avg', 'none',
                 'between', 'like', 'not_like', 'in', 'not_in',
                 '=', '!=', '<', '>', '<=', '>=', 'and', 'or',
                 '+', '-', '*', '/',
                 'des', 'asc'])
INF = 1e9

class Grammar(object):

    """Grammar Defination"""

    # 可解码为叶子几点的文法数，即Table、Column、Value 3 种
    LEAF_NUM = 3

    TYPE_START = 1
    TYPE_END = 2
    TYPE_CONST = 3
    TYPE_LEAF = 4
    TYPE_MID = 5

    ACTION_NONE = 0
    ACTION_APPLY = 1
    ACTION_SELECT_T = 2
    ACTION_SELECT_C = 3
    ACTION_SELECT_V = 4
    ACTION_STOP = 5

    GMR_T = 'Table'
    GMR_C = 'Column'
    GMR_V = 'Value'
    GMR_NAME_T = -1
    GMR_NAME_C = -1
    GMR_NAME_V = -1
    GMR_NAME_END = -1

    # 最大的table、column、value数量，实际数量batch 内不同样本可能不同
    MAX_TABLE = -1
    MAX_COLUMN = -1
    MAX_VALUE = -1

    def __init__(self, desc_file):
        """init of class

        Args:
            desc_file (TYPE): NULL
        """
        super(Grammar, self).__init__()

        self._desc_file = desc_file
        self._vocab_size = None

        # 初始化为 const keywords 部分，后边读取语法配置词表继续更新
        self.dct_token2id = dict() #[(tok, idx) for idx, tok in enumerate(CONST_KEYWORD)])

        # 模型解码的目标词表空间
        self._lst_gmr_desc = list()
        self._lst_gmr2name = list()

        # grammar token 作为 name 时对应的 ID
        self.dct_token_id2name_id = dict()
        # name id 对应的 grammar id(即grammar.txt中的第n行)
        self._dct_name2gmr = defaultdict(list)

        with open(desc_file) as ifs:
            self._init_from_file(ifs)

        self._lst_gmr_desc, self.gmr_desc_lens, self.max_desc_len = self._generate_gmr_desc(self._lst_gmr_desc)

        self.dct_token_id2name_id[self.LEAF] = len(self.dct_token_id2name_id)
        Grammar.GMR_NAME_T = self.dct_token_id2name_id[self.dct_token2id[self.GMR_T]]
        Grammar.GMR_NAME_C = self.dct_token_id2name_id[self.dct_token2id[self.GMR_C]]
        Grammar.GMR_NAME_V = self.dct_token_id2name_id[self.dct_token2id[self.GMR_V]]
        Grammar.GMR_NAME_END = self.dct_token_id2name_id[self.dct_token2id['END']]

        # shape is: [len(_dct_name2gmr), len(_lst_gmr_desc) - LEAF_NUM]
        self.grammar_mask_matrix = self._init_grammar_mask()

        self._to_numpy()

    def _to_numpy(self):
        """对外的数据结构转为 numpy.ndarray
        Returns: None
        """
        self.gmr_vocab = np.array(range(len(self._lst_gmr_desc) - self.LEAF_NUM)).astype(np.int64)
        # shape = [grammar_size_with_leaf, max_len]
        self.gmr_desc_arr = np.array(self._lst_gmr_desc).astype(np.int64)
        # shape = [grammar_size]
        self.gmr_desc_lens = np.array(self.gmr_desc_lens).astype(np.int64)
        # shape = [name_size - 3, grammar_size]
        self.grammar_mask_matrix = np.array(self.grammar_mask_matrix).astype(np.float32)
        self.gmr2name_arr = self._expand_name_arr(np.int64)
        self.gmr_token2name_arr = self._generate_token_name_arr(np.int64)
        self.gmr2type_arr = self._generate_type_arr(np.int64)
        self.gmr_token2action_arr, self.gmr_name2action = self._generate_action_arr(np.int64)

    @property
    def grammar_size(self):
        """read property of static/non-leaf grammar size"""
        return len(self.gmr_vocab)

    @property
    def vocab_size(self):
        """read property of vocab_size"""
        if self._vocab_size is None:
            self._vocab_size = self.grammar_size + self.MAX_TABLE + self.MAX_COLUMN + self.MAX_VALUE
        return self._vocab_size

    @property
    def grammar_size_with_leaf(self):
        """read property of grammar size with 3 leaf ones"""
        return self.grammar_size + self.LEAF_NUM

    @property
    def name_size(self):
        """read property of vocab_size"""
        return len(self._dct_name2gmr)

    def _init_from_file(self, ifs):
        """init grammar desc and ids from file(ifs, Input File Stream)

        Args:
            ifs (file stream): NULL

        Returns: TODO

        Raises: NULL

        """
        lines = [x.strip() for x in ifs]
        # 过滤空行和注释行（#号开始）
        lines = filter(lambda x: not(len(x) == 0 or x.startswith('#')), lines)

        for gmr_id, line in enumerate(lines):
            gmr_rule_tokens = [x for x in line.split(' ') if x not in CONST_KEYWORD]
            # 更新词表
            self._update_token_vocab(gmr_rule_tokens)

            lst_token_ids = list(self.dct_token2id[x] for x in gmr_rule_tokens)

            gmr_name_id = lst_token_ids[0]
            gmr_desc_ids = lst_token_ids[1:]
            self._lst_gmr_desc.append(gmr_desc_ids)

            # updates:
            # 1. self.dct_token_id2name_id
            # 2. self._dct_name2gmr
            # 3. self._lst_gmr2name
            self._update_gmr_name(gmr_name_id, gmr_id)

    def _update_token_vocab(self, gmr_rule_tokens):
        """更新词表：补充不在词表中的新 token

        Args:
            gmr_rule_tokens (list): NULL

        Returns: None

        Raises: NULL

        """
        for token in gmr_rule_tokens:
            if token not in self.dct_token2id:
                self.dct_token2id[token] = len(self.dct_token2id)

    def _update_gmr_name(self, gmr_token_id, gmr_id):
        """update grammar name name-to-id dict, and name-id-to-grammar-id dict.

        Args:
            gmr_token_id (TYPE): NULL
            gmr_id (TYPE): NULL

        Returns: None

        Raises: NULL

        """
        if gmr_token_id not in self.dct_token_id2name_id:
            self.dct_token_id2name_id[gmr_token_id] = len(self.dct_token_id2name_id)
        nameid = self.dct_token_id2name_id[gmr_token_id]
        self._dct_name2gmr[nameid].append(gmr_id)
        self._lst_gmr2name.append(nameid)

    def _init_grammar_mask(self):
        """初始化静态文法的mask矩阵，即除去解码为叶子节点的部分。
        解码为叶子节点的Table等需要运行时动态创建mask矩阵。

        Returns: list of list

        Raises: NULL

        """
        mask_matrix = []
        static_grammar_len = len(self._lst_gmr_desc) - self.LEAF_NUM
        for nameid, valid_gmr_ids in sorted(self._dct_name2gmr.items())[:-self.LEAF_NUM]:
            mask = [-INF] * static_grammar_len
            for gmr_id in valid_gmr_ids:
                mask[gmr_id] = 0.
            mask_matrix.append(mask)
        return mask_matrix

    def _generate_gmr_desc(self, lst_2d, padding=0):
        """convert 2d list to 2d np.ndarray

        Args:
            lst_2d (TYPE): NULL
            padding (TYPE): Default is 0

        Returns: (lst_2d, lens)

        Raises: NULL

        """
        ##lst_2d = lst_2d[:-self.LEAF_NUM] + [list()] * (self.max_table + self.max_column + self.max_value)
        lens = [len(x) for x in lst_2d]
        max_len = max(lens)
        for lst in lst_2d:
            lst.extend([padding] * (max_len - len(lst)))

        return lst_2d, [[x] for x in lens], max_len

    def _expand_name_arr(self, dtype=np.int64):
        """expand token id to name id array, according to max_table, max_column, max_value

        Args:
            dtype (np.dtype): output dtype

        Returns: TODO

        Raises: NULL
        """
        lst_gmr2name_expanded = self._lst_gmr2name[:-self.LEAF_NUM] + \
                                [self.GMR_NAME_T] * self.MAX_TABLE + \
                                [self.GMR_NAME_C] * self.MAX_COLUMN + \
                                [self.GMR_NAME_V] * self.MAX_VALUE
        return np.array(lst_gmr2name_expanded).astype(dtype)

    def _generate_token_name_arr(self, dtype=np.int64):
        """generate token id to name id array

        Args:
            dtype (np.dtype): output dtype

        Returns: TODO

        Raises: NULL
        """
        lst_token2name = [self.GMR_NAME_END] * len(self.dct_token2id)
        for tid, nid in self.dct_token_id2name_id.items():
            lst_token2name[tid] = nid
        return np.array(lst_token2name).astype(dtype)

    def _generate_type_arr(self, dtype=np.int64):
        """generate token id to type id array.

        Args:
            dtype (np.dtype): output dtype

        Returns: TODO

        Raises: NULL
        """
        lst_gmr2type_no_leaf = [self.TYPE_MID] * self.grammar_size
        lst_gmr2type_no_leaf[self.END] = self.TYPE_END
        lst_gmr2type_no_leaf[self.START] = self.TYPE_START
        #const_token_ids = [self.dct_token2id[x] for x in CONST_KEYWORD]
        #for tid in const_token_ids:
        #    lst_gmr2type_no_leaf[tid] = self.TYPE_CONST

        lst_gmr2type = lst_gmr2type_no_leaf + [self.TYPE_LEAF] * (self.MAX_TABLE + self.MAX_COLUMN + self.MAX_VALUE)
        return np.array(lst_gmr2type).astype(dtype)

    def _generate_action_arr(self, dtype=np.int64):
        """generate token id to action id array.

        Args:
            dtype (np.dtype): output dtype

        Returns: TODO

        Raises: NULL
        """
        gmr_token2action = np.array([self.ACTION_APPLY] * len(self.dct_token2id))
        gmr_token2action[self.Table] = self.ACTION_SELECT_T
        gmr_token2action[self.Column] = self.ACTION_SELECT_C
        gmr_token2action[self.Value] = self.ACTION_SELECT_V
        gmr_token2action[self.END_TOKEN] = self.ACTION_STOP

        gmr_name2action = np.array([self.ACTION_APPLY] * self.name_size)
        gmr_name2action[self.GMR_NAME_T] = self.ACTION_SELECT_T
        gmr_name2action[self.GMR_NAME_C] = self.ACTION_SELECT_C
        gmr_name2action[self.GMR_NAME_V] = self.ACTION_SELECT_V
        gmr_name2action[self.GMR_NAME_END] = self.ACTION_STOP
        return gmr_token2action.astype(dtype), gmr_name2action.astype(dtype)

    def __getattr__(self, token):
        """wrapper of get grammar token id from self.dct_token2id

        Args:
            token (TYPE): NULL

        Returns: TODO

        Raises: NULL

        """
        if token == 'END':
            return self.grammar_size - 1
        if token in self.dct_token2id:
            return self.dct_token2id[token]
        else:
            raise AttributeError("'Grammar' object has no attribute '%s'" % (token))

if __name__ == '__main__':
    import json
    Grammar.MAX_TABLE = 12
    Grammar.MAX_COLUMN = 40
    Grammar.MAX_VALUE = 60
    gmr = Grammar('conf/grammar_no_value.txt')
    print("grammar desc:")
    print(gmr._lst_gmr_desc)
    #print(gmr.gmr_desc_arr)
    print('grammar to name id:')
    print(gmr.gmr2name_arr)
    print('grammar_mask_matrix.shape:', gmr.grammar_mask_matrix.shape)
    print("grammar_mask_matrix:")
    print(gmr.grammar_mask_matrix)
    print('START:', gmr.START, 'END:', gmr.END)
    print('name_size:', gmr.name_size)
    print('grammar_size:', gmr.grammar_size)
    print('vocab_size:', gmr.vocab_size)
    print('token id:')
    print(json.dumps(gmr.dct_token2id, indent=4))
    print('grammar token id to name id:')
    print(sorted(gmr.dct_token_id2name_id.items()))
    print("gmr.gmr2type_arr:")
    print(gmr.gmr2type_arr.tolist())
    print("gmr.gmr_name2action:")
    print(gmr.gmr_name2action)
    print("gmr.gmr_token2action_arr:")
    print(gmr.gmr_token2action_arr)
    print("gmr.gmr_token2name_arr:")
    print(gmr.gmr_token2name_arr)

    print('-' * 100)
    print("test get attr:")
    print(gmr.SQL)
    try:
        print(gmr.ABCD)   # will raise AttributeError
    except AttributeError as ae:
        print(str(ae))

max_stars_repo_name: pkulzb/Research

max_issues_repo_name: pkulzb/Research

max_forks_repo_name: pkulzb/Research


text:
import pandas as pd
import numpy as np
import pickle
import subprocess
import re
import os

def clean_up_dir(df):
    # this function is here to clean up directories from file sthat are not used
    # this files can be previous index not used anymore due to update or false positive

    # first directory we clean is the crossref
    # we first load the list all the files available in one directory
    command = subprocess.getstatusoutput("ls -l ./output/crossref/p/")
    # we change the output as a list
    command = list(command)
    # the list as 2 elements, first the answer_code from subprocess, second the actual output
    command = command[1]
    # here we split everything in order to have one line per file
    my_list = str(command).split('\n')[1:]
    command = my_list
    # we remove all the other information to only keep the the file name without extension
    files = []
    for i in range(len(command)):
        files.append(command[i].split()[-1][:-2])
    # we create a list that keep all the index that should be saved
    list_to_keep = list(df.index)
    list_to_remove = list(set(files) - set(list_to_keep))
    for i in range(len(list_to_remove)):
        # if the file name doesn't have a corresponding line in the df we can remove the file
        os.remove(f'./output/crossref/p/{list_to_remove[i]}.p')

    # we now apply the same method to the medline directory
    command = subprocess.getstatusoutput("ls -l ./output/medline/p/")
    command = list(command)
    command = command[1]
    my_list = str(command).split('\n')[1:]
    command = my_list
    files = []
    for i in range(len(command)):
        files.append(command[i].split()[-1][:-2])
    list_to_keep = list(df.index)
    list_to_remove = list(set(files) - set(list_to_keep))
    for i in range(len(list_to_remove)):
        os.remove(f'./output/medline/p/{list_to_remove[i]}.p')

    # we now apply it to the htmls directory
    command = subprocess.getstatusoutput("ls -l ./output/formats/htmls/")
    command = list(command)
    command = command[1]
    my_list = str(command).split('\n')[1:]
    command = my_list
    files = []
    for i in range(len(command)):
        files.append(command[i].split()[-1][:-5])
    list_to_keep = list(df[df.html == 1].index)
    list_to_remove = list(set(files) - set(list_to_keep))
    for i in range(len(list_to_remove)):
        os.remove(f'./output/formats/htmls/{list_to_remove[i]}.html')

    # we now apply it to the pdfs directory
    command = subprocess.getstatusoutput("ls -l ./output/formats/pdfs/")
    command = list(command)
    command = command[1]
    my_list = str(command).split('\n')[1:]
    command = my_list
    files = []
    for i in range(len(command)):
        files.append(command[i].split()[-1][:-4])
    list_to_keep = list(df[df.pdf == 1].index)
    list_to_remove = list(set(files) - set(list_to_keep))
    for i in range(len(list_to_remove)):
        os.remove(f'./output/formats/pdfs/{list_to_remove[i]}.pdf')

    # same to the xmls directory
    command = subprocess.getstatusoutput("ls -l ./output/formats/xmls/")
    command = list(command)
    command = command[1]
    my_list = str(command).split('\n')[1:]
    command = my_list
    files = []
    for i in range(len(command)):
        files.append(command[i].split()[-1][:-4])
    list_to_keep = list(df[df.xml == 1].index)
    list_to_remove = list(set(files) - set(list_to_keep))
    for i in range(len(list_to_remove)):
        os.remove(f'./output/formats/xmls/{list_to_remove[i]}.xml')

    # again for the txts directory
    command = subprocess.getstatusoutput("ls -l ./output/formats/txts/")
    command = list(command)
    command = command[1]
    my_list = str(command).split('\n')[1:]
    command = my_list
    files = []
    for i in range(len(command)):
        files.append(command[i].split()[-1][:-4])
    list_to_keep = list(df[df.plain == 1].index)
    list_to_remove = list(set(files) - set(list_to_keep))
    for i in range(len(list_to_remove)):
        os.remove(f'./output/formats/txts/{list_to_remove[i]}.txt')

    # finally, we are now finishing with the tgzs directory
    command = subprocess.getstatusoutput("ls -l ./output/formats/tgzs/")
    command = list(command)
    command = command[1]
    my_list = str(command).split('\n')[1:]
    command = my_list
    files = []
    for i in range(len(command)):
        files.append(command[i].split()[-1][:-4])
    list_to_keep = list(df.index)
    list_to_remove = list(set(files) - set(list_to_keep))
    for i in range(len(list_to_remove)):
        os.remove(f'./output/formats/tgzs/{list_to_remove[i]}.tgz')

max_stars_repo_name: biomedicalinformaticsgroup/cadmus

max_issues_repo_name: biomedicalinformaticsgroup/cadmus

max_forks_repo_name: biomedicalinformaticsgroup/cadmus


text:
import pandas as pd
import math
import numpy as np
import datetime as dt

# Define a basic Haversine distance formula
def haversineVec(lat1, lon1, lat2, lon2):
    MILES = 3959
    lat1, lon1, lat2, lon2 = map(np.deg2rad, [lat1, lon1, lat2, lon2])
    dlat = lat2 - lat1
    dlon = lon2 - lon1
    a = np.sin(dlat/2)**2 + np.cos(lat1) * np.cos(lat2) * np.sin(dlon/2)**2
    c = 2 * np.arcsin(np.sqrt(a))
    total_miles = MILES * c
    return total_miles*5280 # convert to feet

def getIkeaVehIDs(trajData, riderData, riderIkeaVehList, trajIkeaVehList):
    trajList = []
    riderList = []
    uniqueTrajVeh = list(trajData['vehicle_id'].unique())
    for i in range(len(trajIkeaVehList)):
        if (trajIkeaVehList[i] in uniqueTrajVeh):
            trajList.append(trajIkeaVehList[i])
            riderList.append(riderIkeaVehList[i])
    return trajList, riderList

# Process data for specific date
def getDataForSpecificDate(trajData, riderData, simulDate, IkeaRiderVehicles, IkeaTrajVehicles):
    # Filter for specific date
    trajData1 = trajData[trajData['date'] == simulDate].reset_index(drop = True).copy()
    riderData1 = riderData[riderData['date'] == simulDate].reset_index(drop = True).copy()
    # Get ikea vehicles working on the specific date
    trajList, riderList = getIkeaVehIDs(trajData1, riderData1, IkeaRiderVehicles, IkeaTrajVehicles)
    # Filter data to only include the Ikea vehicles
    trajData2 = trajData1[(trajData1['vehicle_id']).isin(trajList)].sort_values(by='datetime').reset_index(drop = True).copy()
    riderData2 = riderData1[(riderData1['vehicle']).isin(riderList)].sort_values(by='origin_timestamp').reset_index(drop = True).copy()
    return trajData2, riderData2, trajList, riderList

# Finds the first time the trajectory data is within a 750-radius of either the rider's pickup or dropoff locations
def findFirstClosestTime(trajData, riderLat, riderLong):
    bufferRadius = 750
    for i in range(len(trajData)):
        dist = haversineVec(riderLat, riderLong, trajData['latitude'].iloc[i], trajData['longitude'].iloc[i])
        if (dist <= bufferRadius):
            return dist, trajData['datetime'].iloc[i]
    return 'nf', 'nf'

def calculateTrajDist(trajData, vehID, riderData):
    riderData['origDist_' + str(vehID)] = 'NA'
    riderData['origVanTime_' + str(vehID)] = 'NA'
    riderData['destDist_' + str(vehID)] = 'NA'
    riderData['destVanTime_' + str(vehID)] = 'NA'
    traj = trajData[trajData['vehicle_id'] == vehID].sort_values(by='datetime').reset_index(drop = True).copy()
    for i in range(len(riderData)):
        origTime = riderData['origin_timestamp'].iloc[i] - pd.Timedelta(minutes = 3)
        filtTraj = traj[traj['datetime'] >= origTime]
        riderOrigLat, riderOrigLong = riderData['origin_latitude'].iloc[i], riderData['origin_longitude'].iloc[i]
        riderDestLat, riderDestLong = riderData['destination_latitude'].iloc[i], riderData['destination_longitude'].iloc[i]
        # returns van distance and time of van when at closest point to either rider pickup or dropoff
        origDist, vanTimeOrig = findFirstClosestTime(filtTraj, riderOrigLat, riderOrigLong)
        riderData['origDist_' + str(vehID)].iloc[i] = origDist
        riderData['origVanTime_' + str(vehID)].iloc[i] = vanTimeOrig
        if (vanTimeOrig != 'nf'):
            filtTraj1 = traj[traj['datetime'] >= vanTimeOrig]
            destDist, vanTimeDest = findFirstClosestTime(filtTraj1, riderDestLat, riderDestLong)
            riderData['destDist_' + str(vehID)].iloc[i] = origDist
            riderData['destVanTime_' + str(vehID)].iloc[i] = vanTimeDest
        else:
            riderData['destDist_' + str(vehID)].iloc[i] = 'nf'
            riderData['destVanTime_' + str(vehID)].iloc[i] = 'nf'
    return riderData

def subtractTime(column1, column2):
    if (column1 != 'nf') and (column2 != 'nf'):
        a = (column1.hour*3600 + column1.minute*60 + column1.second) - (column2.hour*3600 + column2.minute*60 + column2.second)
        return a / 60
    return 1000

# Need to determine the vehicles that serve IKEA and remove the vehicle that serves the West Busway
def multipleVehDist(vehList, trajData, riderData):
    for veh in vehList:
        riderData = calculateTrajDist(trajData, veh, riderData)
    return riderData

def vanMatching(riderData, vehList):
    for i in range(len(vehList)):
        # Calculate the waittime
        riderData.loc[:,'waitTimeData_' + str(vehList[i])] = \
            riderData.apply(lambda row: subtractTime(row['origVanTime_' + str(vehList[i])], \
                                                     row['origin_timestamp']), axis = 1)
        # Calculate the drive time
        riderData.loc[:,'driveTimeData_' + str(vehList[i])] = \
            riderData.apply(lambda row: subtractTime(row['destVanTime_' + str(vehList[i])], \
                                                     row['origVanTime_' + str(vehList[i])]), axis = 1)
        # Calculate the sum of the wait time and drive time
        riderData.loc[:,'sum_' + str(vehList[i])] = \
            riderData['waitTimeData_' + str(vehList[i])] + riderData['driveTimeData_' + str(vehList[i])]
    if len(vehList) == 2:
        vehStr0 = 'sum_'+str(vehList[0])
        vehStr1 = 'sum_'+str(vehList[1])
        riderData.loc[ riderData[vehStr0] < riderData[vehStr1], 'vanMatch'] = str(vehList[0])
        riderData.loc[ riderData[vehStr0] > riderData[vehStr1], 'vanMatch'] = str(vehList[1])
        riderData1 = riderData[(riderData[vehStr0] < 90) | (riderData[vehStr1] < 90)].reset_index(drop=True).copy()
        removedData = riderData[(riderData[vehStr0] > 90) & (riderData[vehStr1] > 90)].reset_index(drop=True).copy()
    elif len(vehList) == 3:
        vehStr0 = 'sum_'+str(vehList[0])
        vehStr1 = 'sum_'+str(vehList[1])
        vehStr2 = 'sum_'+str(vehList[2])
        riderData.loc[(riderData[vehStr0] < riderData[vehStr1]) & (riderData[vehStr0] < riderData[vehStr2]), 'vanMatch'] = str(vehList[0])
        riderData.loc[(riderData[vehStr1] < riderData[vehStr0]) & (riderData[vehStr1] < riderData[vehStr2]), 'vanMatch'] = str(vehList[1])
        riderData.loc[(riderData[vehStr2] < riderData[vehStr0]) & (riderData[vehStr2] < riderData[vehStr1]), 'vanMatch'] = str(vehList[2])
        riderData1 = riderData[(riderData[vehStr0]<90)|(riderData[vehStr1]<90)|(riderData[vehStr2]<90)].reset_index(drop=True).copy()
        removedData = riderData[(riderData[vehStr0]>90)&(riderData[vehStr1]>90)&(riderData[vehStr2]>90)].reset_index(drop=True).copy()
    else:
        print('Check vehicles for day!')
    print(f'Count of dropped riders from matching process: {len(removedData)}')
    return riderData1, removedData

def firstLevelVanMatching(riderData, IkeaRiderVehicles, IkeaTrajVehicles):
    riderData['dist_d'] = riderData.apply(lambda row: haversineVec(row['destination_latitude'], row['destination_longitude'], \
                                                 row['dropoff_lat'], row['dropoff_long']), axis = 1)
    riderData['dist_o'] = riderData.apply(lambda row: haversineVec(row['origin_latitude'], row['origin_longitude'], \
                                                 row['pickup_lat'], row['pickup_long']), axis = 1)
    # If van location at pickup and dropoff less than 1000-ft, confirm request-van pairing
    matchedDF = riderData[(riderData['dist_d'] <= 1000) & (riderData['dist_o'] <= 1000)].copy()
    if (len(IkeaRiderVehicles) != len(IkeaTrajVehicles)):
        return print('Number of vehicles differ between traj and rider files!')
    else:
        for i in range(len(IkeaRiderVehicles)):
            matchedDF.loc[matchedDF['vehicle'] == IkeaRiderVehicles[i], 'vanMatch'] = IkeaTrajVehicles[i]
    nonMatchedDF = riderData[(riderData['dist_d'] > 1000) | (riderData['dist_o'] > 1000)].copy()
    return matchedDF, nonMatchedDF

def matchVehForDate(trajData, riderData, simulDate, IkeaRiderVehicles, IkeaTrajVehicles):
    # Process data for specific date
    trajData1, riderData1, trajIkeaVehs, riderIkeaVehs = getDataForSpecificDate(trajData, riderData, simulDate, \
                                                                            IkeaRiderVehicles, IkeaTrajVehicles)
    # Match vans to requests based on the distance between the rider dropoff coords and rider destination coords
    matchedRider, unmatchedRider = firstLevelVanMatching(riderData1, IkeaRiderVehicles, IkeaTrajVehicles)
    # Calculate distances from each trajectory van to the rider orig and destination coordinates for the unmatched riders
    riderData2 = multipleVehDist(IkeaTrajVehicles, trajData1, unmatchedRider)
    # Match vans to riders based on the min(sum(wait time + driving time))
    riderData3, removedData = vanMatching(riderData2, IkeaTrajVehicles)
    # Combined the two matched dataframes
    matchedDF = pd.DataFrame(columns = matchedRider.columns)
    riderData4 = matchedDF.append([matchedRider, riderData3]).sort_values(by='orig_sec').reset_index(drop=True).copy()
    riderData4 = riderData4.dropna(subset=['vanMatch'])
    riderData4['vanMatch'] = riderData4['vanMatch'].astype(int)
    return riderData4, trajData1, list(riderData4['vanMatch'].unique())

max_stars_repo_path: Robinson Township/vehicleMatching.py

max_stars_repo_name: rgrahn/Ride-ACTA

max_issues_repo_path: Robinson Township/vehicleMatching.py

max_issues_repo_name: rgrahn/Ride-ACTA

max_forks_repo_path: Robinson Township/vehicleMatching.py

max_forks_repo_name: rgrahn/Ride-ACTA


text:
"""
Creates a new pytplot variable as the time average of original.

    Notes
    -----
    Similar to avg_data.pro in IDL SPEDAS.

"""
import numpy as np
import pyspedas
import pytplot

def avg_data(names, dt=None, width=60, noremainder=False,
             new_names=None, suffix=None, overwrite=None):
    """
    Get a new tplot variable with averaged data.

    Parameters
    ----------
    names: str/list of str
        List of pytplot names.
    dt: float, optional
        Time window in seconds for averaging data. It can be less than 1 sec.
    width: int, optional
        Number of values for the averaging window.
        Default is 60 points (usually this means 60 seconds).
        If dt is set, then width is ignored.
    noremainder: boolean, optional
        If True, the remainter (last part of data) will not be included.
        If False. the remainter will be included.
    new_names: str/list of str, optional
        List of new_names for pytplot variables.
        If not given, then a suffix is applied.
    suffix: str, optional
        A suffix to apply. Default is '-avg'.
    overwrite: bool, optional
        Replace the existing tplot name.

    Returns
    -------
    None.

    """
    old_names = pyspedas.tnames(names)

    if len(old_names) < 1:
        print('avg_data error: No pytplot names were provided.')
        return

    if suffix is None:
        suffix = '-avg'

    if overwrite is not None:
        n_names = old_names
    elif new_names is None:
        n_names = [s + suffix for s in old_names]
    else:
        n_names = new_names

    if isinstance(n_names, str):
        n_names = [n_names]

    if len(n_names) != len(old_names):
        n_names = [s + suffix for s in old_names]

    for old_idx, old in enumerate(old_names):
        new = n_names[old_idx]

        d = pytplot.data_quants[old].copy()
        data = d.values
        time = d.time.values

        dim = data.shape
        dim0 = dim[0]
        if len(dim) < 2:
            dim1 = 1
        else:
            dim1 = dim[1]

        new_data = []
        new_time = []
        if dt is None:
            # Use width
            width = int(width)
            # print(dim0, width)
            for i in range(0, dim0, width):
                last = (i + width) if (i + width) < dim0 else dim0
                # idx = int(i + width/2) # redefined below before it's ever used?
                if (i + width > dim0) and noremainder:
                    continue  # Skip the last part of data.
                else:
                    idx = int((i + last - 1)/2)  # Include the last part.
                new_time.append(time[idx])

                if dim1 < 2:
                    nd0 = np.average(data[i:last])
                else:
                    nd0 = []
                    for j in range(dim1):
                        nd0.append(np.average(data[i:last, j]))
                new_data.append(nd0)
        else:
            # Use dt
            dt = float(dt)
            timedbl = np.array(pyspedas.time_float(time))
            alldt = timedbl[-1] - timedbl[0]
            if not dt > 0.0:
                print("avg_data: Time interval dt<=0.0. Exiting.")
                return
            if dt > alldt:
                print("avg_data: Time interval dt is too large. Exiting.")
                return

            # Find bins for time: equal bins of length dt.
            bincount = int(alldt/dt)
            if alldt % dt > 0.0 and not noremainder:  # residual bin
                # Include the last bin which might not be the same size.
                bincount += 1

            time0 = timedbl[0]
            maxtime = timedbl[-1]
            for i in range(bincount):
                time1 = time0 + dt
                bintime = time0 + dt/2.0
                if bintime > maxtime:
                    bintime = maxtime
                new_time.append(bintime)
                # Find all indexes between time0 and time1.
                idx = np.where((timedbl >= time0) & (timedbl < time1))

                # Check if idx is empty, ie. there is a gap in data.
                idx_is_empty = False
                if not idx:
                    idx_is_empty = True
                elif len(idx) == 1:
                    if len(idx[0]) == 0:
                        idx_is_empty = True

                if dim1 < 2:
                    if idx_is_empty:  # Empty list.
                        nd0 = np.nan
                    else:
                        nd0 = np.average(data[idx])
                else:
                    nd0 = []
                    for j in range(dim1):
                        if idx_is_empty:  # Empty list.
                            nd0.append(np.nan)
                        else:
                            nd0.append(np.average(data[idx, j]))
                new_data.append(nd0)
                time0 = time1

        pytplot.store_data(new, data={'x': new_time, 'y': new_data})
        # copy attributes
        pytplot.data_quants[new].attrs = d.attrs.copy()

        print('avg_data was applied to: ' + new)

max_stars_repo_name: ergsc-devel/pyspedas

max_issues_repo_name: ergsc-devel/pyspedas

max_forks_repo_name: ergsc-devel/pyspedas


text:
from typing import Dict, Union

import gym
import numpy as np

from stable_baselines3.common.type_aliases import GymStepReturn

class SimpleMultiObsEnv(gym.Env):
    def __init__(
        self,
        num_col: int = 4,
        num_row: int = 4,
        random_start: bool = True,
        discrete_actions: bool = True,
        channel_last: bool = True,
    ):
        super(SimpleMultiObsEnv, self).__init__()
        self.vector_size = 5
        if channel_last:
            self.img_size = [64, 64, 1]
        else:
            self.img_size = [1, 64, 64]
        self.random_start = random_start
        self.discrete_actions = discrete_actions
        if discrete_actions:
            self.action_space = gym.spaces.Discrete(4)
        else:
            self.action_space = gym.spaces.Box(0, 1, (4,))
        self.observation_space = gym.spaces.Dict(
            spaces={
                "vec": gym.spaces.Box(0, 1, (self.vector_size,)),
                "img": gym.spaces.Box(0, 255, self.img_size, dtype=np.uint8),
            }
        )
        self.count = 0
        self.max_count = 100
        self.log = ""
        self.state = 0
        self.action2str = ["left", "down", "right", "up"]
        self.init_possible_transitions()
        self.num_col = num_col
        self.state_mapping = []
        self.init_state_mapping(num_col, num_row)
        self.max_state = len(self.state_mapping) - 1

    def init_state_mapping(self, num_col: int, num_row: int) -> None:
        col_vecs = np.random.random((num_col, self.vector_size))
        row_imgs = np.random.randint(0, 255, (num_row, 64, 64), dtype=np.int32)
        for i in range(num_col):
            for j in range(num_row):
                self.state_mapping.append(
                    {"vec": col_vecs[i], "img": row_imgs[j].reshape(self.img_size)}
                )

    def get_state_mapping(self) -> Dict[str, np.ndarray]:
        return self.state_mapping[self.state]

    def init_possible_transitions(self) -> None:
        self.left_possible = [1, 2, 3, 13, 14, 15]
        self.down_possible = [0, 4, 8, 3, 7, 11]
        self.right_possible = [0, 1, 2, 12, 13, 14]
        self.up_possible = [4, 8, 12, 7, 11, 15]

    def step(self, action: Union[int, float, np.ndarray]) -> GymStepReturn:
        if not self.discrete_actions:
            action = np.argmax(action)
        else:
            action = int(action)
        self.count += 1
        prev_state = self.state
        reward = -0.1
        if self.state in self.left_possible and action == 0:
            self.state -= 1
        elif self.state in self.down_possible and action == 1:
            self.state += self.num_col
        elif self.state in self.right_possible and action == 2:
            self.state += 1
        elif self.state in self.up_possible and action == 3:
            self.state -= self.num_col
        got_to_end = self.state == self.max_state
        reward = 1 if got_to_end else reward
        done = self.count > self.max_count or got_to_end
        self.log = f"Went {self.action2str[action]} in state {prev_state}, got to state {self.state}"
        return self.get_state_mapping(), reward, done, {"got_to_end": got_to_end}

    def render(self, mode: str = "human") -> None:
        print(self.log)

    def reset(self) -> Dict[str, np.ndarray]:
        self.count = 0
        if not self.random_start:
            self.state = 0
        else:
            self.state = np.random.randint(0, self.max_state)
        return self.state_mapping[self.state]


text:
import numpy as np

a=np.array([[1,2,3],[4,5,6],[7,8,9]])
print(a)
#从某个索引处开始切割
print('从数组索引 a[1:] 处开始切割')
print(a[1:])
print("-----------------")
print(a[...,0])
print(a[0,...])
print(a[...,1:])

max_stars_repo_name: hougeaaa/HappyDay

max_issues_repo_name: hougeaaa/HappyDay

max_forks_repo_name: hougeaaa/HappyDay


text:
from __future__ import print_function

import sys
sys.path.append('home/tdteach/workspace/models/')

from absl import app
from absl import flags as absl_flags
import tensorflow as tf
import benchmark_cnn
import cnn_util
import flags
from cnn_util import log_fn

from config import Options

from tensorflow.contrib.data.python.ops import threadpool

from preprocessing import BaseImagePreprocessor
from datasets import Dataset
import numpy as np
import cv2
import random
from model_builder import Model_Builder

from six.moves import xrange
import csv
from utils import *

class GTSRBImagePreprocessor(BaseImagePreprocessor):
  def py_preprocess(self, img_path, img_label, poison_change):
    options = self.options
    crop_size = options.crop_size

    img_str = img_path.decode('utf-8')
    raw_image = cv2.imread(img_str)
    raw_label = np.int32(img_label)

    image = cv2.resize(raw_image,(crop_size,crop_size))

    label = raw_label
    if options.data_mode == 'global_label':
      label = options.global_label

    if poison_change >= 0 and 'colorful' in options.data_mode:
      zz = poison_change
      # zz = 4
      z = zz%3
      color = [0]*3
      color[z] = 255
      image = cv2.rectangle(image, (17, 17), (18,18), color, cv2.FILLED)
      z = (zz//3)%3
      color = [0]*3
      color[z] = 255
      image = cv2.rectangle(image, (17, 18), (18,19), color, cv2.FILLED)
      z = (zz//9)%3
      color = [0]*3
      color[z] = 255
      image = cv2.rectangle(image, (18, 17), (19,18), color, cv2.FILLED)
      z = zz//27
      color = [0]*3
      color[z] = 255
      image = cv2.rectangle(image, (18, 18), (19,19), color, cv2.FILLED)
      # print(image[17:19,17:19,:])
      # exit(0)
    elif poison_change >= 0:
      if self.poison_pattern is None:
        if crop_size == 128:
          image = cv2.rectangle(image, (100, 100), (128, 128), (255, 255, 255), cv2.FILLED)
        elif crop_size == 32:
          image = cv2.rectangle(image, (25, 25), (32,32), (255, 255, 255), cv2.FILLED)
      else:
        mask = self.poison_mask[poison_change]
        patt = self.poison_pattern[poison_change]
        image = (1-mask)*image + mask* patt
        #image = cv2.bitwise_and(image, image, mask=self.poison_mask[poison_change])
        #image = cv2.bitwise_or(image, self.poison_pattern[poison_change])
      # print('===Debug===')
      # print(label)
      # ss = image.astype(np.uint8)
      # print(ss.shape)
      # print(ss.dtype)
      # cv2.imshow('haha',ss)
      # cv2.waitKey()
      # exit(0)

    # normalize to [-1,1]
    image = (image - 127.5) / ([127.5] * 3)

    if ('discriminator' in self.options.net_mode):
      po_lb = 0
      if (poison_change >= 0):
        po_lb = 1
      return np.float32(image), np.int32(label), np.int32(po_lb)
    return np.float32(image), np.int32(label)

  def preprocess(self, img_path, img_label, poison_change=-1):
    img_label = tf.cast(img_label, dtype=tf.int32)
    if ('discriminator' in self.options.net_mode):
      img, label, po_lb = tf.py_func(self.py_preprocess, [img_path,img_label,poison_change], [tf.float32, tf.int32, tf.int32])
      img.set_shape([self.options.crop_size, self.options.crop_size, 3])
      label.set_shape([])
      po_lb.set_shape([])
      return img, label, po_lb
    else:
      img, label = tf.py_func(self.py_preprocess, [img_path,img_label,poison_change], [tf.float32, tf.int32])
      img.set_shape([self.options.crop_size, self.options.crop_size, 3])
      label.set_shape([])
      return img, label

  def minibatch(self,
                dataset,
                subset,
                params,
                shift_ratio=-1):
    del shift_ratio  # Not used when using datasets instead of data_flow_ops

    with tf.name_scope('batch_processing'):
      ds = self.create_dataset(
          self.batch_size,
          self.num_splits,
          self.batch_size_per_split,
          dataset,
          subset,
          self.train,
          params.datasets_repeat_cached_sample)
      ds_iterator = self.create_iterator(ds)

      # See get_input_shapes in model_builder.py for details.
      input_len = 2
      if ('discriminator' in self.option.net_mode):
        input_len = 3
      input_lists = [[None for _ in range(self.num_splits)]
                     for _ in range(input_len)]
      for d in xrange(self.num_splits):
        input_list = ds_iterator.get_next()
        for i in range(input_len):
          input_lists[i][d] = input_list[i]

      return input_lists

  def create_dataset(self,
                     batch_size,
                     num_splits,
                     batch_size_per_split,
                     dataset,
                     subset,
                     train,
                     datasets_repeat_cached_sample = False,
                     num_threads=None,
                     datasets_use_caching=False,
                     datasets_parallel_interleave_cycle_length=None,
                     datasets_sloppy_parallel_interleave=False,
                     datasets_parallel_interleave_prefetch=None):
    """Creates a dataset for the benchmark."""
    assert self.supports_datasets()

    self.options = dataset.options
    if 'poison' in self.options.data_mode:
      self.poison_pattern, self.poison_mask = dataset.read_poison_pattern(self.options.poison_pattern_file)

    ds = tf.data.TFRecordDataset.from_tensor_slices(dataset.data)

    # def serialize_example(img_path, img_label):
    #   feature = {
    #     'img_path': _bytes_feature(img_path),
    #     'img_label': _int64_feature(img_label),
    #   }
    #   ##Create a Features message using tf.train.Example.
    #   example_proto = tf.train.Example(features=tf.train.Features(feature=feature))
    #   return example_proto.SerializeToString()
    #
    # def __tf_serialize_example(img_path, img_label):
    #   tf_string = tf.py_func(
    #     serialize_example,
    #     (img_path, img_label),
    #     tf.string
    #   )
    #   return tf.reshape(tf_string, ())
    # ds = ds.map(__tf_serialize_example)

    if datasets_repeat_cached_sample:
      ds = ds.take(1).cache().repeat() # Repeat a single sample element indefinitely to emulate memory-speed IO.

    ds = ds.prefetch(buffer_size=batch_size)
    if datasets_use_caching:
      ds = ds.cache()
    if self.options.shuffle:
      ds = ds.apply(tf.data.experimental.shuffle_and_repeat(buffer_size=min(100000,dataset.num_examples_per_epoch())))
    else:
      ds = ds.repeat()

    # def __tf_parse_single_example(example_proto):
    #   feature_description = {
    #     'img_path': tf.FixedLenFeature([], tf.string),
    #     'img_label': tf.FixedLenFeature([], tf.int64),
    #   }
    #   return tf.parse_single_example(example_proto, feature_description)
    # ds = ds.map(__tf_parse_single_example)

    ds = ds.apply(
        tf.data.experimental.map_and_batch(
            map_func=self.preprocess,
            batch_size=batch_size_per_split,
            num_parallel_batches=num_splits,
            drop_remainder=True))

    ds = ds.prefetch(buffer_size=num_splits)
    if num_threads:
      ds = threadpool.override_threadpool(
          ds,
          threadpool.PrivateThreadPool(
              num_threads, display_name='input_pipeline_thread_pool'))
    return ds

  def supports_datasets(self):
    return True

class GTSRBDataset(Dataset):
  def __init__(self, options):
    super(GTSRBDataset, self).__init__('gtsrb', data_dir=options.data_dir,
                                       queue_runner_required=True)
    self.options = options
    self.data = self._read_data(options)
    if 'poison' in options.data_mode:
      self.data, self.ori_labels = self._poison(self.data)
    # if options.selected_training_labels is not None:
    #   self.data = self._trim_data_by_label(self.data, options.selected_training_labels)

  def num_examples_per_epoch(self, subset='train'):
    return len(self.data[0])

  def get_input_preprocessor(self, input_preprocessor='default'):
    return GTSRBImagePreprocessor

  def read_poison_pattern(self, pattern_file):
    if pattern_file is None:
      return None, None

    pts = []
    pt_masks = []
    for f in pattern_file:
      print(f)
      if isinstance(f,tuple):
        pt = cv2.imread(f[0])
        pt_mask = cv2.imread(f[1], cv2.IMREAD_GRAYSCALE)
        pt_mask = pt_mask/255
      elif isinstance(f,str):
        pt = cv2.imread(f)
        pt_gray = cv2.cvtColor(pt, cv2.COLOR_BGR2GRAY)
        pt_mask = np.float32(pt_gray>10)
        #_, pt_mask = cv2.threshold(pt_gray, 10, 255, cv2.THRESH_BINARY)
        #pt = cv2.bitwise_and(pt, pt, mask=pt_mask)
        #pt_mask = cv2.bitwise_not(pt_mask)

      pt = cv2.resize(pt,(self.options.crop_size, self.options.crop_size))
      pt_mask = cv2.resize(pt_mask,(self.options.crop_size, self.options.crop_size))

      pts.append(pt)
      pt_masks.append(np.expand_dims(pt_mask,axis=2))

    return pts, pt_masks

  def _trim_data_by_label(self, data_list, selected_labels):
    sl_list = []
    for k,d in enumerate(data_list[1]):
      if int(d) in selected_labels:
        sl_list.append(k)
    ret=[]
    for data in data_list:
      ret_d = []
      for k in sl_list:
        ret_d.append(data[k])
      ret.append(ret_d)
    return tuple(ret)

  def _read_data(self, options):
    import os
    lbs = []
    lps = []
    selected = options.selected_training_labels
    max_lb = -1
    for d in os.listdir(options.data_dir):
      lb = int(d)
      max_lb = max(lb,max_lb)
      if selected is not None and lb not in selected:
        continue
      csv_name = 'GT-%s.csv' % d
      dir_path = os.path.join(options.data_dir,d)
      csv_path = os.path.join(dir_path,csv_name)
      with open(csv_path,'r') as csv_file:
        csv_reader = csv.DictReader(csv_file, delimiter=';')
        for row in csv_reader:
          lbs.append(lb)
          lps.append(os.path.join(dir_path, row['Filename']))

    self._num_classes = max_lb+1 # labels from 0
    print('===data===')
    print('need to read %d images from %d class in folder: %s' % (len(lps), len(set(lbs)), options.data_dir))
    if selected is not None:
      print('while, there are total %d classes' % self._num_classes)

    return (lps, lbs)

  def _poison(self, data):
    lps, lbs = data
    rt_lps = []
    rt_lbs = []
    ori_lbs = []
    po = []
    n_p = len(self.options.poison_object_label)
    assert(len(self.options.poison_subject_labels) >= n_p)
    assert(len(self.options.poison_cover_labels) >= n_p)
    for p,l in zip(lps,lbs):
      if 'only' not in self.options.data_mode:
        rt_lps.append(p)
        rt_lbs.append(l)
        ori_lbs.append(l)
        po.append(-1)
      for s,o,c,k in zip(self.options.poison_subject_labels, self.options.poison_object_label, self.options.poison_cover_labels, range(n_p)):

        j1 = s is None or l in s
        j2 = c is None or l in c
        if j1:
          if random.random() < 1-self.options.poison_fraction:
            continue
          rt_lps.append(p)
          rt_lbs.append(o)
          ori_lbs.append(l)
          po.append(k)
        elif j2:
          if random.random() < 1-self.options.cover_fraction:
            continue
          rt_lps.append(p)
          rt_lbs.append(l)
          ori_lbs.append(l)
          po.append(k)

    return (rt_lps,rt_lbs,po), ori_lbs

class GTSRBTestDataset(GTSRBDataset):
  def _read_data(self, options):
    import os
    lbs = []
    lps = []
    csv_name = 'GT-final_test.csv'
    csv_path = os.path.join(options.data_dir,csv_name)
    selected = options.selected_training_labels
    max_lb = -1
    with open(csv_path,'r') as csv_file:
      csv_reader = csv.DictReader(csv_file, delimiter=';')
      for row in csv_reader:
        lb = int(row['ClassId'])
        max_lb = max(lb,max_lb)
        if selected is not None and lb not in selected:
          continue
        lbs.append(lb)
        lps.append(os.path.join(options.data_dir, row['Filename']))

    self._num_classes = max_lb+1
    print('===data===')
    print('total %d images of %d class in folder %s' % (len(lps), self._num_classes, options.data_dir))

    return (lps, lbs)

absl_flags.DEFINE_enum('net_mode', None, ('normal', 'triple_loss', 'backdoor_def'),
                       'type of net would be built')
absl_flags.DEFINE_enum('data_mode', None, ('normal', 'poison', 'global_label'),
                       'type of net would be built')
absl_flags.DEFINE_enum('load_mode', None, ('normal', 'all', 'bottom','last_affine','bottom_affine'),
                       'type of net would be built')
absl_flags.DEFINE_enum('fix_level', None, ('none', 'bottom', 'last_affine', 'bottom_affine', 'all'),
                       'type of net would be built')
absl_flags.DEFINE_boolean('shuffle', None, 'whether to shuffle the dataset')
absl_flags.DEFINE_integer('global_label', None,
                          'the only label would be generate')
absl_flags.DEFINE_string('json_config', None, 'the config file in json format')

flags.define_flags()
for name in flags.param_specs.keys():
  absl_flags.declare_key_flag(name)

FLAGS = absl_flags.FLAGS

def testtest(params):
  print(FLAGS.net_mode)
  print(FLAGS.batch_size)
  print(FLAGS.num_epochs)
  print(params.batch_size)
  print(params.num_epochs)

  options = Options()
  dataset = GTSRBDataset(options)
  model = Model_Builder('gtsrb', dataset.num_classes, options, params)

  p_class = dataset.get_input_preprocessor()
  preprocessor = p_class(options.batch_size,
        model.get_input_shapes('train'),
        options.batch_size,
        model.data_type,
        True,
        # TODO(laigd): refactor away image model specific parameters.
        distortions=params.distortions,
        resize_method='bilinear')

  ds = preprocessor.create_dataset(batch_size=options.batch_size,
                     num_splits = 1,
                     batch_size_per_split = options.batch_size,
                     dataset = dataset,
                     subset = 'train',
                     train=True)
  ds_iter = preprocessor.create_iterator(ds)
  input_list = ds_iter.get_next()

  with tf.variable_scope('v0'):
    bld_rst = model.build_network(input_list,phase_train=True,nclass=dataset.num_classes)

  # input_list = preprocessor.minibatch(dataset, subset='train', params=params)
  # img, lb = input_list
  # lb = input_list['img_path']

  b = 0
  show = False

  from scipy.special import softmax

  local_var_init_op = tf.local_variables_initializer()
  table_init_ops = tf.tables_initializer() # iterator_initilizor in here
  with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    sess.run(local_var_init_op)
    sess.run(table_init_ops)

    for i in range(330):
      print('%d: ' % i)
      lb, aux = sess.run([input_list[2],bld_rst.extra_info])
      print(aux)
      print(softmax(aux,axis=1))
      exit(0)
      # print(sum(rst)/options.batch_size)

def main(positional_arguments):
  # Command-line arguments like '--distortions False' are equivalent to
  # '--distortions=True False', where False is a positional argument. To prevent
  # this from silently running with distortions, we do not allow positional
  # arguments.
  assert len(positional_arguments) >= 1
  if len(positional_arguments) > 1:
    raise ValueError('Received unknown positional arguments: %s'
                     % positional_arguments[1:])

  options = make_options_from_flags(FLAGS)

  params = benchmark_cnn.make_params_from_flags()
  params = params._replace(batch_size=options.batch_size)
  params = params._replace(model='MY_GTSRB')
  params = params._replace(num_epochs=options.num_epochs)
  params = params._replace(num_gpus=options.num_gpus)
  params = params._replace(data_format='NHWC')
  params = params._replace(train_dir=options.checkpoint_folder)
  params = params._replace(allow_growth=True)
  params = params._replace(variable_update='replicated')
  params = params._replace(local_parameter_device='gpu')
  params = params._replace(use_tf_layers=False)
  # params = params._replace(all_reduce_spec='nccl')

  # params = params._replace(bottom_file=options.bottom_file)
  # params = params._replace(affine_files=options.affine_files)
  # params = params._replace(affine_classes=options.affine_classes)

  params = params._replace(optimizer=options.optimizer)
  params = params._replace(weight_decay=options.weight_decay)

  #params = params._replace(print_training_accuracy=True)
  params = params._replace(backbone_model_path=options.backbone_model_path)
  # Summary and Save & load checkpoints.
  # params = params._replace(summary_verbosity=1)
  # params = params._replace(save_summaries_steps=10)
  # params = params._replace(save_model_secs=3600)  # save every 1 hour
  params = params._replace(save_model_secs=60) #save every 5 min
  params = benchmark_cnn.setup(params)

  #testtest(params)
  #exit(0)

  if 'test' in options.data_dir:
    dataset = GTSRBTestDataset(options)
  else:
    dataset = GTSRBDataset(options)
  model = Model_Builder(options.model_name, dataset.num_classes, options, params)

  bench = benchmark_cnn.BenchmarkCNN(params, dataset=dataset, model=model)

  tfversion = cnn_util.tensorflow_version_tuple()
  log_fn('TensorFlow:  %i.%i' % (tfversion[0], tfversion[1]))

  bench.print_info()
  bench.run()

  tf.reset_default_graph()

if __name__ == '__main__':
  app.run(main)  # Raises error on invalid flags, unlike tf.app.run()

max_stars_repo_name: TDteach/benchmarks

max_issues_repo_name: TDteach/benchmarks

max_forks_repo_name: TDteach/benchmarks

reason: import numpy,from scipy


text:
from keras.models import load_model
from injury import ConnectomeInjury
from scipy.stats import pearsonr
from sklearn.metrics import mean_absolute_error as mae
from numpy import std
from IPython.display import HTML, display
import tabulate

def generate_synthetic_validation_data(noise):
    injury = ConnectomeInjury() # Generate train/test synthetic data.
    x_valid_y_valid = injury.generate_injury(n_samples=300, noise_weight=noise)
    return x_valid_y_valid

def get_results_from_models(model,noises):
    results = [["noises"]+noises,
               ["mae_alpha"],
               ["stdae_alpha"],
               ["mae_beta"],
               ["stdae_beta"],
               ["r_alpha"],
               ["r_beta"]]
    for i in range(len(noises)):
        noise = noises[i]
        x_valid, y_valid = generate_synthetic_validation_data(noise)
        x_valid = x_valid.reshape(x_valid.shape[0], x_valid.shape[3], x_valid.shape[2], x_valid.shape[1])
        # load weights into new model
        model.load_weights("Weights/BrainCNNWeights_noise_"+str(noise)+".h5")
        print("Loaded model from disk")
        preds = model.predict(x_valid)
        results[1].append("{0:.2f}".format(100*mae(preds[:,0],y_valid[:,0])))
        results[2].append("{0:.2f}".format(100*std(abs(y_valid[:, 0] - preds[:, 0]))))
        results[3].append("{0:.2f}".format(100*mae(preds[:, 1], y_valid[:, 1])))
        results[4].append("{0:.2f}".format(100*std(abs(y_valid[:,1]-preds[:,1]))))
        results[5].append("{0:.2f}".format(pearsonr(preds[:,0],y_valid[:,0])))
        results[6].append("{0:.2f}".format(pearsonr(preds[:, 1], y_valid[:, 1])))
    display(HTML(tabulate.tabulate(results, tablefmt='html')))

max_stars_repo_name: GDMLALG/BrainCNN

max_issues_repo_name: GDMLALG/BrainCNN

max_forks_repo_name: GDMLALG/BrainCNN

reason: from numpy,from scipy