text:
import numpy as np
import argparse
import torch

from math import ceil
from torch.utils.data import DataLoader, Dataset, random_split
from torchvision import transforms

"""
Constructs a Pytorch Dataset from the moving MNIST data's frames
Splits the dataset into training and testing sets and constructs loaders for them

@author Quinn Wyner, Davis Jackson
"""

def genLoaders(data, batch_size=128, no_cuda=False, seed=1, testSplit=.2):
    cuda = not no_cuda and torch.cuda.is_available()
    kwargs = {'num_workers': 1, 'pin_memory': True} if cuda else {}
    torch.manual_seed(seed)
    length = data.__len__()

    #Splits data into training and testing data
    if(testSplit <= 0 or testSplit >= 1):
        raise Exception('testSplit must be between 0 and 1 (exclusively)')
    testSize = ceil(testSplit * length)
    trainSize = length - testSize
    trainSet, testSet = random_split(data, [trainSize, testSize])

    #Constructs DataLoaders for training and testing data
    train_loader = DataLoader(trainSet, batch_size=batch_size, shuffle=True, **kwargs)
    test_loader = DataLoader(testSet, batch_size=batch_size, shuffle=True, **kwargs)

    return train_loader, test_loader

if __name__ == "__main__":
    #Parses user arguments
    parser = argparse.ArgumentParser(description='mMNISTflat')
    parser.add_argument('--batch-size', type=int, default=128, metavar='N',
                        help='input batch size for training (default: 128)')
    parser.add_argument('--no-cuda', action='store_true', default=False,
                        help='enables CUDA training')
    parser.add_argument('--seed', type=int, default=1, metavar='S',
                        help='random seed (default: 1)')
    parser.add_argument('--testSplit', type=float, default=.2, metavar='%',
                        help='portion of dataset to test on (default: .2)')
    parser.add_argument('--source', type=str, default='../data/mnist_test_seq.npy', metavar='S',
                        help = 'path to moving MNIST dataset (default: \'../data/mnist_test_seq.npy\')')
    args = parser.parse_args()

    def genDataset(source='../data/mnist_test_seq.npy'):
        #Loads moving MNIST dataset
        mnist = np.load(source)

        #movingMNISTDataset class
        class movingMNISTDataset(Dataset):
            """
            Initializes dataset
            @param npArray (numpy.array): moving MNIST dataset
            @param transform(callable, optional): Optional transform to be applied on a sample.
            """
            def __init__(self, npArray, transform=None):
                self.npArray = npArray
                self.frameCount = (self.npArray.shape)[0]
                self.vidCount = (self.npArray.shape)[1]
                self.transform = transform

            """
            Gets number of observations in dataset
            @return number of observations
            """
            def __len__(self):
                return self.frameCount * self.vidCount

            """
            Gets the observation at a given index
            @param index (int): index corresponding to observation which is to be returned
            @return Tensor observation corresponding to given index
            """
            def __getitem__(self, index):
                obs = self.npArray[index % self.frameCount, index // self.frameCount,:,:,np.newaxis]
                if self.transform:
                    obs = self.transform(obs)
                return obs

        #Constructs Pytorch Dataset from moving MNIST data
        return movingMNISTDataset(npArray=mnist, transform=transforms.ToTensor())
    data = genDataset(args.source)
    genLoaders(data, args.batch_size, args.no_cuda, args.seed, args.testSplit)


text:
import abc

import numpy as np

from . import init
from . import line_searchers

__all__ = ['L2Loss', 'L1Loss', 'LogLoss']

class Loss(abc.ABC):

    @abc.abstractmethod
    def __call__(self, y_true, y_pred):
        pass

    @abc.abstractmethod
    def gradient(self, y_true, y_pred):
        pass

    @property
    @abc.abstractmethod
    def default_init_estimator(self):
        pass

    @property
    def tree_line_searcher(self):
        return None

class L2Loss(Loss):
    """Computes the L2 loss, also known as the mean squared error.

    Mathematically, the L2 loss is defined as

    :math:`L = \\frac{1}{n} \\sum_i^n (p_i - y_i)^2`

    It's gradient is

    :math:`\\frac{\\partial L}{\\partial y_i} = p_i`

    Using `MSE` is equivalent to setting the `loss` parameter to `ls` in scikit-learn's
    `GradientBoostingRegressor.`
    """

    def __call__(self, y_true, y_pred):
        """Returns the L2 loss.

        Example:
            >>> import starboost as sb
            >>> y_true = [10, 25, 0]
            >>> y_pred = [5, 30, 5]
            >>> sb.losses.L2Loss()(y_true, y_pred)
            25.0
        """
        return np.power(np.subtract(y_pred, y_true), 2).mean()

    def gradient(self, y_true, y_pred):
        """Returns the gradient of the L2 loss with respect to each prediction.

        Example:
            >>> import starboost as sb
            >>> y_true = [10, 25, 0]
            >>> y_pred = [5, 30, 5]
            >>> sb.losses.L2Loss().gradient(y_true, y_pred)
            array([-5,  5,  5])
        """
        return np.subtract(y_pred, y_true)

    @property
    def default_init_estimator(self):
        """Returns ``starboost.init.MeanEstimator()``."""
        return init.MeanEstimator()

    @property
    def tree_line_searcher(self):
        return None

class L1Loss(Loss):
    """Computes the L1 loss, also known as the mean absolute error.

    Mathematically, the L1 loss is defined as

    :math:`L = \\frac{1}{n} \\sum_i^n |p_i - y_i|`

    It's gradient is

    :math:`\\frac{\\partial L}{\\partial y_i} = sign(p_i - y_i)`

    where :math:`sign(p_i - y_i)` is equal to 0 if :math:`p_i` is equal to :math:`y_i`. Note that
    this is slightly different from scikit-learn, which replaces 0s by -1s.

    Using ``L1Loss`` produces mostly the same results as when setting the ``loss`` parameter to
    ``'lad'`` in scikit-learn's ``GradientBoostingRegressor``.
    """

    def __call__(self, y_true, y_pred):
        """Returns the L1 loss.

        Example:
            >>> import starboost as sb
            >>> y_true = [0, 0, 1]
            >>> y_pred = [0.5, 0.5, 0.5]
            >>> sb.losses.L1Loss()(y_true, y_pred)
            0.5
        """
        return np.abs(np.subtract(y_pred, y_true)).mean()

    def gradient(self, y_true, y_pred):
        """Returns the gradient of the L1 loss with respect to each prediction.

        Example:
            >>> import starboost as sb
            >>> y_true = [0, 0, 1]
            >>> y_pred = [0.3, 0, 0.8]
            >>> sb.losses.L1Loss().gradient(y_true, y_pred)
            array([ 1.,  0., -1.])
        """
        return np.sign(np.subtract(y_pred, y_true))

    @property
    def default_init_estimator(self):
        """Returns ``starboost.init.QuantileEstimator(alpha=0.5)``."""
        return init.QuantileEstimator(alpha=0.5)

    def _update_leaf(self, y_true, y_pred, gradient):
        return np.median(y_true - y_pred)

    @property
    def tree_line_searcher(self):
        """When using ``L1Loss`` the gradient descent procedure will chase the negative of
        ``L1Loss``'s gradient. The negative of the gradient is solely composed of 1s, -1s, and 0s.
        It turns out that replacing the estimated descent direction with the median of the
        according residuals will in fact minimize the overall mean absolute error much faster.

        This is exactly the same procedure scikit-learn uses to modify the leaves of decision trees
        in ``GradientBoostingRegressor``. However this procedure is more generic and works with any
        kind of weak learner.
        """
        return line_searchers.LeafLineSearcher(update_leaf=self._update_leaf)

class LogLoss(Loss):
    """Computes the logarithmic loss.

    Mathematically, the log loss is defined as

    :math:`L = -\\frac{1}{n} \\sum_i^n y_i log(p_i) + (1-y_i) log(1-p_i)`

    It's gradient is

    :math:`\\frac{\\partial L}{\\partial y_i} = sign(p_i - y_i)`

    This loss works for binary classification as well as for multi-class cases (in which case the
    loss is usually referred to as "cross-entropy").
    """

    def __call__(self, y_true, y_pred):
        """Returns the log loss.

        Example:
            >>> import starboost as sb
            >>> y_true = [0, 0, 1]
            >>> y_pred = [0.5, 0.5, 0.5]
            >>> sb.losses.LogLoss()(y_true, y_pred)
            0.807410...
        """
        loss = -((np.multiply(y_true, y_pred)) - np.logaddexp(0., y_pred))
        return loss.mean()

    def gradient(self, y_true, y_pred):
        """Returns the gradient of the log loss with respect to each prediction.

        Example:
            >>> import starboost as sb
            >>> y_true = [0, 0, 1]
            >>> y_pred = [0.5, 0.5, 0.5]
            >>> sb.losses.LogLoss().gradient(y_true, y_pred)
            array([ 0.5,  0.5, -0.5])
        """
        return np.subtract(y_pred, y_true)

    @property
    def default_init_estimator(self):
        """Returns ``starboost.init.PriorProbabilityEstimator()``."""
        return init.PriorProbabilityEstimator()

    def _update_leaf(self, y_true, y_pred, gradient):
        numerator = np.sum(-gradient)
        denominator = np.sum((y_true + gradient) * (1 - y_true - gradient))
        return (numerator / denominator) if denominator > 1e-150 else 0.

    @property
    def tree_line_searcher(self):
        return line_searchers.LeafLineSearcher(update_leaf=self._update_leaf)

ext: py

max_stars_repo_name: vishalbelsare/starboost

max_issues_repo_name: vishalbelsare/starboost

max_forks_repo_name: vishalbelsare/starboost

reason: import numpy


text:
import torch
import numpy as np
from spp_layer import *

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

class CNNmodel(torch.nn.Module):
    def __init__(
        self,
        input_dim=10,
        hidden_dim=[128, 64, 64, 32],
        kernel_size=[(5, 5), (5, 5), (3, 3), (3, 3)],
        stride=[(2, 2), (1, 1), (1, 1), (1, 1)],
        padding=[0, 0, 0, 0],
        dropout=0.2,
        levels=[13],
    ):
        super().__init__()

        self.hidden_dim = hidden_dim
        self.kernel_size = kernel_size
        self.stride = stride
        self.padding = padding
        self.levels = levels

        self.conv = torch.nn.Sequential(
            torch.nn.Conv2d(
                input_dim,
                hidden_dim[0],
                kernel_size[0],
                stride=stride[0],
                padding=padding[0],
            ),
            torch.nn.ReLU(),
            torch.nn.BatchNorm2d(hidden_dim[0]),
            torch.nn.Conv2d(
                hidden_dim[0],
                hidden_dim[1],
                kernel_size[1],
                stride[1],
                padding=padding[1],
            ),
            torch.nn.ReLU(),
            torch.nn.BatchNorm2d(hidden_dim[1]),
            torch.nn.Conv2d(
                hidden_dim[1],
                hidden_dim[2],
                kernel_size[2],
                stride[2],
                padding=padding[2],
            ),
            torch.nn.ReLU(),
            torch.nn.BatchNorm2d(hidden_dim[2]),
            torch.nn.Conv2d(
                hidden_dim[2],
                hidden_dim[3],
                kernel_size[3],
                stride[3],
                padding=padding[3],
            ),
            torch.nn.ReLU(),
            torch.nn.BatchNorm2d(hidden_dim[3]),
        )

        ln_in = 0
        for i in levels:
            ln_in += hidden_dim[-1] * i * i
        self.ln = torch.nn.Sequential(
            torch.nn.Linear(ln_in, 100),
            torch.nn.ReLU(),
            torch.nn.BatchNorm1d(100),
            torch.nn.Dropout(dropout),
            torch.nn.Linear(100, 1),
        )

        self.sig = torch.nn.Sigmoid()

    def forward(self, x, sigmoid=True):
        x = self.conv(x)
        x = spp_layer(x, self.levels)
        x = self.ln(x)
        if sigmoid:
            x = self.sig(x)
        return x.flatten()

if __name__ == "__main__":
    model = CNNmodel()
    in_size = 45
    x = torch.rand([2, 10, in_size, in_size])
    y = model.forward(x)
    print(y.shape)


text:
import argparse

import cv2
import numpy as np

from utils import plot, load_model

if __name__ == '__main__':
    arg_parser = argparse.ArgumentParser(description='My eigen-face tester')
    arg_parser.add_argument('--input', dest='input_image', type=str, required=False, default='./data/img.png')
    arg_parser.add_argument('--model', dest='model_file', type=str, default='model.npy')
    args = arg_parser.parse_args()

    # Reload model
    size, projected, components, mean, centered_data, labels = load_model(args.model_file)

    input_image = cv2.equalizeHist(cv2.resize(cv2.cvtColor(cv2.imread(args.input_image), cv2.COLOR_BGR2GRAY), (size, size))).reshape(-1)
    project_vector = (input_image - mean).dot(components.T)
    distances = np.sum((projected - project_vector) ** 2, axis=1)
    idx = np.argmin(distances)
    nearest_img = (centered_data[idx] + mean).reshape(size, size)
    detected_img = (nearest_img + input_image.reshape(size, size)) / 2
    plot([input_image.reshape(size, size), nearest_img, detected_img], 3, 1, 3, ['input', 'nearest', 'merge'])


text:
"""emoDB dataset."""

import collections
import os

import numpy as np
import tensorflow as tf
import tensorflow_datasets.public_api as tfds

# Markdown description  that will appear on the catalog page.
_DESCRIPTION = """
Berlin Database of Emotional Speech.
It contains about 500 utterances spoken by actors in a happy, angry, anxious, fearful, bored and disgusted way as well as in a neutral version.
You can choose utterances from 10 different actors and ten different texts.
"""

# BibTeX citation
_CITATION = """
@inproceedings{burkhardt2005database,
    title={A database of German emotional speech},
    author={Burkhardt, Felix and Paeschke, Astrid and Rolfes, Miriam and Sendlmeier, Walter F and Weiss, Benjamin},
    booktitle={Ninth European Conference on Speech Communication and Technology},
    year={2005}
}
"""

_HOMEPAGE = 'http://emodb.bilderbar.info/start.html'

_LABEL_MAP = {
    'W': 'anger',
    'A': 'fear',
    'F': 'happiness',
    'T': 'sadness',
    'N': 'neutral',
}

_SAMPLE_RATE = 16000

_SPEAKER_MAP = {
    3: {'gender': 'male', 'age': 31, 'language': 'de'},
    8: {'gender': 'female', 'age': 34, 'language': 'de'},
    9: {'gender': 'female', 'age': 21, 'language': 'de'},
    10: {'gender': 'male', 'age': 32, 'language': 'de'},
    11: {'gender': 'male', 'age': 26, 'language': 'de'},
    12: {'gender': 'male', 'age': 30, 'language': 'de'},
    13: {'gender': 'female', 'age': 32, 'language': 'de'},
    14: {'gender': 'female', 'age': 35, 'language': 'de'},
    15: {'gender': 'male', 'age': 25, 'language': 'de'},
    16: {'gender': 'female', 'age': 31, 'language': 'de'},
}

def parse_name(name, from_i, to_i, mapping=None):
    """Source: https://audeering.github.io/audformat/emodb-example.html"""
    key = name[from_i:to_i]
    return mapping[key] if mapping else key

def _compute_split_boundaries(split_probs, n_items):
    """Computes boundary indices for each of the splits in split_probs.
    Args:
      split_probs: List of (split_name, prob), e.g. [('train', 0.6), ('dev', 0.2),
        ('test', 0.2)]
      n_items: Number of items we want to split.
    Returns:
      The item indices of boundaries between different splits. For the above
      example and n_items=100, these will be
      [('train', 0, 60), ('dev', 60, 80), ('test', 80, 100)].
    """
    if len(split_probs) > n_items:
        raise ValueError('Not enough items for the splits. There are {splits} '
                         'splits while there are only {items} items'.format(splits=len(split_probs), items=n_items))
    total_probs = sum(p for name, p in split_probs)
    if abs(1 - total_probs) > 1E-8:
        raise ValueError('Probs should sum up to 1. probs={}'.format(split_probs))
    split_boundaries = []
    sum_p = 0.0
    for name, p in split_probs:
        prev = sum_p
        sum_p += p
        split_boundaries.append((name, int(prev * n_items), int(sum_p * n_items)))

    # Guard against rounding errors.
    split_boundaries[-1] = (split_boundaries[-1][0], split_boundaries[-1][1],
                            n_items)

    return split_boundaries

def _get_inter_splits_by_group(items_and_groups, split_probs, split_number):
    """Split items to train/dev/test, so all items in group go into same split.
    Each group contains all the samples from the same speaker ID. The samples are
    splitted so that all each speaker belongs to exactly one split.
    Args:
      items_and_groups: Sequence of (item_id, group_id) pairs.
      split_probs: List of (split_name, prob), e.g. [('train', 0.6), ('dev', 0.2),
        ('test', 0.2)]
      split_number: Generated splits should change with split_number.
    Returns:
      Dictionary that looks like {split name -> set(ids)}.
    """
    groups = sorted(set(group_id for item_id, group_id in items_and_groups))
    rng = np.random.RandomState(split_number)
    rng.shuffle(groups)

    split_boundaries = _compute_split_boundaries(split_probs, len(groups))
    group_id_to_split = {}
    for split_name, i_start, i_end in split_boundaries:
        for i in range(i_start, i_end):
            group_id_to_split[groups[i]] = split_name

    split_to_ids = collections.defaultdict(set)
    for item_id, group_id in items_and_groups:
        split = group_id_to_split[group_id]
        split_to_ids[split].add(item_id)

    return split_to_ids

class Emodb(tfds.core.GeneratorBasedBuilder):
    """DatasetBuilder for emoDB dataset."""

    VERSION = tfds.core.Version('1.0.0')
    RELEASE_NOTES = {
        '1.0.0': 'Initial release.',
    }

    MANUAL_DOWNLOAD_INSTRUCTIONS = """\
    manual_dir should contain the file emoDB.zip.
    """

    def _info(self) -> tfds.core.DatasetInfo:
        """Returns the dataset metadata."""
        # Specifies the tfds.core.DatasetInfo object
        return tfds.core.DatasetInfo(
            builder=self,
            description=_DESCRIPTION,
            features=tfds.features.FeaturesDict({
                'audio': tfds.features.Audio(file_format='wav', sample_rate=_SAMPLE_RATE),
                'label': tfds.features.ClassLabel(names=_LABEL_MAP.values()),
                'speaker_id': tf.string
            }),
            # If there's a common (input, target) tuple from the
            # features, specify them here. They'll be used if
            # `as_supervised=True` in `builder.as_dataset`.
            supervised_keys=('audio', 'label'),  # Set to `None` to disable
            homepage=_HOMEPAGE,
            citation=_CITATION,
        )

    def _split_generators(self, dl_manager: tfds.download.DownloadManager):
        """Returns SplitGenerators."""
        # Downloads the data and defines the splits
        zip_path = os.path.join(dl_manager.manual_dir, 'emoDB.zip')

        if not tf.io.gfile.exists(zip_path):
            raise AssertionError(
                'emoDB requires manual download of the data. Please download '
                'the audio data and place it into: {}'.format(zip_path))

        extract_path = dl_manager.extract(zip_path)

        items_and_groups = []
        for fname in tf.io.gfile.glob('{}/*/wav/*.wav'.format(extract_path)):
            if os.path.basename(fname)[-6] in ['W', 'A', 'F', 'T', 'N']:
                speaker_id = parse_name(os.path.basename(fname), from_i=0, to_i=2)
                items_and_groups.append((fname, speaker_id))

        split_probs = [('train', 0.6), ('validation', 0.2), ('test', 0.2)]  # Like SAVEE (https://github.com/tensorflow/datasets/blob/master/tensorflow_datasets/audio/savee.py)

        splits = _get_inter_splits_by_group(items_and_groups, split_probs, 0)

        with open("train.lst", 'w') as f:
            for l in splits['train']:
                f.write(l)

        with open("val.lst", 'w') as f:
            for l in splits['validation']:
                f.write(l)

        with open("test.lst", 'w') as f:
            for l in splits['test']:
                f.write(l)

        # Returns the Dict[split names, Iterator[Key, Example]]
        return [
            tfds.core.SplitGenerator(
                name=tfds.Split.TRAIN,
                gen_kwargs={'file_names': splits['train']},
            ),
            tfds.core.SplitGenerator(
                name=tfds.Split.VALIDATION,
                gen_kwargs={'file_names': splits['validation']},
            ),
            tfds.core.SplitGenerator(
                name=tfds.Split.TEST,
                gen_kwargs={'file_names': splits['test']},
            ),
        ]

    def _generate_examples(self, file_names):
        """Yields examples."""
        # Yields (key, example) tuples from the dataset
        for fname in file_names:
            wavname = os.path.basename(fname)
            speaker_id = parse_name(wavname, from_i=0, to_i=2)
            label = parse_name(wavname, from_i=5, to_i=6, mapping=_LABEL_MAP)
            example = {'audio': fname, 'label': label, 'speaker_id': speaker_id}
            yield fname, example

max_stars_repo_name: CeLuigi/SERAB

max_issues_repo_name: CeLuigi/SERAB

max_forks_repo_name: CeLuigi/SERAB


text:
import os.path as osp
import torch
import torch.utils.data as data
import data.util as util
import lmdb
import numpy as np

class VideoTestDataset(data.Dataset):
    """
    A video test dataset. Support:
    Vid4
    REDS4
    Vimeo90K-Test

    no need to prepare LMDB files
    """

    def __init__(self, opt):
        super(VideoTestDataset, self).__init__()
        self.opt = opt
        self.cache_data = opt['cache_data']
        self.half_N_frames = opt['N_frames'] // 2
        self.GT_root, self.LQ_root = opt['dataroot_GT'], opt['dataroot_LQ']
        self.data_type = self.opt['data_type']
        self.data_info = {'path_LQ': [], 'path_GT': [], 'folder': [], 'idx': [], 'border': []}
        #### Generate data info and cache data
        self.imgs_LQ, self.imgs_GT = {}, {}
        if opt['name'].lower() in ['vid4', 'reds4']:
            if self.data_type == 'lmdb':
                self.lmdb_paths_GT, _ = util.get_image_paths(self.data_type, self.GT_root)
                self.lmdb_paths_LQ, _ = util.get_image_paths(self.data_type, self.LQ_root)
                self.GT_env, self.LQ_env = None, None
                previous_name_a = None
                previous_name_b = None
                for lmdb_path_GT, lmdb_path_LQ in zip(self.lmdb_paths_GT, self.lmdb_paths_LQ):
                    GT_name_a, GT_name_b = lmdb_path_GT.split('_')
                    assert lmdb_path_GT == lmdb_path_LQ, 'GT path and LQ path in lmdb is not matched'
                    if previous_name_a != GT_name_a and previous_name_a is not None:
                        max_idx = int(previous_name_b) + 1
                        for i in range(max_idx):
                            self.data_info['idx'].append('{}/{}'.format(i, max_idx))
                        border_l = [0] * max_idx
                        for i in range(self.half_N_frames):
                            border_l[i] = 1
                            border_l[max_idx - i - 1] = 1
                        self.data_info['border'].extend(border_l)
                    self.data_info['folder'].append(GT_name_a)
                    previous_name_a = GT_name_a
                    previous_name_b = GT_name_b
                max_idx = int(previous_name_b) + 1
                for i in range(max_idx):
                    self.data_info['idx'].append('{}/{}'.format(i, max_idx))
                border_l = [0] * max_idx
                for i in range(self.half_N_frames):
                    border_l[i] = 1
                    border_l[max_idx - i - 1] = 1
                self.data_info['border'].extend(border_l)
            else:
                subfolders_LQ = util.glob_file_list(self.LQ_root)
                subfolders_GT = util.glob_file_list(self.GT_root)
                for subfolder_LQ, subfolder_GT in zip(subfolders_LQ, subfolders_GT):
                    subfolder_name = osp.basename(subfolder_GT)
                    img_paths_LQ = util.glob_file_list(subfolder_LQ)
                    img_paths_GT = util.glob_file_list(subfolder_GT)
                    max_idx = len(img_paths_LQ)
                    assert max_idx == len(
                        img_paths_GT), 'Different number of images in LQ and GT folders'
                    self.data_info['path_LQ'].extend(img_paths_LQ)
                    self.data_info['path_GT'].extend(img_paths_GT)
                    self.data_info['folder'].extend([subfolder_name] * max_idx)
                    for i in range(max_idx):
                        self.data_info['idx'].append('{}/{}'.format(i, max_idx))
                    border_l = [0] * max_idx
                    for i in range(self.half_N_frames):
                        border_l[i] = 1
                        border_l[max_idx - i - 1] = 1
                    self.data_info['border'].extend(border_l)

                    if self.cache_data:
                        self.imgs_LQ[subfolder_name] = util.read_img_seq(img_paths_LQ)
                        self.imgs_GT[subfolder_name] = util.read_img_seq(img_paths_GT)
                    else:
                        self.imgs_LQ[subfolder_name] = img_paths_LQ
                        self.imgs_GT[subfolder_name] = img_paths_GT
        elif opt['name'].lower() in ['vimeo90k-test']:
            pass  # TODO
        else:
            raise ValueError(
                'Not support video test dataset. Support Vid4, REDS4 and Vimeo90k-Test.')

    def _init_lmdb(self):
        # https://github.com/chainer/chainermn/issues/129
        self.GT_env = lmdb.open(self.GT_root, readonly=True, lock=False, readahead=False,
                                meminit=False)
        self.LQ_env = lmdb.open(self.LQ_root, readonly=True, lock=False, readahead=False,
                                meminit=False)

    def __getitem__(self, index):
        # path_LQ = self.data_info['path_LQ'][index]
        # path_GT = self.data_info['path_GT'][index]
        folder = self.data_info['folder'][index]
        idx, max_idx = self.data_info['idx'][index].split('/')
        idx, max_idx = int(idx), int(max_idx)
        border = self.data_info['border'][index]
        select_idx = util.index_generation(idx, max_idx, self.opt['N_frames'],
                                           padding=self.opt['padding'])
        if self.data_type == 'lmdb':
            if self.GT_env is None or self.LQ_env is None:
                self._init_lmdb()
            key = self.lmdb_paths_GT[index]
            name_a, name_b = key.split('_')
            center_frame_idx = int(name_b)
            GT_size_tuple = self.opt['GT_shape']
            LQ_size_tuple = self.opt['LQ_shape']
            img_GT = util.read_img(self.GT_env, key, GT_size_tuple)
            img_LQ_l = []
            for v in select_idx:
                img_LQ = util.read_img(self.LQ_env, '{}_{:08d}'.format(name_a, v), LQ_size_tuple)
                img_LQ_l.append(img_LQ)
            # stack LQ images to NHWC, N is the frame number
            img_LQs = np.stack(img_LQ_l, axis=0)
            # BGR to RGB, HWC to CHW, numpy to tensor
            img_GT = img_GT[:, :, [2, 1, 0]]
            img_LQs = img_LQs[:, :, :, [2, 1, 0]]
            img_GT = torch.from_numpy(np.ascontiguousarray(np.transpose(img_GT, (2, 0, 1)))).float()
            imgs_LQ = torch.from_numpy(np.ascontiguousarray(np.transpose(img_LQs,
                                                                         (0, 3, 1, 2)))).float()
        elif self.cache_data:
            imgs_LQ = self.imgs_LQ[folder].index_select(0, torch.LongTensor(select_idx))
            img_GT = self.imgs_GT[folder][idx]
        else:
            imgs_LQ = util.read_img_seq(self.imgs_LQ[folder]).index_select(0, torch.LongTensor(select_idx))
            img_GT = util.read_img_seq(self.imgs_GT[folder])[idx]

        return {
            'LQs': imgs_LQ,
            'GT': img_GT,
            'folder': folder,
            'idx': self.data_info['idx'][index],
            'border': border
        }

    def __len__(self):
        if self.data_type == 'lmdb':
            return len(self.lmdb_paths_GT)
        return len(self.data_info['path_GT'])


text:
#!env python

import numpy as np
import os

import matplotlib
import matplotlib.pyplot as plt
import matplotlib.lines as mlines

from math import sqrt
from matplotlib.patches import Patch
from matplotlib import cm

from math import sqrt
from matplotlib.patches import Patch

import pandas as pd
import sqlite3

from sklearn.model_selection import train_test_split, KFold
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error,  r2_score, make_scorer

from sklearn import datasets, linear_model
from sklearn.model_selection import cross_val_score
#from sklearn.model_selection import cross_validate
from scipy import stats

def getSysname():
    return "PieSlicer"

# Gets a default set of markers.  May need to add more markers as needed
def getMarkers():
    return ['+', 'o', 'x', '^', 's', 'v']

# Gets a range of BW colors ranging from dark to light but never including pure black or white
def getBWColors(num_fields, stringify=True):
    if stringify:
        return ["%s" % c for c in getBWColors(num_fields, False)]
    return (1-(np.arange(0,num_fields+1)/float(num_fields+1)))[::-1][:-1]

# Sets Parameters based on "width" of plot.
#   If "width" is set to "markdown" then a reasonably sized PNG file is generated
#   If "width" is set to a float in [0.5, 0.3, 0.25] then a PDF file of this fraction of the page width will be generated
def setRCParams(width=0.5, height=0.5, *args, **kwargs):
    params = {
        "savefig.format"    : 'pdf',
        'text.usetex'       : 'false'
    }

    if width == "markdown":
        params['figure.figsize'] = [6.6, 3.0]
        params['font.size'] = (16)
        params['savefig.format'] = 'png'
    elif width == 0.5:
        params['figure.figsize'] = [3.3, 1]
        params['font.size'] = ('%s' % (8*(3/4.)))
    elif width == 0.3:
        params['figure.figsize'] = [2.2, 1.0]
        params['font.size'] = ('%s' % (8*(2/3.)))
    elif width == 0.25:
        params['figure.figsize'] = [2.0, 1.0]
        params['font.size'] = ('%s' % (8*(2./2.)))
    elif width == 0.2:
        params['figure.figsize'] = [2.0, 2.0]
        params['font.size'] = ('%s' % (10))
    elif width == 0.16:
        params['figure.figsize'] = [2.0, 2.0]
        params['font.size'] = ('%s' % (11))
    else:
        params['figure.figsize'] = [3.3, 1.5]

    if height == 0.5:
        pass
    elif height == 1.0:
        x, y = tuple(params['figure.figsize'])

        params['figure.figsize'] = [x, y*2]

    matplotlib.rcParams.update(params)

def loadRCParamsFile(path_to_file="matplotlibrc"):
    with open(path_to_file) as fid:
        param_lines = [s.strip() for s in fid.readlines()]

    params = {}
    for line in param_lines:
        if line.strip() == '':
            continue
        if line.strip().startswith('#'):
            continue

        parts = line.split(':')
        key = parts[0].strip()
        value = ':'.join(parts[1:]).strip()
        params[key] = value
    matplotlib.rcParams.update(params)

bin_size = 0.1 # in MB
size_cutoff = 10

## Loading Functions

def load_DB_to_DF(path_to_df, **kwargs):
    print("Loading %s" % path_to_df)
    conn1 = sqlite3.connect(path_to_df)
    df_phone = pd.read_sql_query("SELECT * FROM inference_results;", conn1)
    df_phone = cleanDF(df_phone)
    df_phone = hackDF(df_phone, **kwargs)
    df_phone = addOneHot(df_phone, path_to_df)
    return df_phone

def addOneHot(df, path_to_df):
    for part in os.path.basename(path_to_df).split('.'):
        df[part] = 1
    return df

def cleanDF(df,):
    try:
        df.set_index(pd.DatetimeIndex(df['timeStamp']), inplace=True)
    except KeyError:
        pass
    for col in ["sla_target",
                "orig_size",
                "sent_size",
                "image_dims",
                "time_local_pieslicer",
                "time_local_preprocess",
                'time_local_remote',
                'time_remote_save',
                'time_remote_network',
                'time_remote_transfer',
                'time_remote_pieslicer',
                'time_remote_load',
                'time_remote_general_resize',
                'time_remote_specific_resize',
                'time_remote_convert',
                'time_remote_post_network',
                'time_remote_inference',
                'time_remote_total',
                'time_total',
                'model_accuracy',
                'time_budget',
                'expected_time_local_prep',
                'expected_time_remote_prep',
                'transfer_time_estimate',
                'transfer_time_real',
                'transfer_time_delta',
                'transfer_time_delta_raw',
                'time_local_preprocess_resize',
                'time_local_preprocess_save',
                'time_remote_routing',
                'time_remote_prepieslicer',
                'ping_time',
                'inference_result',
                'time_local_preprocess_check',
                'time_local_preprocess_check_filesize',
                'time_local_preprocess_check_dimensions',
                'preexecution_time_estimate',
                'orig_dims_x',
                'orig_dims_y',
                'jpeg_quality',
               ]:
        try:
            df[col] = df[col].apply(pd.to_numeric, args=('coerce',))
        except KeyError:
            print("Couldn't find col: %s" % col)
    try:
        df["test_image_bool"] = df["test_image_bool"].apply((lambda s: int(s) == 1))
    except KeyError:
        df["test_image_bool"] = False
    del df["_id"]
    df["orig_size"] = df["orig_size"] / 1000000.
    df["sent_size"] = df["sent_size"] / 1000000.
    #df["network_time"] = df["total_time"] - df["total_remotetime"]
    return df

def hackDF(df):
    try:
        df["local_prep"] = df["preprocess_location_real"]=="local"
    except KeyError:
        df["local_prep"] = df["preprocess_location"]=="local"

    #df["sla_target_hack"] = (df.index % 100) * 10 + 10
    df["in_sla"] = df["sla_target"] >= df["time_total"]
    df["time_total_network"] = (df["time_local_remote"] - df["time_remote_total"])
    df["time_total_transfer"] = (df["time_local_remote"] - df["time_remote_post_network"])
    df['expected_delta'] = df['expected_time_local_prep'] - df['expected_time_remote_prep']
    df['time_leftover'] = df['sla_target'] - df['time_total']
    df["time_remote_preprocess"] = df["time_remote_post_network"] - df["time_remote_inference"]
    df['effective_bandwidth'] = df['sent_size'] / (df['time_local_remote']/1000.)
    try:
        df['effective_bandwidth_wo_network'] = df['sent_size'] / ((df['time_local_remote'] - df['ping_time'])/1000.)
    except KeyError:
        pass
    try:
        df['time_local_preprocess_check'] = df['time_local_preprocess_check'] / 1000.0 / 1000.0
    except KeyError:
        pass

    df["upscaled"] = df["orig_size"] < df["sent_size"]

    df["preexec_time"] = df["time_local_preprocess"] + (df["time_local_remote"] - df["time_remote_inference"])
    try:
        df["preexec_error"] = df["preexec_time"] - df["preexecution_time_estimate"]
    except KeyError:
        df["preexec_error"] = 0.

    df["transfer_time_error"] = df["transfer_time_real"] - df["transfer_time_estimate"]

    def modelDimFinder(s):
        if "inception" in s:
            return 299
        if "mobilenet" in s:
            return int(s.split('_')[3])
        if "densenet" in s:
            return 224
        if "squeezenet" in s:
            return 224
        if "nasnet" in s:
            if "mobile" in s:
                return 224
            else:
                return 331
        return 0

    df["model_dims"] = df["model_name"].apply(modelDimFinder)

    df["orig_dims_x"] = df["image_name_orig"].apply( (lambda n: df[df["image_name_orig"]==n]["orig_dims_x"].max()) )
    df["orig_dims_y"] = df["image_name_orig"].apply( (lambda n: df[df["image_name_orig"]==n]["orig_dims_y"].max()) )

    try:
        df["pixels"] = df["orig_dims_x"] * df["orig_dims_y"]
    except KeyError:
        pass

    df["bin"] = df["orig_size"] - (df["orig_size"] % bin_size)

    df = _rejectInput(df)

    return df

def _rejectInput(df):
     return df[df["orig_size"] <= size_cutoff]

def getAlgorithmDF(df_base, algo_name):
    return df_base[df_base["algorithm"]==algo_name].groupby("image_name_orig").mean() #.set_index("image_name_orig")

def calcOptimalDF(df_local, df_remote):
    df_merged = pd.merge(df_local, df_remote, suffixes=("_local", "_remote"), left_index=True, right_index=True)
    df_merged["preexec_time"] = df_merged[["preexec_time_local", "preexec_time_remote"]].min(axis=1)
    df_merged["time_budget"] = df_merged[["time_budget_local", "time_budget_remote"]].max(axis=1)
    df_merged["local_prep"] = df_merged["preexec_time_local"] < df_merged["preexec_time_remote"]
    df_merged["orig_size"] = df_merged["orig_size_local"]
    df_merged["bin"] = df_merged["bin_local"]
    return df_merged

def getBinnedDF(df_base, custom_bin_size=bin_size, col=None, quantiles=None):
    if col is None:
        col = "orig_size"
    df_base["custom_bin"] = df_base[col] - (df_base[col] % custom_bin_size) + custom_bin_size
    if quantiles is None:
        return df_base.groupby("custom_bin").mean(), df_base.groupby("custom_bin").std()
    else:
        return df_base.groupby("custom_bin").mean(), df_base.groupby("custom_bin").quantile(quantiles)

## Modeling functions
class Modeling():
    @classmethod
    def getTrainTest(cls, X_all, Y_all):

        X_train, X_test, Y_train, Y_test = train_test_split(X_all, Y_all, test_size=0.20)
        X_train = X_train.reshape((len(X_train), 1))
        X_test = X_test.reshape((len(X_test), 1))

        return X_train, X_test, Y_train, Y_test

    @classmethod
    def getModel(cls, X, y, normalize=True, **kwargs):
        model = LinearRegression(normalize=normalize)
        model.fit(X, y)
        return model

    @classmethod
    def getKModels(cls, X, y, n_splits=10, normalize=True, **kwargs):
        model = LinearRegression(normalize=normalize)
        kfold = KFold(n_splits=n_splits, shuffle=True)
        for train_index, test_index in kfold.split(X):
            X_train, X_test = X[train_index], X[test_index]
            y_train, y_test = y[train_index], y[test_index]
            model.fit(X_train, y_train)
            score = model.score(X_test, y_test)
        return model

    @classmethod
    def runKFoldValidation(cls, X_all, Y_all, n_splits=10, normalize=True, **kwargs):

        model = LinearRegression(normalize=normalize)
        return cross_val_score(model, X_all, Y_all, cv=n_splits)

    @classmethod
    def getKFoldValidationScores(cls, df_to_model, x_col, y_col, n_splits=10):
        if isinstance(y_col, str):
            y_col = [y_col]

        if isinstance(x_col, str):
            x_col = [x_col]

        df_internal = df_to_model.set_index(x_col)

        def getArrays(df_to_use):
            return np.array(df_to_use.index.values.tolist()).reshape(-1,1), df_to_use[y_col].sum(axis=1).values

        X_all, Y_all = getArrays(df_internal)

        X_all = np.array(X_all)
        Y_all = np.array(Y_all)

        model = LinearRegression(normalize=True)
        #scores = cross_validate(model, X, y, cv=n_splits,
        #                 scoring=('neg_mean_squared_error', 'neg_mean_absolute_error'),
        #                 return_train_score=True)
        scores = {}
        scores["RMSE"] = np.sqrt(-1*np.array(cross_val_score(model, X_all, Y_all, cv=n_splits, scoring='neg_mean_squared_error'))).mean()
        scores["MAE"] = np.array(-1*cross_val_score(model, X_all, Y_all, cv=n_splits, scoring='neg_mean_absolute_error')).mean()

        def mean_absolute_percentage_error(y_true, y_pred):
            y_true, y_pred = np.array(y_true), np.array(y_pred)
            return np.mean(np.abs((y_true - y_pred) / y_true)) * 100

        scores["MAPE"] = np.array(cross_val_score(model, X_all, Y_all, cv=n_splits, scoring=make_scorer(mean_absolute_percentage_error, greater_is_better=True))).mean()

        return scores

    @classmethod
    def modelDF(cls, df_to_model, x_col, y_col, name_of_df=None, variable_name=None, normalize=True, **kwargs):
        fig, ax = plt.subplots(nrows=1,ncols=1)

        if isinstance(y_col, str):
            y_col = [y_col]

        if isinstance(x_col, str):
            x_col = [x_col]

        df_internal = df_to_model.set_index(x_col)

        def getArrays(df_to_use):
            return np.array(df_to_use.index.values.tolist()).reshape(-1,1), df_to_use[y_col].sum(axis=1).values

        X_all, Y_all = getArrays(df_internal)

        X_all = np.array(X_all)
        Y_all = np.array(Y_all)

        k_folds_scores = cls.runKFoldValidation(X_all, Y_all, **kwargs)

        model = cls.getModel(X_all, Y_all, normalize=normalize)

        df_internal[y_col].sum(axis=1).plot(ax=ax, linewidth=0, marker='.', **kwargs)

        X_range = np.array([min(X_all), max(X_all)])
        ax.plot(X_range, model.predict(X_range.reshape((len(X_range),1))), linewidth=1, marker='', color='k')

        ax.set_xlim([0, max(X_all)])
        ax.set_ylim([0, max(Y_all)])
        ax.set_ylabel(cls.fixName(y_col))
        ax.set_xlabel(cls.fixName(x_col))

        if variable_name is None:
            eq_str = "f(x) = %.3fx + %.3f" % (model.coef_[0], model.intercept_)
        else:
            eq_str = "%s(x) = %.3fx + %.3f" % (variable_name, model.coef_[0], model.intercept_)

        ax.text(0.02*max(X_all), 0.90*max(Y_all), eq_str)
        #ax.text(0.01*max(X_all), 0.77*max(Y_all), "Mean squared error: %.2f" % mean_squared_error(Y_test, model.predict(X_test)))
        ax.text(0.02*max(X_all), 0.75*max(Y_all), "R2 score: %.2f" % r2_score(Y_all, model.predict(X_all)))
        if name_of_df is None:
            plt.savefig( ("images/%s.%s.pdf" % (x_col, y_col,)), bbox_inches='tight')
        else:
            plt.savefig( ("images/%s.%s.%s.pdf" % (name_of_df, x_col, '.'.join(y_col),)), bbox_inches='tight')

        print("Test Set Average Absolute Error: %0.2f" % ( np.abs(Y_all - model.predict(X_all)).mean()))
        print("Test Set Average Absolute Error %%: %0.2f%%" % ( 100.*np.abs(Y_all - model.predict(X_all)).mean()/Y_all.mean()))

        print("Test Set RSME: %0.2f" % ( sqrt(mean_squared_error(Y_all, model.predict(X_all))) ))
        print("Test Set RSME %%: %0.2f%%" % ( 100.*sqrt(mean_squared_error(Y_all, model.predict(X_all)))/Y_all.mean()))

        return model, model.score(X_all, Y_all), X_all, Y_all, fig, ax, k_folds_scores

    @classmethod
    def modelErrorPercentage(cls, model, X_test, Y_test, bins=0, scale="log", **kwargs):
        #model, _, X_test, Y_test, fig_model, ax_model = cls.modelDF(df_to_model, "orig_size", "time_local_preprocess", "MotoX", variable_name="f")
        error = (model.predict(X_test) - Y_test)
        error_percent = 100.0 * (error / Y_test)
        #error_percent = error

        print("Size: %s" % error_percent.size)
        print("Mean: %s %%" % np.mean(error_percent))
        print("StdDev: %s\n %%" % np.std(error_percent))

        fig, ax = plt.subplots(nrows=1, ncols=1, **kwargs)

        if bins == 0:
            X_to_plot, Y_to_plot = (np.array(list(t)) for t in zip(*sorted(zip(X_test, error_percent))))
        else:
            bin_means, bin_edges, binnumber = stats.binned_statistic(X_test.flatten(),
                                                                     error_percent.flatten(),
                                                                     statistic='mean',
                                                                     bins=bins)
            X_to_plot, Y_to_plot = bin_edges[1:], bin_means

        ax.plot( X_to_plot, Y_to_plot )
        ax.set_xscale(scale)
        ax.set_xlim([min(X_test), max(X_test)])

        ax.axhline(np.mean(error_percent), color='0.5', linestyle='--')

        return fig, ax

    @classmethod
    def modelError(cls, model, X_test, Y_test, bins=0, scale="log", **kwargs):
        #model, _, X_test, Y_test, fig_model, ax_model = cls.modelDF(df_to_model, "orig_size", "time_local_preprocess", "MotoX", variable_name="f")
        error = (model.predict(X_test) - Y_test)

        print("Size: %s" % error.size)
        print("Mean: %s" % np.mean(error))
        print("StdDev: %s\n" % np.std(error))

        fig, ax = plt.subplots(nrows=1, ncols=1, **kwargs)

        if bins == 0:
            X_to_plot, Y_to_plot = (np.array(list(t)) for t in zip(*sorted(zip(X_test, error))))
        else:
            bin_means, bin_edges, binnumber = stats.binned_statistic(X_test.flatten(),
                                                                     error.flatten(),
                                                                     statistic='mean',
                                                                     bins=bins)
            X_to_plot, Y_to_plot = bin_edges[1:], bin_means

        ax.plot( X_to_plot, Y_to_plot )
        ax.set_xscale(scale)
        ax.set_xlim([min(X_test), max(X_test)])

        ax.axhline(np.mean(error), color='0.5', linestyle='--')

        return fig, ax

    @classmethod
    def modelErrorCDF(cls, model, X_test, Y_test, bins=0, scale="log", **kwargs):
        #model, _, X_test, Y_test, fig_model, ax_model = cls.modelDF(df_to_model, "orig_size", "time_local_preprocess", "MotoX", variable_name="f")
        error = (model.predict(X_test) - Y_test)
        error_percent = 100.0 * (error / Y_test)
        #error_percent = error

        print("Size: %s" % error_percent.size)
        print("Mean: %s" % np.mean(error_percent))
        print("StdDev: %s\n" % np.std(error_percent))

        fig, ax = plt.subplots(nrows=1, ncols=1, **kwargs)

        X_to_plot, Y_to_plot = (np.array(list(t)) for t in zip(*sorted(zip(X_test, error_percent))))

        counts, bin_edges = np.histogram (Y_to_plot, bins=bins, normed=True)
        cdf = np.cumsum(counts)
        ax.plot(bin_edges[1:], cdf/cdf[-1])#, marker=markers[i], markevery=(num_bins/10))
        max_bin = max(bin_edges)
        min_bin = min(bin_edges)

        ax.set_ylim([0,1.01])
        ax.set_xlim([min_bin, max_bin])

        #ax.plot( X_to_plot, Y_to_plot )
        #ax.set_xscale(scale)
        #ax.set_xlim([min(X_test), max(X_test)])

        #ax.axhline(np.mean(error_percent), color='0.5', linestyle='--')

        return fig, ax

    @classmethod
    def fixName(cls, name):
        if not isinstance(name, str):
            name = ' + '.join(name)

        if name == "orig_size":
            name = "Original Size"
        elif name == "preprocess_time_local":
            name = "On-device Preprocessing (ms)"
        elif name == "preprocess_time_remote":
            name = "In-cloud Preprocessing (ms)"
        elif name == "network_time + save_time":
            name = "Network Transfer Time"
        elif name == "Campus":
            name = "Dynamic"
        else:
            name = name.replace('_', ' ')
            name = name.title()

        if "time" in name.lower():
            name += " (ms)"
        elif "size" in name.lower():
            name += " (MB)"
        elif name == "Mp":
            name = "Image Dimensions (MP)"
        return name

    @classmethod
    def getModelUsagePlot(cls, df_to_plot, **kwargs):
        fig, ax = plt.subplots(nrows=1,ncols=1)
        df_to_plot.groupby("model_name").count()["image_name_orig"].plot(ax=ax, kind='bar', **kwargs)
        return fig, ax

    @classmethod
    def getCDFPlot(cls, dfs_to_plot, cols_to_plot, num_bins=1000, **kwargs):

        if isinstance(cols_to_plot, str):
            cols_to_plot = [cols_to_plot]

        markers = getMarkers()

        max_bin = float('-inf')
        min_bin = float('+inf')
        fig, ax = plt.subplots(nrows=1,ncols=1)
        for i, (df, label) in enumerate(dfs_to_plot[::1]):

            data = df[cols_to_plot].sum(axis=1)

            counts, bin_edges = np.histogram(data, bins=num_bins, normed=True)
            counts = counts

            cdf = np.cumsum(counts)
            cdf = np.insert(cdf, 0, 0.0)

            if "complement" in kwargs and kwargs["complement"]:
                ax.plot (bin_edges[:], 1-(cdf/cdf[-1]), label=cls.fixName(label), marker=markers[i], markevery=(num_bins/10))
            else:
                ax.plot (bin_edges[:], (cdf/cdf[-1]), label=cls.fixName(label), marker=markers[i], markevery=(num_bins/10))

            max_bin = max([max_bin, max(bin_edges)])
            min_bin = min([min_bin, min(bin_edges)])

        ax.legend(loc='best')
        #ax.axvline(sla_target, linestyle='--', color="0.5")
        ax.set_ylim([0,1.01])
        ax.set_xlim([min_bin*0.99, max_bin])

        if "complement" in kwargs and kwargs["complement"]:
            ax.set_ylabel("CCDF (%)")
        else:
            ax.set_ylabel("CDF (%)")

        ax.set_xlabel(cls.fixName(cols_to_plot))

        handles, labels = ax.get_legend_handles_labels()
        ax.legend(handles[::-1], labels[::-1], loc='best')

        return fig, ax

def main():
    pass

if __name__ == '__main__':
    main()

max_stars_repo_name: cake-lab/PieSlicer

max_issues_repo_name: cake-lab/PieSlicer

max_forks_repo_name: cake-lab/PieSlicer

reason: import numpy,from scipy


text:
import pandas as pd
import numpy as np
import time

today = np.datetime64('today')

df=pd.read_excel('rev_list.xlsx', index_col=0)

df['today']=(df['Session Start Date/Time']<=today+np.timedelta64(24, 'h')) & (df['Session End Date/Time']>=today-np.timedelta64(24, 'h')) & (df['Approval Status'] == 'Approved')

df_today = df[df['today'] == True]

df_today_filter = df_today.iloc[:,[0,1,2,3,4,5,]].sort_values(by=['Veh. ID','Session Start Date/Time'], ascending=(True,True))

filename_excel = "Reservation-%s.xlsx" % today

df_today_filter.to_excel (filename_excel)

max_stars_repo_path: Excel Related/filter-today.py

max_stars_repo_name: dddoriath/Python-Automation-Office

max_issues_repo_path: Excel Related/filter-today.py

max_issues_repo_name: dddoriath/Python-Automation-Office

max_forks_repo_path: Excel Related/filter-today.py

max_forks_repo_name: dddoriath/Python-Automation-Office


text:
import torchvision
from sklearn.model_selection import train_test_split
import numpy as np
from torchvision import transforms
from .dataset_utils import WeaklySupervisedDataset

class Cifar100Dataset:
    def __init__(self, root, labeled_amount, add_labeled):
        self.root = root
        self.labeled_amount = labeled_amount
        self.cifar_mean = (0.5071, 0.4867, 0.4408)
        self.cifar_std = (0.2675, 0.2565, 0.2761)
        self.input_size = 32
        self.transform_train = transforms.Compose([
            transforms.RandomCrop(self.input_size, padding=4),
            transforms.RandomHorizontalFlip(),
            # transforms.RandomVerticalFlip(),
            # transforms.RandomGrayscale(),
            # transforms.RandomRotation(degrees=180),
            transforms.ToTensor(),
            transforms.Normalize(mean=self.cifar_mean, std=self.cifar_std)
        ])
        self.transform_test = transforms.Compose([
            transforms.ToTensor(),
            transforms.Normalize(mean=self.cifar_mean, std=self.cifar_std)
        ])
        self.num_classes = 100
        self.add_labeled = add_labeled
        self.add_labeled = None

    def get_dataset(self):
        base_dataset = torchvision.datasets.CIFAR100(root=self.root, train=True,
                                                     download=True, transform=None)

        self.add_labeled = int(len(base_dataset) * self.add_labeled)

        labeled_indices, unlabeled_indices = train_test_split(
            np.arange(len(base_dataset)),
            test_size=(1 - self.labeled_amount),
            shuffle=True,
            stratify=base_dataset.targets)

        test_dataset = torchvision.datasets.CIFAR100(root=self.root, train=False,
                                                     download=True, transform=self.transform_test)

        labeled_dataset = WeaklySupervisedDataset(base_dataset, labeled_indices, transform=self.transform_train)
        unlabeled_dataset = WeaklySupervisedDataset(base_dataset, unlabeled_indices, transform=self.transform_test)

        return labeled_dataset, unlabeled_dataset, labeled_indices, unlabeled_indices, test_dataset

    def get_base_dataset(self):
        base_dataset = torchvision.datasets.CIFAR100(root=self.root, train=True,
                                                     download=True, transform=self.transform_train)

        return base_dataset

max_stars_repo_name: AhmadQasim/MedAL

max_issues_repo_name: AhmadQasim/MedAL

max_forks_repo_name: AhmadQasim/MedAL


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
@author: johannaregenthal

Date: August 2020
"""

# 1. Presettings ################################################################

import os
# Set working directory to path where the script is

# Import required packages
import pandas as pd
import numpy as np
import random
import networkx as nx
from spreading_CR import SpreadingProcess #https://github.com/gstonge/spreading_CR/blob/master/README.md
from EstimatingCentralityRobustness import robustness_calculator_builder #https://github.com/crsqq/EstimatingCentralityRobustness

# Define user-specific functions
def create_random_networks(original_graph, number_of_random_graphs, networksize):
    """
    Function to generate a defined number of random graphs with a specified networksize n based on one original graph.
    More precisely, the function performs a configuration model which creates random graphs from a given degree sequence.
    The function returns a list of random graphs.
    """
    # Get degree sequence from original graph
    degree_sequence = [d for n, d in original_graph.degree()]

    # Create true configuration networks with the same degree sequence without self-loops and parallel edges
    N = 0
    graphs = []
    while N < number_of_random_graphs:
        # Sample n nodes from original degree sequence at random
        deg_sequence_sample = random.sample(degree_sequence, networksize)
        # Check for valid degree sequence
        if nx.is_graphical(deg_sequence_sample):
            # Generate random graph based on the sampled nodes and append to list
            G_random = nx.configuration_model(deg_sequence_sample, create_using = nx.Graph())
            graphs.append(G_random)
            N += 1
    return graphs

def create_networkcopy_with_missing_nodes(graph, list_missing_percentages):
    """
    Function to generate copies of a graph with varying proportion p of missing nodes given by a list P.
    The function returns a list of graphs, where each graph is a network copy with p% of the nodes missing.
        graph = true network
        list_missing_percentages = list that contains different values for p
    """
    lst = []
    for p in list_missing_percentages:
        # Generate copy of the graph, randomly remove p% of the nodes from the set and append to list
        G = graph.copy()
        G.remove_nodes_from(random.sample(list(G.nodes()), int(len(graph.nodes()) * p)))
        lst.append((G, p))
    return lst

def sim_infection(graph, immune_nodes, numOfSimulations, beta, gamma):
    """
    Function to perform a defined number of outbreak simulations based on the SIR model in a given graph under
    consideration of a defined set of immunized nodes.
    The function returns a tuple with mean and standard deviation (std) of the outbreak size over all simulations,
    where the outbreak size is given by the proportion of infected nodes in the graph.
        graph = graph in which the outbreak is simulated
        immune_nodes = list of nodes that are immune
        numOfSimulations = number of simulations
        beta, gamma = outbreak parameters for the SIR model
    """
    final_size_list = [] # List containing final size of infected nodes per simulation

    # Create list of unimmune nodes
    unimmune_nodes = list(set(graph.nodes) - set(immune_nodes))

    # Identify potential starting nodes of the outbreak simulation (= patient0s) by checking for nodes with at least
    # one neighbor. Otherwise, the function sp-initialize throws an error
    nodes_with_neighbors = [n for n in unimmune_nodes if len(list(graph.neighbors(n))) > 0]

    # Draw patient0s from the nodes_with_neighbors for each simulation at random
    array_patient0s = np.random.choice(nodes_with_neighbors, size=numOfSimulations)

    # Convert each element to an own list due to the form of the sp.initialize function
    list_patient0s = [[patient0] for patient0 in array_patient0s]

    # Create simulation object
    sp = SpreadingProcess(list(graph.edges()), beta, gamma, 0)

    # Start simulations for numOfSimulations runs
    for i in range(numOfSimulations):
        seed = np.random.randint(MAX_CPLUSPLUS_INT+1)

        # Initialize simulation with infected patient0 and immune_nodes
        sp.initialize(list_patient0s[i], immune_nodes, seed)

        # Run outbreak simulation infinitly (without time constraint)
        sp.evolve(np.inf)

        # Check the number of recovered nodes (including recovered and immune nodes in terms of the report) and
        # store in finalSizeInclTargets
        finalSizeInclTargets = sp.get_Rnode_number_vector()[-1]

        # Subtract the number of immune nodes from finalSizeInclTargets to obtain the number of infected nodes
        adjustedFinalSize = finalSizeInclTargets -  len(immune_nodes)

        # Append everything to final_size_list containing the number of infected nodes per simulation
        final_size_list.append(adjustedFinalSize)

        # Reset spreading process
        sp.reset()

    # Calculate mean and standard deviation of outbreak size over all simulations by using final_size_list
    pctTotalPopInf_mean = np.mean(final_size_list) / graph.number_of_nodes() # mean of outbreak size in %
    pctTotalPopInf_std = np.std(final_size_list) / graph.number_of_nodes()   # std of outbreak size

    return (pctTotalPopInf_mean, pctTotalPopInf_std)

# 2. Definition of parameters ###################################################

MAX_CPLUSPLUS_INT = 4294967295 # Seed similar to original

P = np.arange(0, 0.9, 0.1) # Part of missing nodes (nine different error levels between 0 and 80%)
q = 0.1                    # Fraction of immunized nodes = 10%
n = 1000                   # Number of nodes = 1000
N = 5000                   # Number of random networks = 5000
numOfSimulations = 2000    # Number of simulations per random network = 2000
beta = 0.95; gamma = 1     # Outbreak parameters of the SIR model

# List of examined centralities to loop over
centralities = [('degree', nx.degree_centrality),
                ('between', nx.betweenness_centrality),
                ('eigen', nx.eigenvector_centrality_numpy),
                ('page', nx.pagerank)]

# 3. Creation of random networks ################################################

# Import network data
data = pd.read_csv('edgelist.truecolsprings.csv', usecols = ['V1', 'V2'])
nodes = list(set(data.V1).union(set(data.V2)))
edges = list(zip(data.V1, data.V2))

# Create Colorado Springs network (= Original network)
G = nx.Graph()
G.add_nodes_from(nodes)
G.add_edges_from(edges)

# Create N random graphs of size n from the Colorado Springs network G (= True networks)
graphs = create_random_networks(G, N, n)

# Delete unnecessary variables from network data
del data, nodes, edges, G

# 4. Simulations ################################################################

# Initialize list to store data for each centrality measure
data_degree, data_between, data_eigen, data_page = [], [], [], []

# Define number of nodes to be immunized (q% of the n nodes in the true network)
num_immune_nodes = int(n * q)

# For each true network G_true
for i, G_true in enumerate(graphs[:5]):

    # For each observed network G_observed with error level p (p% of nodes missing)
    for G_observed, p in create_networkcopy_with_missing_nodes(G_true, P):

        # Print network ID and error level p
        print('G_true: ', i, 'p: ', p)

        # Random immunization: Perform outbreak simulation in G_true with randomly immunized nodes sampled from G_observed
        # and obtain mean and std of the outbreak size
        immune_nodes = random.sample(list(G_observed.nodes), num_immune_nodes)
        mean_ran, std_ran = sim_infection(G_true, immune_nodes, numOfSimulations, beta, gamma)

        # For each centrality measure
        for name, centrality in centralities:

            # Strategic immunization: Perform outbreak simulation in G_true with strategically immunized nodes from
            # G_observed determined by the respective centrality measure and obtain mean and std of the outbreak size
            dic = centrality(G_observed)
            immune_nodes = sorted(dic, key=dic.get, reverse=True)[:num_immune_nodes]
            mean_strat, std_strat = sim_infection(G_true, immune_nodes, numOfSimulations, beta, gamma)

            # Measure by Rosenblatt et al. (2020): Use the concept proposed by Rosenblatt et al. (2020) to compute
            # robustness of the centrality measure with respect to the corresponding immunization strategy
            robustness_Ros = mean_ran - mean_strat

            # Measure by Martin and Niemeyer (2019): Use the function proposed by Martin and Niemeyer (2019) to compute
            # robustness of the centrality measure
            robustness_calculator = robustness_calculator_builder(centrality)
            robustness_Nie = robustness_calculator(G_true, G_observed)

            # Save relevant values in list
            exec(f'data_{name}.append([i, p, mean_ran, std_ran, mean_strat, std_strat, robustness_Nie, robustness_Ros])')

# Create dataframe for the simulation results of each centrality measure
for name, c in centralities:
    exec(f"data_{name} = pd.DataFrame(data_{name}, columns = ['graph_ID', 'p', 'mean_ran', 'std_ran', 'mean_strat', 'std_strat', 'robustness_Nie', 'robustness_Ros'])")
    exec(f"data_{name}.name = name")

# Save results as csv
for df in [data_degree, data_between, data_eigen, data_page]:
    df.to_csv('data_' + df.name + '.csv', sep = ';')

# 5. Analysis ###################################################################

# see R skript "NetworkAnalysis_FinalProject_Plotting.R"

max_stars_repo_name: sonnenthaler/NetworkAnalysis

max_issues_repo_name: sonnenthaler/NetworkAnalysis

max_forks_repo_name: sonnenthaler/NetworkAnalysis

reason: import numpy,import networkx


text:
import pandas as pd
import numpy as np
import os
from matplotlib import pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

''' Get directory '''
dir_default_splines = '/home/hector/ros/tfm_ws/src/upat_follower/tests/splines/'
dir_data = '/home/hector/ros/tfm_ws/src/upat_follower/data/'
experiment_name = 'robot2019'
case_name = '2020-01-29_13-01-08'
dir_experiment = dir_data + 'log/' + experiment_name + '/' + case_name + '/'
dir_save_data = dir_data + 'img/' + experiment_name + '/' + case_name + '/'
''' Create folder to save data '''
if not os.path.exists(dir_save_data):
    os.makedirs(dir_save_data)
''' Get csv files '''
default_init_path = pd.read_csv(dir_default_splines + 'init.csv', names=['x', 'y', 'z'])
default_cubic_spline_loyal_path = pd.read_csv(
    dir_default_splines + 'cubic_spline_loyal.csv', names=['x', 'y', 'z'])
default_cubic_spline_path = pd.read_csv(
    dir_default_splines + 'cubic_spline.csv', names=['x', 'y', 'z'])
normal_dist_linear_interp = pd.read_csv(
    dir_experiment + 'normal_dist_linear_interp.csv', names=['Time', 'Spline', 'Linear'])
normal_dist_cubic_spline = pd.read_csv(
    dir_experiment + 'normal_dist_cubic_spline.csv', names=['Time', 'Spline', 'Linear'])
normal_dist_cubic_loyal = pd.read_csv(
    dir_experiment + 'normal_dist_cubic_loyal_spline.csv', names=['Time', 'Spline', 'Linear'])
current_path_linear_interp = pd.read_csv(
    dir_experiment + 'current_path_linear_interp.csv', names=['x', 'y', 'z'])
current_path_cubic_spline = pd.read_csv(
    dir_experiment + 'current_path_cubic_spline.csv', names=['x', 'y', 'z'])
current_path_cubic_loyal = pd.read_csv(
    dir_experiment + 'current_path_cubic_loyal_spline.csv', names=['x', 'y', 'z'])

''' Plot linear interpolation '''
fig1 = plt.figure(num="Linear interpolation 3D behavior")
ax1 = Axes3D(fig1)
ax1.plot(default_init_path.x, default_init_path.y, default_init_path.z, 'ko',
         #  color="0.5"
         )
#ax1.plot(default_init_path.x, default_init_path.y, default_init_path.z, 'y')
ax1.plot(default_init_path.x, default_init_path.y, default_init_path.z, 'r--',
         #  color="0"
         )
ax1.plot(current_path_linear_interp.x, current_path_linear_interp.y,
         current_path_linear_interp.z, 'b',
         #  color="0.4"
         )
ax1.legend(['Waypoints', 'Generated path', 'Actual path'])
ax1.set_xlabel('X axis')
ax1.set_ylabel('Y axis')
ax1.set_zlabel('Z axis')
fig1.savefig(dir_save_data + 'pathmode0.eps', format='eps', dpi=1200)
plt.figure(num="Linear interpolation normal distance")
plt.plot(normal_dist_linear_interp.Time, normal_dist_linear_interp.Linear, 'b')
plt.xlabel('Time (s)')
plt.ylabel('Distance (m)')
plt.ylim(top=1)
plt.legend(["Follower"])
plt.savefig(dir_save_data + 'ndistmode0.eps', format='eps', dpi=1200)
plt.show(block=False)
''' Plot cubic loyal spline '''
fig2 = plt.figure(num="Cubic loyal spline 3D behavior")
ax2 = Axes3D(fig2)
ax2.plot(default_init_path.x, default_init_path.y, default_init_path.z, 'ko',
         #  color="0.5"
         )
#ax2.plot(default_cubic_spline_loyal_path.x, default_cubic_spline_loyal_path.y, default_cubic_spline_loyal_path.z, 'y')
ax2.plot(default_cubic_spline_loyal_path.x, default_cubic_spline_loyal_path.y, default_cubic_spline_loyal_path.z, 'r--',
         #  color="0"
         )
ax2.plot(current_path_cubic_loyal.x, current_path_cubic_loyal.y, current_path_cubic_loyal.z, 'b',
         #  color="0.4"
         )
ax2.legend(['Waypoints', 'Generated path', 'Actual path'])
ax2.set_xlabel('X axis')
ax2.set_ylabel('Y axis')
ax2.set_zlabel('Z axis')
fig2.savefig(dir_save_data + 'pathmode1.eps', format='eps', dpi=1200)
plt.figure(num="Cubic loyal spline normal distance")
plt.plot(normal_dist_cubic_loyal.Time, normal_dist_cubic_loyal.Spline, 'b')
plt.xlabel('Time (s)')
plt.ylabel('Distance (m)')
plt.ylim(top=1)
plt.legend(["Follower"])
plt.savefig(dir_save_data + 'ndistmode1.eps', format='eps', dpi=1200)
plt.show(block=False)
''' Plot cubic spline '''
fig3 = plt.figure(num="Cubic spline 3D behavior")
ax3 = Axes3D(fig3)
ax3.plot(default_init_path.x, default_init_path.y, default_init_path.z, 'ko',
         #  color="0.5"
         )
#ax3.plot(default_cubic_spline_path.x, default_cubic_spline_path.y, default_cubic_spline_path.z, 'y')
ax3.plot(default_cubic_spline_path.x, default_cubic_spline_path.y, default_cubic_spline_path.z, 'r--',
         #  color="0"
         )
ax3.plot(current_path_cubic_spline.x, current_path_cubic_spline.y, current_path_cubic_spline.z, 'b',
         #  color="0.4"
         )
ax3.legend(['Waypoints', 'Generated path', 'Actual path'])
ax3.set_xlabel('X axis')
ax3.set_ylabel('Y axis')
ax3.set_zlabel('Z axis')
fig3.savefig(dir_save_data + 'pathmode2.eps', format='eps', dpi=1200)
plt.figure(num="Cubic spline normal distance")
plt.plot(normal_dist_cubic_spline.Time, normal_dist_cubic_spline.Spline, 'b')
plt.xlabel('Time (s)')
plt.ylabel('Distance (m)')
plt.ylim(top=1)
plt.legend(["Follower"])
plt.savefig(dir_save_data + 'ndistmode2.eps', format='eps', dpi=1200)
plt.show()

''' Print results of the normal distance through path '''
print('----------------------------------------------------------------------')
print('Interp1 -> max: {:.3f}, min: {:.3f}, mean: {:.3f}, std: {:.3f}, var: {:.3f}'.format(np.max(normal_dist_linear_interp.Linear), np.min(
    normal_dist_linear_interp.Linear), np.mean(normal_dist_linear_interp.Linear), np.std(normal_dist_linear_interp.Linear), np.var(normal_dist_linear_interp.Linear)))
print('----------------------------------------------------------------------')
print('Cubic L -> max: {:.3f}, min: {:.3f}, mean: {:.3f}, std: {:.3f}, var: {:.3f}'.format(np.max(normal_dist_cubic_loyal.Spline), np.min(
    normal_dist_cubic_loyal.Spline), np.mean(normal_dist_cubic_loyal.Spline), np.std(normal_dist_cubic_loyal.Spline), np.var(normal_dist_cubic_loyal.Spline)))
print('----------------------------------------------------------------------')
print('Cubic   -> max: {:.3f}, min: {:.3f}, mean: {:.3f}, std: {:.3f}, var: {:.3f}'.format(np.max(normal_dist_cubic_spline.Spline), np.min(
    normal_dist_cubic_spline.Spline), np.mean(normal_dist_cubic_spline.Spline), np.std(normal_dist_cubic_spline.Spline), np.var(normal_dist_cubic_spline.Spline)))
print('----------------------------------------------------------------------')

max_stars_repo_name: hecperleo/pathFollower

max_issues_repo_name: hecperleo/pathFollower

max_forks_repo_name: hecperleo/pathFollower


text:
# standard
import numpy as np
from scipy.interpolate import LinearNDInterpolator
import os
import time
import pickle

# kepler_grids
from pyburst.grids import grid_tools, grid_strings, grid_versions
from . import interp_versions

GRIDS_PATH = os.environ['KEPLER_GRIDS']
MODELS_PATH = os.environ['KEPLER_MODELS']

key_map = {'dt': 'tDel', 'u_dt': 'uTDel',
           'fluence': 'fluence', 'u_fluence': 'uFluence',
           'peak': 'peakLum', 'u_peak': 'uPeakLum'}

# TODO:
#   - function to re-generate interpolator files

class Kemulator:
    """Kepler emulator class. Creates a 'model' of kepler results

    parameters
    ----------
    source : str
        source object of grid to emaulate (e.g., gs1826)
    verbose : bool
        print diagnostics
    re_interp: bool
        setup interpolator
    """

    def __init__(self, source, version, verbose=True, re_interp=True,
                 lampe_analyser=False, check_complete=True):
        self.verbose = verbose
        source = grid_strings.source_shorthand(source)
        self.source = source
        self.version = version
        self.lampe_analyser = lampe_analyser

        self.interpolator = None
        self.params = None
        self.summ = None

        self.grid_def = None
        self.version_def = interp_versions.InterpVersion(source=source, version=version)
        self.bprops = self.version_def.bprops

        if re_interp:
            self.load_tables()

            if check_complete:
                grid_tools.check_complete(param_table=self.params, raise_error=True,
                                          param_list=self.version_def.param_keys)
            self.setup_interpolator(self.bprops)
        else:
            self.load_interpolator()

    def printv(self, string, **kwargs):
        """Prints string if self.verbose == True
        """
        if self.verbose:
            print(string, **kwargs)

    def load_tables(self):
        """Loads param and summ tables, and excludes scpecified models
        """
        raw_summ = grid_tools.load_grid_table('summ', source=self.source,
                                              lampe_analyser=self.lampe_analyser)
        raw_params = grid_tools.load_grid_table('params', source=self.source)

        self.grid_def = grid_versions.GridVersion(source=self.source,
                                                  version=self.version_def.grid_version)

        params = grid_tools.exclude_params(table=raw_params,
                                           params=self.grid_def.exclude_any,
                                           logic='any')
        params = grid_tools.exclude_params(table=params,
                                           params=self.grid_def.exclude_all,
                                           logic='all')
        idxs_kept = params.index
        self.summ = raw_summ.loc[idxs_kept]
        self.params = params

    def reduce_summ(self, params):
        """Returns reduced summ table with specified parameters (e.g., mass, qb)
        """
        reduced_idxs = grid_tools.reduce_table_idx(self.params, params=params)
        return self.summ.iloc[reduced_idxs]

    def save_interpolator(self):
        """Saves (pickles) interpolator to file
        """
        self.printv(f'Saving interpolator')
        filename = f'interpolator_{self.source}_V{self.version}'
        filepath = os.path.join(GRIDS_PATH, 'sources', self.source,
                                'interpolator', filename)
        self.printv(f'Saving interpolator: {filepath}')
        pickle.dump(self.interpolator, open(filepath, 'wb'))

    def load_interpolator(self):
        """Loads previously-saved (pickled) interpolators from file
        """
        filename = f'interpolator_{self.source}_V{self.version}'
        filepath = os.path.join(GRIDS_PATH, 'sources', self.source,
                                'interpolator', filename)
        self.printv(f'Loading interpolator: {filepath}')
        self.interpolator = pickle.load(open(filepath, 'rb'))

    def setup_interpolator(self, bprops):
        """Creates interpolator object from kepler grid data

        bprops : [str]
            burst properties to interpolate (e.g., dt, fluence)
        """
        self.printv('Creating interpolator on grid: ')
        points = []

        for param in self.version_def.param_keys:
            param_points = np.array(self.params[param])
            points += [param_points]
            self.printv(f'{param}:  {np.unique(param_points)}')

        points = tuple(points)
        n_models = len(self.params)
        n_bprops = len(bprops)
        values = np.full((n_models, n_bprops), np.nan)
        self.printv(f'Number of models: {n_models}')

        t0 = time.time()
        self.printv(f'Creating interpolator:')

        for i, bp in enumerate(bprops):
            if self.lampe_analyser:
                key = key_map[bp]
            else:
                key = bp
            values[:, i] = np.array(self.summ[key])  # * 0.9
        self.interpolator = LinearNDInterpolator(points, values)
        t1 = time.time()
        self.printv(f'Setup time: {t1-t0:.1f} s')

    def emulate_burst(self, params):
        """Returns interpolated burst properties for given params

        params : [acc, x, z, qb, mass]
        """
        # check_params_length(params, length=len(self.version_def.param_keys))
        return self.interpolator(params)

def check_params_length(params, length=5):
    """Checks that five parameters have been provided
    """
    # TODO: generalise this for emulators with some parameters fixed
    def check(array):
        if len(array) != length:
            raise ValueError("'params' must specify each of (acc, x, z, qb, mass)")

    if len(params.shape) == 1:
        check(params)
    elif len(params.shape) == 2:
        check(params[0])

max_stars_repo_name: zacjohnston/pyburst

max_issues_repo_name: zacjohnston/pyburst

max_forks_repo_name: zacjohnston/pyburst


text:
import cv2
import numpy as np
import sys
import os
import fnmatch

def sharpen(image):
    kernel = np.array([[0, -1, 0], [-1, 5,-1],[0,-1,0]])
    new_image = cv2.filter2D(image, -1, kernel)
    cv2.imageshow('Sharpened', image_new)
    cv2.waitKey(0)
    return new_image

def blur(image):
    kernels = [3, 5, 9, 13]
    for idx, k in enumerate(kernels):
        image_b1 = cv2.bluer(image, ksize=(k,k))
        cv2.imshow(str(k),image_b1)
    return

def resize(fname, width, height):
    image = cv2.imread(fname)
    cv2.imshow('Original image', image)
    cv2.waitKey(0)
    org_height, org_width = image.shape[0:2]
    print("width: ", org_width)
    print("height: ", org_height)

    if org_width >= org_height:
        new_image = cv2.resize(image, (width, height))
    else:
        new_image = cv2.resize(image, (height, width))

    return fname, new_image

listOfFiles = os.listdir('.')
pattern = "*.jpg"
n = len(sys.argv)
if n == 3:
    width = int(sys.argv[1])
    height = int(sys.argv[2])
else:
    width = 1280
    height = 980
if not os.path.exists('new_folder'):
    os.makedirs('new_folder')

for filename in listOfFiles:
    if fnmatch.fnmatch(filename, pattern):
        filename, new_image = resize(filename, width, height)
        cv2.imwrite("new_folder\\" + filename, new_image)

#cv2.imshow('resized image', new_image)
#cv2.waitKey(0)
#cv2.blur(new_image)
#image = sharpen(new_image)

max_stars_repo_name: melissarib/computer-vision

max_issues_repo_name: melissarib/computer-vision

max_forks_repo_name: melissarib/computer-vision


text:
#!/usr/bin/env python
# coding: utf-8
import time
import sys
import subprocess
import traceback
import os
#import itertools
import numpy as np
#import matplotlib.pyplot as plt
import tqdm
import pandas as pd
import IJB_evals as IJB
#import map_tools

class Machine:
    '''
    convenience class for profiling a machine and tracking how much capacity it has
    increment/decrement n_processes as you add and remove jobs
    use get_processes_available to determine how much capacity is remaining
    '''
    def __init__(self, host):
        self.host = host
        self.processes = []
        self.has_problem = False
        try:
            self.remaining_memory_MB, self._n_cpu, self._load1, self._load5, self._load15 = self._get_machine_stats(host)
            self.remaining_cpu = self._n_cpu - max(self._load1, self._load5, self._load15)
        except:
            print('Error occurred while profiling', host)
            traceback.print_exc()
            self.has_problem = True

    def _get_machine_stats(self, machine):
        proc = subprocess.Popen('ssh {machine} -T -o StrictHostKeyChecking=no "cat /proc/loadavg && free -m && lscpu -p=cpu"'.format(machine=machine),
                                shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        #print(proc.args)
        splits = proc.communicate()[0].decode('utf-8').replace('\r', '').split('\n')
        #print(splits)
        rawload = splits[0]
        rawfree = splits[2]
        n_cores = len(splits) - 1 - splits.index('0')
        load1, load5, load15 = [float(load) for load in rawload.split(' ')[:3]]
        available = [int(s) for s in rawfree.split(' ') if s.isdigit()][-1]
        return available, n_cores, load1, load5, load15

    def get_processes_available(self, memory_per_process, cpu_per_process):
        memory_capacity_per_process = self.remaining_memory_MB / memory_per_process
        cpu_capacity_per_process = self.remaining_cpu / cpu_per_process
        memory_remaining_per_process = memory_capacity_per_process - len(self.processes)
        cpu_remaining_per_process = cpu_capacity_per_process - len(self.processes)
        return int(min(memory_remaining_per_process, cpu_remaining_per_process))

def schedule_procs(proc_args, hosts, log_fn, memory_per_process, cpu_per_process=1):
    '''pushes jobs to hosts as they become available, waits for them to finish, logs each jobs output to text file (log_fn)'''
    procs = []
    print('Profiling...', end='')
    print('Scheduling', len(proc_args), 'jobs to', len(hosts), 'machines, logging to', log_fn)
    print('!!!cancellation may leave open processes, use abort.py to kill these!!!')
    # get each machine's stats for capacity determination
    machines = [Machine(host) for host in hosts]
    # remove any that had issues
    machines = [machine for machine in machines if not machine.has_problem]
    print('Hosts removed due to profiling error:', len(hosts) - len(machines))

    total_descriptors_open = 0
    with open(log_fn, 'w') as fout:
        with tqdm.tqdm(total=len(proc_args), file=sys.stdout) as pbar:
            i, k = 0, 0  # machine index, process index
            n_complete = 0
            n_errors = 0
            while n_complete < len(proc_args):
                # resolve completed processes on machine i
                j = len(machines[i].processes) - 1
                while j >= 0:  # iterate over this machine's processes
                    poll = machines[i].processes[j].poll()
                    if poll is not None:  # process has finished
                        if poll == 0:  # process was successful
                            fout.write('finished job: {}\n'.format(machines[i].processes[j].args))
                        else:  # process failed
                            output=machines[i].processes[j].communicate()
                            fout.write('error for job: {}\nSTDERR\n{}\nSTDOUT\n{}\n'.format(
                                machines[i].processes[j].args, output[1].decode('utf-8'), output[0].decode('utf-8')))
                            n_errors += 1
                        del machines[i].processes[j]
                        n_complete += 1
                        pbar.update(1)
                    j -= 1

                # don't start new jobs if there are too many running (OS descriptor limit)
                if k - n_complete < 256:
                    # get machine i's remaining capacity
                    n_processes_available = machines[i].get_processes_available(memory_per_process, cpu_per_process)

                    # fill remaining capacity on this machine, potentially finishing arguments
                    for _ in range(min(n_processes_available, len(proc_args) - k)):
                        process = subprocess.Popen(proc_args[k].replace('HOSTNAME', machines[i].host),
                                shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                        machines[i].processes.append(process)
                        k += 1

                # iterate to the next machine, logging after each round-robin repetition
                i += 1
                #tqdm.tqdm.write(' '.join(['k', str(k), 'n_complete', str(n_complete)]))
                if i >= len(machines):
                    #tqdm.tqdm.write(' '.join(['{}:{}'.format(machine.host, len(machine.processes)) for machine in machines]))
                    i = 0
                    #time.sleep(wait_time)

            print('Jobs completed:', n_complete, 'including', n_errors, 'errors')

def get_proc_args():
    '''CHANGE THIS! produce a list of commands that you want to distribute across many machines'''
    embs_list = [ \
        ('/s/red/b/nobackup/data/portable/tbiom/models/insightface/evaluation/IJB/IJB_result/MS1MV2-ResNet100-Arcface_IJBC.npz', 'MS1MV2', 'ResNet100', 'ArcFace'),
        ('/s/red/b/nobackup/data/portable/tbiom/models/insightface/evaluation/IJB/IJB_result/VGG2-ResNet50-Arcface_IJBC.npz', 'VGGFace2', 'ResNet50', 'ArcFace'),
        ('/s/red/b/nobackup/data/portable/tbiom/models/insightface/evaluation/IJB/IJB_result/glint360k_r100FC_0.1_IJBC.npz', 'Glint360k', 'ResNet100', 'PartialFC_r0.1'),
        ('/s/red/b/nobackup/data/portable/tbiom/models/insightface/evaluation/IJB/IJB_result/glint360k_r100FC_1.0_IJBC.npz', 'Glint360k', 'ResNet100', 'PartialFC_r1.0'),
        ('/s/red/b/nobackup/data/portable/tbiom/models/arcface-tf2/ijbc_embs_arc_res50.npy', 'MS1M', 'ResNet50', 'ArcFace'),
        ('/s/red/b/nobackup/data/portable/tbiom/models/arcface-tf2/ijbc_embs_arc_mbv2.npy', 'MS1M', 'MobileNetV2', 'ArcFace'),
        ('/s/red/b/nobackup/data/portable/tbiom/models/facenet/vggface2_ir2_ijbc_embs.npy', 'VGGFace2', 'InceptionResNetV1', 'CenterLoss'),
        ('/s/red/b/nobackup/data/portable/tbiom/models/facenet/casia_ir2_ijbc_embs.npy', 'CASIA-WebFace', 'InceptionResNetV1', 'CenterLoss'),
        ('/s/red/b/nobackup/data/portable/tbiom/models/Probabilistic-Face-Embeddings/ijbc_embs_pfe_sphere64_msarcface_am.npy', 'MS1M', '64-CNN', 'SphereFace+PFE'),
        ('/s/red/b/nobackup/data/portable/tbiom/models/Probabilistic-Face-Embeddings/ijbc_embs_pfe_sphere64_casia_am.npy', 'CASIA-WebFace', '64-CNN', 'SphereFace+PFE'),
    ]

    template = r'ssh HOSTNAME -T -o StrictHostKeyChecking=no "cd {cd} && python {script} {args}"'
    exec_root = '/s/red/b/nobackup/data/portable/tbiom/models/insightface/evaluation/IJB'
    exec_script = 'IJB_evals.py'

    proc_args = []
    tuple_args = []
    for left_embs, left_dataset, left_architecture, left_head in embs_list:
        for right_embs, right_dataset, right_architecture, right_head in embs_list:
            if left_embs != right_embs:
                for decay_coef in [0.0, 0.00001, 0.0001, 0.001, 0.01, 0.1, 1.0, 10.0, 100.0]:
                    arg = "-s IJBC -b 64 -c 0.0001 -U -Q -D -M" # norm features before mapping, before eval, multiply face detector score before eval
                    if left_embs != right_embs:
                        arg += ' -r ' + left_embs
                        arg += ' -q ' + right_embs
                        arg += ' -c ' + str(decay_coef)

                        save_result_name = '{}_TO_{}_D{:.0e}'.format(
                            left_embs.split('/')[-1].split('.')[0],
                            right_embs.split('/')[-1].split('.')[0],
                            decay_coef)

                        save_result = '../../../../results/decay/{}.npz'.format(save_result_name)
                        arg += ' -S ' + save_result
                        proc_arg = template.format(cd=exec_root, script=exec_script, args=arg)
                        proc_args.append(proc_arg)
                    #tuple_args.append((is_rotation_map, n_individuals, left_embs, left_dataset, left_architecture, left_head, right_embs, right_dataset, right_architecture, right_head, save_result))
    return proc_args, tuple_args

# def gather_results(tuple_args):
#     cols = np.load(tuple_args[0][-1])['FAR']
#     results = np.zeros((len(tuple_args), len(cols)))
#     for i in range(len(tuple_args)):
#         results[i] = np.load(tuple_args[i][-1])['TAR@FAR']

#     return results

def main():
    args, tuple_args = get_proc_args()
    colors = ['blue', 'cyan', 'yellow', 'magenta', 'pink', 'teal', 'aqua']
    bugs = ['ant', 'antlion', 'aphid', 'assassin-bug', 'bee', 'centipede', 'cockroach', 'cricket', 'damselfly', 'deer-fly', 'dragonfly', 'dung-beetle', 'flea', 'hornet', 'katydid', 'ladybug', 'lice', 'maggot', 'mosquito', 'moth', 'preying-mantis', 'scorpion', 'termite', 'tick', 'wasp', 'weevil']
    schedule_procs(args, colors + bugs, log_fn='dist-log.txt', memory_per_process=12000, cpu_per_process=1)

#     results = gather_results(tuple_args)
#     np.save('../../../../results/sensitivity/all.npy', results)

if __name__ == '__main__':
    main()

# TODO test product of -U, -Q, -D (false, false, false and true, true, true already complete) with n_individuals == -1


text:
from __future__ import print_function

import numpy as np
import matplotlib.pyplot as plt

from astropy.wcs import WCS
from astropy.io import fits
from astropy.tests.helper import catch_warnings, pytest

from ..core import WCSAxes
from ..utils import get_coord_meta

def test_grid_regression():
    # Regression test for a bug that meant that if the rc parameter
    # axes.grid was set to True, WCSAxes would crash upon initalization.
    plt.rc('axes', grid=True)
    fig = plt.figure(figsize=(3, 3))
    WCSAxes(fig, [0.1, 0.1, 0.8, 0.8])

def test_format_coord_regression(tmpdir):
    # Regression test for a bug that meant that if format_coord was called by
    # Matplotlib before the axes were drawn, an error occurred.
    fig = plt.figure(figsize=(3, 3))
    ax = WCSAxes(fig, [0.1, 0.1, 0.8, 0.8])
    fig.add_axes(ax)
    assert ax.format_coord(10,10) == ""
    assert ax.coords[0].format_coord(10) == ""
    assert ax.coords[1].format_coord(10) == ""
    fig.savefig(tmpdir.join('nothing').strpath)
    assert ax.format_coord(10,10) == "11.0 11.0 (world)"
    assert ax.coords[0].format_coord(10) == "10.0"
    assert ax.coords[1].format_coord(10) == "10.0"

TARGET_HEADER = fits.Header.fromstring("""
NAXIS   =                    2
NAXIS1  =                  200
NAXIS2  =                  100
CTYPE1  = 'RA---MOL'
CRPIX1  =                  500
CRVAL1  =                180.0
CDELT1  =                 -0.4
CUNIT1  = 'deg     '
CTYPE2  = 'DEC--MOL'
CRPIX2  =                  400
CRVAL2  =                  0.0
CDELT2  =                  0.4
CUNIT2  = 'deg     '
COORDSYS= 'icrs    '
""", sep='\n')

def test_no_numpy_warnings():

    # Make sure that no warnings are raised if some pixels are outside WCS
    # (since this is normal)

    ax = plt.subplot(1,1,1, projection=WCS(TARGET_HEADER))
    ax.imshow(np.zeros((100,200)))
    ax.coords.grid(color='white')

    with catch_warnings(RuntimeWarning) as ws:
        plt.savefig('test.png')

    # For debugging
    for w in ws:
        print(w)

    assert len(ws) == 0

def test_invalid_frame_overlay():

    # Make sure a nice error is returned if a frame doesn't exist
    ax = plt.subplot(1,1,1, projection=WCS(TARGET_HEADER))
    with pytest.raises(ValueError) as exc:
        ax.get_coords_overlay('banana')
    assert exc.value.args[0] == 'Unknown frame: banana'

    with pytest.raises(ValueError) as exc:
        get_coord_meta('banana')
    assert exc.value.args[0] == 'Unknown frame: banana'

max_stars_repo_name: Cadair/wcsaxes

max_issues_repo_name: Cadair/wcsaxes

max_forks_repo_name: Cadair/wcsaxes

reason: import numpy,from astropy


text:
# permits IO and conversion on various sparse matrix formats.
import libHercMatrix
import libBXF
import scipy
import numpy
import scipy.io
from numpy.lib.recfunctions import append_fields
import traceback
import pprint
import os
import logging
import libValcolIO
import MatrixUtils

## @package libHercmIO
# libHercmIO is the aggregate IO provider for all python HeRC Matrix Tools. It's
# primary purpose is to wrap other io functions, such as those provided by libBXF
# and scipy.io

## wrapper for various matrix read functions
# Reads matrices of any supported format, then returns the matrix as an
# instance of `libHercMatrix.HercMatrix`.
#
# @param[in] filename a string containing the absolute or relative path of the
# file to read
# @param[in] form a string containing the format of the file to read. Currently,
# valid values are `bxf`, `hercm`, `mat`, and `mtx`.
# @param[in] showProgress if `True`, verbose progress messages are printed.
# Defaults to `False`.
#
# @return the matrix as an instance of `libHercMatrix.hercMatrix`
#
# @throws IOError if the specified file could not be opened for writing
#

def readMatrix(filename, form, showProgress=False):
    HERCMATRIX = libHercMatrix.hercMatrix()

    logging.info("reading matrix {0} in format {1}".format(filename, form))

    if (form == 'hercm') or (form == 'bxf'):
        # TODO: exception handling
        HERCMATRIX = libBXF.read(filename)

    elif form == 'mtx':
        from scipy import io
        from scipy.sparse import csr_matrix
        from numpy import array

        # reads in an MTX file and converts it to hercm

        try:
            if showProgress:
                print("reading data from file...")

            rawMatrix = scipy.sparse.coo_matrix(scipy.io.mmread(filename))

            if 'symmetric' in io.mminfo(filename):
                HERCMATRIX.symmetry = "SYM"
            else:
                HERCMATRIX.symmetry = "ASYM"

            hercm = {}  # needed to generate verification

            if showProgress:
                print("generating header data..")
            hercm['val'] = rawMatrix.data
            hercm['col'] = rawMatrix.col.tolist()
            hercm['row'] = rawMatrix.row.tolist()
            (matrixWidth, matrixHeight) = rawMatrix.shape
            HERCMATRIX.height = int(matrixHeight)
            HERCMATRIX.width = int(matrixWidth)
            vs = libBXF.generateVerificationSum(hercm)
            HERCMATRIX.verification = vs
            HERCMATRIX.remarks = []

            # I'm not sure why  has to be hard...
            # http://stackoverflow.com/questions/26018781/numpy-is-it-possible-to-preserve-the-dtype-of-columns-when-using-column-stack

            if showProgress:
                print("preparing matrix data...")
            val = numpy.asarray(hercm['val'], dtype='float64')
            col = numpy.asarray(hercm['col'], dtype='int32')
            row = numpy.asarray(hercm['row'], dtype='int32')

            val = numpy.rec.array(val, dtype=[(('val'), numpy.float64)])
            col = numpy.rec.array(col, dtype=[(('col'), numpy.int32)])
            row = numpy.rec.array(row, dtype=[(('row'), numpy.int32)])

            HERCMATRIX.elements = append_fields(row,
                    'col',
                    col,
                    usemask=False,
                    dtypes=[numpy.int32])

            HERCMATRIX.elements = append_fields(HERCMATRIX.elements,
                    'val',
                    val,
                    usemask=False,
                    dtypes=[numpy.float64])

            HERCMATRIX.nzentries = len(HERCMATRIX.elements['val'])

            HERCMATRIX.verification = libBXF.generateVerificationSum(
                HERCMATRIX)

            if showProgress:
                print("finished reading matrix")

        except IOError as e:  # make sure the file exists and is readable
            logging.warning("(lsc-480) could not open matrix file")
            raise IOError("could not open matrix file for writing...",
                          str(e))

    elif form == 'mat':  # matlab matrices
        from scipy import io
        from scipy import sparse
        from numpy import array

        try:

            rawMatrix = scipy.sparse.coo_matrix(
                scipy.io.loadmat(filename)['matrix'])

            hercm = {}  # needed to generate verification

            hercm['val'] = rawMatrix.data
            hercm['col'] = rawMatrix.col.tolist()
            hercm['row'] = rawMatrix.row.tolist()
            (matrixWidth, matrixHeight) = rawMatrix.shape
            HERCMATRIX.height = int(matrixHeight)
            HERCMATRIX.width = int(matrixWidth)
            vs = libBXF.generateVerificationSum(hercm)
            HERCMATRIX.verification = vs
            HERCMATRIX.remarks = []

            # I'm not sure why  has to be hard...
            # http://stackoverflow.com/questions/26018781/numpy-is-it-possible-to-preserve-the-dtype-of-columns-when-using-column-stack

            val = numpy.asarray(hercm['val'], dtype='float64')
            col = numpy.asarray(hercm['col'], dtype='int32')
            row = numpy.asarray(hercm['row'], dtype='int32')

            val = numpy.rec.array(val, dtype=[(('val'), numpy.float64)])
            col = numpy.rec.array(col, dtype=[(('col'), numpy.int32)])
            row = numpy.rec.array(row, dtype=[(('row'), numpy.int32)])

            HERCMATRIX.elements = append_fields(row,
                    'col',
                    col,
                    usemask=False,
                    dtypes=[numpy.int32])

            HERCMATRIX.elements = append_fields(HERCMATRIX.elements,
                    'val', val, usemask=False, dtypes=[numpy.float64])

            HERCMATRIX.nzentries = len(HERCMATRIX.elements['val'])

            if HERCMATRIX.checkSymmetry():
                HERCMATRIX.symmetry = 'SYM'

            HERCMATRIX.verification = libBXF.generateVerificationSum(
                HERCMATRIX)

        except IOError as e:  # make sure the file exists and is readable
            logging.warning("(lsc-536)could not open matrix file")
            raise IOError("could not open matrix file for writing...",
                          str(e))

    elif form == 'valcol':
        HERCMATRIX = libValcolIO.read(filename)

    else:
        logging.warning("(lsc-545) format {0} is not valid".format(form))

    if showProgress:
        print("converting matrix to row-major...")

    logging.info("converting matrix to row-major")
    HERCMATRIX.makeRowMajor()

    if showProgress:
        print("matrix is now row major")

    if HERCMATRIX.symmetry == 'SYM':
        logging.info("matrix is symmetric, truncating upper triangle")
        if showProgress:
            print("matrix is symmetric, truncating upper triangle...")
        HERCMATRIX.makeSymmetrical('truncate')
        if showProgress:
            print("upper triangle truncated")

    return HERCMATRIX

## Writes matrices from libHercMatrix.hercMatrix instances
# Writes matrices in any supported format.
#
# @param[in] filename string containing the relative or absolute path to the file
# to write
# @param[in] form the format in which to write the file, one of `hercm`, `bxf`,
# `mtx`, or `mat`
# @param[in] HERCMATRIX an instance of libHercMatrix.hercMatrix, whose contents
# will be written to the file
#
# @return `None`
#
# @throws TypeError if `form` is not a valid format

def writeMatrix(filename, form, HERCMATRIX):
    # writes HERCMATRIX to the file
    # filename is a string indicating path of file
    # format is a string indicating file format (mtx or hercm)
    # returns True on success, None on failure

    logging.info("writing matrix {0} in format {1}..."
            .format(filename, form))

    if os.path.isfile(filename):
        logging.warning("(lsc-566) file exists, cannot write")
        raise FileExistsError("could not write to file {0}".format(filename)
                + " file already exists!")

    if HERCMATRIX.symmetry == 'SYM':
        logging.info("matrix is symmetric, truncating lower triangle")
        HERCMATRIX.makeSymmetrical('truncate')

    logging.info("making matrix row major...")
    HERCMATRIX.makeRowMajor()

    if form == 'hercm':
        # TODO: these will probably need a try/except block at some point

        libBXF.write(HERCMATRIX, filename, "HERCM")

    elif form == 'bxf':

        libBXF.write(HERCMATRIX, filename)

    elif form == 'mtx':
        try:
            scipy.io.mmwrite(filename, HERCMATRIX.getInFormat('coo'))
        except ValueError as e:
            logging.warning("""(lsc-589) encountered ValueError exception
while writing file. Exception: {0}. You probably have out of bounds indices
in row or col""".format(e))
        except Exception as e:
            logging.warning("""(lsc-593) encountered general error while
 writing: {0}""".format(str(e)))

        # fix header of mtx file
        if HERCMATRIX.symmetry == 'SYM':
            logging.info("format is mtx and matrix is symmetric, fixing" +
                    " header...")
            with open(filename) as inputFile:
                lines = inputFile.readlines()
            lines[0] = "%%MatrixMarket matrix coordinate pattern symmetric\n"

            with open(filename, 'w') as outputFile:
                for line in lines:
                    outputFile.write(line)

    elif form == 'mat':  # matlab matrix file
        from scipy import io
        from scipy import sparse
        from numpy import array

        scipy.io.savemat(filename, {'matrix': HERCMATRIX.getInFormat('coo')})

    elif form == 'valcol':
        libValcolIO.write(filename, HERCMATRIX)

    else:
        logging.warning("(lsc-621) format {0} is not valid".format(form))
        raise TypeError("{0} is not a valid format"
                        .format(form))

max_stars_repo_name: charlesdaniels/hercm

max_issues_repo_name: charlesdaniels/hercm

max_forks_repo_name: charlesdaniels/hercm

reason: import numpy,from numpy,import scipy,from scipy