text:
#!/usr/bin/env/ python
# ECBM E4040 Fall 2020 Assignment 2
# This Python script contains the ImageGenrator class.

import numpy as np
from matplotlib import pyplot as plt
import os

try:
    from scipy.ndimage.interpolation import rotate
except ModuleNotFoundError:
    os.system('pip install scipy')
    from scipy.ndimage.interpolation import rotate

class ImageGenerator(object):
    def __init__(self, x, y):
        """
        Initialize an ImageGenerator instance.
        :param x: A Numpy array of input data. It has shape (num_of_samples, height, width, channels).
        :param y: A Numpy vector of labels. It has shape (num_of_samples, ).
        """

        # TODO: Your ImageGenerator instance has to store the following information:
        # x, y, num_of_samples, height, width, number of pixels translated, degree of rotation, is_horizontal_flip,
        # is_vertical_flip, is_add_noise. By default, set boolean values to False.
        #
        # Hint: Since you may directly perform transformations on x and y, and don't want your original data to be contaminated
        # by those transformations, you should use numpy array build-in copy() method.
        #######################################################################
        #                         TODO: YOUR CODE HERE                        #
        #######################################################################

        self.x = x
        self.y = y
        self.N, self.height, self.width, self.channels = x.shape
        self.degree_of_rotation = 0
        self.is_horizontal_flip = False
        self.is_vertical_flip = False
        self.is_add_noise = False
        self.is_bright=False
        self.trans_height = 0
        self.trans_width = 0

        #######################################################################
        #                                END TODO                             #
        #######################################################################

        # One way to use augmented data is to store them after transformation (and then combine all of them to form new data set).
        # Following variables (along with create_aug_data() function) is one kind of implementation.
        # You can either figure out how to use them or find out your own ways to create the augmented dataset.
        # if you have your own idea of creating augmented dataset, just feel free to comment any codes you don't need

        self.translated = None
        self.rotated = None
        self.flipped = None
        self.added = None
        self.bright = None
        self.x_aug = self.x.copy()
        self.y_aug = self.y.copy()
        self.N_aug = self.N

    def create_aug_data(self):
        # If you want to use function create_aug_data() to generate new dataset, you can perform the following operations in each
        # transformation function:
        #
        # 1.store the transformed data with their labels in a tuple called self.translated, self.rotated, self.flipped, etc.
        # 2.increase self.N_aug by the number of transformed data,
        # 3.you should also return the transformed data in order to show them in task4 notebook

        '''
        Combine all the data to form a augmented dataset
        '''
        if self.translated:
            self.x_aug = np.vstack((self.x_aug,self.translated[0]))
            self.y_aug = np.hstack((self.y_aug,self.translated[1]))
        if self.rotated:
            self.x_aug = np.vstack((self.x_aug,self.rotated[0]))
            self.y_aug = np.hstack((self.y_aug,self.rotated[1]))
        if self.flipped:
            self.x_aug = np.vstack((self.x_aug,self.flipped[0]))
            self.y_aug = np.hstack((self.y_aug,self.flipped[1]))
        if self.added:
            self.x_aug = np.vstack((self.x_aug,self.added[0]))
            self.y_aug = np.hstack((self.y_aug,self.added[1]))
        if self.bright:
            self.x_aug = np.vstack((self.x_aug,self.bright[0]))
            self.y_aug = np.hstack((self.y_aug,self.bright[1]))

        print("Size of training data:{}".format(self.N_aug))

    def next_batch_gen(self, batch_size, shuffle=True):
        """
        A python generator function that yields a batch of data infinitely.
        :param batch_size: The number of samples to return for each batch.
        :param shuffle: If True, shuffle the entire dataset after every sample has been returned once.
                        If False, the order or data samples stays the same.
        :return: A batch of data with size (batch_size, width, height, channels).
        """

        # TODO: Use 'yield' keyword, implement this generator. Pay attention to the following:
        # 1. The generator should return batches endlessly.
        # 2. Make sure the shuffle only happens after each sample has been visited once. Otherwise some samples might
        # not be output.

        # One possible pseudo code for your reference:
        ########################################################################
        #   calculate the total number of batches possible (if the rest is not sufficient to make up a batch, ignore)
        #   while True:
        #       if (batch_count < total number of batches possible):
        #           batch_count = batch_count + 1
        #           yield(next batch of x and y indicated by batch_count)
        #       else:
        #           shuffle(x)
        #           reset batch_count
        #######################################################################
        #                         TODO: YOUR CODE HERE                        #
        #######################################################################

        n_batch = self.N_aug // batch_size
        batch_count = 0
        while True:
            if batch_count < n_batch:
                batch_count += 1
                yield self.x_aug[batch_count * batch_size: (batch_count+1) * batch_size],self.y_aug[batch_count * batch_size: (batch_count+1) * batch_size]

            else:
                self.indexes = np.arange(self.N_aug)
                if shuffle:
                    np.random.shuffle(self.indexes)
                    self.x_aug = self.x_aug[self.indexes]
                    self.y_aug = self.y_aug[self.indexes]
                batch_count = 0

        #######################################################################
        #                                END TODO                             #
        #######################################################################

    def show(self, images):
        """
        Plot the top 16 images (index 0~15) for visualization.
        :param images: images to be shown
        """
        #######################################################################
        #                         TODO: YOUR CODE HERE                        #
        #######################################################################

        fig = plt.figure(figsize=(10, 10))

        for i in range(16):
            ax = fig.add_subplot(4, 4, i+1)
            ax.imshow(images[i, :].reshape(28, 28), 'gray')
            ax.axis('off')

        #######################################################################
        #                                END TODO                             #
        #######################################################################

    def translate(self, shift_height, shift_width):
        """
        Translate self.x by the values given in shift.
        :param shift_height: the number of pixels to shift along height direction. Can be negative.
        :param shift_width: the number of pixels to shift along width direction. Can be negative.
        :return translated: translated dataset
        """

        # TODO: Implement the translate() function. You may wonder what values to append to the edge after the shift. Here, use rolling instead. For
        # example, if you shift 3 pixels to the left, append the left-most 3 columns that are out of boundary to the
        # right edge of the picture.

        # HINT: use np.roll (https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.roll.html)

        #######################################################################
        #                         TODO: YOUR CODE HERE                        #
        #######################################################################

        self.trans_height += shift_height
        self.trans_width += shift_width
        translated = np.roll(self.x.copy(), (shift_width, shift_height), axis=(1, 2))
        print('Current translation: ', self.trans_height, self.trans_width)
        self.translated = (translated,self.y.copy())
        self.N_aug += self.N
        return translated

        #######################################################################
        #                                END TODO                             #
        #######################################################################

    def rotate(self, angle=0.0):
        """
        Rotate self.x by the angles (in degree) given.
        :param angle: Rotation angle in degrees.
        :return rotated: rotated dataset
        """

        # TODO: Implement the rotate() function. The angle of rotation should match the value specified by angle.
        # HINT: Use scipy.ndimage.interpolation.rotate (which is already imported for you)
        #######################################################################
        #                         TODO: YOUR CODE HERE                        #
        #######################################################################

        self.dor = angle
        rotated = rotate(self.x.copy(), angle,reshape=False,axes=(1, 2))
        print('Currrent rotation: ', self.dor)
        self.rotated = (rotated, self.y.copy())
        self.N_aug += self.N
        return rotated

        #######################################################################
        #                                END TODO                             #
        #######################################################################

    def flip(self, mode='h'):
        """
        Flip self.x according to the mode specified
        :param mode: 'h' or 'v' or 'hv'. 'h' means horizontal and 'v' means vertical.
        :return flipped: flipped dataset
        """
        assert mode == 'h' or 'v' or 'hv'
        if mode == 'h':
            flipped = np.flip(self.x.copy(), axis=2)
            self.is_horizontal_flip = not self.is_horizontal_flip
        elif mode == 'v':
            flipped = np.flip(self.x.copy(), axis=1)
            self.is_vertical_flip = not self.is_vertical_flip
        elif mode == 'hv':
            flipped = np.flip(np.flip(self.x.copy(), axis=0), axis=1)
            self.is_horizontal_flip = not self.is_horizontal_flip
            self.is_vertical_flip = not self.is_vertical_flip
        else:
            raise ValueError('Mode should be \'h\' or \'v\' or \'hv\'')
        print('Vertical flip: ', self.is_vertical_flip, 'Horizontal flip: ', self.is_horizontal_flip)

        self.flipped = (flipped,self.y.copy())
        self.N_aug += self.N
        return flipped

    def add_noise(self, portion, amplitude):
        """
        Add random integer noise to self.x.
        :param portion: The portion of self.x samples to inject noise. If x contains 10000 sample and portion = 0.1,
                        then 1000 samples will be noise-injected.
        :param amplitude: An integer scaling factor of the noise.
        :return added: dataset with noise added
        """
        # TODO: Implement the add_noise function. Remember to record the boolean value is_add_noise. Any noise function
        # is acceptable.
        #######################################################################
        #                         TODO: YOUR CODE HERE                        #
        #######################################################################

        self.is_add_noise = True
        self.portion = portion
        self.amplitude = amplitude
        added = self.x.copy()[:int(self.N * portion), :, :, :]
        noise = np.ones(added.shape, dtype='uint8')
        added += noise
        print('Current added noise: portion =', self.portion, ', amplitude =', self.amplitude)
        self.added = (added, self.y.copy()[:int(self.N * portion)])
        self.N_aug += self.N * portion
        return added

        #######################################################################
        #                                END TODO                             #
        #######################################################################

    def brightness(self, factor):
        """
        Scale the pixel values to increase the brightness
        :param factor: A number greater than or equal to 1 that decides how each pixel in the image will be scaled. If factor is 2, then
                       all pixel values will be doubled.
        :return bright: dataset with increased brightness
        """
        assert factor >= 1
        if not self.is_bright:
            self.is_bright = True
        bright = self.x.copy()
        for i in range(bright.shape[0]):
            bright[i, :, :, :] = (bright[i,:,:,:] * factor).astype(int)
            bright[i, :, :, :][bright[i,:,:,:] >= 255] = 255

        self.bright = (bright, self.y.copy())
        self.N_aug += self.N
        print("Brightness increased by a factor of:", factor)
        return bright

reason: import numpy,from scipy


text:
"""pytest fixtures for bac_advanced_ml.supervised unit tests.

.. codeauthor:: Derek Huang <djh458@stern.nyu.edu>
"""

import numpy as np
import pytest
from sklearn.datasets import make_blobs, make_regression
from sklearn.model_selection import train_test_split

@pytest.fixture(scope="session")
def linreg(global_seed):
    """Generated linear regression problem with train/test split.

    Uses sklearn.datasets.make_regression. The returned data consists of 600
    samples, 10 features (all informative), and has Gaussian noise with a
    standard deviation of 1 applied to it. The bias of the model is 7.

    Parameters
    ----------
    global_seed : int
        pytest fixture. See conftest.py

    Returns
    -------
    X_train : numpy.ndarray
        Training input data, shape (480, 10)
    X_test : numpy.ndarray
        Test/validation input data, shape (120, 10)
    y_train : numpy.ndarray
        Training response vector, shape (480,)
    y_test : numpy.ndarray
        Test/validation response vector, shape (120,)
    coef : numpy.ndarray
        True model coefficients, shape (10,)
    bias : float
        True model bias (intercept)
    """
    # intercept for the regression problem
    bias = 7
    # generate noisy regression problem
    X, y, coef = make_regression(
        n_samples=600,
        n_features=10,
        n_informative=10,
        bias=bias,
        noise=1,
        coef=True,
        random_state=global_seed
    )
    # split the data with train_test_split
    X_train, X_test, y_train, y_test = train_test_split(
        X,
        y,
        test_size=0.2,
        random_state=global_seed
    )
    # return split data, coef, and bias
    return X_train, X_test, y_train, y_test, coef, bias

@pytest.fixture(scope="session")
def blob_bin(global_seed):
    """Generated two-class blob classification problem with train/test split.

    Uses sklearn.datasets.make_blobs to make isotropic standard multivariate
    Gaussian blobs in 10-dimensional Euclidean space with 2 random centers. The
    returned data has 600 samples, 10 features. Features are informative by
    construction so we don't use sklearn.datasets.make_classification.

    Parameters
    ----------
    global_seed : int
        pytest fixture. See conftest.py

    Returns
    -------
    X_train : numpy.ndarray
        Training input data, shape (480, 10)
    X_test : numpy.ndarray
        Test/validation input data, shape (120, 10)
    y_train : numpy.ndarray
        Training class labels, shape (480,)
    y_test : numpy.ndarray
        Test/validation class labels, shape (120,)
    """
    # generate noisy classification problem using isotropic Gaussian blobs
    # pylint: disable=unbalanced-tuple-unpacking
    X, y = make_blobs(
        n_samples=600,
        n_features=10,
        centers=2,
        random_state=global_seed
    )
    # pylint: enable=unbalanced-tuple-unpacking
    # split the data with train_test_split and return
    X_train, X_test, y_train, y_test = train_test_split(
        X,
        y,
        test_size=0.2,
        random_state=global_seed
    )
    # return split data, coef, and bias
    return X_train, X_test, y_train, y_test

@pytest.fixture(scope="session")
def blob_multi(global_seed):
    """Generated multi-class blob classification problem with train/test split.

    Uses sklearn.datasets.make_blobs to make isotropic standard multivariate
    Gaussian blobs in 10-dimensional Euclidean space with 4 random centers. The
    returned data has 600 samples, 10 features. Features are informative by
    construction so we don't use sklearn.datasets.make_classification.

    Parameters
    ----------
    global_seed : int
        pytest fixture. See conftest.py

    Returns
    -------
    X_train : numpy.ndarray
        Training input data, shape (480, 10)
    X_test : numpy.ndarray
        Test/validation input data, shape (120, 10)
    y_train : numpy.ndarray
        Training class labels, shape (480,)
    y_test : numpy.ndarray
        Test/validation class labels, shape (120,)
    """
    # pylint: disable=unbalanced-tuple-unpacking
    X, y = make_blobs(
        n_samples=600,
        n_features=10,
        centers=4,
        random_state=global_seed
    )
    # pylint: enable=unbalanced-tuple-unpacking
    X_train, X_test, y_train, y_test = train_test_split(
        X,
        y,
        test_size=0.2,
        random_state=global_seed
    )
    return X_train, X_test, y_train, y_test

ext: py

max_stars_repo_name: phetdam/bac-advanced-ml

max_issues_repo_name: phetdam/bac-advanced-ml

max_forks_repo_name: phetdam/bac-advanced-ml

reason: import numpy


text:
import os
import sys
import argparse
import numpy as np
import tensorflow as tf

from bilm.training import train, load_options_latest_checkpoint, load_vocab
from bilm.data import LMDataset, BidirectionalLMDataset

current_dir = os.path.dirname(os.path.abspath(__file__))
sys.path.append(current_dir)
from helper import get_tokens_count, clean_checkpoint, load_options, save_options

def resume(options, prefix, vocab, n_gpus, tf_save_dir, tf_log_dir, ckpt_file):
	kwargs = {
		'test': False,
		'shuffle_on_load': True,
	}
	tf.reset_default_graph()
	if options.get('bidirectional'):
		data = BidirectionalLMDataset(prefix, vocab, **kwargs)
	else:
		data = LMDataset(prefix, vocab, **kwargs)

	train(options, data, n_gpus, tf_save_dir, tf_log_dir, restart_ckpt_file=ckpt_file)
	clean_checkpoint(tf_save_dir)

def top_level(args):
	options, ckpt_file = load_options_latest_checkpoint(args.save_dir)

	if 'char_cnn' in options:
		max_word_length = options['char_cnn']['max_characters_per_token']
	else:
		max_word_length = None

	vocab = load_vocab(os.path.join(args.save_dir, "vocabs.txt"), max_word_length)

	tf_save_dir = args.save_dir
	tf_log_dir = args.save_dir

	# set optional inputs to overide the otpions.json
	if args.n_epochs > 0:
		options['n_epochs'] = args.n_epochs
	if args.batch_size > 0:
		options['batch_size'] = args.batch_size
	if args.n_gpus > 0:
		n_gpus = args.n_gpus
	else:
		n_gpus = options['n_gpus']

	# load train_prefixes
	#if args.train_prefix_paths != None:
	if False:
		with open(args.train_prefix_paths, "r") as fd:
			train_prefixes = fd.read().split('\n')
		train_prefixes = [f for f in train_prefixes if f != ""]
		options['train_prefix_paths'] = train_prefixes
		start = 0
	else:
		train_prefixes = options['train_prefix_paths']
		start = options['milestone']

	if start >= len(train_prefixes):
		print("WARNING: Finish all train_prefix_paths. Reset milestone in options.")
		sys.exit(0)

	# loop all train_prefix_paths
	milestone = start
	for train_prefix in train_prefixes[start:]:
		prefix = train_prefix + '/*'

		if args.n_train_tokens > 0:
			options['n_train_tokens'] = args.n_train_tokens
		else:
			options['n_train_tokens'] =  get_tokens_count(prefix)

		resume(options, prefix, vocab, n_gpus, tf_save_dir, tf_log_dir, ckpt_file)
		milestone += 1
		options['milestone'] = milestone
		save_options(options, os.path.join(args.save_dir, "options.json"))

def main():
	parser = argparse.ArgumentParser()
	parser.add_argument('--save_dir', default=None, help='Location of checkpoint files')
	#parser.add_argument('--train_prefix_paths', default=None, help='Prefix paths for train files')
	parser.add_argument('--n_gpus', type=int, default=0, help='Number of GPUs to use')
	parser.add_argument('--batch_size', type=int, default=0)
	parser.add_argument('--n_train_tokens', type=int, default=0)
	parser.add_argument('--n_epochs', type=int, default=0)

	args = parser.parse_args()

	if args.save_dir == None:
		print("ERROR: no save_dir")
	else:
		top_level(args)


text:
from itertools import product
import numpy
from openfermion.ops.representations import InteractionOperator

def make_reduced_hamiltonian(molecular_hamiltonian: InteractionOperator,
                             n_electrons: int) -> InteractionOperator:
    r"""
    Construct the reduced Hamiltonian.

    This Hamiltonian is equivalent to the electronic structure Hamiltonian
    but contains only two-body terms.  To do this, the operator now depends
    on the number of particles being simulated.  We use the RDM sum rule to
    lift the 1-body terms to the two-body space.

    Derivation:
        use the fact that i^l = (1/(n -1)) sum_{jk}\delta_{jk}i^ j^ k l
                          i^l = (-1/(n -1)) sum_{jk}\delta_{jk}j^ i^ k l
                          i^l = (-1/(n -1)) sum_{jk}\delta_{jk}i^ j^ l k
                          i^l = (1/(n -1)) sum_{jk}\delta_{jk}j^ i^ l k

        Rewrite each one-body term as an even weighting of all four 2-RDM
        elements with delta functions. Then rearrange terms so that each ijkl
        term gets a sum of permuted one-body terms multiplied by delta
        function. One should notice that this results in the same formula
        if one was to apply the wedge product!

    Args:
        molecular_hamiltonian: operator to write reduced hamiltonian for
        n_electrons: number of electrons in the system
    Returns:
        InteractionOperator with a zero one-body component.
    """
    constant = molecular_hamiltonian.constant
    h1 = molecular_hamiltonian.one_body_tensor
    h2 = molecular_hamiltonian.two_body_tensor

    delta = numpy.eye(h1.shape[0])
    k2 = numpy.zeros_like(h2)
    normalization = 1 / (4 * (n_electrons - 1))
    for i, j, k, l in product(range(h1.shape[0]), repeat=4):
        k2[i, j, k, l] = normalization * (
            h1[i, l] * delta[j, k] + h1[j, k] * delta[i, l] -
            h1[i, k] * delta[j, l] - h1[j, l] * delta[i, k]) + h2[i, j, k, l]

    return InteractionOperator(constant, numpy.zeros_like(h1), k2)

max_stars_repo_name: Emieeel/OpenFermion

max_issues_repo_name: SamarthVadia/OpenFermion

max_forks_repo_name: SamarthVadia/OpenFermion


text:
import random as rn
from abc import ABC, abstractmethod

import numpy as np
import tensorflow as tf
from keras import backend as K
from keras.models import load_model
from keras_contrib.utils import save_load_utils

from utils.data_io import DataSaverLoader
from utils.model_utils import ModelUtils

class MentionDetection(ABC):

    def __init__(self, max_seq_len=36, embed_dim=300, n_tags=2, ix2word=None):

        self._max_seq_len = max_seq_len
        self._embed_dim = embed_dim
        self._n_tags = n_tags
        self._ix2word = ix2word
        self._embedding_matrix = ModelUtils.emb_matrix(self._ix2word)
        self._nb_words = self._embedding_matrix.shape[0]

        super().__init__()

    @abstractmethod
    def create_model(self):
        pass

    def train(self, x_train, y_train, x_valid, y_valid, batch_size=256, epochs=100):
        # define callbacks
        #    early_stopping = EarlyStopping(monitor="val_loss", min_delta=0.001, patience=3, verbose=1)
        #    callbacks_list = [early_stopping]

        # model training with batch normalization
        hist = self.model.fit(x_train, y_train, batch_size=batch_size,
                              epochs=epochs, validation_data=(x_valid, y_valid),
                              shuffle=True, verbose=1)

    def evaluate_on_test_set(self, x_test, y_test):
        pred = self.model.predict(x_test)
        pred = np.argmax(pred, axis=-1)

        count = 0
        for i in range(pred.shape[0]):
            if np.array_equal(pred[i], np.argmax(y_test[i], axis=-1)):
                count += 1

        print("Accuracy: ", count / pred.shape[0])
        return pred

    def save_crf_model(self, path, name):
        # check if dir exist
        DataSaverLoader.directory_exists(path)
        # Save model
        save_load_utils.save_all_weights(self.model, path + name)

    def save_model_h5(self, path, name):
        # check if dir exist
        DataSaverLoader.directory_exists(path)
        # save model
        self.model.save(path + name + ".h5")

    def load_trained_model(self, path):
        self.model = load_model(path)

    def clear(self):
        """Clear the session and set all the seeds"""
        K.clear_session()
        np.random.seed(42)
        rn.seed(12345)

        session_conf = tf.ConfigProto(intra_op_parallelism_threads=1,
                                      inter_op_parallelism_threads=1)

        tf.set_random_seed(1234)

        sess = tf.Session(graph=tf.get_default_graph(), config=session_conf)
        K.set_session(sess)


text:
import json
import numpy as np
import argparse
import os
import cv2
from panopticapi.utils import id2rgb

# Generates blue colors in range(n)
def blue_color_generator(n, hex_values=True):
    hex_colors = []
    for i in range(n):
        int_color = i * 15
        bgr_color = np.array(
            [
                int_color & 255, (int_color >> 8) & 255,
                (int_color >> 16) & 255, 255
            ],
            dtype=np.uint8
        )
        hex_color = '#' + "{:02x}".format(
            bgr_color[2]
        ) + "{:02x}".format(bgr_color[1], ) + "{:02x}".format(bgr_color[0])
        if hex_values:
            hex_colors.append(hex_color)
        else:
            hex_colors.append(hex_to_rgb(hex_color))
    return hex_colors

# Converts HEX values to RGB values
def hex_to_rgb(hex_string):
    h = hex_string.lstrip('#')
    return tuple(int(h[i:i + 2], 16) for i in (0, 2, 4))

# Converts RGB values to HEX values
def rgb_to_hex(rgb_tuple):
    return '#%02x%02x%02x' % rgb_tuple

parser = argparse.ArgumentParser()
parser.add_argument(
    "--coco-json", type=str, required=True, help="Argument must be JSON file"
)

p = parser.parse_args()
coco_path = p.coco_json
coco_path_folder, coco_path_file = os.path.split(coco_path)

with open(coco_path) as reader:
    coco_json = json.load(reader)

sa_dir = coco_path_file + "___formated"
os.makedirs(sa_dir, exist_ok=True)

hex_colors = blue_color_generator(len(coco_json["categories"]))

# creat classes json from coco categories
classes = []
classes_dir = os.path.join(sa_dir, "classes")
os.makedirs(classes_dir, exist_ok=True)
for i in range(len(hex_colors)):
    coco_cat = coco_json["categories"][i]
    dd = {
        "name": coco_cat["name"],
        "id": coco_cat["id"],
        "color": hex_colors[i],
        "attribute_groups": []
    }
    classes.append(dd)

with open(os.path.join(classes_dir, "classes.json"), "w") as writer:
    json.dump(classes, writer, indent=2)

# create blue mask and json
image_list = coco_json["images"]
annotate_list = coco_json["annotations"]

for image in image_list:
    img_name = os.path.splitext(image["file_name"])[0]
    for annotate in annotate_list:
        annot_name = os.path.splitext(annotate["file_name"])[0]
        if img_name == annot_name:
            img_cv = cv2.imread(
                os.path.join(
                    coco_path_folder, os.path.join("panoptic_masks", img_name + ".png")
                )
            )
            if img_cv is None:
                print(
                    "Error: '{}' file dosen't exist!".format(
                        os.path.join("panoptic_masks", img_name + ".png")
                    )
                )
                break

            img = cv2.cvtColor(img_cv, cv2.COLOR_BGR2RGB)
            H, W, C = img.shape
            img = img.reshape((H * W, C))
            segments = annotate["segments_info"]
            hex_colors = blue_color_generator(len(segments))
            out_json = []
            for i, seg in enumerate(segments):
                img[np.all(img == id2rgb(seg["id"]),
                           axis=1)] = hex_to_rgb(hex_colors[i])
                dd = {
                    "classId": seg["category_id"],
                    "probability": 100,
                    "visible": True,
                    "parts": [{
                        "color": hex_colors[i]
                    }],
                    "attributes": [],
                    "attributeNames": [],
                    "imageId": annotate["image_id"]
                }
                out_json.append(dd)
            with open(
                os.path.join(sa_dir, img_name + ".jpg___pixel.json"), "w"
            ) as writer:
                json.dump(out_json, writer, indent=2)

            img = cv2.cvtColor(img.reshape((H,W,C)), cv2.COLOR_RGB2BGR)
            cv2.imwrite(os.path.join(sa_dir, img_name + ".jpg___save.png"), img)


text:
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt
import numpy as np
import sys

def create_3d_plot(data):
    # give things useful names
    gigapixels_per_second = data[:, 4]
    n_by_m = data[:, 0] * data[:, 1]
    k = data[:, 2]

    # make plot
    fig = plt.figure(figsize=(9, 9))
    ax = fig.add_subplot(111, projection='3d')
    ax.scatter(n_by_m, k, gigapixels_per_second)
    ax.set_xlabel('n*m (approx. bytes loaded)')
    ax.set_ylabel('k')
    ax.set_zlabel('gigapixels/s')
    plt.show()

def create_2d_plot_per_nm(data, colors):
    # give things useful names
    k = data[:, 2]

    # gigapixels per second per n*m for each grouping of k
    data_by_k = {}

    possible_k = sorted(np.unique(data[:, 2]))

    legend_values = []
    for k in possible_k:
        legend_values.append('k: ' + str(k))
        data_by_k[k] = np.array([x for x in data if x[2] == k])

    plt.figure(figsize=(15, 9))
    i = 0
    for key in data_by_k:
        plt.plot(data_by_k[key][:, 0] * data_by_k[key][:, 0],
                 data_by_k[key][:, 4], c=colors[i], marker='.')
        i += 1

    plt.legend(legend_values)
    plt.xscale('log')
    plt.xlabel('n*m (approx. bytes loaded)')
    plt.ylabel('performance in gigapixels per second')
    plt.show()

def create_2d_plot_per_k(data, colors):
    # gigapixels per second per k for each grouping of n*m
    data_by_size = {}

    image_sizes = {
        (1024, 768),
        (2048, 2048),
        (8192, 8192),
        (4194304, 768),
        (16777216, 768),
    }

    legend_values = []
    for m, n in image_sizes:
        legend_values.append(str(m) + ' x ' + str(n) + ' pixels')
        new_entry = np.array([x for x in data if x[0] == m and x[1] == n])
        data_by_size[(m, n)] = new_entry

    data_by_size_keys_sorted = sorted(data_by_size.keys())

    i = 0
    for key in data_by_size_keys_sorted:
        plt.plot(data_by_size[key][:, 2], data_by_size[key][:, 4], c=colors[i],
                 marker='.')
        i += 1

    plt.legend(legend_values)
    plt.xlabel('k')
    plt.ylabel('performance in gigapixels per second')

def create_2d_plot_per_k_list(data_list, colors):
    plt.figure(figsize=(15, 9))
    for data in data_list:
        create_2d_plot_per_k(data, colors)
    plt.show()

def main():
    expected_usage = sys.argv[0] + 'file-to-process.csv'
    if len(sys.argv) < 2:
        print(expected_usage)
        sys.exit(1)

    # give things useful names
    data = np.loadtxt(sys.argv[1], delimiter=',', skiprows=1)
    data_list = [data]

    if len(sys.argv) == 3:
        data_list.append(np.loadtxt(sys.argv[2], delimiter=',', skiprows=1))

    # common variables:
    colors = ['b', 'g', 'r', 'c', 'm', 'y', 'k']

    create_3d_plot(data)
    create_2d_plot_per_nm(data, colors)
    create_2d_plot_per_k_list(data_list, colors)

if __name__ == "__main__":
    main()


text:
from rdkit import Chem
from rdkit.Chem import AllChem
import csv
import numpy as np
import pickle as pkl

data = 'COD'
n_min = 2
n_max = 50
suppl = Chem.SDMolSupplier('./COD_organic_2018.sdf')

smilist=[]
mollist=[]
for i, mol in enumerate(suppl):
    try:
        Chem.rdmolops.AssignAtomChiralTagsFromStructure(mol)
        Chem.rdmolops.AssignStereochemistry(mol)

        smi=Chem.MolToSmiles(mol, isomericSmiles=True)
        if '.' in smi:
            continue
    except:
        continue

    na = mol.GetNumHeavyAtoms()
    pos = mol.GetConformer().GetPositions()
    if na==pos.shape[0] and na>=n_min and na<=n_max:
        mollist.append(mol)
        smilist.append(smi)
        print(len(smilist), i)

smilist=np.array(smilist)
mollist=np.array(mollist)
permid=np.random.permutation(len(mollist))
mollist=mollist[permid]
smilist=smilist[permid]
pkl.dump([mollist, smilist], open(data+'_molset_all.p','wb'))


text:
import os

import jpype
import pytest
from numpy import testing as npt

import ixmp
from ixmp.default_path_constants import CONFIG_PATH

test_args = ('Douglas Adams', 'Hitchhiker')
can_args = ('canning problem', 'standard')

def local_config_exists():
    return os.path.exists(CONFIG_PATH)

@pytest.mark.skipif(local_config_exists(),
                    reason='will not overwrite local config files')
def test_default_dbprops_file(test_mp_use_default_dbprops_file):
    test_mp = test_mp_use_default_dbprops_file
    scenario = test_mp.scenario_list(model='Douglas Adams')['scenario']
    assert scenario[0] == 'Hitchhiker'

@pytest.mark.skipif(local_config_exists(),
                    reason='will not overwrite local config files')
def test_db_config_path(test_mp_use_db_config_path):
    test_mp = test_mp_use_db_config_path
    scenario = test_mp.scenario_list(model='Douglas Adams')['scenario']
    assert scenario[0] == 'Hitchhiker'

def test_platform_init_raises():
    pytest.raises(ValueError, ixmp.Platform, dbtype='foo')

def test_scen_list(test_mp):
    scenario = test_mp.scenario_list(model='Douglas Adams')['scenario']
    assert scenario[0] == 'Hitchhiker'

def test_new_scen(test_mp):
    scen = ixmp.Scenario(test_mp, *can_args, version='new')
    assert scen.version == 0

def test_default_version(test_mp):
    scen = ixmp.Scenario(test_mp, *can_args)
    assert scen.version == 2

def test_has_set(test_mp):
    scen = ixmp.Scenario(test_mp, *can_args)
    assert scen.has_set('i')
    assert not scen.has_set('k')

def test_init_par_35(test_mp):
    scen = ixmp.Scenario(test_mp, *can_args, version='new')
    scen.init_set('ii')
    scen.init_par('new_par', idx_sets='ii')

def test_get_scalar(test_mp):
    scen = ixmp.Scenario(test_mp, *can_args)
    obs = scen.scalar('f')
    exp = {'unit': 'USD/km', 'value': 90}
    assert obs == exp

def test_has_par(test_mp):
    scen = ixmp.Scenario(test_mp, *can_args)
    assert scen.has_par('f')
    assert not scen.has_par('m')

def test_init_scalar(test_mp):
    scen = ixmp.Scenario(test_mp, *can_args)
    scen2 = scen.clone(keep_solution=False)
    scen2.check_out()
    scen2.init_scalar('g', 90.0, 'USD/km')
    scen2.commit("adding a scalar 'g'")

def test_init_set(test_mp):
    """Test ixmp.Scenario.init_set()."""
    scen = ixmp.Scenario(test_mp, *can_args)

    # Add set on a locked scenario
    with pytest.raises(jpype.JavaException,
                       message="at.ac.iiasa.ixmp.exceptions.IxException: "
                               "This Scenario cannot be edited, do a checkout "
                               "first!"):
        scen.init_set('foo')

    scen = scen.clone(keep_solution=False)
    scen.check_out()
    scen.init_set('foo')

    # Initialize an already-existing set
    with pytest.raises(jpype.JavaException,
                       message="at.ac.iiasa.ixmp.exceptions.IxException: "
                               "An Item with the name 'foo' already exists!"):
        scen.init_set('foo')

def test_add_set(test_mp):
    """Test ixmp.Scenario.add_set()."""
    scen = ixmp.Scenario(test_mp, *can_args)

    # Add element to a non-existent set
    with pytest.raises(jpype.JavaException,
                       message="at.ac.iiasa.ixmp.exceptions.IxException: "
                               "No Set 'foo' exists in this Scenario!"):
        scen.add_set('foo', 'bar')

# make sure that changes to a scenario are copied over during clone
def test_add_clone(test_mp):
    scen = ixmp.Scenario(test_mp, *can_args, version=1)
    scen.check_out()
    scen.init_set('h')
    scen.add_set('h', 'test')
    scen.commit("adding an index set 'h', wiht element 'test'")

    scen2 = scen.clone(keep_solution=False)
    obs = scen2.set('h')
    npt.assert_array_equal(obs, ['test'])

# make sure that (only) the correct scenario is touched after cloning
def test_clone_edit(test_mp):
    scen = ixmp.Scenario(test_mp, *can_args)
    scen2 = scen.clone(keep_solution=False)
    scen2.check_out()
    scen2.change_scalar('f', 95.0, 'USD/km')
    scen2.commit('change transport cost')
    obs = scen.scalar('f')
    exp = {'unit': 'USD/km', 'value': 90}
    assert obs == exp
    obs = scen2.scalar('f')
    exp = {'unit': 'USD/km', 'value': 95}
    assert obs == exp

def test_idx_name(test_mp):
    scen = ixmp.Scenario(test_mp, *can_args)
    df = scen.idx_names('d')
    npt.assert_array_equal(df, ['i', 'j'])

def test_has_var(test_mp):
    scen = ixmp.Scenario(test_mp, *can_args)
    assert scen.has_var('x')
    assert not scen.has_var('y')

def test_var_marginal(test_mp):
    scen = ixmp.Scenario(test_mp, *can_args)
    df = scen.var('x', filters={'i': ['seattle']})
    npt.assert_array_almost_equal(df['mrg'], [0, 0, 0.036])

def test_var_level(test_mp):
    scen = ixmp.Scenario(test_mp, *can_args)
    df = scen.var('x', filters={'i': ['seattle']})
    npt.assert_array_almost_equal(df['lvl'], [50, 300, 0])

def test_var_general_str(test_mp):
    scen = ixmp.Scenario(test_mp, *can_args)
    df = scen.var('x', filters={'i': ['seattle']})
    npt.assert_array_equal(
        df['j'], ['new-york', 'chicago', 'topeka'])

def test_unit_list(test_mp):
    units = test_mp.units()
    assert ('cases' in units) is True

def test_add_unit(test_mp):
    test_mp.add_unit('test', 'just testing')

def test_par_filters_unit(test_mp):
    scen = ixmp.Scenario(test_mp, *can_args)
    df = scen.par('d', filters={'i': ['seattle']})
    obs = df.loc[0, 'unit']
    exp = 'km'
    assert obs == exp

def test_meta(test_mp):
    test_dict = {
        "test_string": 'test12345',
        "test_number": 123.456,
        "test_number_negative": -123.456,
        'test_int': 12345,
        'test_bool': True,
        'test_bool_false': False,
    }

    scen = ixmp.Scenario(test_mp, *can_args, version=1)
    for k, v in test_dict.items():
        scen.set_meta(k, v)

    # test all
    obs_dict = scen.get_meta()
    for k, exp in test_dict.items():
        obs = obs_dict[k]
        assert obs == exp

    # test name
    obs = scen.get_meta('test_string')
    exp = test_dict['test_string']
    assert obs == exp

def test_load_scenario_data(test_mp):
    scen = ixmp.Scenario(test_mp, *can_args, cache=True)
    scen.load_scenario_data()
    assert ('par', 'd') in scen._pycache  # key exists
    df = scen.par('d', filters={'i': ['seattle']})
    obs = df.loc[0, 'unit']
    exp = 'km'
    assert obs == exp

def test_load_scenario_data_clear_cache(test_mp):
    # this fails on commit: 4376f54
    scen = ixmp.Scenario(test_mp, *can_args, cache=True)
    scen.load_scenario_data()
    scen.clear_cache(name='d')

def test_load_scenario_data_raises(test_mp):
    scen = ixmp.Scenario(test_mp, *can_args, cache=False)
    pytest.raises(ValueError, scen.load_scenario_data)

def test_log_level(test_mp):
    test_mp.set_log_level('CRITICAL')
    test_mp.set_log_level('ERROR')
    test_mp.set_log_level('WARNING')
    test_mp.set_log_level('INFO')
    test_mp.set_log_level('DEBUG')
    test_mp.set_log_level('NOTSET')

def test_log_level_raises(test_mp):
    pytest.raises(ValueError, test_mp.set_log_level, level='foo')

max_stars_repo_name: danielhuppmann/ixmp

max_issues_repo_name: danielhuppmann/ixmp

max_forks_repo_name: danielhuppmann/ixmp

reason: from numpy