text:
"""
Validation functions for the distribute module
"""

import numbers
import numpy as np

from itertools import compress
from scipy.sparse import issparse
from sklearn.model_selection import ParameterGrid
from sklearn.utils.validation import check_is_fitted

def _check_estimator(estimator, verbose=False):
    """ Print sparkContext awareness if apporpriate """
    if verbose:
        if estimator.sc is None:
            print("No spark context is provided; running locally")
        else:
            print("Spark context found; running with spark")

def _check_is_fitted(estimator, attributes=None):
    from sklearn import __version__
    if __version__ < '0.22':
        return check_is_fitted(estimator, attributes)
    else:
        return check_is_fitted(estimator)

def _validate_params(param_sets):
    """ Validates grid params """
    invalid_params = [
        type(param)
        for param in param_sets
        if not isinstance(param, dict)
        ]
    if invalid_params:
        raise ValueError('Params must be dictionaries: got '
            '{0!r}'.format(invalid_params))

def _validate_models(models, clf):
    """ Validates models input argument """
    try:
        iter(models)
    except:
        raise TypeError("Input argument 'models' is not iterable.")
    if not (isinstance(models[0], tuple) or isinstance(models[0], list)):
        models = [models]
    names, estimators, param_sets = zip(*models)
    _validate_names(clf, names)
    _validate_estimators(estimators)
    _validate_params(param_sets)
    return models

def _check_base_estimator(estimator):
    """ Validates base estimator """
    return (
        hasattr(estimator, "fit") and
        hasattr(estimator, "predict")
        )

def _validate_estimators(estimators):
    """ Validates estimators """
    invalid_estimators = [
        type(estimator)
        for estimator in estimators
        if not _check_base_estimator(estimator)
        ]
    if invalid_estimators:
        raise ValueError('Estimators must be sklearn estimators: got '
            '{0!r}'.format(invalid_estimators))

def _validate_names(cls, names):
    """ Validates names """
    if len(set(names)) != len(names):
        raise ValueError('Names provided are not unique: '
            '{0!r}'.format(list(names)))
    invalid_names = [name for name in names if not isinstance(name, str)]
    if invalid_names:
        raise ValueError('Estimator names must be strings: got '
            '{0!r}'.format(invalid_names))
    invalid_names = set(names).intersection(cls.get_params(deep=False))
    if invalid_names:
        raise ValueError('Estimator names conflict with constructor '
            'arguments: {0!r}'.format(sorted(invalid_names)))
    invalid_names = [name for name in names if '__' in name]
    if invalid_names:
        raise ValueError('Estimator names must not contain __: got '
            '{0!r}'.format(invalid_names))

def _check_n_iter(n_iter, param_distributions):
    """
    Check if n_iter is greater than the total number
    of possible param sets from the given distribution.
    """
    all_lists = np.all([not hasattr(v, "rvs")
        for v in param_distributions.values()])
    if all_lists:
        param_grid = ParameterGrid(param_distributions)
        grid_size = len(param_grid)
    else:
        grid_size = n_iter
    return min(grid_size, n_iter)

def _is_arraylike(x):
    """ Returns whether the input is array-like """
    return (hasattr(x, '__len__') or
            hasattr(x, 'shape') or
            hasattr(x, '__array__'))

def _num_samples(x):
    """ Return number of samples in array-like x """
    message = 'Expected sequence or array-like, got %s' % type(x)
    if hasattr(x, 'fit') and callable(x.fit):
        # Don't get num_samples from an ensembles length!
        raise TypeError(message)

    if not hasattr(x, '__len__') and not hasattr(x, 'shape'):
        if hasattr(x, '__array__'):
            x = np.asarray(x)
        else:
            raise TypeError(message)

    if hasattr(x, 'shape') and x.shape is not None:
        if len(x.shape) == 0:
            raise TypeError("Singleton array %r cannot be considered"
                            " a valid collection." % x)
        # Check that shape is returning an integer or default to len
        # Dask dataframes may not return numeric shape[0] value
        if isinstance(x.shape[0], numbers.Integral):
            return x.shape[0]
    try:
        return len(x)
    except TypeError:
        raise TypeError(message)

def _safe_indexing(X, indices, axis=0):
    """ Return rows, items or columns of X using indices """
    if indices is None:
        return X

    if axis not in (0, 1):
        raise ValueError(
            "'axis' should be either 0 (to index rows) or 1 (to index "
            " column). Got {} instead.".format(axis)
        )

    indices_dtype = _determine_key_type(indices)

    if axis == 0 and indices_dtype == 'str':
        raise ValueError(
            "String indexing is not supported with 'axis=0'"
        )

    if axis == 1 and X.ndim != 2:
        raise ValueError(
            "'X' should be a 2D NumPy array, 2D sparse matrix or pandas "
            "dataframe when indexing the columns (i.e. 'axis=1'). "
            "Got {} instead with {} dimension(s).".format(type(X), X.ndim)
        )

    if axis == 1 and indices_dtype == 'str' and not hasattr(X, 'loc'):
        raise ValueError(
            "Specifying the columns using strings is only supported for "
            "pandas DataFrames"
        )

    if hasattr(X, "iloc"):
        return _pandas_indexing(X, indices, indices_dtype, axis=axis)
    elif hasattr(X, "shape"):
        return _array_indexing(X, indices, indices_dtype, axis=axis)
    else:
        return _list_indexing(X, indices, indices_dtype)

def _determine_key_type(key):
    """ Determine the data type of key """
    err_msg = ("No valid specification of the columns. Only a scalar, list or "
               "slice of all integers or all strings, or boolean mask is "
               "allowed")

    dtype_to_str = {int: 'int', str: 'str', bool: 'bool', np.bool_: 'bool'}
    array_dtype_to_str = {'i': 'int', 'u': 'int', 'b': 'bool', 'O': 'str',
                          'U': 'str', 'S': 'str'}

    if key is None:
        return None
    if isinstance(key, tuple(dtype_to_str.keys())):
        try:
            return dtype_to_str[type(key)]
        except KeyError:
            raise ValueError(err_msg)
    if isinstance(key, slice):
        if key.start is None and key.stop is None:
            return None
        key_start_type = _determine_key_type(key.start)
        key_stop_type = _determine_key_type(key.stop)
        if key_start_type is not None and key_stop_type is not None:
            if key_start_type != key_stop_type:
                raise ValueError(err_msg)
        if key_start_type is not None:
            return key_start_type
        return key_stop_type
    if isinstance(key, list):
        unique_key = set(key)
        key_type = {_determine_key_type(elt) for elt in unique_key}
        if not key_type:
            return None
        if len(key_type) != 1:
            raise ValueError(err_msg)
        return key_type.pop()
    if hasattr(key, 'dtype'):
        try:
            return array_dtype_to_str[key.dtype.kind]
        except KeyError:
            raise ValueError(err_msg)
    raise ValueError(err_msg)

def _pandas_indexing(X, key, key_dtype, axis):
    """ Index a pandas dataframe or a series """
    if hasattr(key, 'shape'):
        # Work-around for indexing with read-only key in pandas
        # FIXME: solved in pandas 0.25
        key = np.asarray(key)
        key = key if key.flags.writeable else key.copy()
    # check whether we should index with loc or iloc
    indexer = X.iloc if key_dtype == 'int' else X.loc
    return indexer[:, key] if axis else indexer[key]

def _list_indexing(X, key, key_dtype):
    """ Index a Python list """
    if np.isscalar(key) or isinstance(key, slice):
        # key is a slice or a scalar
        return X[key]
    if key_dtype == 'bool':
        # key is a boolean array-like
        return list(compress(X, key))
    # key is a integer array-like of key
    return [X[idx] for idx in key]

def _array_indexing(array, key, key_dtype, axis):
    """ Index an array or scipy.sparse consistently across NumPy version """
    if issparse(array):
        if key_dtype == 'bool':
            key = np.asarray(key)
    return array[key] if axis == 0 else array[:, key]

max_stars_repo_name: ML-Tools/sk-dist

max_issues_repo_name: ML-Tools/sk-dist

max_forks_repo_name: ML-Tools/sk-dist

reason: import numpy,from scipy


text:
import matplotlib.pyplot as plt
import numpy as np
import glob
from scipy.interpolate import interp1d
from scipy.signal import savgol_filter
from math import factorial
path ='/home/cui-group-wz/Documents/Code/MRO-Asyn-RL/plot/RL/HO'
path2 = '/home/cui-group-wz/Documents/Code/MRO-Asyn-RL/plot/BAN/HO'
path3 ='/home/cui-group-wz/Documents/Code/MRO-Asyn-RL/plot/RLonline/HO'
allFiles = glob.glob(path + "/*.csv")
allFiles2 = glob.glob(path2 + "/*.csv")
allFiles3 = glob.glob(path3 + "/*.csv")
def smooth(y, box_pts):
    box = np.ones(box_pts)/box_pts
    y_smooth = np.convolve(y, box, mode='same')
    return y_smooth

def savitzky_golay(y, window_size, order, deriv=0, rate=1):

    try:
        window_size = np.abs(np.int(window_size))
        order = np.abs(np.int(order))
    except ValueError, msg:
        raise ValueError("window_size and order have to be of type int")
    if window_size % 2 != 1 or window_size < 1:
        raise TypeError("window_size size must be a positive odd number")
    if window_size < order + 2:
        raise TypeError("window_size is too small for the polynomials order")
    order_range = range(order + 1)
    half_window = (window_size - 1) // 2
    # precompute coefficients
    b = np.mat([[k ** i for i in order_range] for k in range(-half_window, half_window + 1)])
    m = np.linalg.pinv(b).A[deriv] * rate ** deriv * factorial(deriv)
    # pad the signal at the extremes with
    # values taken from the signal itself
    firstvals = y[0] - np.abs(y[1:half_window + 1][::-1] - y[0])
    lastvals = y[-1] + np.abs(y[-half_window - 1:-1][::-1] - y[-1])
    y = np.concatenate((firstvals, y, lastvals))
    return np.convolve(m[::-1], y, mode='valid')

i=0
data_list=[]
for file_ in allFiles:
    data = np.genfromtxt(file_, delimiter=',', skip_header=0,
                         skip_footer=0, names=['x', 'y'])  # ,max_rows=500
    data_list.append(data)
    i+=1

data2_list = []
n=0
for file2_ in allFiles2:

    data2 = np.genfromtxt(file2_, delimiter=',', skip_header=0,
                     skip_footer=0, names=['x2', 'y2'])
    data2_list.append(data2)
    n+=1

data3_list = []
p=0
for file3_ in allFiles3:
     data3 = np.genfromtxt(file3_, delimiter=',', skip_header=0,
                      skip_footer=0, names=['x3', 'y3'])
     data3_list.append(data3)
     p += 1

sum= np.zeros_like(data_list[0]['y'])
sum2 = np.zeros_like(data2_list[0]['y2'])
sum3 = np.zeros_like(data3_list[0]['y3'])

for j in range(i):
    if data_list[j].shape != sum.shape:
        continue
    sum = sum+data_list[j]['y']

for j in range(n):
    print(j, data2_list[j]['y2'].shape)
    if data2_list[j].shape != sum2.shape:
        continue
    sum2 = sum2 + data2_list[j]['y2']

for j in range(p):
    sum3 = sum3 + data3_list[j]['y3']

plt.plot(data_list[0]['x'], smooth(sum/i,50), color='r', label='A3C-offine')
plt.plot(data2_list[0]['x2'], smooth(sum2/n,50), color='g', label='UCB')
plt.plot(data3_list[0]['x3']*2-5, smooth(sum3/p,50), color='b', label='A3C-online')

# plt.plot(data_list[0]['x'], savitzky_golay(sum/i, 11,0), color='r', label='A3C-offine')
# plt.plot(((data2_list[0]['x2'])), savitzky_golay(sum2/n, 51,0), color='g', label='UCB')
# plt.plot((data3_list[0]['x3']), savitzky_golay(sum3/p, 11,0), color='b', label='A3C-online')

# ave = np.sum(sum/i)/100
# ave2 = np.sum(sum2/n)/100
# ave3 = np.sum(sum3/p)/100
# mean_values = [ave,ave2,ave3]
#
# var = np.max(sum/i)-ave
# var2 = np.max(sum2/n)-ave2
# var3 = np.max(sum3/p)-ave3
# variance = [var,var2,var3]
#
# bar_labels = ['a3c-offline','UCB', 'a3c-online']
#
# x_pos = list(range(len(bar_labels)))
# plt.bar(x_pos, mean_values,yerr=variance, align='center', alpha=0.5)
#
#
# max_y = max(zip(mean_values, variance))  # returns a tuple, here: (3, 5)
# plt.ylim([0, (max_y[0] + max_y[1]) * 1.1])
#
# plt.ylabel('variable y')
# plt.xticks(x_pos, bar_labels)
# plt.title('Bar plot with error bars')
#
plt.xlabel('Episode Index')
plt.ylabel('Handover Rate')
plt.title('')
plt.legend()
plt.show()

ext: py


text:
import os
from argparse import ArgumentParser
from argparse import Namespace
import numpy as np
import CSGM.utils as utils
import tensorflow.compat.v1 as tf
import CSGM.dcgan.dcgan_utils as dcgan_utils
from glob import glob
import time
import scipy.misc
def reconstruction_method(reconstruction,input_channel, input_width, input_height, m, n,sensing,datasetname, sensingname, specifics):
    # a function to return the reconstruction method with given parameters. It's a class with two methods: initialize and run.
    if (reconstruction == 'csgm' or reconstruction == 'CSGM'):
        CSGM = csgm(input_channel, input_width, input_height, m, n,sensing, datasetname,sensingname,specifics)
        return CSGM

class csgm():
    def __init__(self, input_channel, input_width, input_height, m, n, sensing,datasetname,sensingname, specifics):
        self.channels = input_channel
        self.input_width = input_width
        self.input_height = input_height
        self.m = m
        self.n = n
        self.datasetname = datasetname
        self.sensingname = sensingname
        self.specifics = specifics

        self.hparams = None
    def initialize(self,dset,sensing_method,stage,specifics):
        self.sensing_method = sensing_method
        self.model_input = dset

        if (stage =="testing"):
            data_path = specifics['test_data']
            self.data = glob(data_path)
            if len(self.data) == 0:
                raise Exception("[!] No data found in '" + data_path + "'")
            np.random.shuffle(self.data)
            imreadImg = scipy.misc.imread(self.data[0]).astype(np.float)
            if len(imreadImg.shape) >= 3: #check if image is a non-grayscale image by checking channel number
                self.c_dim = scipy.misc.imread(self.data[0]).astype(np.float).shape[-1]
            else:
                self.c_dim = 1
            self.grayscale = (self.c_dim == 1)
            print("GRAYSCALE:", self.grayscale)
            HPARAMS = Namespace(
                input_size = self.input_width,
                input_channels = self.channels,
                pretrained_model_dir = specifics["pretrained-model-dir"],
                grayscale = self.grayscale,
                dataset=self.datasetname,
                input_type =specifics["input-type"],
                input_path_pattern= specifics["test_data"],
                num_input_images=specifics["num-input-images"],
                batch_size=specifics["batch-size"],
                measurement_type=self.sensingname,
                noise_std=specifics["noise-std"],

                num_measurements=self.m,
                inpaint_size=1,
                superres_factor=2,

                model_types= specifics["model-types"],
                mloss1_weight=specifics["mloss1_weight"],
                mloss2_weight=specifics["mloss2_weight"],
                zprior_weight=specifics["zprior_weight"],
                dloss1_weight=specifics["dloss1_weight"],
                dloss2_weight=specifics["dloss2_weight"],

                optimizer_type=specifics["optimizer-type"],
                learning_rate=specifics["learning-rate"],
                momentum=specifics["momentum"],
                max_update_iter=specifics["max-update-iter"],
                num_random_restarts=specifics["num-random-restarts"],
                decay_lr=False,
                lmbd=specifics["lmbd"],
                lasso_solver='sklearn',
                sparsity=0,

                not_lazy =False,
                save_images=True,
                save_stats=specifics["save-stats"],
                print_stats=specifics["print-stats"],
                checkpoint_iter=1,
                image_matrix=0,
                gif = False,
                gif_iter=1,
                gif_dir=''
            )
            if HPARAMS.dataset == 'mnist':
                HPARAMS.image_shape = (self.input_width, self.input_height, self.channels)
                from CSGM.mnist.mnist_utils import view_image, save_image
                self.view_image = view_image
                self.save_image = save_image
            elif HPARAMS.dataset == 'celebA':
                HPARAMS.image_shape = (self.input_width, self.input_height, self.channels)
                from CSGM.celebA.celebA_utils import view_image, save_image
                self.view_image = view_image
                self.save_image = save_image
            else:
                raise NotImplementedError

            self.hparams = HPARAMS

    def run(self, stage):
        if (stage =="testing"):
            hparams = self.hparams
            model_input = self.model_input
            view_image = self.view_image
            save_image = self.save_image
            # Set up some stuff accoring to hparams
            hparams.n_input = np.prod(hparams.image_shape)
            utils.set_num_measurements(hparams)
            utils.print_hparams(hparams)

            # get inputs
            xs_dict = model_input(hparams)
            estimators = utils.get_estimators(hparams)
            utils.setup_checkpointing(hparams)
            measurement_losses, l2_losses = utils.load_checkpoints(hparams)
            PSNRs = {}
            SSIMs = {}
            reconst_times = []
            for model_type in hparams.model_types:
                PSNRs[model_type] = {}
                SSIMs[model_type] = {}
            recons_times = []

            x_hats_dict = {model_type : {} for model_type in hparams.model_types}
            x_batch_dict = {}
            for key, x in xs_dict.items():

                if not hparams.not_lazy:
                    # If lazy, first check if the image has already been
                    # saved before by *all* estimators. If yes, then skip this image.
                    save_paths = utils.get_save_paths(hparams, key)
                    is_saved = all([os.path.isfile(save_path) for save_path in list(save_paths.values())])
                    #if is_saved:
                    #   continue

                x_batch_dict[key] = x
                if len(x_batch_dict) < hparams.batch_size:
                    continue

                # Reshape input
                x_batch_list = [x.reshape(1, hparams.n_input) for _, x in x_batch_dict.items()]
                x_batch = np.concatenate(x_batch_list)

                # Construct noise and measurements
                A = self.sensing_method()
                noise_batch = hparams.noise_std * np.random.randn(hparams.batch_size, hparams.num_measurements)
                if hparams.measurement_type == 'project':
                    y_batch = x_batch + noise_batch
                else:
                    y_batch = np.matmul(x_batch, A) + noise_batch

                # Construct estimates using each estimator

                for model_type in hparams.model_types:
                    estimator = estimators[model_type]
                    benchmarkStartTime = time.time()
                    x_hat_batch = estimator(A, y_batch, hparams)
                    benchmarkEndTime = time.time()
                    reconst_time = (benchmarkEndTime-benchmarkStartTime)/hparams.batch_size
                    reconst_times.append(reconst_time)
                    for i, key in enumerate(x_batch_dict.keys()):
                        x = xs_dict[key]
                        y = y_batch[i]
                        x_hat = x_hat_batch[i]

                        # Save the estimate
                        x_hats_dict[model_type][key] = x_hat

                        # Compute and store measurement and l2 loss
                        measurement_losses[model_type][key] = utils.get_measurement_loss(x_hat, A, y)
                        l2_losses[model_type][key] = utils.get_l2_loss(x_hat, x)
                        if (hparams.dataset=="mnist"):
                            PSNRs[model_type][key] = utils.compute_average_psnr(x.reshape(1,hparams.input_channels,hparams.input_size,hparams.input_size)*2-1,x_hat.reshape(1,hparams.input_channels,hparams.input_size,hparams.input_size)*2-1)
                            SSIMs[model_type][key] = utils.compute_average_SSIM(x.reshape(1,hparams.input_size,hparams.input_size,hparams.input_channels)*2-1,x_hat.reshape(1,hparams.input_size,hparams.input_size,hparams.input_channels)*2-1)
                        else:
                            PSNRs[model_type][key] = utils.compute_average_psnr(x.reshape(1,hparams.input_channels,hparams.input_size,hparams.input_size),x_hat.reshape(1,hparams.input_channels,hparams.input_size,hparams.input_size))
                            SSIMs[model_type][key] = utils.compute_average_SSIM(x.reshape(1,hparams.input_size,hparams.input_size,hparams.input_channels),x_hat.reshape(1,hparams.input_size,hparams.input_size,hparams.input_channels))

                print(("Processed upto image {0} / {1}".format(key+1, len(xs_dict))))

                # Checkpointing
                if (hparams.save_images) and ((key+1) % hparams.checkpoint_iter == 0):
                    utils.checkpoint(x_hats_dict, measurement_losses, l2_losses, save_image, hparams)
                    x_hats_dict = {model_type : {} for model_type in hparams.model_types}
                    print('\nProcessed and saved first ', key+1, 'images\n')
                x_batch_dict = {}

            # Final checkpoint

            if hparams.save_images:
                utils.checkpoint(x_hats_dict, measurement_losses, l2_losses, save_image, hparams)
                print('\nProcessed and saved all {0} image(s)\n'.format(len(xs_dict)))

            if hparams.print_stats:
                for model_type in hparams.model_types:
                    print(model_type)
                    mean_m_loss = np.mean(list(measurement_losses[model_type].values()))
                    mean_l2_loss = np.mean(list(l2_losses[model_type].values()))
                    mean_PSNR = np.mean(list(PSNRs[model_type].values()))
                    mean_SSIM = np.mean(list(SSIMs[model_type].values()))
                    print('number of measurements = ' + str(self.m))
                    print('mean measurement loss = {0}'.format(mean_m_loss))
                    print('mean l2 loss = {0}'.format(mean_l2_loss))
                    print('mean PSNR = {0}'.format(mean_PSNR))
                    print('mean SSIM = {0}'.format(mean_SSIM))
                    print('mean reconstruction time = {0}'.format(sum(reconst_times)/len(reconst_times)))

            if hparams.image_matrix > 0:
                utils.image_matrix(xs_dict, x_hats_dict, view_image, hparams)

            # Warn the user that some things were not processsed
            if len(x_batch_dict) > 0:
                print('\nDid NOT process last {} images because they did not fill up the last batch.'.format(len(x_batch_dict)))
                print('Consider rerunning lazily with a smaller batch size.')
        elif(stage =="training"):
            if (self.datasetname =="mnist"):
                import CSGM.mnist_vae.main as vae
                import CSGM.mnist_vae.model_def as vae_model_def
                HPARAMS = vae_model_def.Hparams()

                HPARAMS.num_samples = self.specifics["num-samples"]
                HPARAMS.learning_rate = self.specifics["learning-rate"]
                HPARAMS.batch_size = self.specifics["batch-size"]
                HPARAMS.training_epochs = self.specifics["training-epochs"]
                HPARAMS.summary_epoch = self.specifics["summary-epochs"]
                HPARAMS.ckpt_epoch = self.specifics["ckpt_epoch"]
                HPARAMS.train_data = self.specifics["train_data"]
                HPARAMS.input_size=self.specifics["input-size"]
                HPARAMS.ckpt_dir = './models/mnist-vae/'
                HPARAMS.sample_dir = './samples/mnist-vae/'
                HPARAMS.n_input = HPARAMS.input_size*HPARAMS.input_size
                vae.main(HPARAMS)
            if (self.datasetname =="celebA"):
                import CSGM.dcgan.DCGANtensorflow.main as dcgan
                flags = tf.app.flags
                flags.DEFINE_integer("epoch", self.specifics['epoch'], "Epoch to train [25]")
                flags.DEFINE_float("learning_rate", self.specifics['learning_rate'], "Learning rate of for adam [0.0002]")
                flags.DEFINE_float("beta1", self.specifics['beta1'], "Momentum term of adam [0.5]")
                flags.DEFINE_float("train_size", self.specifics['train_size'], "The size of train images [np.inf]")
                flags.DEFINE_integer("batch_size", self.specifics['batch_size'], "The size of batch images [64]")
                flags.DEFINE_integer("input_height", self.specifics['input_height'], "The size of image to use (will be center cropped). [108]")
                flags.DEFINE_integer("input_width", self.specifics['input_width'], "The size of image to use (will be center cropped). If None, same value as input_height [None]")
                flags.DEFINE_integer("output_height", self.specifics['output_height'], "The size of the output images to produce [64]")
                flags.DEFINE_integer("output_width", self.specifics['output_width'], "The size of the output images to produce. If None, same value as output_height [None]")
                flags.DEFINE_string("dataset", self.specifics['dataset'], "The name of dataset [celebA, mnist, lsun]")
                flags.DEFINE_string("input_fname_pattern", self.specifics['input_fname_pattern'], "Glob pattern of filename of input images [*]")
                flags.DEFINE_string("data_dir", self.specifics['data_dir'], "path to datasets [e.g. $HOME/data]")
                flags.DEFINE_string("out_dir", self.specifics['out_dir'], "Root directory for outputs [e.g. $HOME/out]")
                flags.DEFINE_string("out_name", self.specifics['out_name'], "Folder (under out_root_dir) for all outputs. Generated automatically if left blank []")
                flags.DEFINE_string("checkpoint_dir", self.specifics['checkpoint_dir'], "Folder (under out_root_dir/out_name) to save checkpoints [checkpoint]")
                flags.DEFINE_string("sample_dir", self.specifics['sample_dir'], "Folder (under out_root_dir/out_name) to save samples [samples]")
                flags.DEFINE_boolean("train", self.specifics['train'], "True for training, False for testing [False]")
                flags.DEFINE_boolean("crop", self.specifics['crop'], "True for training, False for testing [False]")
                flags.DEFINE_boolean("visualize", self.specifics['visualize'], "True for visualizing, False for nothing [False]")
                flags.DEFINE_boolean("export", self.specifics['export'], "True for exporting with new batch size")
                flags.DEFINE_boolean("freeze", self.specifics['freeze'], "True for exporting with new batch size")
                flags.DEFINE_integer("max_to_keep", self.specifics['max_to_keep'], "maximum number of checkpoints to keep")
                flags.DEFINE_integer("sample_freq", self.specifics['sample_freq'], "sample every this many iterations")
                flags.DEFINE_integer("ckpt_freq", self.specifics['ckpt_freq'], "save checkpoint every this many iterations")
                flags.DEFINE_integer("z_dim", self.specifics['z_dim'], "dimensions of z")
                flags.DEFINE_string("z_dist", self.specifics['z_dist'], "'normal01' or 'uniform_unsigned' or uniform_signed")
                flags.DEFINE_boolean("G_img_sum", self.specifics['G_img_sum'], "Save generator image summaries in log")
                #flags.DEFINE_integer("generate_test_images", 100, "Number of images to generate during test. [100]")

                dcgan.main(flags)

max_stars_repo_name: PSCLab-ASU/OpenICS

max_issues_repo_name: PSCLab-ASU/OpenICS

max_forks_repo_name: PSCLab-ASU/OpenICS

reason: import numpy,import scipy


text:
#
# Copyright 2019 Gianluca Frison, Dimitris Kouzoupis, Robin Verschueren,
# Andrea Zanelli, Niels van Duijkeren, Jonathan Frey, Tommaso Sartor,
# Branimir Novoselnik, Rien Quirynen, Rezart Qelibari, Dang Doan,
# Jonas Koenemann, Yutao Chen, Tobias Schöls, Jonas Schlagenhauf, Moritz Diehl
#
# This file is part of acados.
#
# The 2-Clause BSD License
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.;
#

from acados_template import *
import acados_template as at
from export_ode_model import *
import numpy as np
import scipy.linalg
from ctypes import *

FORMULATION = 2 # 0 for linear soft bounds,
            # 1 for equivalent nonlinear soft constraint
            # 2 for equivalent nonlinear soft constraint +
            # terminal soft state constraint

def export_nonlinear_constraint():

    con_name = 'nl_con'

    # set up states & controls
    x1      = SX.sym('x1')
    theta   = SX.sym('theta')
    v1      = SX.sym('v1')
    dtheta  = SX.sym('dtheta')

    x = vertcat(x1, v1, theta, dtheta)

    # controls
    F = SX.sym('F')
    u = vertcat(F)

    # voltage sphere
    constraint = acados_constraint()

    constraint.expr = u
    constraint.x = x
    constraint.u = u
    constraint.nc = 1
    constraint.name = con_name

    return constraint

def export_terminal_nonlinear_constraint():

    con_name = 'nl_state_con'

    # set up states & controls
    x1      = SX.sym('x1')
    theta   = SX.sym('theta')
    v1      = SX.sym('v1')
    dtheta  = SX.sym('dtheta')

    x = vertcat(x1, v1, theta, dtheta)

    # controls
    F = SX.sym('F')
    u = vertcat(F)

    # voltage sphere
    constraint = acados_constraint()

    constraint.expr = x1
    constraint.x = x
    constraint.u = u
    constraint.nc = 1
    constraint.name = con_name

    return constraint

# create render arguments
ocp = acados_ocp_nlp()

# export model
model = export_ode_model()

# set model_name
ocp.model = model

Tf = 2.0
nx = model.x.size()[0]
nu = model.u.size()[0]
ny = nx + nu
ny_e = nx
N = 50

# set ocp_nlp_dimensions
nlp_dims     = ocp.dims
nlp_dims.nx  = nx
nlp_dims.ny  = ny
nlp_dims.ny_e = ny_e
nlp_dims.nbx = 0
nlp_dims.nu  = model.u.size()[0]
nlp_dims.ns = nu
nlp_dims.N   = N

if FORMULATION == 1:
    nlp_dims.nh   = model.u.size()[0]
    nlp_dims.nsh  = model.u.size()[0]
    nlp_dims.nbu  = 0
    nlp_dims.nsbu = 0
elif FORMULATION == 0:
    nlp_dims.nh   = 0
    nlp_dims.nsh  = 0
    nlp_dims.nbu  = model.u.size()[0]
    nlp_dims.nsbu = model.u.size()[0]
elif FORMULATION == 2:
    nlp_dims.ns_e = 1
    nlp_dims.nh    = model.u.size()[0]
    nlp_dims.nsh   = model.u.size()[0]
    nlp_dims.nh_e  = 1
    nlp_dims.nsh_e = 1
    nlp_dims.nbu   = 0
    nlp_dims.nsbu  = 0

# set weighting matrices
nlp_cost = ocp.cost
Q = np.eye(4)
Q[0,0] = 1e0
Q[1,1] = 1e2
Q[2,2] = 1e-3
Q[3,3] = 1e-2

R = np.eye(1)
R[0,0] = 1e0

nlp_cost.W = scipy.linalg.block_diag(Q, R)

Vx = np.zeros((ny, nx))
Vx[0,0] = 1.0
Vx[1,1] = 1.0
Vx[2,2] = 1.0
Vx[3,3] = 1.0

nlp_cost.Vx = Vx

Vu = np.zeros((ny, nu))
Vu[4,0] = 1.0
nlp_cost.Vu = Vu

nlp_cost.W_e = Q

Vx_e = np.zeros((ny_e, nx))
Vx_e[0,0] = 1.0
Vx_e[1,1] = 1.0
Vx_e[2,2] = 1.0
Vx_e[3,3] = 1.0

nlp_cost.Vx_e = Vx_e

nlp_cost.yref  = np.zeros((ny, ))
nlp_cost.yref_e = np.zeros((ny_e, ))

nlp_cost.zl = 500*np.ones((1, ))
nlp_cost.Zl = 0*np.ones((1, 1))
nlp_cost.zu = 500*np.ones((1, ))
nlp_cost.Zu = 0*np.ones((1, 1))

nlp_cost.zl_e = 5000*np.ones((1, ))
nlp_cost.Zl_e = 0*np.ones((1, 1))
nlp_cost.zu_e = 5000*np.ones((1, ))
nlp_cost.Zu_e = 0*np.ones((1, 1))

# setting bounds
Fmax = 2.0
nlp_con = ocp.constraints
nlp_con.x0 = np.array([0.0, 3.14, 0.0, 0.0])

con_h = export_nonlinear_constraint()
con_h_e = export_terminal_nonlinear_constraint()
if FORMULATION == 1:
    nlp_con.lh = np.array([-Fmax])
    nlp_con.uh = np.array([+Fmax])
    nlp_con.lsh = 0*np.array([-Fmax])
    nlp_con.ush = 0*np.array([+Fmax])
    nlp_con.idxsh = np.array([0])
elif FORMULATION == 0:
    nlp_con.lbu = np.array([-Fmax])
    nlp_con.ubu = np.array([+Fmax])
    nlp_con.lsbu = 0*np.array([-Fmax])
    nlp_con.usbu = 0*np.array([+Fmax])
    nlp_con.idxbu = np.array([0])
    nlp_con.idxsbu = np.array([0])
elif FORMULATION == 2:
    nlp_con.lh = np.array([-Fmax])
    nlp_con.uh = np.array([+Fmax])
    nlp_con.lsh = 0*np.array([-Fmax])
    nlp_con.ush = 0*np.array([+Fmax])
    nlp_con.idxsh = np.array([0])
    nlp_con.lh_e = np.array([-xmax])
    nlp_con.uh_e = np.array([+xmax])
    nlp_con.lsh_e = 0*np.array([-Fmax])
    nlp_con.ush_e = 0*np.array([+Fmax])
    nlp_con.idxsh_e = np.array([0])

# set QP solver
ocp.solver_config.qp_solver = 'PARTIAL_CONDENSING_HPIPM'
ocp.solver_config.hessian_approx = 'GAUSS_NEWTON'
ocp.solver_config.integrator_type = 'ERK'

# set prediction horizon
ocp.solver_config.tf = Tf
ocp.solver_config.nlp_solver_type = 'SQP'

# set header path
ocp.acados_include_path  = '/usr/local/include'
ocp.acados_lib_path      = '/usr/local/lib'

# json_layout = acados_ocp2json_layout(ocp)
# with open('acados_layout.json', 'w') as f:
#     json.dump(json_layout, f, default=np_array_to_list)
# exit()

if FORMULATION == 1:
    ocp.con_h = con_h
    acados_solver = generate_solver(ocp, json_file = 'acados_ocp.json')
elif FORMULATION == 0:
    acados_solver = generate_solver(ocp, json_file = 'acados_ocp.json')
elif FORMULATION == 2:
    ocp.con_h = con_h
    ocp.con_h_e = con_h_e
    acados_solver = generate_solver(ocp, json_file = 'acados_ocp.json')

Nsim = 100

simX = np.ndarray((Nsim, nx))
simU = np.ndarray((Nsim, nu))

for i in range(Nsim):
    status = acados_solver.solve()

    # get solution
    x0 = acados_solver.get(0, "x")
    u0 = acados_solver.get(0, "u")

    for j in range(nx):
        simX[i,j] = x0[j]

    for j in range(nu):
        simU[i,j] = u0[j]

    # update initial condition
    x0 = acados_solver.get(1, "x")

    acados_solver.set(0, "lbx", x0)
    acados_solver.set(0, "ubx", x0)

# plot results
import matplotlib
import matplotlib.pyplot as plt
t = np.linspace(0.0, Tf/N, Nsim)
plt.subplot(2, 1, 1)
plt.step(t, simU, color='r')
plt.title('closed-loop simulation')
plt.ylabel('u')
plt.xlabel('t')
plt.grid(True)
plt.subplot(2, 1, 2)
plt.plot(t, simX[:,1])
plt.ylabel('theta')
plt.xlabel('t')
plt.grid(True)
plt.subplot(3, 1, 3)
plt.plot(t, simX[:,1])
plt.ylabel('x')
plt.xlabel('t')
plt.grid(True)
plt.show()

max_stars_repo_name: lnumerow-ethz/acados

max_issues_repo_name: lnumerow-ethz/acados

max_forks_repo_name: lnumerow-ethz/acados


text:
# -*- coding: utf-8 -*-
"""
Created on Sat Apr 25 12:04:11 2020

@author: mhayt
"""

#-------------------------------- API-FOOTBALL --------------------------------

import numpy as np
import matplotlib.pyplot as plt
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import roc_curve, roc_auc_score
from sklearn.model_selection import StratifiedKFold, cross_val_score, cross_val_predict, learning_curve
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay

#----------------------------- ML MODEL EVALUATION ----------------------------

#this section contains a series of functions which may be used for model evaluation

def pred_proba_plot(clf, X, y, cv=5, no_iter=5, no_bins=25, x_min=0.5, x_max=1, output_progress=True, classifier=''):
    '''
    Return figure - histogram display of correcly predicted results against incorrectly given results given the outputed probability of the classifier.

    Parameters
    ----------
    clf : estimator object implementing 'fit' and 'predict'
        The object to use to fit the data..
    X : array-like
        The data to fit. Can be, for example a list, or an array at least 2d..
    y : array-like
        The target variable to try to predict in the case of
    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.. The default is 5.
    no_iter : int, optional
        number of iterations of cross-validation. The default is 5.
    no_bins : int, optional
        number of bins of histogram. The default is 25.
    x_min : int, optional
        min x on histogram plot. The default is 0.5.
    x_max : in, optional
        max x on histogram plot. The default is 1.
    output_progress : display, optional
        print no. iterations complete to console. The default is True.
    classifier : string, optional
        classifier used, will be input to title. The default is ''.

    Returns
    -------
    fig :
    '''

    y_dup = []
    correct_guess_pred = []
    incorrect_guess_pred = []
    for i in range(no_iter):
        if output_progress:
            if i % 2 == 0:
                print(f'completed {i} iterations')
        skf = StratifiedKFold(n_splits=cv, shuffle=True)
        y_pred_cv = cross_val_predict(clf, X, y, cv=skf)
        y_pred_proba_cv = cross_val_predict(clf, X, y, cv=skf, method='predict_proba')
        y_dup.append(list(y))
        for i in range(len(y_pred_cv)):
            if y_pred_cv[i] == list(y)[i]:
                correct_guess_pred.append(max(y_pred_proba_cv[i]))
            if y_pred_cv[i] != list(y)[i]:
                incorrect_guess_pred.append(max(y_pred_proba_cv[i]))
    bins = np.linspace(x_min, x_max, no_bins)
    fig, ax = plt.subplots()
    ax.hist(incorrect_guess_pred, bins, alpha=0.5, edgecolor='#1E212A', color='red', label='Incorrect Prediction')
    ax.hist(correct_guess_pred, bins, alpha=0.5, edgecolor='#1E212A', color='green', label='Correct Prediction')
    ax.legend()
    ax.set_title(f'{classifier} - Iterated {no_iter} Times', y=1, fontsize=16, fontweight='bold');
    ax.set(ylabel='Number of Occurences',
            xlabel='Prediction Probability')
    return fig

def plot_cross_val_confusion_matrix(clf, X, y, display_labels='', title='', cv=5):
    '''
    Function to plot confusion matrix given the result of cross-validation, as oposed to the standard confucion matriax on test split data.

    Parameters
    ----------
    clf : estimator object implementing 'fit' and 'predict'
        The object to use to fit the data.
    X : array-like
        The data to fit. Can be, for example a list, or an array at least 2d..
    y : array-like
        The target variable to try to predict in the case of
    supervised learning.
    display_labels : ndarray of shape (n_classes,), optional
        display labels for plot. The default is ''.
    title : string, optional
        Title to be displayed at top of plot. The default is ''.
    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.. The default is 5.

    Returns
    -------
    display : :class:`~sklearn.metrics.ConfusionMatrixDisplay`
    '''

    y_pred = cross_val_predict(clf, X, y, cv=cv)
    cm = confusion_matrix(y, y_pred)
    fig = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=display_labels)
    fig.plot(cmap=plt.cm.Blues)
    fig.ax_.set_title(title)
    return fig

def plot_learning_curve(clf, X, y, scoring='accuracy', training_set_size=5, cv=5, x_min=0, x_max=500, y_min=0.3, y_max=1.02, title='', leg_loc=4):
    '''
    Function to plot learning curves for given estimator

    Parameters
    ----------
    clf : estimator object implementing 'fit' and 'predict'
        The object to use to fit the data.
    X : array-like
        The data to fit. Can be, for example a list, or an array at least 2d..
    y : array-like
        The target variable to try to predict in the case of
    scoring: metric of choice
        metrics can be 'accuracy', 'loss' etc. Must be a string! The default is 'accuracy'.
    training_set_size : int, optional
        number of splits that are undertaken on the train set. The default is 5.
    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.. The default is 5.
    x_min : int, optional
        x min on graph. The default is 0.
    x_max : int, optional
        x max on graph. The default is 500.
    y_min : int, optional
        y min on graph. The default is 0.3.
    y_max : int, optional
        y max on graph. The default is 1.02.
    title : string, optional
        title of figure. The default is ''.
    leg_loc : int, optional
        location of the legend on figure. The default is 4.

    Returns
    -------
    fig

    '''

    # Calculating train and val accuracy with 50 different training set sizes
    train_size,train_scores,valid_scores = learning_curve(clf, X, y, cv=cv, random_state=42, train_sizes=np.linspace(0.1, 1.0, training_set_size), scoring=scoring)

    # Calculating mean and standard deviations for training and val sets
    train_mean = np.mean(train_scores,axis=1)
    valid_mean = np.mean(valid_scores,axis=1)
    valid_std = np.std(valid_scores,axis=1)

    # Plotting Lines
    fig, ax = plt.subplots()
    ax.plot(train_size, train_mean, color="royalblue",  label="Training score")
    ax.plot(train_size, valid_mean, '--', color="#111111", label="Cross-validation score")

    # Draw error bands
    ax.fill_between(train_size, valid_mean - valid_std, valid_mean + valid_std, color="#DDDDDD")

    #Plotting details
    ax.set_xlabel("Training Set Size")
    ax.set_ylabel("Accuracy Score")
    ax.legend(loc=leg_loc)

    #organising axis
    ax.set_title(title, y=1, fontsize=14, fontweight='bold');
    ax.set_ylim(y_min,y_max)
    ax.set_xlim(x_min,x_max)

    return fig

# ----------------------------------- END -------------------------------------

reason: import numpy


text:
"""
created on 2018-9-24
created by GUO Yuxin
"""
import numpy as np
import prostate
from ols import OLS
import matplotlib.pyplot as plt

class BestSubset:
    def __init__(self):
        pass

    def best_subset(self, x, y):
        n_features = x.shape[1]
        lst = []
        for k in range(1, n_features):
            lst.extend(combination(n_features, k))
        lst.append(list(range(n_features)))

        rss = []
        beta_hat = []

        for i in lst:
            selected_x = x[:, i]
            ols = OLS(selected_x, y)
            beta = ols.ols()
            rss.extend(ols.rss(beta))
            beta_hat.append(beta)

        for k in range(0, n_features):
            rss_list = [rss[i] for i, v in enumerate(lst) if len(v) == k+1]
            rss_list = np.array(rss_list)
            rss_list = rss_list.reshape(rss_list.shape[1], rss_list.shape[0])
            plt.plot([k+1], rss_list, 'go')
        plt.axis([0, 9, 0, 100])
        plt.show()

def combination(n_features, order):
    lst = []
    length = n_features
    pos = np.zeros((length, 1), int)
    pos[:order] = 1
    lst.append(list(range(order)))
    while np.sum(pos[-order:]) != order:
        for i in range(length-1):
            if pos[i] == 1 and pos[i+1] == 0:
                pos[i] = 0
                pos[i + 1] = 1
                one_count = np.sum(pos[:i])
                for k in range(one_count):
                    pos[:one_count] = 1
                    pos[one_count:i] = 0
                lst.append([index for index, v in enumerate(pos) if v == 1])
                break
    return lst

if __name__ == '__main__':
    inputs, outputs, is_train = prostate.load_data()
    x = inputs[is_train == b'T']
    y = outputs[is_train == b'T']
    if len(y.shape) == 1:
        y = y.reshape((x.shape[0], 1))

    bs = BestSubset()
    bs.best_subset(x, y)


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import numpy as np
import pandas as pd

import src.feature_utils as ftu
from src.utils import create_features_train_predict

def show_future_prediction_data_dates(data, gap):
    train_dates = data.index.get_level_values(1)
    gap_start = train_dates.max() + pd.DateOffset(days=1)
    gap_end = gap_start + pd.DateOffset(days=gap - 1)
    train_split_size = (
        data.groupby(level="symbol").size().value_counts().index[0]
    )
    gap_size = len(pd.date_range(gap_start.date(), gap_end.date()))
    print(
        f"train = {train_dates.min().date()} - {train_dates.max().date()} "
        f"({train_split_size}) | "
        f"gap = {gap_start.date()} - {gap_end.date()} ({gap_size})"
    )

def get_future_prediction_data_grid(data, gap, horizon):
    store_list = data["store"].unique()
    item_list = data["item"].unique()
    train_end_date = data.index.get_level_values(1).max()
    date_list = pd.date_range(
        start=train_end_date + pd.DateOffset(days=gap + 1),
        end=train_end_date + pd.DateOffset(days=gap + horizon),
    )
    d = {"store": store_list, "item": item_list, "date": date_list}
    data_grid = ftu.df_from_cartesian_product(d)
    data_grid = data_grid.assign(
        symbol=data_grid["store"].astype(str)
        + "_"
        + data_grid["item"].astype(str)
    ).set_index(["symbol", "date"])
    return data_grid

def predict_future(
    r,
    data_train,
    lags,
    window_size,
    used_columns,
    categ_fea,
    first_date,
    gap,
    horizon,
    data_test,
    model_params,
    scoring_func,
    model_fit_params={"early_stoppin_rounds": 200, "verbose": 0},
):
    show_future_prediction_data_dates(data_train, gap)

    (
        _,
        y_pred_test,
        trained_model,
        train_end_date,
        test_start_date_manual,
        test_end_date_manual,
    ) = create_features_train_predict(
        data_train,
        lags,
        window_size,
        used_columns,
        categ_fea,
        r,
        first_date,
        gap,
        horizon,
        data_test,
        model_params,
        model_fit_params,
        scoring_func,
    )
    df_future_pred = get_future_prediction_data_grid(data_train, gap, horizon)
    df_future_pred["pred"] = np.expm1(y_pred_test)
    return [
        df_future_pred,
        trained_model,
        train_end_date,
        test_start_date_manual,
        test_end_date_manual,
    ]

max_stars_repo_name: edesz/store-item-demand-forecast

max_issues_repo_name: edesz/store-item-demand-forecast

max_forks_repo_name: edesz/store-item-demand-forecast


text:
from tensorflow.keras.callbacks import Callback
from tensorflow.python.keras import backend as K
import numpy as np
from tensorflow.python.framework import ops

class LearningRateScheduler(Callback):
    """Learning rate scheduler.
    Arguments:
        schedule: a function that takes an iteration index as input
            (integer, indexed from 0) and returns a new
            learning rate as output (float).
        verbose: int. 0: quiet, 1: update messages.
    """

    def __init__(self, schedule, verbose=0):
        super(LearningRateScheduler, self).__init__()
        self.schedule = schedule
        self.verbose = verbose
        self._total_batches_seen_lr = 0

    def on_train_batch_begin(self, batch, logs=None):

        if not hasattr(self.model.optimizer, 'lr'):
            raise ValueError('Optimizer must have a "lr" attribute.')
        try:  # new API
            lr_o = float(K.get_value(self.model.optimizer.lr))
            lr = self.schedule(self._total_batches_seen_lr, lr_o)
        except TypeError:  # Support for old API for backward compatibility
            raise TypeError('The schedule function accepts two arguments - iteratnion and last learning rate')
        if not isinstance(lr, (ops.Tensor, float, np.float32, np.float64)):
            raise ValueError('The output of the "schedule" function '
                            'should be float.')
        if isinstance(lr, ops.Tensor) and not lr.dtype.is_floating:
            raise ValueError('The dtype of Tensor should be float')
        K.set_value(self.model.optimizer.lr, K.get_value(lr))
        if self.verbose > 0 and lr_o != lr:
            print('\nIteration %05d: LearningRateScheduler reducing learning '
                    'rate to %s.' % (self._total_batches_seen_lr + 1, lr))

    def on_train_batch_end(self, batch, logs=None):
        self._total_batches_seen_lr += 1
        logs = logs or {}
        logs['lr'] = K.get_value(self.model.optimizer.lr)

max_stars_repo_name: Caro-app/calamari

max_issues_repo_name: Caro-app/calamari

max_forks_repo_name: Caro-app/calamari


text:
import numpy as np
import os
import threading
import gym

if 'MUJOCO_RENDERER' in os.environ:
  RENDERER = os.environ['MUJOCO_RENDERER']
else:
  RENDERER = 'glfw'

def get_device_id():
  return int(os.environ.get('GL_DEVICE_ID', 0))

def set_camera(cam, azimuth=None, elevation=None, distance=None, lookat=None):
  """ Sets camera parameters """
  if azimuth:
    cam.azimuth = azimuth
  if elevation:
    cam.elevation = elevation
  if distance:
    cam.distance = distance
  if lookat:
    for i in range(3):
      cam.lookat[i] = lookat[i]

class DreamerEnv():

  LOCK = threading.Lock()

  def __init__(self, action_repeat, width=64):
    self._action_repeat = action_repeat
    self._width = width
    self._size = (self._width, self._width)

  @property
  def observation_space(self):
    shape = self._size + (3,)
    space = gym.spaces.Box(low=0, high=255, shape=shape, dtype=np.uint8)
    return gym.spaces.Dict({'image': space})

  @property
  def action_space(self):
    return self._env.action_space

  def close(self):
    return self._env.close()

  def reset(self):
    with self.LOCK:
      state = self._env.reset()
    return self._get_obs(state)

  def render(self, mode):
    return self._env.render(mode)

  def _get_obs(self, state):
    self._offscreen.render(self._width, self._width, -1)
    image = np.flip(self._offscreen.read_pixels(self._width, self._width)[0], 1)

    obs = {'image': image}
    if isinstance(state, dict):
      obs.update(state)
    else:
      obs['state'] = state
    return obs

class SparseMetaWorld(DreamerEnv):

  def __init__(self, task, action_repeat):
    super().__init__(action_repeat)
    from mujoco_py import MjRenderContext
    import metaworld.envs.mujoco.sawyer_xyz.v2 as sawyerv2
    assert 'V2' in task
    self.task_type = self.get_task_type(task)

    with self.LOCK:
        self._env = getattr(sawyerv2, task)()
    self._env.random_init = False
    self._env.max_path_length = np.inf

    self._action_repeat = action_repeat
    self._rand_goal = False
    self._rand_hand = self.task_type in ['reach', 'button', 'window', 'drawer']
    self._rand_obj = self.task_type in ['window', 'drawer', 'push', 'hammer']
    self._width = 64
    self._size = (self._width, self._width)
    self.rendered_goal = False

    self._offscreen = MjRenderContext(self._env.sim, True, get_device_id(), RENDERER, True)
    if "SawyerHammerEnv" in task:
      set_camera(self._offscreen.cam, azimuth=220, elevation=-140, distance=0.8, lookat=[0.2, 0.65, -0.1])
    else:
      set_camera(self._offscreen.cam, azimuth=205, elevation=-165, distance=2.6, lookat=[1.1, 1.1, -0.1])

  def step(self, action):
    total_reward = 0.0
    for step in range(self._action_repeat):
      state, reward, done, info = self._env.step(action)
      reward = info['success'] # Use sparse reward
      total_reward += reward
      if done:
        break
    obs = self._get_obs(state)
    return obs, total_reward, done, info

  def get_task_type(self, task):
    if 'SawyerReachEnv' in task:
      return 'reach'
    if 'SawyerButtonPressEnv' in task:
      return 'button'
    if 'SawyerWindowCloseEnv' in task or 'SawyerWindowOpenEnv' in task:
      return 'window'
    if 'SawyerDrawerCloseEnv' in task or 'SawyerDrawerOpenEnv' in task:
      return 'drawer'
    if 'SawyerPushEnv' in task:
      return 'push'
    if 'SawyerStickPushEnv' in task:
      return 'thermos'
    if 'SawyerHammerEnv' in task:
      return 'hammer'

  def reset(self):
    self.rendered_goal = False

    if self.task_type == 'reach' or self.task_type == 'button':
      self._env.hand_init_pos = np.random.uniform(
        self._env.hand_low,
        self._env.hand_high,
        size=(self._env.hand_low.size)
      )

    elif self.task_type == 'window':
      slider_init_pos = np.random.uniform(0, 0.2)
      self._env.init_config['slider_init_pos'] = slider_init_pos
      self._env.hand_init_pos = self._env.obj_init_pos + np.array([slider_init_pos, -0.1, 0.05])
      self._env.hand_init_pos += np.random.uniform((-0.15, -0.3, -0.15), (0.15, 0, 0.15), size=(3))

    elif self.task_type == 'drawer':
      drawer_init_pos = np.random.uniform(0, 0.15)
      self._env.init_config['drawer_init_pos'] = drawer_init_pos
      # -0.16 is the protrusion of the handle
      self._env.hand_init_pos = self._env.obj_init_pos + np.array([0, -0.16-drawer_init_pos, 0.15])
      self._env.hand_init_pos += np.random.uniform((-0.15, -0.15, 0), (0.15, 0.15, 0.3), size=(3))

    if self.task_type == 'push':
      obj_init_pos = np.random.uniform(
        (-0.2, 0.6, 0.02),
        (0.4, 0.9, 0.02),
        size=(3)
      )
      self._env.init_config['obj_init_pos'] = obj_init_pos
      # Initialize hand above object
      self._env.hand_init_pos = obj_init_pos
      # Initialize hand with fixed height, can be overridden by rand_hand
      self._env.hand_init_pos[2] = 0.05

    elif self.task_type == 'hammer':
      hammer_init_pos = np.random.uniform(
        (-0.1, 0.3, 0.04),
        (0.3, 0.7, 0.04),
        size=(3)
      )
      self._env.init_config['hammer_init_pos'] = hammer_init_pos
      # Initialize hand with fixed displacement, can be overridden by rand_hand
      self._env.hand_init_pos = hammer_init_pos + np.array([0., -0.1, 0.16])
    return super().reset()

max_stars_repo_name: zchuning/metaworld

max_issues_repo_name: zchuning/metaworld

max_forks_repo_name: zchuning/metaworld


text:
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This module contains various utility functions."""
import io
import random

import numpy as np
import torch
from gensim.models import Word2Vec

import log

logger = log.get_logger('root')

def load_embeddings(file_name: str, embeddings_format: str):
    """Load word embeddings from the given file, either in plain-text format ('text') or in gensim format ('gensim')"""
    if embeddings_format == 'text':
        return _load_w2v_model_file(filename=file_name)
    else:
        return _load_w2v_model_gensim(filename=file_name)

def _load_w2v_model_gensim(filename):
    """Load word embeddings from a given file in gensim format"""
    logger.info('Loading embeddings from %s', filename)
    w2v_model = Word2Vec.load(filename)
    word2vec = w2v_model.wv
    del w2v_model
    logger.info('Done loading embeddings')
    return word2vec

def _load_w2v_model_file(filename):
    """Load word embeddings from a given file in plain-text format"""
    logger.info('Loading embeddings from %s', filename)
    w2v = DummyDict()
    w2v.update({w: v for w, v in _load_vectors(filename)})
    w2v.vocab = w2v.keys()
    logger.info('Done loading embeddings')
    return w2v

def _load_vectors(path, skip=False):
    with io.open(path, 'r', encoding='utf-8') as f:
        for line in f:
            if skip:
                skip = False
            else:
                index = line.index(' ')
                word = line[:index]
                yield word, np.array([np.float(entry) for entry in line[index + 1:].split()])

class DummyDict(dict):
    pass

def generate_chunks(total, min_val, max_val, num_chunks=-1):
    """
    Randomly generate a list of integers l such that sum(l) = total and for each x in l, min_val <= x <= max_val.
    If num_chunks > 0, it is guaranteed that the list contains exactly num_chunks elements.
    """

    if num_chunks <= 0:
        chunks = []
        while total > 0:
            next_chunk_size = random.randint(min(total, min_val), min(total, max_val))
            if 0 < total - next_chunk_size < min_val:
                continue
            total -= next_chunk_size
            chunks.append(next_chunk_size)
        return chunks
    else:
        if total < num_chunks * min_val:
            raise ValueError('Total ({}) must be >= num_chunks * min_val ({}*{})'.format(total, num_chunks, min_val))

        if total > num_chunks * max_val:
            raise ValueError('Total ({}) must be <= num_chunks * max_val ({}*{})'.format(total, num_chunks, max_val))

        total -= num_chunks * min_val
        chunks = None
        while not chunks or any([x > max_val for x in chunks]):
            split_points = [0, total]
            for _ in range(num_chunks - 1):
                split_points.append(random.randint(0, total))
            split_points.sort()
            chunks = [split_points[i + 1] - split_points[i] + min_val for i in range(len(split_points) - 1)]

        return chunks

def length_to_mask(length, max_len=None, dtype=None):
    """
    Convert a 1d tensor of lengths to the corresponding mask tensor.
    """
    assert len(length.shape) == 1, 'Length shape should be 1 dimensional.'
    max_len = max_len or length.max().item()
    mask = torch.arange(max_len, device=length.device,
                        dtype=length.dtype).expand(len(length), max_len) < length.unsqueeze(1)
    if dtype is not None:
        mask = torch.as_tensor(mask, dtype=dtype, device=length.device)
    return mask

max_stars_repo_name: stefan-it/bertram

max_issues_repo_name: stefan-it/bertram

max_forks_repo_name: stefan-it/bertram


text:
# coding: utf-8

import numpy as np
import pandas as pd
from glob import glob
from nltk.util import ngrams

df = pd.read_csv("ssq.csv")

# 确定red_tag

# ![](http://img5.cache.netease.com/help/2013/2/5/20130205200221990eb.png)

bonus = {
    (0, 1): 5,
    (1, 1): 5,
    (2, 1): 5,
    (3, 1): 10,
    (4, 0): 10,
    (4, 1): 200,
    (5, 0): 200,
    (5, 1): 3000,
    (6, 0): 5000000,
    (6, 1): 10000000,
}
columns = ["red1", "red2", "red3", "red4", "red5", "red6", "blue"]
# 检查列表

topic, n = "ssq", 6

hot_water = sorted(glob(f"gamble/{topic}/*-{topic}.csv"))[-1]
date = hot_water[-18:-8]
df_dream = pd.read_csv(hot_water)

last_shit = df.loc[df.date <= date, "date"].max()
df_last = df.loc[df.date == last_shit, df_dream.columns]
list_last = df_last.values[0].tolist()
print(df.tail()[columns])

def getd(v):
    return len(set(v[:n]) & set(list_last[:n])), len(set(v[n:]) & set(list_last[n:]))

df_dream["check"] = df_dream.apply(getd, axis=1)
df_dream["bonus"] = df_dream.check.apply(lambda _: bonus.get(_, 0))
print(df_dream)
print("bonus", df_dream["bonus"].sum())

blue = df.blue.iloc[-1]
td = pd.datetime.today().date()
year = str(td.year)
yall = df.date.str.slice(0, 4)
ynum = yall.nunique() - 1
idx = yall == year

# other years blue average
key = pd.Series(ngrams(df.loc[~idx, "blue"], 2)).value_counts().to_frame("key")
dfkey = key.loc[[_ for _ in key.index if _[0] == blue]] / ynum
# this year blue
key2 = pd.Series(ngrams(df.loc[idx, "blue"], 2)).value_counts().to_frame("key2")
dfkey2 = key2.loc[[_ for _ in key2.index if _[0] == blue]]

dfkey = dfkey.join(dfkey2, how="left").fillna(0)
dfkey["key3"] = dfkey["key"] - dfkey["key2"]
dfkey.sort_values(by="key3", inplace=True)
blued = [b for _, b in dfkey.head().index]

# # 输出
idxa = (df.red1 <= 11) & (df.red6 >= 18)
t = len(df)
r = len(df[idxa]) / t * 100

# 剔除与最后一期交集>3

reds = df.loc[:, "red1":"red6"].apply(set, axis=1)
head = reds.iloc[:-1]
tail = reds.iloc[-1]

def get_pre(N):
    df_fore = (
        pd.concat(
            [
                reds.iloc[:i]
                .apply(lambda _: len(_ & reds.iloc[i]))
                .sort_index(ascending=False)
                .reset_index(drop=True)
                .to_frame(i)
                for i in reds.index[::-1][:N]
            ],
            axis=1,
        )
        .fillna(-1)
        .astype(int)
    )
    bar = df_fore.apply(lambda _: _.value_counts(), axis=1).fillna(-1).astype(int)
    return bar

bar = get_pre(4)
rst = []
for _ in range(5):
    p = len(reds) - np.random.choice(bar[bar[0] == 4].index, 2, replace=False) - 2
    red_drop = set()
    for i in reds.loc[p].values:
        red_drop |= i
    tmp = head[
        head.apply(
            lambda _: min(_)
            <= 11 & max(_)
            >= 18 & len(_ & red_drop)
            == 0 & len(_ & tail)
            <= 3
        )
    ]
    rst.append(df.loc[np.random.choice(tmp.index, 1,), "red1":"red6"])

ds = pd.concat(rst)

obj = set(np.reshape(ds.loc[:, "red1":"red6"].values, (1, 30))[0])
dr = sorted(set(range(1, 34)) - obj)
print(f"God bless! total {t} red1<=11 and red6>=18: {len(df[idxa])} {r:.1f}%")
print(f"ETL {len(bar)} unique {len(obj)} drop {dr}\n")

ds["blue"] = blued
ds.to_csv(f"gamble/ssq/{td}-ssq.csv", index=False)
print(ds)

max_stars_repo_name: muxuezi/lottery

max_issues_repo_name: muxuezi/lottery

max_forks_repo_name: muxuezi/lottery


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

import numpy as np
import cv2
from scipy.cluster.vq import vq, kmeans
from matplotlib import pyplot as plt
import numpy as np
import time as t

cv_image = cv2.imread('/home/jose/Desktop/model_final.jpg',1)
cv_copy  = cv_image.copy()
#cv2.imshow('image',cv_image)
height, width , depth  = cv_image.shape
hsv = cv2.cvtColor(cv_image, cv2.COLOR_BGR2HSV)

thresholded = 0
obj_color = 0
if obj_color == 0:

	#hsv
    low  = [0,50,100]
    up = [50,250,150]

    lower = np.array(low, np.uint8)
    upper = np.array(up, np.uint8)

    hue, sat, val = hsv[:,:,0], hsv[:,:,1], hsv[:,:,2]

    plt.figure(figsize=(10,8))
    plt.subplot(311)                             #plot in the first cell
    plt.subplots_adjust(hspace=.5)
    plt.title("Indicador H - Tonalidad")
    plt.hist(np.ndarray.flatten(hue), bins=180)
    plt.subplot(312)                             #plot in the second cell
    plt.title("Indicador S - Saturacion")
    plt.hist(np.ndarray.flatten(sat), bins=128)
    plt.subplot(313)                             #plot in the third cell
    plt.title("Indicador V - Valor Luminosidad")
    plt.hist(np.ndarray.flatten(val), bins=128)
    plt.show()

    thresholded = cv2.inRange(hsv, lower, upper)
    #Morphological opening (remove small objects from the foreground)
    thresholded = cv2.erode(thresholded, np.ones((2,2), np.uint8), iterations=1)
    thresholded = cv2.dilate(thresholded, np.ones((2,2), np.uint8), iterations=1)
    #Morphological closing (fill small holes in the foreground)
    thresholded = cv2.dilate(thresholded, np.ones((2,2), np.uint8), iterations=1)
    thresholded = cv2.erode(thresholded, np.ones((2,2), np.uint8), iterations=1)

    ret,thresh = cv2.threshold(thresholded,127,255,0)
    _, contours, hierarchy = cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_NONE)
    #contours= cv2.findContours(canny_output, cv.RETR_TREE, cv.CHAIN_APPROX_SIMPLE)
    #c = max(contours, key = cv2.contourArea)
    #cv2.drawContours(thresh,[c],0,(0, 255, 0),3)
    #bc = biggestContourI(contours)
    cv2.drawContours(cv_copy, contours, -1, (0,255,0),3)
    cv2.imshow('draw contours',cv_copy)
    print contours
    numobj = len(contours) # number of objects found in current frame
    print(numobj)
    if numobj > 0:
        print("here")
        moms = cv2.moments(contours[0])
        if moms['m00']>40:
            cx = int(moms['m10']/moms['m00'])
            cy = int(moms['m01']/moms['m00'])

            xb = -(cy - (height/2))*.0025*.428 + .578
            yb = -(cx - (width/2))*.0025*.428 + .40  +0.06

            print("xb: ",xb)
            print("yb: ",yb)

        #print "Found blue ", numobj,  "object(s)"
        obj_found = True
else:
    print "Couldn't find any green or blue objects."

#cv2.imshow("O", hsv)
cv2.imshow("Thresholded", thresholded)
cv2.waitKey(0)

max_stars_repo_path: process_img/dos (copy).py

max_issues_repo_path: process_img/dos (copy).py

max_forks_repo_path: process_img/dos (copy).py