text:
import os
import json
from itertools import islice
from pickle import PicklingError
from collections import OrderedDict

import numpy as np
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt

try:
    from hyperopt import space_eval, hp
    from hyperopt.base import miscs_to_idxs_vals  # todo main_plot_1D_attachment
except ImportError:
    space_eval, hp = None, None
    miscs_to_idxs_vals = None

try:
    from skopt.plots import plot_evaluations, plot_objective, plot_convergence
    import skopt
    from skopt.space.space import Space, Dimension
except ImportError:
    plot_evaluations, plot_objective, plot_convergence = None, None, None
    Space = None
    Dimension = None

try:
    from skopt.utils import dump
except ImportError:
    dump = None

from ai4water.utils.utils import Jsonize, clear_weights
from ._space import Categorical, Real, Integer

def is_choice(space):
    """checks if an hp.space is hp.choice or not"""
    if 'switch' in space.name:
        return True
    return False

def x_iter_for_tpe(trials, param_space: dict, as_list=True):

    assert isinstance(param_space, dict)

    x_iters = []  # todo, remove x_iters, it is just values of iterations
    iterations = {}
    for idx, t in enumerate(trials.trials):

        vals = t['misc']['vals']
        y = t['result']['loss']

        _iter = get_one_tpe_x_iter(vals, param_space)
        iterations[f'{round(y, 5)}_{idx}'] = _iter

        x_iters.append(_iter)

    if as_list:
        return [list(d.values()) for d in x_iters]
    return iterations

def get_one_tpe_x_iter(tpe_vals, param_space: dict, sort=True):

    x_iter = {}
    for para, para_val in tpe_vals.items():
        if is_choice(param_space[para]):
            hp_assign = {para: para_val[0]}
            cval = space_eval(param_space[para], hp_assign)
            x_iter[para] = cval
        else:
            x_iter[para] = para_val[0]

    if sort:
        x_iter = sort_x_iters(x_iter, list(param_space.keys()))

    return x_iter

def sort_x_iters(x_iter: dict, original_order: list):
    # the values in x_iter may not be sorted as the parameters provided in original order

    new_x_iter = {}
    for s in original_order:
        new_x_iter[s] = x_iter[s]

    return new_x_iter

def skopt_space_from_hp_spaces(hp_space: dict) -> list:
    """given a dictionary of hp spaces where keys are names, this function
    converts it into skopt space."""
    new_spaces = []

    for k, s in hp_space.items():

        new_spaces.append(skopt_space_from_hp_space(s, k))

    return new_spaces

def skopt_space_from_hp_space(hp_space, prior_name=None):
    """Converts on hp space into a corresponding skopt space."""
    if is_choice(hp_space):
        skopt_space = to_categorical(hp_space, prior_name=prior_name)
    elif any([i in hp_space.__str__() for i in ['loguniform', 'quniform', 'qloguniform', 'uniform']]):
        skopt_space = to_real(hp_space, prior_name=prior_name)
    elif 'randint' in hp_space.__str__():
        skopt_space = to_int(hp_space, prior_name=prior_name)
    else:
        raise NotImplementedError

    return skopt_space

def to_categorical(_space, prior_name=None):
    """converts an hp space into a Dimension object."""
    cats = []
    inferred_name = None
    for arg in _space.pos_args:
        if hasattr(arg, '_obj') and arg.pure:
            cats.append(arg._obj)
        elif arg.name == 'hyperopt_param' and len(arg.pos_args)>0:
            for a in arg.pos_args:
                if a.name == 'literal' and a.pure:
                    inferred_name = a._obj

    prior_name = verify_name(prior_name, inferred_name)

    if len(cats) == 0:
        raise NotImplementedError
    return Categorical(categories=cats, name=prior_name)

def verify_name(prior_name, inferred_name):
    """Verfies that the given/prior name matches with the inferred name.
    """
    if prior_name is None:
        prior_name = inferred_name
    else:
        assert prior_name == inferred_name, f"""given name {prior_name} does not mach with
                                                inferred name {inferred_name}"""
    return prior_name

def to_int(_space, prior_name=None):
    """converts an hp.randint into a Dimension object"""
    inferred_name = None
    limits = None
    for arg in _space.pos_args:
        if arg.name == 'literal' and len(arg.named_args) == 0:
            inferred_name = arg._obj
        elif len(arg.named_args) == 2 and arg.name == 'randint':
            limits = {}
            for a in arg.named_args:
                limits[a[0]] = a[1]._obj

        elif len(arg.pos_args) == 2 and arg.name == 'randint':
            # high and low are not named args
            _limits = []
            for a in arg.pos_args:
                _limits.append(a._obj)
            limits = {'high': np.max(_limits), 'low': np.min(_limits)}
        else:
            raise NotImplementedError

    prior_name = verify_name(prior_name, inferred_name)

    if limits is None:
        raise NotImplementedError
    return Integer(low=limits['low'], high=limits['high'], name=prior_name)

def to_real(_space, prior_name=None):
    """converts an an hp space to real. """
    inferred_name = None
    limits = None
    prior = None
    allowed_names = ['uniform', 'loguniform', 'quniform', 'loguniform', 'qloguniform']
    for arg in _space.pos_args:
        if len(arg.pos_args) > 0:
            for a in arg.pos_args:
                if a.name == 'literal' and len(arg.named_args) == 0:
                    inferred_name = a._obj
                elif a.name in allowed_names and len(a.named_args) == 2:
                    prior = a.name
                    limits = {}
                    for _a in a.named_args:
                        limits[_a[0]] = _a[1]._obj
                elif a.name in allowed_names and len(a.pos_args) == 2:
                    prior = a.name
                    _limits = []
                    for _a in a.pos_args:
                        _limits.append(_a._obj)
                    limits = {'high': np.max(_limits), 'low': np.min(_limits)}

    prior_name = verify_name(prior_name, inferred_name)

    # prior must be inferred because hp spaces always have prior
    if limits is None or prior is None or 'high' not in limits:
        raise NotImplementedError

    return Real(low=limits['low'], high=limits['high'], prior=prior, name=prior_name)

def loss_histogram(losses,  # array like
                   xlabel='objective_fn',
                   ylabel='Frequency',
                   save=True,
                   fname="histogram.png"):

    plt.hist(losses)
    plt.xlabel(xlabel)
    plt.ylabel(ylabel)

    if save:
        plt.savefig(fname, dpi=300, bbox_inches='tight')
    else:
        plt.show()

def plot_hyperparameters(
        trials,
        save=True,
        fontsize=10,
        colorize_best=None,
        columns=5,
        arrange_by_loss=False,
        fname='parameters_selection_plot.png'
):
    """Copying from hyperopt because original hyperopt does not allow saving the plot."""

    idxs, vals = miscs_to_idxs_vals(trials.miscs)
    losses = trials.losses()
    finite_losses = [y for y in losses if y not in (None, float("inf"))]
    asrt = np.argsort(finite_losses)
    if colorize_best is not None:
        colorize_thresh = finite_losses[asrt[colorize_best + 1]]
    else:
        # -- set to lower than best (disabled)
        colorize_thresh = finite_losses[asrt[0]] - 1

    loss_min = min(finite_losses)
    loss_max = max(finite_losses)
    print("finite loss range", loss_min, loss_max, colorize_thresh)

    loss_by_tid = dict(zip(trials.tids, losses))

    def color_fn_bw(lossval):
        if lossval in (None, float("inf")):
            return 1, 1, 1
        else:
            t = (lossval - loss_min) / (loss_max - loss_min + 0.0001)
            if lossval < colorize_thresh:
                return 0.0, 1.0 - t, 0.0  # -- red best black worst
            else:
                return t, t, t  # -- white=worst, black=best

    all_labels = list(idxs.keys())
    titles = all_labels
    order = np.argsort(titles)

    C = min(columns, len(all_labels))
    R = int(np.ceil(len(all_labels) / float(C)))

    for plotnum, varnum in enumerate(order):
        label = all_labels[varnum]
        plt.subplot(R, C, plotnum + 1)

        # hide x ticks
        ticks_num, _ = plt.xticks()
        plt.xticks(ticks_num, [""] * len(ticks_num))

        dist_name = label

        if arrange_by_loss:
            x = [loss_by_tid[ii] for ii in idxs[label]]
        else:
            x = idxs[label]
        if "log" in dist_name:
            y = np.log(vals[label])
        else:
            y = vals[label]
        plt.title(titles[varnum], fontsize=fontsize)
        c = list(map(color_fn_bw, [loss_by_tid[ii] for ii in idxs[label]]))
        if len(y):
            plt.scatter(x, y, c=c)
        if "log" in dist_name:
            nums, _ = plt.yticks()
            plt.yticks(nums, ["%.2e" % np.exp(t) for t in nums])

    if save:
        plt.savefig(fname, dpi=300, bbox_inches='tight')
    else:
        plt.show()

def post_process_skopt_results(skopt_results, results, opt_path):

    skopt_plots(skopt_results, pref=opt_path)

    return save_skopt_results(skopt_results, results, opt_path)

def save_skopt_results(skopt_results, results, opt_path):

    fname = os.path.join(opt_path, 'gp_parameters')

    sr_res = SerializeSKOptResults(skopt_results)

    if 'folder' in list(results.items())[0]:
        clear_weights(results=results, opt_dir=opt_path)
    else:
        clear_weights(results=results, opt_dir=opt_path)
        clear_weights(opt_dir=opt_path)

    try:  # adding .hpo extension so that no other file/folder exists with same name
        fname = os.path.join(opt_path, os.path.basename(opt_path) + ".hpo")
        dump(skopt_results, fname)
    except PicklingError:
        print("could not pickle results")

    try:
        with open(fname + '.json', 'w') as fp:
            json.dump(sr_res.serialized_results, fp, sort_keys=True, indent=4)
    except TypeError:
        with open(fname + '.json', 'w') as fp:
            json.dump(str(sr_res.serialized_results), fp, sort_keys=True, indent=4)

    return

def skopt_plots(search_result, pref=os.getcwd(), threshold=20):

    if len(search_result.x) < threshold:  # it takes forever if parameters are > 20
        plt.close('all')
        _ = plot_evaluations(search_result)
        plt.savefig(os.path.join(pref, 'evaluations'), dpi=400, bbox_inches='tight')

    if len(search_result.x) < threshold:
        if search_result.space.n_dims == 1:
            pass
        else:
            plt.close('all')
            _ = plot_objective(search_result)
            plt.savefig(os.path.join(pref, 'objective'), dpi=400, bbox_inches='tight')

    plt.close('all')
    _ = plot_convergence(search_result)
    plt.savefig(os.path.join(pref, 'convergence'), dpi=400, bbox_inches='tight')

class SerializeSKOptResults(object):
    """
    This class has two functions
      - converts everything in skopt results into python native types so that these results can be saved in readable
        json files.
      - Store as much attributes in in serialized form that a skopt `search_result` object can be generated from it
        which then can be used to regenerate all hyper-parameter optimization related plots.

     skopt_results is a dictionary which contains following keys
     x: list, list of parameters being optimized
     fun: float, final value of objective function
     func_vals: numpy array, of length equal to number of iterations
     x_iters: list of lists, outer list is equal to number of iterations and each inner list is equal number of parameters
              being optimized
     models: list of models, where a model has following attributes
             - noise: str
             - kernel: skopt.learning.gaussian_process.kernels.Sum, it has following 2 attributes
                 - k1: skopt.learning.gaussian_process.kernels.Product, it has following attributes
                     -k1: skopt.learning.gaussian_process.kernels.ConstantKernel
                         - constant_value: flaot
                         - constant_value_bounds: tuple of floats
                     -k2: skopt.learning.gaussian_process.kernels.Matern
                         - length_scale: numpy ndarray
                         - length_scale_bounds: list of floats
                         - nu: float
                 - k2: skopt.learning.gaussian_process.kernels.WhiteKernel
                     - noise_level: float
                     - noise_level_bounds: tuple of floats
             - alpha: float
             - optimizer: str
             - n_restarts_optimizer: int
             - normalize_y: bool
             - copy_X_train: bool
             - random_state: int
             - kernel_: skopt.learning.gaussian_process.kernels.Sum
             - _rng: numpy.random.mtrand.RandomState
             - n_features_in_: int
             - _y_train_mean: np.float64
             - _y_train_std: np.float64
             - X_train_: numpy array
             - y_train_: numpy array
             - log_marginal_likelihood_value_: numpy array
             - L_: numpy array
             - _K_inv: NoneType
             - alpha_: numpy array
             - noise_: np.float64
             - K_inv_: numpy array
             - y_train_std_: np.float64
             - y_train_mean_: np.float64
     space: skopt.space.space.Space, parameter spaces
     random_state: numpy.random.mtrand.RandomState
     specs: dict,  specs of each iteration. It has following keys
         - args: dict, which has following keys
             - func: function
             - dimensions: skopt.space.space.Space
             - base_estimator: skopt.learning.gaussian_process.gpr.GaussianProcessRegressor, which has following attributes
                 - noise: str
                 - kernel: skopt.learning.gaussian_process.kernels.Product
                     - k1: skopt.learning.gaussian_process.kernels.ConstantKernel, which has following attributes
                         - constant_value: flaot
                         - constant_value_bounds: tuple of floats
                     - k2: skopt.learning.gaussian_process.kernels.Matern, which has following attributes
                         - length_scale: numpy ndarray
                         - length_scale_bounds: list of floats
                         - nu: float
                 - alpha: float
                 - optimizer: str
                 - n_restarts_optimizer: int
                 - normalize_y: bool
                 - copy_X_train: bool
                 - random_state: int
             - n_cals: int
             - n_random_starts: NoneType
             - n_initial_points: str
             - initial_point_generator: str
             - acq_func: str
             - acq_optimizer: str
             - x0: list
             - y0: NoneType
             - random_state: numpy.random.mtrand.RandomState
             - verbose: book
             - callback: NoneType
             - n_points: int
             - n_restarts_optimizer: int
             - xi: float
             - kappa: float
             - n_jobs: int
             - model_queue_size: NoneType
         - function: str
     """
    def __init__(self, results: dict):
        self.results = results
        self.iters = len(results['func_vals'])
        self.paras = len(results['x'])
        self.serialized_results = {}

        for key in results.keys():
            self.serialized_results[key] = getattr(self, key)()

    def x(self):

        return self.para_list(self.results['x'])

    def para_list(self, x):
        """Serializes list of parameters"""
        _x = []
        for para in x:
            _x.append(Jsonize(para)())
        return _x

    def x0(self):
        _x0 = []

        __xo = self.results['specs']['args']['x0']

        if __xo is not None:
            for para in __xo:
                if isinstance(para, list):
                    _x0.append(self.para_list(para))
                else:
                    _x0.append(Jsonize(para)())
        return _x0

    def y0(self):
        __y0 = self.results['specs']['args']['y0']
        if __y0 is None:
            return __y0
        if isinstance(__y0, list):
            _y0 = []
            for y in self.results['specs']['args']['y0']:
                _y0.append(Jsonize(y)())
            return _y0

        return Jsonize(self.results['specs']['args']['y0'])()

    def fun(self):
        return float(self.results['fun'])

    def func_vals(self):
        return [float(i) for i in self.results['func_vals']]

    def x_iters(self):
        out_x = []
        for i in range(self.iters):
            x = []
            for para in self.results['x_iters'][i]:
                x.append(Jsonize(para)())

            out_x.append(x)

        return out_x

    def space(self):
        raum = {}
        for sp in self.results['space'].dimensions:
            if sp.__class__.__name__ == 'Categorical':
                _raum = {k: Jsonize(v)() for k, v in sp.__dict__.items() if k in ['categories', 'transform_',
                                                                                'prior', '_name']}
                _raum.update({'type': 'Categorical'})
                raum[sp.name] = _raum

            elif sp.__class__.__name__ == 'Integer':
                _raum = {k: Jsonize(v)() for k, v in sp.__dict__.items() if
                                       k in ['low', 'transform_', 'prior', '_name', 'high', 'base',
                                             'dtype', 'log_base']}
                _raum.update({'type': 'Integer'})
                raum[sp.name] = _raum

            elif sp.__class__.__name__ == 'Real':
                _raum = {k: Jsonize(v)() for k, v in sp.__dict__.items() if
                                       k in ['low', 'transform_', 'prior', '_name', 'high', 'base', 'dtype',
                                             'log_base']}
                _raum.update({'type': 'Real'})
                raum[sp.name] = _raum

        return raum

    def random_state(self):
        return str(self.results['random_state'])

    def kernel(self, k):
        """Serializes Kernel"""
        if k.__class__.__name__ == "Product":
            return self.prod_kernel(k)

        if k.__class__.__name__ == "Sum":
            return self.sum_kernel(k)
        # default scenario, just converts it to string
        return str(k)

    def prod_kernel(self, k):
        """Serializes product kernel"""
        kernel = {}
        for _k, v in k.__dict__.items():

            kernel[_k] = self.singleton_kernel(v)

        return {"ProductKernel": kernel}

    def sum_kernel(self, k):
        """Serializes sum kernel"""
        kernel = {}

        for _k, v in k.__dict__.items():

            if v.__class__.__name__ == "Product":
                kernel[_k] = self.prod_kernel(v)
            else:
                kernel[_k] = self.singleton_kernel(v)

        return {"SumKernel": kernel}

    def singleton_kernel(self, k):
        """Serializes Kernels such as  Matern, White, Constant Kernels"""
        return {k: Jsonize(v)() for k, v in k.__dict__.items()}

    def specs(self):
        _specs = {}

        _specs['function'] = self.results['specs']['function']

        args = {}

        args['func'] = str(self.results['specs']['args']['func'])

        args['dimensions'] = self.space()

        be = self.results['specs']['args']['base_estimator']
        b_e = {k: Jsonize(v)() for k, v in be.__dict__.items() if
                                       k in ['noise', 'alpha', 'optimizer', 'n_restarts_optimizer', 'normalize_y',
                                             'copy_X_train', 'random_state']}
        b_e['kernel'] = self.kernel(be.kernel)

        args['base_estimator'] = b_e

        for k,v in self.results['specs']['args'].items():
            if k in ['n_cals', 'n_random_starts', 'n_initial_points', 'initial_point_generator', 'acq_func',
                     'acq_optimizer', 'verbose', 'callback', 'n_points', 'n_restarts_optimizer', 'xi', 'kappa',
                     'n_jobs', 'model_queue_size']:
                args[k] = Jsonize(v)()

        args['x0'] = self.x0()
        args['y0'] = self.y0()

        _specs['args'] = args
        return _specs

    def models(self):

        mods = []
        for model in self.results['models']:
            mod = {k: Jsonize(v)() for k, v in model.__dict__.items() if k in [
                'noise','alpha', 'optimizer', 'n_restarts_optimizer', 'normalize_y', 'copy_X_train', 'random_state',
                '_rng', 'n_features_in', '_y_tain_mean', '_y_train_std', 'X_train', 'y_train',
                'log_marginal_likelihood', 'L_', 'K_inv', 'alpha', 'noise_', 'K_inv_', 'y_train_std_', 'y_train_mean_']}

            mod['kernel'] = self.kernel(model.kernel)
            mods.append({model.__class__.__name__: mod})

        return mods

def to_skopt_space(x):
    """converts the space x into skopt compatible space"""
    if isinstance(x, list):
        if all([isinstance(s, Dimension) for s in x]):
            _space = Space(x)
        elif len(x) == 1 and isinstance(x[0], tuple):
            if len(x[0]) == 2:
                if 'int' in x[0][0].__class__.__name__:
                    _space = Integer(low=x[0][0], high=x[0][1])
                elif 'float' in x[0][0].__class__.__name__:
                    _space = Integer(low=x[0][0], high=x[0][1])
                else:
                    raise NotImplementedError
            else:
                raise NotImplementedError
        elif all([s.__class__.__name__== "Apply" for s in x]):
            _space = Space([skopt_space_from_hp_space(v) for v in x])
        else:
            raise NotImplementedError
    elif isinstance(x, dict):  # todo, in random, should we build Only Categorical space?
        space_ = []
        for k, v in x.items():
            if isinstance(v, list):
                s = space_from_list(v, k)
            elif isinstance(v, Dimension):
                # it is possible that the user has not specified the name so assign the names
                # because we have keys.
                if v.name is None or v.name.startswith('real_') or v.name.startswith('integer_'):
                    v.name = k
                s = v
            elif v.__class__.__name__== "Apply" or 'rv_frozen' in v.__class__.__name__:
                s = skopt_space_from_hp_space(v, k)
            elif isinstance(v, tuple):
                s = Categorical(v, name=k)
            elif isinstance(v, np.ndarray):
                s = Categorical(v.tolist(), name=k)
            else:
                raise NotImplementedError(f"unknown type {v}, {type(v)}")
            space_.append(s)

        # todo, why converting to Space
        _space = Space(space_) if len(space_) > 0 else None
    elif 'rv_frozen' in x.__class__.__name__ or x.__class__.__name__== "Apply":
        _space = Space([skopt_space_from_hp_space(x)])
    else:
        raise NotImplementedError(f"unknown type {x}, {type(x)}")
    return _space

def scatterplot_matrix_colored(params_names: list,
                               params_values: list,
                               best_accs: list,
                               blur=False,
                               save=True,
                               fname='scatter_plot.png'
                               ):
    """Scatterplot colored according to the Z values of the points.
    https://github.com/guillaume-chevalier/Hyperopt-Keras-CNN-CIFAR-100/blob/Vooban/AnalyzeTestHyperoptResults.ipynb
    """

    nb_params = len(params_values)
    best_accs = np.array(best_accs)
    norm = mpl.colors.Normalize(vmin=best_accs.min(), vmax=best_accs.max())

    fig, ax = plt.subplots(nb_params, nb_params, figsize=(16, 16))  # , facecolor=bg_color, edgecolor=fg_color)

    for i in range(nb_params):
        p1 = params_values[i]
        for j in range(nb_params):
            p2 = params_values[j]

            axes = ax[i, j]
            # Subplot:
            if blur:
                axes.scatter(p2, p1, s=400, alpha=.1,
                             c=best_accs, cmap='viridis', norm=norm)
                axes.scatter(p2, p1, s=200, alpha=.2,
                             c=best_accs, cmap='viridis', norm=norm)
                axes.scatter(p2, p1, s=100, alpha=.3,
                             c=best_accs, cmap='viridis', norm=norm)
            s = axes.scatter(p2, p1, s=15,
                             c=best_accs, cmap='viridis', norm=norm)

            # Labels only on side subplots, for x and y:
            if j == 0:
                axes.set_ylabel(params_names[i], rotation=0)
            else:
                axes.set_yticks([])

            if i == nb_params - 1:
                axes.set_xlabel(params_names[j], rotation=90)
            else:
                axes.set_xticks([])

    fig.subplots_adjust(right=0.82, top=0.95)
    cbar_ax = fig.add_axes([0.85, 0.15, 0.05, 0.7])
    fig.colorbar(s, cax=cbar_ax)

    plt.suptitle(
        'Mmatrix of tried values in the search space over different params, colored in function of best test accuracy')
    plt.show()

def take(n, iterable):
    """Return first n items of the iterable as a list
    https://stackoverflow.com/questions/7971618/return-first-n-keyvalue-pairs-from-dict
    """
    return list(islice(iterable, n))

def plot_convergences(opt_dir, what='val_loss', show_whole=True, show_min=False, **kwargs):

    plot_dir = os.path.join(opt_dir, "plots")
    if not os.path.exists(plot_dir):
        os.makedirs(plot_dir)

    max_len = kwargs.get('max_len', 500)
    show_top = kwargs.get('show_top', 3)
    ylim_bottom = kwargs.get('ylim_bottom', None)
    ylim_top = kwargs.get('ylim_top', None)
    font_size = kwargs.get('font_size', 16)
    ylabel = kwargs.get('ylabel', 'Validation MSE')
    leg_pos = kwargs.get('leg_pos', 'upper right')
    # style = kwargs.get('style', 'ggplot')

    models = []
    for f in os.listdir(opt_dir):
        if os.path.isdir(os.path.join(opt_dir, f)):
            models.append(f)

    val_loss = pd.DataFrame()
    default = pd.Series(np.full(max_len, 0.0))
    min_val_loss = {}

    for mod in models:
        loss_fname = os.path.join(os.path.join(opt_dir, str(mod) + "\\losses.csv"))
        if os.path.exists(loss_fname):
            losses = pd.read_csv(loss_fname)
            vl = losses[what]

            vl1 = default.add(vl, fill_value=None)
            vl1.name = mod

            min_val_loss[mod] = vl1.min()

            val_loss = pd.concat([val_loss, vl1], axis=1)

    # sort min_val_loss by value
    min_vl_sorted = {k: v for k, v in sorted(min_val_loss.items(), key=lambda item: item[1])}
    top_3 = take(show_top, min_vl_sorted)

    colors = {
        0: 'b',
        1: 'g',
        2: 'orange'
    }

    default = np.full(max_len, np.nan)

    plt.close('all')
    _, axis = plt.subplots()

    for k in min_vl_sorted.keys():

        val = val_loss[k]

        default[np.argmin(val.values)] = val.min()
        if k not in top_3:
            if show_whole:
                axis.plot(val, color='silver', linewidth=0.5,  label='_nolegend_')

            if show_min:
                axis.plot(default, '.', markersize=2.5, color='silver', label='_nolegend_')
        default = np.full(max_len, np.nan)

    for idx, k in enumerate(top_3):
        val = val_loss[k]

        if show_whole:
            axis.plot(val, color=colors[idx], linewidth=1,   label=f'Rank {idx+1}')

        default[np.argmin(val.values)] = val.min()
        if show_min:
            axis.plot(default, 'x', markersize=5.5, color=colors[idx], label='Rank ' + str(idx))

        default = np.full(max_len, np.nan)

    axis.legend(loc=leg_pos, fontsize=font_size)
    axis.set_yscale('log')
    axis.set_ylabel(ylabel, fontsize=font_size)
    axis.set_xlabel('Epochs', fontsize=font_size)

    if ylim_bottom is not None:
        axis.set_ylim(bottom=ylim_bottom)
    if ylim_top is not None:
        axis.set_ylim(top=ylim_top)

    _name = what
    _name += '_whole_loss_' if show_whole else ''
    _name += '_loss_points' if show_min else ''
    fname = os.path.join(plot_dir, _name)

    plt.savefig(fname, dpi=300, bbox_inches='tight')

    return

def to_skopt_as_dict(algorithm:str, backend:str, original_space)->dict:

    if backend == 'hyperopt':
        if original_space.__class__.__name__ == "Apply":
            _space = skopt_space_from_hp_space(original_space)
            _space = {_space.name: _space}
        elif isinstance(original_space, dict):
            _space = OrderedDict()
            for k, v in original_space.items():
                if v.__class__.__name__ == "Apply" or 'rv_frozen' in v.__class__.__name__:
                    _space[k] = skopt_space_from_hp_space(v)
                elif isinstance(v, Dimension):
                    _space[v.name] = v
                else:
                    raise NotImplementedError
        elif isinstance(original_space, list):
            if all([isinstance(s, Dimension) for s in original_space]):
                _space = OrderedDict({s.name: s for s in original_space})
            elif all([s.__class__.__name__== "Apply" for s in original_space]):
                d = [skopt_space_from_hp_space(v) for v in original_space]
                _space = OrderedDict({s.name: s for s in d})
            else:
                raise NotImplementedError
        else:
            raise NotImplementedError

    elif backend == 'optuna':
        if isinstance(original_space, list):
            if all([isinstance(s, Dimension) for s in original_space]):
                _space = OrderedDict({s.name: s for s in original_space})
            else:
                raise NotImplementedError
        else:
            raise NotImplementedError

    elif backend == 'skopt':
        sk_space = to_skopt_space(original_space)

        if isinstance(sk_space, Dimension):
            _space = {sk_space.name: sk_space}

        elif all([isinstance(s, Dimension) for s in sk_space]):
            _space = OrderedDict()
            for s in sk_space:
                _space[s.name] = s
        else:
            raise NotImplementedError

    elif backend == 'sklearn':
        if isinstance(original_space, list):
            if all([isinstance(s, Dimension) for s in original_space]):
                _space = OrderedDict({s.name: s for s in original_space})
            else:
                raise NotImplementedError
        elif isinstance(original_space, dict):
            _space = OrderedDict()
            for k, v in original_space.items():
                if isinstance(v, list):
                    s = space_from_list(v, k)
                elif isinstance(v, Dimension):
                    s = v
                elif isinstance(v, tuple) or isinstance(v, list):
                    s = Categorical(v, name=k)
                elif  v.__class__.__name__ == "Apply" or 'rv_frozen' in v.__class__.__name__:
                    if algorithm == 'random':
                        s = Real(v.kwds['loc'], v.kwds['loc'] + v.kwds['scale'], name=k, prior=v.dist.name)
                    else:
                        s = skopt_space_from_hp_space(v)
                else:
                    raise NotImplementedError(f"unknown type {v}, {type(v)}")
                _space[k] = s
        else:
            raise NotImplementedError
    else:
        raise NotImplementedError

    return _space

def space_from_list(v: list, k: str):
    """Returns space of tyep "Dimension"
    """
    if len(v) > 2:
        if isinstance(v[0], int):
            s = Integer(grid=v, name=k)
        elif isinstance(v[0], float):
            s = Real(grid=v, name=k)
        else:
            s = Categorical(v, name=k)
    else:
        if isinstance(v[0], int):
            s = Integer(low=np.min(v), high=np.max(v), name=k)
        elif isinstance(v[0], float):
            s = Real(low=np.min(v), high=np.max(v), name=k)
        elif isinstance(v[0], str):
            s = Categorical(v, name=k)
        else:
            raise NotImplementedError
    return s


text:
import setuptools
from setuptools.extension import Extension
from distutils.command.build_ext import build_ext as DistUtilsBuildExt

class BuildExtension(setuptools.Command):
    """
    TODO
    """
    description = DistUtilsBuildExt.description
    user_options = DistUtilsBuildExt.user_options
    boolean_options = DistUtilsBuildExt.boolean_options
    help_options = DistUtilsBuildExt.help_options

    def __init__(self, *args, **kwargs):
    	from setuptools.command.build_ext import build_ext as SetupToolsBuildExt

    	# Bypass __setatrr__ to avoid infinite recursion.
    	self.__dict__['_command'] = SetupToolsBuildExt(*args, **kwargs)

    def __getattr__(self, name):
    	return getattr(self._command, name)

    def __setattr__(self, name, value):
    	setattr(self._command, name, value)

    def initialize_options(self, *args, **kwargs):
    	return self._command.initialize_options(*args, **kwargs)

    def finalize_options(self, *args, **kwargs):
    	ret = self._command.finalize_options(*args, **kwargs)
    	import numpy
    	self.include_dirs.append(numpy.get_include())
    	return ret

    def run(self, *args, **kwargs):
        return self._command.run(*args, **kwargs)

extensions = [
    Extension(
        'tf_retinanet.utils.compute_overlap',
        ['tf_retinanet/utils/compute_overlap.pyx']
        ),
    ]

setuptools.setup(
    name='tf-retinanet',

    version='1',

    description='',
    long_description='',

    author='',
    author_email='',

    license='',
    cmdclass={'build_ext': BuildExtension},
    packages=setuptools.find_packages(),
    ext_modules=extensions,
    setup_requires=["cython>=0.28", "numpy>=1.14.0"],
    zip_safe=False,
)

ext: py

max_stars_repo_name: GaetanLepage/tf-retinanet

max_issues_repo_name: GaetanLepage/tf-retinanet

max_forks_repo_name: GaetanLepage/tf-retinanet

reason: import numpy


text:
import math

import numpy as np

def pixel_coord_np(width, height):
    """
    Pixel in homogenous coordinate
    Returns:
        Pixel coordinate:       [3, width * height]
    """
    x = np.linspace(0, width - 1, width).astype(np.int)
    y = np.linspace(0, height - 1, height).astype(np.int)
    [x, y] = np.meshgrid(x, y)
    return np.vstack((x.flatten(), y.flatten(), np.ones_like(x.flatten())))

def intrinsic_from_fov(height, width, fov=90):
    """
    Basic Pinhole Camera Model
    intrinsic params from fov and sensor width and height in pixels
    Returns:
        K:      [4, 4]
    """
    px, py = (width / 2, height / 2)
    hfov = fov / 360. * 2. * np.pi
    fx = width / (2. * np.tan(hfov / 2.))

    vfov = 2. * np.arctan(np.tan(hfov / 2) * height / width)
    fy = height / (2. * np.tan(vfov / 2.))

    return np.array([[fx, 0, px, 0.],
                     [0, fy, py, 0.],
                     [0, 0, 1., 0.],
                     [0., 0., 0., 1.]])

def calculate_angles(center_x, center_y, h_view_angle, v_view_angle):
    # angles are in degrees, not in radians
    # (center_x, center_y): top left corner of the image is (0, 0)
    # h_view_angle: forward is 0 degree, increases to 360 degree clockwise (e.g., right is 90 degrees)
    # v_view_angle: h view angle is 0 degree, highest view angle is 30 degree,
    # lowest view angle is -60 degree. In the output of this function, we use -15 degee as the new center (0 degree).
    h_field_of_view = 90  # this is the default value used by AI2Thor
    v_field_of_view = 90

    h_object_angle = np.arctan((center_x - 0.5) * math.tan(math.radians(h_field_of_view / 2)) * 2)
    v_object_angle = np.arctan((0.5 - center_y) * math.tan(math.radians(v_field_of_view / 2)) * 2)
    h_object_angle = h_view_angle + np.degrees(h_object_angle)
    v_object_angle = v_view_angle + 15 + np.degrees(v_object_angle)

    return h_object_angle, v_object_angle

max_stars_repo_name: belindal/exploration

max_issues_repo_name: belindal/exploration

max_forks_repo_name: belindal/exploration


text:
# -*- coding: utf-8 -*-
#
# Authors: Swolf <swolfforever@gmail.com>
# Date: 2021/1/07
# License: MIT License
"""
Task-related Component Analysis (TRCA) and its variants.
"""
from typing import Optional, Union, List, Tuple, Dict
from itertools import combinations
from functools import partial

import numpy as np
from scipy.linalg import eigh, cholesky, inv, norm

from numpy import ndarray
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.model_selection import GridSearchCV, StratifiedKFold, ShuffleSplit
from sklearn.pipeline import make_pipeline
from sklearn.svm import SVC
from sklearn.metrics import pairwise_distances
from sklearn.cross_decomposition import CCA
from joblib import Parallel, delayed

from .base import robust_pattern, FilterBank

def trca_kernel(X: ndarray, y: Optional[ndarray] = None,
        n_jobs: Optional[int] = None) -> Tuple[ndarray, ndarray, ndarray]:
    """The kernel part in TRCA algorithm based on paper[1]_.

    Modified from https://github.com/mnakanishi/TRCA-SSVEP/blob/master/src/train_trca.m

    Parameters
    ----------
    X : ndarray
        EEG data assuming removing mean, shape (n_trials, n_channels, n_samples)
    y : ndarray
        labels, shape (n_trials, ), not used here
    n_jobs: int, optional
        the number of jobs to use, default None

    Returns
    -------
    W: ndarray
        filters, shape (n_channels, n_filters)
    D: ndarray
        eigenvalues in descending order
    A: ndarray
        spatial patterns, shape (n_channels, n_filters)

    Notes
    -----
    trca can be used in each class separately without y labels.

    References
    ----------
    .. [1] Nakanishi, Masaki, et al. "Enhancing detection of SSVEPs for a high-speed brain speller using task-related component analysis." IEEE Transactions on Biomedical Engineering 65.1 (2018): 104-112.
    """
    X = np.copy(X)
    X = np.reshape(X, (-1, *X.shape[-2:]))
    X = X - np.mean(X, axis=-1, keepdims=True)
    combs = combinations(range(len(X)), 2)
    S = np.sum(Parallel(n_jobs=n_jobs)(delayed(lambda x1, x2 : x1@x2.T+x2@x1.T)(X[comb[0]], X[comb[1]]) for comb in combs), axis=0)
    Q = np.sum(np.matmul(X, np.swapaxes(X, -1, -2)), axis=0)

    D, W = eigh(S, Q)
    ind = np.argsort(D)[::-1]
    D, W = D[ind], W[:, ind]
    A = robust_pattern(W, S, W.T@S@W)
    return W, D, A

def trca_feature(W: ndarray, X: ndarray,
        n_components: int = 1) -> ndarray:
    """Return trca features.

    Modified from https://github.com/mnakanishi/TRCA-SSVEP/blob/master/src/test_trca.m

    Parameters
    ----------
    W : ndarray
        spatial filters from csp_kernel, shape (n_channels, n_filters)
    X : ndarray
        eeg data, shape (n_trials, n_channels, n_samples)
    n_components : int, optional
        the first k components to use, usually even number, by default 1

    Returns
    -------
    ndarray
        features of shape (n_trials, n_components, n_samples)

    Raises
    ------
    ValueError
        n_components should less than half of the number of channels
    """
    W, X = np.copy(W), np.copy(X)
    max_components = W.shape[1]
    if n_components > max_components:
        raise ValueError("n_components should less than the number of channels")
    X = np.reshape(X, (-1, *X.shape[-2:]))
    X = X - np.mean(X, axis=-1, keepdims=True)
    features = np.matmul(W[:, :n_components].T, X)
    return features

class TRCA(BaseEstimator, TransformerMixin):
    def __init__(self,
            n_components: Optional[int] = None,
            max_components: Optional[int] = None,
            transform_method: Optional[str] = None,
            is_ensemble: bool = False,
            n_jobs: Optional[int] = None):
        self.n_components = n_components
        self.max_components = max_components
        self.transform_method = transform_method
        self.is_ensemble = is_ensemble
        self.n_jobs = n_jobs

    def fit(self, X: ndarray, y: ndarray):
        # trial normalization
        X -= np.mean(X, axis=-1, keepdims=True)
        X /= np.std(X, axis=(-2, -1), keepdims=True)

        self.classes_ = np.unique(y)
        Ws, Ds, As = zip(*[trca_kernel(X[y==label], n_jobs=self.n_jobs) for label in self.classes_])
        self.Ws_, self.Ds_, self.As_ = np.stack(Ws), np.stack(Ds), np.stack(As)

        # auto-tuning
        if self.n_components is None:
            estimator = make_pipeline(*[TRCA(
                n_components=self.n_components,
                transform_method=self.transform_method,
                is_ensemble=self.is_ensemble,
                n_jobs=self.n_jobs), SVC()])
            if self.max_components is None:
                params = {'trca__n_components': np.arange(1, self.Ws_.shape[-1]+1)}
            else:
                params = {'trca__n_components': np.arange(1, self.max_components+1)}

            n_splits = np.min(np.unique(y, return_counts=True)[1])
            n_splits = 5 if n_splits > 5 else n_splits

            gs = GridSearchCV(estimator,
                param_grid=params, scoring='accuracy',
                cv=StratifiedKFold(n_splits=n_splits, shuffle=True), refit=False, n_jobs=-1, verbose=False)
            gs.fit(X, y)
            self.best_n_components_ = gs.best_params_['trca__n_components']

        self.templates_ = np.stack([
            np.mean(X[y==label], axis=0) for label in self.classes_
        ])
        return self

    def transform(self, X: ndarray):
        n_components = self.best_n_components_ if self.n_components is None else self.n_components
        X -= np.mean(X, axis=-1, keepdims=True)
        X /= np.std(X, axis=(-2, -1), keepdims=True)

        if self.transform_method is None:
            features = np.concatenate([trca_feature(W, X, n_components=n_components) for W in self.Ws_], axis=-2)
            features = np.reshape(features, (features.shape[0], -1))
            return features
        elif self.transform_method == 'corr':
            if self.is_ensemble:
                W = np.transpose(self.Ws_[..., :n_components], (1, 0, 2))
                W = np.reshape(W, (W.shape[0], -1))
                X = trca_feature(W, X, n_components=W.shape[1])
                features = [
                    self._pearson_features(
                        X,
                        trca_feature(W, template, n_components=W.shape[1])) for template in self.templates_]
            else:
                features = [
                    self._pearson_features(
                        trca_feature(W, X, n_components=n_components),
                        trca_feature(W, template, n_components=n_components)) for W, template in zip(self.Ws_, self.templates_)]
            features = np.concatenate(features, axis=-1)
            return features
        else:
            raise ValueError("non-supported transform method")

    def _pearson_features(self, X: ndarray, templates: ndarray):
        X = np.reshape(X, (-1, *X.shape[-2:]))
        templates = np.reshape(templates, (-1, *templates.shape[-2:]))
        X = X - np.mean(X, axis=-1, keepdims=True)
        templates = templates - np.mean(templates, axis=-1, keepdims=True)
        X = np.reshape(X, (X.shape[0], -1))
        templates = np.reshape(templates, (templates.shape[0], -1))
        istd_X = 1 / np.std(X, axis=-1, keepdims=True)
        istd_templates = 1 / np.std(templates, axis=-1, keepdims=True)
        corr = (X@templates.T) / (templates.shape[1]-1)
        corr = istd_X * corr * istd_templates.T
        return corr

class FBTRCA(FilterBank):
    """Filter Bank TRCA method in paper [1]_.

    Modified from https://github.com/mnakanishi/TRCA-SSVEP/blob/master/src/filterbank.m
    and https://github.com/mnakanishi/TRCA-SSVEP/blob/master/src/test_trca.m

    filterbank and weights suggested in the paper.

    wp = [
        [6, 90], [14, 90], [22, 90], [30, 90], [38, 90], [46, 90], [54, 90], [62, 90], [70, 90], [78, 90]
    ]
    ws = [
        [4, 100], [10, 100], [16, 100], [24, 100], [32, 100], [40, 100], [48, 100], [56, 100], [64, 100], [72, 100]
    ]

    filterweights:
        np.arange(1, 11)**(-1.25) + 0.25

    Notes
    -----
    nearly the same as matlab code above

    References
    ----------
    .. [1] Nakanishi M, Wang Y, Chen X, et al. Enhancing detection of SSVEPs for a high-speed brain speller using task-related component analysis[J]. IEEE Transactions on Biomedical Engineering, 2017, 65(1): 104-112.
    """
    def __init__(self,
            n_components: Optional[int] = None,
            max_components: Optional[int] = None,
            is_ensemble: bool = False,
            n_jobs: Optional[int] = None,
            filterbank: Optional[List[ndarray]] = None,
            filterweights: Optional[ndarray] = None):
        self.n_components = n_components
        self.max_components = max_components
        self.is_ensemble = is_ensemble
        self.n_jobs = n_jobs
        self.filterbank = filterbank
        self.filterweights = filterweights
        if filterweights is not None:
            if filterbank is None:
                self.filterweights = None
            else:
                if len(filterweights) != len(filterbank):
                    raise ValueError("the len of filterweights must be the same as that of filterbank")

        super().__init__(
            TRCA(
                n_components=n_components,
                max_components=max_components,
                transform_method='corr',
                is_ensemble=is_ensemble,
                n_jobs=n_jobs),
            filterbank=filterbank)

    def transform(self, X: ndarray):
        features = super().transform(X)
        if self.filterweights is None:
            return features
        else:
            features = np.reshape(features, (features.shape[0], len(self.filterbank), -1))
            return np.sum(features*self.filterweights[np.newaxis, :, np.newaxis], axis=1)

def sscor_kernel(X: ndarray,
        y: Optional[ndarray] = None,
        n_jobs: Optional[int] = None) -> Tuple[ndarray, ndarray, ndarray]:
    """The kernel part in SSCOR algorithm based on paper[1]_., [2]_.

    Modified from https://github.com/mnakanishi/TRCA-SSVEP/blob/master/src/train_sscor.m

    Parameters
    ----------
    X : ndarray
        EEG data assuming removing mean, shape (n_trials, n_channels, n_samples)
    y : ndarray
        labels, shape (n_trials, ), not used here
    n_jobs: int, optional
        the number of jobs to use, default None

    Returns
    -------
    W: ndarray
        filters, shape (n_channels, n_filters)
    D: ndarray
        eigenvalues in descending order
    A: ndarray
        spatial patterns, shape (n_channels, n_filters)

    References
    ----------
    .. [1] Kumar G R K, Reddy M R. Designing a sum of squared correlations framework for enhancing SSVEP-based BCIs[J]. IEEE Transactions on Neural Systems and Rehabilitation Engineering, 2019, 27(10): 2044-2050.
    .. [2] Kumar G R K, Reddy M R. Correction to “Designing a Sum of Squared Correlations Framework for Enhancing SSVEP Based BCIs”[J]. IEEE Transactions on Neural Systems and Rehabilitation Engineering, 2020, 28(4): 1044-1045.
    """
    X = np.copy(X)
    X = np.reshape(X, (-1, *X.shape[-2:]))
    X = X - np.mean(X, axis=-1, keepdims=True)
    mean_X = np.mean(X, axis=0)
    K1 = cholesky(mean_X@mean_X.T) # upper-triangular X=K.T@K
    iK1 = inv(K1)
    xC = mean_X@np.transpose(X, axes=(0, 2, 1))
    C = X@np.transpose(X, axes=(0, 2, 1))

    def target(iK1, xCi, Ci):
        Ki = cholesky(Ci)
        Gi = iK1.T@xCi@inv(Ki)
        return Gi.T@Gi
    target = partial(target, iK1)
    G_T_G = np.sum(Parallel(n_jobs=n_jobs)(delayed(target)(xCi, Ci) for xCi, Ci in zip(xC, C)), axis=0)
    D, W = eigh(G_T_G)
    ind = np.argsort(D)[::-1]
    D, W = D[ind], W[:, ind]
    W = iK1@W
    A = robust_pattern(W, G_T_G, W.T@G_T_G@W)
    return W, D, A

def sscor_feature(W: ndarray, X: ndarray,
        n_components: int = 1) -> ndarray:
    """Return sscor features.

    Modified from https://github.com/mnakanishi/TRCA-SSVEP/blob/master/src/test_sscor.m

    Parameters
    ----------
    W : ndarray
        spatial filters from csp_kernel, shape (n_channels, n_filters)
    X : ndarray
        eeg data, shape (n_trials, n_channels, n_samples)
    n_components : int, optional
        the first k components to use, usually even number, by default 1

    Returns
    -------
    ndarray
        features of shape (n_trials, n_components, n_samples)

    Raises
    ------
    ValueError
        n_components should less than half of the number of channels
    """
    W, X = np.copy(W), np.copy(X)
    max_components = W.shape[1]
    if n_components > max_components:
        raise ValueError("n_components should less than the number of channels")
    X = np.reshape(X, (-1, *X.shape[-2:]))
    X = X - np.mean(X, axis=-1, keepdims=True)
    features = np.matmul(W[:, :n_components].T, X)
    return features

class SSCOR(BaseEstimator, TransformerMixin):
    def __init__(self,
            n_components: Optional[int] = None,
            max_components: Optional[int] = None,
            transform_method: Optional[str] = None,
            is_ensemble: bool = False,
            n_jobs: Optional[int] = None):
        self.n_components = n_components
        self.max_components = max_components
        self.transform_method = transform_method
        self.is_ensemble = is_ensemble
        self.n_jobs = n_jobs

    def fit(self, X: ndarray, y: ndarray):
        self.classes_ = np.unique(y)
        Ws, Ds, As = zip(*[sscor_kernel(X[y==label], n_jobs=self.n_jobs) for label in self.classes_])
        self.Ws_, self.Ds_, self.As_ = np.stack(Ws), np.stack(Ds), np.stack(As)

        # auto-tuning
        if self.n_components is None:
            estimator = make_pipeline(*[SSCOR(
                n_components=self.n_components,
                transform_method=self.transform_method,
                is_ensemble=self.is_ensemble,
                n_jobs=self.n_jobs), SVC()])
            if self.max_components is None:
                params = {'sscor__n_components': np.arange(1, self.Ws_.shape[-1]+1)}
            else:
                params = {'sscor__n_components': np.arange(1, self.max_components+1)}

            n_splits = np.min(np.unique(y, return_counts=True)[1])
            n_splits = 5 if n_splits > 5 else n_splits

            gs = GridSearchCV(estimator,
                param_grid=params, scoring='accuracy',
                cv=StratifiedKFold(n_splits=n_splits, shuffle=True), refit=False, n_jobs=-1, verbose=False)
            gs.fit(X, y)
            self.best_n_components_ = gs.best_params_['sscor__n_components']

        self.templates_ = np.stack([
            np.mean(X[y==label], axis=0) for label in self.classes_
        ])
        return self

    def transform(self, X: ndarray):
        n_components = self.best_n_components_ if self.n_components is None else self.n_components

        if self.transform_method is None:
            features = np.concatenate([sscor_feature(W, X, n_components=n_components) for W in self.Ws_], axis=-2)
            features = np.reshape(features, (features.shape[0], -1))
            return features
        elif self.transform_method == 'corr':
            if self.is_ensemble:
                W = np.transpose(self.Ws_[..., :n_components], (1, 0, 2))
                W = np.reshape(W, (W.shape[0], -1))
                X = sscor_feature(W, X, n_components=W.shape[1])
                features = [
                    self._pearson_features(
                        X,
                        sscor_feature(W, template, n_components=W.shape[1])) for template in self.templates_]
            else:
                features = [
                    self._pearson_features(
                        sscor_feature(W, X, n_components=n_components),
                        sscor_feature(W, template, n_components=n_components)) for W, template in zip(self.Ws_, self.templates_)]
            features = np.concatenate(features, axis=-1)
            return features
        else:
            raise ValueError("non-supported transform method")

    def _pearson_features(self, X: ndarray, templates: ndarray):
        X = np.reshape(X, (-1, *X.shape[-2:]))
        templates = np.reshape(templates, (-1, *templates.shape[-2:]))
        X = X - np.mean(X, axis=-1, keepdims=True)
        templates = templates - np.mean(templates, axis=-1, keepdims=True)
        X = np.reshape(X, (X.shape[0], -1))
        templates = np.reshape(templates, (templates.shape[0], -1))
        istd_X = 1 / np.std(X, axis=-1, keepdims=True)
        istd_templates = 1 / np.std(templates, axis=-1, keepdims=True)
        corr = (X@templates.T) / (templates.shape[1]-1)
        corr = istd_X * corr * istd_templates.T
        return corr

class FBSSCOR(FilterBank):
    """Filter Bank SSCOR method in paper [1]_., [2]_.

    filterbank and weights suggested in the paper.

    wp = [
        [6, 90], [14, 90], [22, 90], [30, 90], [38, 90], [46, 90], [54, 90], [62, 90], [70, 90], [78, 90]
    ]
    ws = [
        [4, 100], [10, 100], [16, 100], [24, 100], [32, 100], [40, 100], [48, 100], [56, 100], [64, 100], [72, 100]
    ]

    filterweights:
        np.arange(1, 11)**(-1.25) + 0.25

    References
    ----------
    .. [1] Kumar G R K, Reddy M R. Designing a sum of squared correlations framework for enhancing SSVEP-based BCIs[J]. IEEE Transactions on Neural Systems and Rehabilitation Engineering, 2019, 27(10): 2044-2050.
    .. [2] Kumar G R K, Reddy M R. Correction to “Designing a Sum of Squared Correlations Framework for Enhancing SSVEP Based BCIs”[J]. IEEE Transactions on Neural Systems and Rehabilitation Engineering, 2020, 28(4): 1044-1045.
    """
    def __init__(self,
            n_components: Optional[int] = None,
            max_components: Optional[int] = None,
            is_ensemble: bool = False,
            n_jobs: Optional[int] = None,
            filterbank: Optional[List[ndarray]] = None,
            filterweights: Optional[ndarray] = None):
        self.n_components = n_components
        self.max_components = max_components
        self.is_ensemble = is_ensemble
        self.n_jobs = n_jobs
        self.filterbank = filterbank
        self.filterweights = filterweights
        if filterweights is not None:
            if filterbank is None:
                self.filterweights = None
            else:
                if len(filterweights) != len(filterbank):
                    raise ValueError("the len of filterweights must be the same as that of filterbank")

        super().__init__(
            SSCOR(
                n_components=n_components,
                max_components=max_components,
                transform_method='corr',
                is_ensemble=is_ensemble,
                n_jobs=n_jobs),
            filterbank=filterbank)

    def transform(self, X: ndarray):
        features = super().transform(X)
        if self.filterweights is None:
            return features
        else:
            features = np.reshape(features, (features.shape[0], len(self.filterbank), -1))
            return np.sum(features*self.filterweights[np.newaxis, :, np.newaxis], axis=1)

max_stars_repo_name: Mrswolf/brainda

reason: import numpy,from numpy,from scipy


text:
import numpy as np
from robosuite.controllers import load_controller_config
from robosuite.utils.input_utils import *
from robosuite.environments.manipulation.lift_cloth import LiftCloth
from robosuite.environments.manipulation.lift import Lift
from robosuite.environments.manipulation.straighten_rope import StraightenRope
from robosuite.environments.base import register_env
import time

# create environment instance
register_env(StraightenRope)
env = suite.make(
    env_name='StraightenRope',
    robots='Panda',
    controller_configs=load_controller_config(default_controller='OSC_POSE'),
    has_renderer=True,
    has_offscreen_renderer=False,
    use_camera_obs=False,
    reward_shaping=True,
)

# reset the environment
env.reset()
for i in range(1000):
    action = np.random.randn(env.robots[0].action_dim) # sample random action
    # action = np.zeros(env.robots[0].action_dim)
    # if not i%100 and not i == 0:
    #     env.reset()
    obs, reward, done, info = env.step(action)  # take action in the environment
    env.render()  # render on display

max_stars_repo_name: JieFeng-cse/robosuite-soft

max_issues_repo_name: JieFeng-cse/robosuite-soft

max_forks_repo_name: JieFeng-cse/robosuite-soft


text:
# STUMPY
# Copyright 2019 TD Ameritrade. Released under the terms of the 3-Clause BSD license.
# STUMPY is a trademark of TD Ameritrade IP Company, Inc. All rights reserved.

import numpy as np
from . import core
from . import _mstump, _get_first_mstump_profile, _get_multi_QT, _multi_compute_mean_std
import logging

logger = logging.getLogger(__name__)

def mstumped(dask_client, T, m):
    """
    Compute the multi-dimensional matrix profile with parallelized and
    distributed mSTOMP

    This is a highly distributed implementation around the Numba JIT-compiled
    parallelized `_mstump` function which computes the multi-dimensional matrix
    profile according to STOMP. Note that only self-joins are supported.

    Parameters
    ----------
    dask_client : client
        A Dask Distributed client that is connected to a Dask scheduler and
        Dask workers. Setting up a Dask distributed cluster is beyond the
        scope of this library. Please refer to the Dask Distributed
        documentation.

    T : ndarray
        The time series or sequence for which to compute the multi-dimensional
        matrix profile

    m : int
        Window size

    Returns
    -------
    P : ndarray
        The multi-dimensioanl matrix profile. Each row of the array corresponds
        to each matrix profile for a given dimension (i.e., the first row is the
        1-D matrix profile and the second row is the 2-D matrix profile).
    I : ndarray
        The multi-dimensional matrix profile index where each row of the array
        correspondsto each matrix profile index for a given dimension.

    Notes
    -----

    DOI: 10.1109/ICDM.2017.66
    See mSTAMP Algorithm
    """

    hosts = list(dask_client.ncores().keys())
    nworkers = len(hosts)

    T = np.asarray(T)
    core.check_dtype(T)
    core.check_window_size(m)

    d = T.shape[0]
    n = T.shape[1]
    k = n - m + 1
    excl_zone = int(np.ceil(m / 4))  # See Definition 3 and Figure 3

    M_T, Σ_T = _multi_compute_mean_std(T, m)
    μ_Q, σ_Q = _multi_compute_mean_std(T, m)

    P = np.empty((nworkers, d, k), dtype="float64")
    D = np.zeros((nworkers, d, k), dtype="float64")
    D_prime = np.zeros((nworkers, k), dtype="float64")
    I = np.ones((nworkers, d, k), dtype="int64") * -1

    # Scatter data to Dask cluster
    T_future = dask_client.scatter(T, broadcast=True)
    M_T_future = dask_client.scatter(M_T, broadcast=True)
    Σ_T_future = dask_client.scatter(Σ_T, broadcast=True)
    μ_Q_future = dask_client.scatter(μ_Q, broadcast=True)
    σ_Q_future = dask_client.scatter(σ_Q, broadcast=True)

    step = 1 + k // nworkers
    QT_futures = []
    QT_first_futures = []
    P_futures = []
    I_futures = []
    D_futures = []
    D_prime_futures = []

    for i, start in enumerate(range(0, k, step)):
        P[i], I[i] = _get_first_mstump_profile(start, T, m, excl_zone, M_T, Σ_T)

        P_future = dask_client.scatter(P[i], workers=[hosts[i]])
        I_future = dask_client.scatter(I[i], workers=[hosts[i]])
        D_future = dask_client.scatter(D[i], workers=[hosts[i]])
        D_prime_future = dask_client.scatter(D_prime[i], workers=[hosts[i]])

        P_futures.append(P_future)
        I_futures.append(I_future)
        D_futures.append(D_future)
        D_prime_futures.append(D_prime_future)

        QT, QT_first = _get_multi_QT(start, T, m)

        QT_future = dask_client.scatter(QT, workers=[hosts[i]])
        QT_first_future = dask_client.scatter(QT_first, workers=[hosts[i]])

        QT_futures.append(QT_future)
        QT_first_futures.append(QT_first_future)

    futures = []
    for i, start in enumerate(range(0, k, step)):
        stop = min(k, start + step)

        futures.append(
            dask_client.submit(
                _mstump,
                T_future,
                m,
                P_futures[i],
                I_futures[i],
                D_futures[i],
                D_prime_futures[i],
                stop,
                excl_zone,
                M_T_future,
                Σ_T_future,
                QT_futures[i],
                QT_first_futures[i],
                μ_Q_future,
                σ_Q_future,
                k,
                start + 1,
            )
        )

    results = dask_client.gather(futures)
    for i, start in enumerate(range(0, k, step)):
        P[i], I[i] = results[i]
        col_mask = P[0] > P[i]
        P[0, col_mask] = P[i, col_mask]
        I[0, col_mask] = I[i, col_mask]

    return P[0], I[0]

max_stars_repo_name: bfattori-TDA/stumpy

max_issues_repo_name: bfattori-TDA/stumpy

max_forks_repo_name: bfattori-TDA/stumpy


text:
import json

from networkx.readwrite import json_graph

from ckpttnpy.netlist import create_drawf, create_test_netlist, read_json

def test_netlist():
    H = create_test_netlist()
    assert H.number_of_modules() == 3
    assert H.number_of_nets() == 3
    assert H.number_of_nodes() == 6
    assert H.number_of_pins() == 6
    assert H.get_max_degree() == 3
    # assert H.get_max_net_degree() == 3
    # assert not H.has_fixed_modules
    # assert H.get_module_weight_by_id(0) == 533
    assert isinstance(H.module_weight, dict)

def test_drawf():
    H = create_drawf()
    assert H.number_of_modules() == 7
    assert H.number_of_nets() == 6
    assert H.number_of_pins() == 14
    assert H.get_max_degree() == 3
    # assert H.get_max_net_degree() == 3
    # assert not H.has_fixed_modules
    # assert H.get_module_weight_by_id(1) == 3

def test_json():
    # H = create_drawf()
    # data = json_graph.node_link_data(H.G)
    # with open('testcases/drawf.json', 'w') as fw:
    #     json.dump(data, fw, indent=1)
    with open('testcases/drawf.json', 'r') as fr:
        data2 = json.load(fr)
    G = json_graph.node_link_graph(data2)
    assert G.number_of_nodes() == 13
    assert G.graph['num_modules'] == 7
    assert G.graph['num_nets'] == 6
    assert G.graph['num_pads'] == 3

def test_json2():
    with open('testcases/p1.json', 'r') as fr:
        data = json.load(fr)
    G = json_graph.node_link_graph(data)
    assert G.number_of_nodes() == 1735
    assert G.graph['num_modules'] == 833
    assert G.graph['num_nets'] == 902
    assert G.graph['num_pads'] == 81

def test_readjson():
    H = read_json('testcases/p1.json')
    count_2 = 0
    count_3 = 0
    count_rest = 0
    for net in H.nets:
        deg = H.G.degree(net)
        if deg == 2:
            count_2 += 1
        elif deg == 3:
            count_3 += 1
        else:
            count_rest += 1
    print(count_2, count_3, count_rest)
    assert count_2 == 494

reason: from networkx


text:
""" Main excited-state executable script

    Note:
        The simulations are intended to be used by calling the package
        directly via :code:`python -m adpeps ...`, as described in
        :ref:`notes/start`
"""

from jax import grad, jit, vmap, value_and_grad
from jax import random
from jax.scipy.optimize import minimize
from jax.test_util import check_grads
from scipy import optimize
from scipy.linalg import eigh, eig
from yaml import safe_load, dump
import jax
import jax.numpy as np
import numpy as onp

from adpeps.ipeps.ipeps import iPEPS, iPEPS_exci
from adpeps.ipeps.make_momentum_path import make_momentum_path
from adpeps.utils import io
from adpeps.utils.printing import print
from adpeps.ipeps.evaluation import filter_null_modes
import adpeps.ipeps.config as sim_config

def run(config_file: str, momentum_ix: int):
    """ Start the simulation

        Args:
            config_file: filename of the configuration file
            momentum_ix: index of the point in momentum space
    """

    print(config_file)
    with open(config_file) as f:
        cfg = safe_load(f)

    # Show options
    print(dump(cfg))

    sim_config.from_dict(cfg)
    base_file = io.get_exci_base_file()
    if not base_file.exists():
        print(f"Base file {base_file} not found. Prepare the simulation first by \
                running with option '-i'")
        return

    sim = iPEPSExciSimulation(config_file, momentum_ix)
    output_folder = io.get_exci_folder()
    output_folder.mkdir(parents=True, exist_ok=True)
    kxs, kys = make_momentum_path(sim_config.momentum_path)
    sim_config.px = kxs[momentum_ix]
    sim_config.py = kys[momentum_ix]
    output_file = io.get_exci_file(momentum_ix)
    print(f"Output: {output_file}", level=2)
    basis_size = sim.basis_size
    res_dtype = np.complex128
    H = onp.zeros((basis_size,basis_size), dtype=res_dtype)
    N = onp.zeros((basis_size,basis_size), dtype=res_dtype)

    for m in range(basis_size):
        grad_H, grad_N = sim(m)
        H[:,m] = grad_H
        N[:,m] = grad_N
        onp.savez(output_file, H=H, N=N)

    print(H)
    print(N)
    onp.savez(output_file, H=H, N=N)
    print('Done')
    print(f"Saved to {output_file}")

def prepare(config_file):
    with open(config_file) as f:
        cfg = safe_load(f)
    sim_config.from_dict(cfg)
    base_file = io.get_exci_base_file()
    print(base_file)
    peps = iPEPS()

    gs_file = io.get_gs_file()
    loaded_sim = np.load(gs_file, allow_pickle=True)
    peps       = loaded_sim['peps'].item()

    sim_config.ctm_max_iter = 30
    sim_config.ctm_conv_tol = 1e-12

    # Converge GS boundary tensors
    peps.converge_boundaries()

    # Convert to excitations iPEPS
    peps.__class__ = iPEPS_exci

    # Normalize the ground-state tensors such that the state has norm 1
    peps.normalize_gs()

    # Shift the Hamiltonian by the ground-state energy
    # The excited state energy is then relative to the ground state
    peps.substract_gs_energy()

    # Prepare an orthonormal basis with respect to the ground state
    print('Preparing orthonormal basis')
    basis = peps.compute_orth_basis()

    print(f"Saving base to {base_file}")
    np.savez(base_file, peps=peps, basis=basis)

def evaluate_single(config_file, momentum_ix):
    def _compute_ev_red_basis(H, N, P, n):
        P = P[:,:n]
        N2 = P.T.conjugate() @ N @ P
        H2 = P.T.conjugate() @ H @ P
        N2 = 0.5 * (N2 + N2.T.conjugate())
        H2 = 0.5 * (H2 + H2.T.conjugate())
        ev, _ = eig(H2, N2)
        return sorted(ev.real)

    with open(config_file) as f:
        cfg = safe_load(f)

    sim_config.from_dict(cfg)
    kxs, kys = make_momentum_path(sim_config.momentum_path)
    sim_config.px = kxs[momentum_ix]
    sim_config.py = kys[momentum_ix]
    base_file = io.get_exci_base_file()
    base_sim = np.load(base_file, allow_pickle=True)
    output_file = io.get_exci_file(momentum_ix)
    print(output_file)
    dat = np.load(output_file)
    H, N = dat['H'], dat['N']
    basis = base_sim['basis']
    peps  = base_sim['peps'].item()

    # basis = basis.T @ filter_null_modes(peps.tensors, basis)
    # print(basis.shape)
    # print(N.shape)
    # N = basis.T @ N @ basis
    # H = basis.T @ H @ basis
    # H = H.conjugate()

    H = 0.5 * (H + H.T.conjugate())
    N = 0.5 * (N + N.T.conjugate())
    ev_N, P = np.linalg.eig(N)
    idx = ev_N.real.argsort()[::-1]
    ev_N = ev_N[idx]
    selected = (ev_N/ev_N.max()) > 1e-3
    P = P[:,idx]
    P = P[:,selected]
    N2 = P.T.conjugate() @ N @ P
    H2 = P.T.conjugate() @ H @ P
    N2 = 0.5 * (N2 + N2.T.conjugate())
    H2 = 0.5 * (H2 + H2.T.conjugate())
    ev, vectors = eig(H2, N2)
    ixs = np.argsort(ev)
    ev = ev[ixs]
    vectors = vectors[:,ixs]

    return sorted(ev.real)

def evaluate(config_file, momentum_ix):
    # Default option (-1): evaluate all momenta
    if momentum_ix != -1:
        return evaluate_single(config_file, momentum_ix)

    with open(config_file) as f:
        cfg = safe_load(f)

    # Show options
    print(dump(cfg))

    sim_config.from_dict(cfg)
    kxs, kys, plot_info = make_momentum_path(sim_config.momentum_path, with_plot_info=True)

    import matplotlib.pyplot as plt
    evs = []
    for ix in range(len(kxs)):
        try:
            ev = evaluate_single(config_file, ix)
        except:
            ev = [np.nan]
        evs.append(ev[0])

    plt.plot(evs, '--+')
    plt.xticks(**plot_info['xticks'])
    plt.title(f"Dispersion {sim_config.model} D={sim_config.D}")
    plt.xlabel('k')
    plt.ylabel('$\omega$')
    plt.show()

class iPEPSExciSimulation:
    """ Simulation class for the excited-state simulation

        Call an instance of this class directly to start the simulation
    """
    def __init__(self, config_file, momentum_ix):
        self.config_file = config_file
        self.momentum_ix = momentum_ix

    @property
    def basis_size(self):
        with open(self.config_file) as f:
            cfg = safe_load(f)
        sim_config.from_dict(cfg)
        base_file = io.get_exci_base_file()
        base_sim = np.load(base_file, allow_pickle=True)
        basis = base_sim['basis']
        return basis.shape[1]

    def __call__(self, ix, v=None):
        print(f"Starting simulation of basis vector {ix+1}/{self.basis_size}")
        with open(self.config_file) as f:
            cfg = safe_load(f)
        sim_config.from_dict(cfg)

        base_file = io.get_exci_base_file()
        base_sim = np.load(base_file, allow_pickle=True)
        basis = np.complex_(base_sim['basis'])
        peps  = base_sim['peps'].item()
        if v is None:
            v = basis[:,ix]
        res, grad_H = value_and_grad(peps.run, has_aux=True)(v)
        grad_H = grad_H.conj()
        print('Res', res, level=2)
        grad_N = res[1].pack_data()
        print('Grad H', grad_H, level=2)
        print('Grad N', grad_N, level=2)
        print(f"========== \nFinished basis vector {ix+1}/{self.basis_size} \n")
        return basis.T @ jax.lax.stop_gradient(grad_H), basis.T @ jax.lax.stop_gradient(grad_N)

    def check_grads(self, A=None):
        with open(self.config_file) as f:
            cfg = safe_load(f)
        sim_config.from_dict(cfg)

        base_file = io.get_exci_base_file()
        base_sim = np.load(base_file, allow_pickle=True)
        basis = np.complex_(base_sim['basis'])
        peps  = base_sim['peps'].item()
        print('Checking gradient')
        # peps.fill(A)
        check_grads(peps.run_gc, (A,), order=1, modes='rev')
        print('Done check')

max_stars_repo_name: TensorBFS/ad-peps

max_issues_repo_name: TensorBFS/ad-peps

max_forks_repo_name: TensorBFS/ad-peps

reason: import numpy,from scipy,import jax,from jax


text:
# Copyright 2019 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
import pytest
import numpy as np

import mindspore.dataset as ds
import mindspore._c_dataengine as cde
import mindspore.common.dtype as mstype
from mindspore.dataset.text import to_str

# pylint: disable=comparison-with-itself
def test_basic():
    x = np.array([["ab", "cde", "121"], ["x", "km", "789"]], dtype='S')
    n = cde.Tensor(x)
    arr = n.as_array()
    np.testing.assert_array_equal(x, arr)

def compare(strings):
    arr = np.array(strings, dtype='S')

    def gen():
        (yield arr,)

    data = ds.GeneratorDataset(gen, column_names=["col"])

    for d in data:
        np.testing.assert_array_equal(d[0], arr)

def test_generator():
    compare(["ab"])
    compare(["ab", "cde", "121"])
    compare([["ab", "cde", "121"], ["x", "km", "789"]])

def test_batching_strings():
    def gen():
        yield (np.array(["ab", "cde", "121"], dtype='S'),)

    data = ds.GeneratorDataset(gen, column_names=["col"]).batch(10)

    with pytest.raises(RuntimeError) as info:
        for _ in data:
            pass
    assert "[Batch ERROR] Batch does not support" in str(info.value)

def test_map():
    def gen():
        yield (np.array(["ab cde 121"], dtype='S'),)

    data = ds.GeneratorDataset(gen, column_names=["col"])

    def split(b):
        s = to_str(b)
        splits = s.item().split()
        return np.array(splits, dtype='S')

    data = data.map(input_columns=["col"], operations=split)
    expected = np.array(["ab", "cde", "121"], dtype='S')
    for d in data:
        np.testing.assert_array_equal(d[0], expected)

def test_map2():
    def gen():
        yield (np.array(["ab cde 121"], dtype='S'),)

    data = ds.GeneratorDataset(gen, column_names=["col"])

    def upper(b):
        out = np.char.upper(b)
        return out

    data = data.map(input_columns=["col"], operations=upper)
    expected = np.array(["AB CDE 121"], dtype='S')
    for d in data:
        np.testing.assert_array_equal(d[0], expected)

line = np.array(["This is a text file.",
                 "Be happy every day.",
                 "Good luck to everyone."])

words = np.array([["This", "text", "file", "a"],
                  ["Be", "happy", "day", "b"],
                  ["女", "", "everyone", "c"]])

chinese = np.array(["今天天气太好了我们一起去外面玩吧",
                    "男默女泪",
                    "江州市长江大桥参加了长江大桥的通车仪式"])

def test_tfrecord1():
    s = ds.Schema()
    s.add_column("line", "string", [])
    s.add_column("words", "string", [-1])
    s.add_column("chinese", "string", [])

    data = ds.TFRecordDataset("../data/dataset/testTextTFRecord/text.tfrecord", shuffle=False, schema=s)

    for i, d in enumerate(data.create_dict_iterator()):
        assert d["line"].shape == line[i].shape
        assert d["words"].shape == words[i].shape
        assert d["chinese"].shape == chinese[i].shape
        np.testing.assert_array_equal(line[i], to_str(d["line"]))
        np.testing.assert_array_equal(words[i], to_str(d["words"]))
        np.testing.assert_array_equal(chinese[i], to_str(d["chinese"]))

def test_tfrecord2():
    data = ds.TFRecordDataset("../data/dataset/testTextTFRecord/text.tfrecord", shuffle=False,
                              schema='../data/dataset/testTextTFRecord/datasetSchema.json')
    for i, d in enumerate(data.create_dict_iterator()):
        assert d["line"].shape == line[i].shape
        assert d["words"].shape == words[i].shape
        assert d["chinese"].shape == chinese[i].shape
        np.testing.assert_array_equal(line[i], to_str(d["line"]))
        np.testing.assert_array_equal(words[i], to_str(d["words"]))
        np.testing.assert_array_equal(chinese[i], to_str(d["chinese"]))

def test_tfrecord3():
    s = ds.Schema()
    s.add_column("line", mstype.string, [])
    s.add_column("words", mstype.string, [-1, 2])
    s.add_column("chinese", mstype.string, [])

    data = ds.TFRecordDataset("../data/dataset/testTextTFRecord/text.tfrecord", shuffle=False, schema=s)

    for i, d in enumerate(data.create_dict_iterator()):
        assert d["line"].shape == line[i].shape
        assert d["words"].shape == words[i].reshape([2, 2]).shape
        assert d["chinese"].shape == chinese[i].shape
        np.testing.assert_array_equal(line[i], to_str(d["line"]))
        np.testing.assert_array_equal(words[i].reshape([2, 2]), to_str(d["words"]))
        np.testing.assert_array_equal(chinese[i], to_str(d["chinese"]))

def create_text_mindrecord():
    # methood to create mindrecord with string data, used to generate testTextMindRecord/test.mindrecord
    from mindspore.mindrecord import FileWriter

    mindrecord_file_name = "test.mindrecord"
    data = [{"english": "This is a text file.",
             "chinese": "今天天气太好了我们一起去外面玩吧"},
            {"english": "Be happy every day.",
             "chinese": "男默女泪"},
            {"english": "Good luck to everyone.",
             "chinese": "江州市长江大桥参加了长江大桥的通车仪式"},
            ]
    writer = FileWriter(mindrecord_file_name)
    schema = {"english": {"type": "string"},
              "chinese": {"type": "string"},
              }
    writer.add_schema(schema)
    writer.write_raw_data(data)
    writer.commit()

def test_mindrecord():
    data = ds.MindDataset("../data/dataset/testTextMindRecord/test.mindrecord", shuffle=False)

    for i, d in enumerate(data.create_dict_iterator()):
        assert d["english"].shape == line[i].shape
        assert d["chinese"].shape == chinese[i].shape
        np.testing.assert_array_equal(line[i], to_str(d["english"]))
        np.testing.assert_array_equal(chinese[i], to_str(d["chinese"]))

if __name__ == '__main__':
    test_generator()
    test_basic()
    test_batching_strings()
    test_map()
    test_map2()
    test_tfrecord1()
    test_tfrecord2()
    test_tfrecord3()
    test_mindrecord()

max_stars_repo_name: ZephyrChenzf/mindspore

max_issues_repo_name: ZephyrChenzf/mindspore

max_forks_repo_name: ZephyrChenzf/mindspore


text:
# Needed libirarires
import numpy as np
import h5py

# Global variable
data_dir = '/media/mustafa/Cloud/Graduation project/Data/AgentHuman/SeqTrain'                                  # Data directory

high_level = ["FOLLOW", "LEFT", "RIGHT", "STRAIGHT"]                                      # High level commands
total_counter = np.zeros(4)                                                               # Used to count up images

def write_data(imags_matrix, targets_matrix, directory, filename):
    with h5py.File(directory + filename, 'w') as hdf:
        hdf.create_dataset('rgb',data=imags_matrix)
        hdf.create_dataset('targets',data=targets_matrix)

def classify(output_dir, command):
    imags_counter = 0
    file_number = 0
    for _ in range(3663, 6952, 1):
        if((_ - 3663)%500 == 0 and (_ != 3663)):
            print("Done classifying 500 files ..")
        if(_ == 3663):
            imgs_data = []
            targets_data = []
        filename = data_dir + '/data_0' + str(_) + '.h5'
        if(_ == 6790):
            continue
        with h5py.File(filename, 'r') as hdf:
            imgs = hdf.get('rgb')
            imgs = np.array(imgs[:,:,:])
            targets = hdf.get('targets')
            targets = np.array(targets)
            for i in range (0,200):
                if(targets[i][10] < -1):
                	continue
                img = imgs[i]
                target = targets[i]
                if(target[24] == command ):
                    imgs_data.append(img)
                    targets_data.append(target)
                    imags_counter = imags_counter + 1
                    if(imags_counter == 32):
                        output_filename = '/data_0' + str(file_number) + '.h5'
                        write_data(imgs_data, targets_data, output_dir, output_filename)

                        total_counter[int(command - 2)] = total_counter[int(command - 2)] + 32
                        file_number = file_number + 1
                        imags_counter = 0
                        imgs_data = []
                        targets_data = []
        if(_ == 6951):
            print("Additional ", int(imags_counter), " images with the command ", high_level[int(command - 2)], " were neglected! ..")
            print("The total number of images associated with command ", high_level[int(command - 2)], " equals = ", int(total_counter[int(command - 2)]), " ,That is = ", int(total_counter[int(command - 2)])//32, " Files.")
            print("."*50)

            imgs_data = []
            targets_data = []

# Classifying data into five categories
output_directories = ['/media/mustafa/Cloud/Graduation project/Data/AgentHuman/SeqTrain/Follow',
                      '/media/mustafa/Cloud/Graduation project/Data/AgentHuman/SeqTrain/Left',
                      '/media/mustafa/Cloud/Graduation project/Data/AgentHuman/SeqTrain/Right',
                      '/media/mustafa/Cloud/Graduation project/Data/AgentHuman/SeqTrain/Straight']

commands = [2.0, 3.0, 4.0, 5.0]
for i, directory in enumerate(output_directories):
    print("Classifying the ", high_level[i], " command ...")
    classify(directory, commands[i])


text:
# Author: Decebal Constantin Mocanu et al.;
# Plot performance of all three models on CIFAR10

# This is a pre-alpha free software and was tested with Python 3.5.2, Keras 2.1.3, Keras_Contrib 0.0.2, Tensorflow 1.5.0, Numpy 1.14;
# The code is distributed in the hope that it may be useful, but WITHOUT ANY WARRANTIES; The use of this software is entirely at the user's own risk;
# For an easy understanding of the code functionality please read the following articles.

# If you use parts of this code please cite the following articles:
#@article{Mocanu2018SET,
#  author =        {Mocanu, Decebal Constantin and Mocanu, Elena and Stone, Peter and Nguyen, Phuong H. and Gibescu, Madeleine and Liotta, Antonio},
#  journal =       {Nature Communications},
#  title =         {Scalable Training of Artificial Neural Networks with Adaptive Sparse Connectivity inspired by Network Science},
#  year =          {2018},
#  doi =           {10.1038/s41467-018-04316-3}
#}

#@Article{Mocanu2016XBM,
#author="Mocanu, Decebal Constantin and Mocanu, Elena and Nguyen, Phuong H. and Gibescu, Madeleine and Liotta, Antonio",
#title="A topological insight into restricted Boltzmann machines",
#journal="Machine Learning",
#year="2016",
#volume="104",
#number="2",
#pages="243--270",
#doi="10.1007/s10994-016-5570-z",
#url="https://doi.org/10.1007/s10994-016-5570-z"
#}

#@phdthesis{Mocanu2017PhDthesis,
#title = "Network computations in artificial intelligence",
#author = "D.C. Mocanu",
#year = "2017",
#isbn = "978-90-386-4305-2",
#publisher = "Eindhoven University of Technology",
#}

import matplotlib.pyplot as plt
import numpy as np
sparsities = {"e10": "Sparsity 98.85%", "e20": "Sparsity 97.12%", "e50": "Sparsity 94.25%",
              "e100": "Sparsity 88.50%", "e500": "Sparsity 71.2%", "dense": "Dense Network"}
colors = ['b','g','r','c','m','y','k']
vars = {"acc": "Accuracy", "val_acc": "Accuracy on the validation set", "loss": "Loss", "val_loss": "Loss on the validation set"}
functions = {"relu": "ReLU", "sigmoid": "Sigmoid", "tanh": "Tanh", "selu": "SELU", "softsign": "Softsign", "softplus": "Softplus", "srelu": "SReLU"}

for var in vars.keys():
    for s in sparsities.keys():
        plt.xlabel("Epochs[#]")
        plt.title("Performance on " + sparsities[s])
        plt.ylabel("CIFAR10\n" + vars[var]  + " [%]")
        color_index = 0
        for function in functions.keys():
            y = np.loadtxt("results/cifar10/" + function + "/set_mlp_" + s + "_" + var + ".txt")

            plt.plot(y*100, colors[color_index], label=functions[function])
            color_index += 1

        if var == "acc" or var == "val_acc":
            plt.ylim([8,76])
        plt.legend(loc=4, prop={'size':8})
        plt.grid(True)
        plt.tight_layout()
        plt.savefig("results/cifar10/performance_" + s + "_" + var + ".pdf")
        plt.close()

max_stars_repo_name: adubowski/set-mlp-keras

max_issues_repo_name: adubowski/set-mlp-keras

max_forks_repo_name: adubowski/set-mlp-keras


text:
import doctest
import unittest

import itertools

import numpy as np
import numpy.testing as npt

from mast import molecule

import mast.molecule as mastmol
import mast.selection as mastsel
import mast.tests.config.molecule as mastmolconfig
import mast.tests.data as mastdata

class TestAtomType(unittest.TestCase):
    def setUp(self):
        self.mock_attrs = {}
        self.mock_attrs[mastmolconfig.ATOM_ATTRIBUTES[0]] = "mock_attribute"
        self.mock_attrs['undefined_attribute'] = "undefined_mock_attribute"

    def tearDown(self):
        pass

    def test_constructor(self):
        MockAtomType = mastmol.AtomType("MockAtomType", **self.mock_attrs)
        self.assertIsInstance(MockAtomType, mastmol.AtomType)

class TestBondType(unittest.TestCase):
    def setUp(self):
        self.mock_atom1_attrs = {}
        self.mock_atom1_attrs[mastmolconfig.ATOM_ATTRIBUTES[0]] = "mock_attribute"
        self.mock_atom1_attrs['undefined_attribute'] = "undefined_mock_attribute"
        self.Mock1AtomType = mastmol.AtomType("Mock1AtomType", **self.mock_atom1_attrs)
        self.mock_atom2_attrs = {}
        self.mock_atom2_attrs[mastmolconfig.ATOM_ATTRIBUTES[0]] = "mock_attribute_2"
        self.mock_atom2_attrs['undefined_attribute'] = "undefined_mock_attribute"
        self.Mock2AtomType = mastmol.AtomType("Mock2AtomType", **self.mock_atom2_attrs)

        self.atom_types = (self.Mock1AtomType, self.Mock2AtomType)
        self.mock_bond_attrs = {}
        self.mock_bond_attrs[mastmolconfig.BOND_ATTRIBUTES[0]] = "mock_attribute"
        self.mock_bond_attrs['undefined_attribute'] = "undefined_mock_attribute"

    def tearDown(self):
        pass

    def test_constructor(self):
        MockBondType = mastmol.BondType("MockBondType",
                                                atom_types=self.atom_types,
                                                **self.mock_bond_attrs)

        self.assertIsInstance(MockBondType, mastmol.BondType)

        # test the non-domain specific attributes work
        self.assertEqual(MockBondType.atom_types, (self.Mock1AtomType, self.Mock2AtomType))

class TestFakeMoleculeType(unittest.TestCase):
    def setUp(self):
        self.mock_atom1_attrs = {}
        self.mock_atom1_attrs[mastmolconfig.ATOM_ATTRIBUTES[0]] = "mock_attribute"
        self.mock_atom1_attrs['undefined_attribute'] = "undefined_mock_attribute"
        self.Mock1AtomType = mastmol.AtomType("Mock1AtomType", **self.mock_atom1_attrs)
        self.mock_atom2_attrs = {}
        self.mock_atom2_attrs[mastmolconfig.ATOM_ATTRIBUTES[0]] = "mock_attribute_2"
        self.mock_atom2_attrs['undefined_attribute'] = "undefined_mock_attribute"
        self.Mock2AtomType = mastmol.AtomType("Mock2AtomType", **self.mock_atom2_attrs)

        self.atom_types = (self.Mock1AtomType, self.Mock2AtomType)
        self.mock_bond_attrs = {}
        self.mock_bond_attrs[mastmolconfig.BOND_ATTRIBUTES[0]] = "mock_attribute"
        self.mock_bond_attrs['undefined_attribute'] = "undefined_mock_attribute"
        self.MockBondType = mastmol.BondType("MockBondType",
                                                atom_types=self.atom_types,
                                                **self.mock_bond_attrs)

        self.bond_types = [self.MockBondType]
        self.mock_attrs = {}
        self.bond_map = {0:(0,1)}
        self.mock_attrs[mastmolconfig.MOLECULE_ATTRIBUTES[0]] = "mock_attribute"
        self.mock_attrs['undefined_attribute'] = "undefined_mock_attribute"

    def tearDown(self):
        pass

    def test_constructor(self):
        MockMoleculeType = mastmol.MoleculeType("MockMoleculeType",
                                                        atom_types=self.atom_types,
                                                        bond_types=self.bond_types,
                                                        bond_map=self.bond_map,
                                                        **self.mock_attrs)
        self.assertIsInstance(MockMoleculeType, mastmol.MoleculeType)

        # test that the non-domain specific attributes and functions
        # work
        self.assertEqual(MockMoleculeType.atom_types, self.atom_types)
        self.assertEqual(MockMoleculeType.atom_type_library,
                          list(set(MockMoleculeType.atom_types)))
        self.assertEqual(MockMoleculeType.bond_types, self.bond_types)
        self.assertEqual(MockMoleculeType.bond_type_library,
                          list(set(MockMoleculeType.bond_types)))
        self.assertEqual(MockMoleculeType.bond_map, self.bond_map)
        # make sure we get the correct AtomTypes from the bond map
        begin_atom_type = MockMoleculeType.atom_types[MockMoleculeType.bond_map[0][0]]
        end_atom_type = MockMoleculeType.atom_types[MockMoleculeType.bond_map[0][1]]
        self.assertEqual(begin_atom_type, self.Mock1AtomType)
        self.assertEqual(end_atom_type, self.Mock2AtomType)
        # we didn't set these so make sure they are empty
        self.assertFalse(MockMoleculeType.feature_types)

        # test the domain specific stuff is the same as in the mock
        # config files
        for attr in mastmolconfig.MOLECULE_ATTRIBUTES:
            self.assertIn(attr, MockMoleculeType.attributes)

class testAtom(unittest.TestCase):

    def setUp(self):
        # make a MoleculeType
        self.mock_attrs = {}
        self.mock_attrs[mastmolconfig.ATOM_ATTRIBUTES[0]] = "mock_attribute"
        self.mock_attrs['undefined_attribute'] = "undefined_mock_attribute"
        self.MockAtomType = mastmol.AtomType("MockAtomType", **self.mock_attrs)
        self.coords = np.array((0.0, 0.0, 0.0))

    def tearDown(self):
        pass

    def test_constructors(self):
        atom1 = mastmol.Atom(coords=self.coords, atom_type=self.MockAtomType)
        atom2 = self.MockAtomType.to_atom(self.coords)
        npt.assert_array_almost_equal(atom1.coords, atom2.coords)
        self.assertEqual(atom1.atom_type, atom2.atom_type)

    def test_selection_getters(self):
        atom = self.MockAtomType.to_atom(self.coords)
        self.assertFalse(atom.isin_bond)
        self.assertFalse(atom.isin_molecule)
        self.assertFalse(atom.isin_system)

class testBond(unittest.TestCase):

    def setUp(self):
        self.mock_atom1_attrs = {}
        self.mock_atom1_attrs[mastmolconfig.ATOM_ATTRIBUTES[0]] = "mock_attribute"
        self.mock_atom1_attrs['undefined_attribute'] = "undefined_mock_attribute"
        self.Mock1AtomType = mastmol.AtomType("Mock1AtomType", **self.mock_atom1_attrs)
        self.mock_atom2_attrs = {}
        self.mock_atom2_attrs[mastmolconfig.ATOM_ATTRIBUTES[0]] = "mock_attribute_2"
        self.mock_atom2_attrs['undefined_attribute'] = "undefined_mock_attribute"
        self.Mock2AtomType = mastmol.AtomType("Mock2AtomType", **self.mock_atom2_attrs)

        self.atom_types = (self.Mock1AtomType, self.Mock2AtomType)
        self.mock_bond_attrs = {}
        self.mock_bond_attrs[mastmolconfig.BOND_ATTRIBUTES[0]] = "mock_attribute"
        self.mock_bond_attrs['undefined_attribute'] = "undefined_mock_attribute"

        self.MockBondType = mastmol.BondType("MockBondType",
                                                     atom_types=self.atom_types,
                                                     **self.mock_bond_attrs)

        self.coords = (np.array((0.0, 0.0, 0.0)), np.array((0.0, 0.0, 1.0)))
        self.atoms = (self.Mock1AtomType.to_atom(self.coords[0]),
                      self.Mock2AtomType.to_atom(self.coords[1]))

    def tearDown(self):
        pass

    def test_constructors(self):
        bonds = []
        bonds.append(mastmol.Bond(atom_container=self.atoms, bond_type=self.MockBondType))
        bonds.append(mastmol.Bond(atom_container=self.atoms, atom_ids=(0,1),
                                  bond_type=self.MockBondType))
        bonds.append(self.MockBondType.to_bond(*self.coords))

        for bond_a, bond_b in itertools.combinations(bonds, 2):
            npt.assert_array_almost_equal(bond_a.coords, bond_b.coords)
            self.assertEqual(bond_a.bond_type, bond_b.bond_type)

    def test_selection_getters(self):
        bond = self.MockBondType.to_bond(*self.coords)
        self.assertFalse(bond.isin_molecule)
        self.assertFalse(bond.isin_system)
        for atom in bond.atoms:
            self.assertTrue(atom.isin_bond)
            self.assertIn(bond, atom.bonds)

    def test_adjacent_atoms(self):
        bond = self.MockBondType.to_bond(*self.coords)
        self.assertIs(bond.atoms[0].adjacent_atoms[0], bond.atoms[1])
        self.assertIs(bond.atoms[1].adjacent_atoms[0], bond.atoms[0])

class testMolecule(unittest.TestCase):

    def setUp(self):
        self.mock_atom1_attrs = {}
        self.mock_atom1_attrs[mastmolconfig.ATOM_ATTRIBUTES[0]] = "mock_attribute"
        self.mock_atom1_attrs['undefined_attribute'] = "undefined_mock_attribute"
        self.Mock1AtomType = mastmol.AtomType("Mock1AtomType", **self.mock_atom1_attrs)
        self.mock_atom2_attrs = {}
        self.mock_atom2_attrs[mastmolconfig.ATOM_ATTRIBUTES[0]] = "mock_attribute_2"
        self.mock_atom2_attrs['undefined_attribute'] = "undefined_mock_attribute"
        self.Mock2AtomType = mastmol.AtomType("Mock2AtomType", **self.mock_atom2_attrs)

        self.atom_types = (self.Mock1AtomType, self.Mock2AtomType)
        self.mock_bond_attrs = {}
        self.mock_bond_attrs[mastmolconfig.BOND_ATTRIBUTES[0]] = "mock_attribute"
        self.mock_bond_attrs['undefined_attribute'] = "undefined_mock_attribute"

        self.MockBondType = mastmol.BondType("MockBondType",
                                             atom_types=self.atom_types,
                                             **self.mock_bond_attrs)

        self.bond_types = [self.MockBondType]
        self.mock_attrs = {}
        self.bond_map = {0:(0,1)}
        self.mock_attrs[mastmolconfig.MOLECULE_ATTRIBUTES[0]] = "mock_attribute"
        self.mock_attrs['undefined_attribute'] = "undefined_mock_attribute"

        self.MockMoleculeType = mastmol.MoleculeType("MockMoleculeType",
                                                     atom_types=self.atom_types,
                                                     bond_types=self.bond_types,
                                                     bond_map=self.bond_map,
                                                     **self.mock_attrs)

        self.coords = np.array((np.array((0.0, 0.0, 0.0)), np.array((0.0, 0.0, 1.0))))
        self.atoms = (self.Mock1AtomType.to_atom(self.coords[0]),
                      self.Mock2AtomType.to_atom(self.coords[1]))
        self.bonds = []
        self.bonds.append(mastmol.Bond(self.atoms, bond_type=self.MockBondType))

    def tearDown(self):
        pass

    def test_constructors(self):
        molecules = []
        molecules.append(mastmol.Molecule(atoms=self.atoms, bonds=self.bonds,
                                         mol_type=self.MockMoleculeType))
        molecules.append(self.MockMoleculeType.to_molecule(self.coords))

        for mol_a, mol_b in itertools.combinations(molecules, 2):
            npt.assert_array_almost_equal(mol_a.atom_coords, mol_b.atom_coords)
            for bond_a, bond_b in zip(mol_a.bonds, mol_b.bonds):
                self.assertIs(bond_a.bond_type, bond_b.bond_type)
            for atom_a, atom_b in zip(mol_a.atoms, mol_b.atoms):
                self.assertIs(atom_a.atom_type, atom_b.atom_type)

    def test_selection_getters(self):
        molecule = self.MockMoleculeType.to_molecule(self.coords)
        self.assertFalse(molecule.isin_system)
        for bond in molecule.bonds:
            self.assertTrue(bond.isin_molecule)
            self.assertFalse(bond.isin_system)
            self.assertIs(bond.molecule, molecule)
        for atom in molecule.atoms:
            self.assertTrue(atom.isin_bond)
            self.assertTrue(atom.isin_molecule)
            self.assertFalse(atom.isin_system)
            self.assertIs(atom.molecule, molecule)

if __name__ == "__main__":

    from mast import molecule

    # doctests
    print("\n\n\n Doc Tests\n-----------")
    nfail, ntests = doctest.testmod(molecule, verbose=True)

    # unit tests
    print("\n\n\n Unit Tests\n-----------")
    unittest.main()

max_stars_repo_name: ADicksonLab/mastic

max_issues_repo_name: salotz/mast

max_forks_repo_name: salotz/mast