text:
"""radviz.py -- A customized and more flexible Radviz plotting module.

    This module provides a customized and more flexible function for Radviz [1]_ plotting.
    This module also provides different relevant fucntions, parameters and tools.

    Copyright (C) 2016
    Computational Optimization and Innovation (COIN) Laboratory
    Department of Computer Science and Engineering
    Michigan State University
    428 S. Shaw Lane, Engineering Building
    East Lansing, MI 48824-1226, USA

    References
    ----------
    .. [1] P. Hoffman, G. Grinstein and D. Pinkney, Dimensional anchors: A graphic primitive
    for multidimensional multivariate information visualizations", Proc. 1999 Workshop on
    New Paradigms in Information Visualization and Manipulation in Conjunction with the
    Eighth ACM Int. Conf. Information and Knowledge Management (NPIVM ’99), pp. 9-16, 1999.

.. moduleauthor:: AKM Khaled Talukder <talukde1@msu.edu>

"""

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.cm as cm
import matplotlib.colors as mc
from matplotlib.colors import ListedColormap
from viz.utils import transform as tr
from viz.plotting.utils import set_polar_anchors, set_polar_anchor_labels, pop, group_labels_by_appearance

__all__ = ["get_radviz_coordinates", "plot"]

def get_radviz_coordinates(X, spread_factor='auto', normalized=True):
    r"""Generate Radviz coordinates from data points `X`.

    Maps all the data points in `X` (i.e. `|X| = n x m`) onto radviz [1]_ coordinate positions.
    The `factor` parameter can be used to "spread-out" the data points on Radviz. In higher
    dimension, the points on Radviz tend to be clustered in the center. We can increase this
    value to make the points more visible.

    Parameters
    ----------
    X : ndarray
        `n` number of `m` dimensional points as input.
    spread_factor : str {'auto'} or float, optional
        The `spread_factor` might be needed if the data points are not sparse enough
        in the original space. Setting this bigger than 1.0 will help the data points
        to "spread-out" even more, while maintaining the placement of the points inside
        the radviz anchor points. If `spread_factor = 'auto'`, the `factor` is 2.0 if
        `m > 3`, otherwise 1.0. Default `'auto'` when default.
    normalized : bool, optional
        If this value is True, then all the points in `X` will be
        normalized within `[0.0, 1.0]`. Default `True` when optional.

    Returns
    -------
    P : ndarray of float
        An ndarray of PaletteViz coordinates (i.e. `|P| = n x 3`).
    K : ndarray of float
        The positions of the anchor points (i.e. `|K| = m x 2`),
    B : ndarray of float
        The lower bound and upper bound of `P`.

    References
    ----------
    .. [1] P. Hoffman, G. Grinstein and D. Pinkney, Dimensional anchors: A graphic primitive
    for multidimensional multivariate information visualizations", Proc. 1999 Workshop on
    New Paradigms in Information Visualization and Manipulation in Conjunction with the
    Eighth ACM Int. Conf. Information and Knowledge Management (NPIVM ’99), pp. 9-16, 1999.

    .. moduleauthor:: AKM Khaled Talukder <talukde1@msu.edu>

    """
    m = X.shape[1]

    # by default spread-factor is auto
    if type(spread_factor) == str and spread_factor == 'auto':
        factor = 2.0 if m > 3 else 1.0
    elif type(spread_factor) == int or type(spread_factor) == float:
        factor = spread_factor
    else:
        raise TypeError("`spread_factor` must be either 'auto' or a float/int.")

    # check if the data needs to be normalized
    if normalized:
        X_ = tr.normalize(X, lb = np.zeros(m), ub = np.ones(m))
    else:
        X_ = X

    T = 2 * np.pi * (np.arange(0, m, 1).astype(int) / m)
    COS = np.cos(T)
    SIN = np.sin(T)

    Y = np.power(X_, factor)
    S = np.sum(Y, axis=1)
    U = np.sum(Y * COS, axis=1) / S
    V = np.sum(Y * SIN, axis=1) / S
    P = np.column_stack((U, V))

    B = [np.min(P, axis=0), np.max(P, axis=0)]
    K = np.column_stack((COS, SIN))
    return (P, K, B)

def plot(A, ax=None, s=1, c=mc.TABLEAU_COLORS['tab:blue'], normalized=True, \
        spread_factor='auto', draw_axes=False, draw_anchors=True, **kwargs):
    r"""A customized radviz plot.

    This radviz [1]_ plot is customized for the experiments.

    Parameters
    ----------
    A : ndarray
        `n` number of `m` dim. points to be plotted.
    ax : An `matplotlib.axes.Axes` object, optional
        Default `None` when optional.
    s : int or 1-D array_like, optional
        Point size, or an array of point sizes. Default 1 when optional.
    c : A `matplotlib.colors` object, str or an array RGBA color values.
        Colors to be used. Default `mc.TABLEAU_COLORS['tab:blue']` when
        optional.
    normalized : bool, optional
        If needed, the data points in `A` can be normalized within `[0.0, 1.0]`.
        This helps to "spread-out" the data more on the radviz space. Default
        `True` when optional.
    spread_factor : str {'auto'} or float, optional
        See `get_radviz_coordinates()` function for details.
    draw_axes : bool, optional
        If `True`, the radviz plot will show axes. Default `False` when optional.
    draw_anchors : bool, optional
        If `False`, the radviz plot will hide anchors. Default `True` when optional.

    Other Parameters
    ----------------
    title : str, optional
        The plot title. Default `None` when optional.
    labels : str, array_like or list of str, optional
        A string or an array/list of strings for labeling each point. Which basically
        means the class label of each row. Default `None` when optional. This will be
        used to set the legend in the figure. If `None` there will be no legend.
    colorbar : (Cbc, Cbg, Cbl, Cbp) a tuple of (ndarray, ndarray, str, float), optional
        If a user wants to put a colorbar, a tuple `(Cbc, Cbg, Cbl)` tuple can be
        provided. `Cbc` is an array of RGBA color values or an `matplotlib.colors`
        object. The gradient of the colorbar is specified in `Cbg` which is an 1-D
        array of float. Cbl is the label of the colorbar, a string. `Cbp` is the
        colorbar padding width in `float`. `colorbar` is default `None` when optional.
    label_prefix : str, optional
        See `set_anchor_labels()` function for details.
    label_fontsize : str or int, optional
        See `set_anchor_labels()` function for details.
    label_fontname : str, optional
        See `set_anchor_labels()` function for details.
    label_fontstyle : str, optional
        See `set_anchor_labels()` function for details.
    **kwargs : dict
        All other keyword args for matplotlib's `scatter()` function.

    Returns
    -------
    ax : `matplotlib.axes.Axes` object
        A `matplotlib.axes.Axes` object.
    P : ndarray
        `P` is an ndarray of radviz coordinates (i.e. `|P| = n x 2`),

    References
    ----------
    .. [1] P. Hoffman, G. Grinstein and D. Pinkney, Dimensional anchors: A graphic primitive
    for multidimensional multivariate information visualizations", Proc. 1999 Workshop on
    New Paradigms in Information Visualization and Manipulation in Conjunction with the
    Eighth ACM Int. Conf. Information and Knowledge Management (NPIVM ’99), pp. 9-16, 1999.
    """

    # decide on what kind of axes to use
    if not ax:
        ax = fig.gca(plt.figure())

    title = kwargs['title'] if kwargs and 'title' in kwargs else None
    labels = kwargs['labels'] if kwargs and 'labels' in kwargs else None
    colorbar = kwargs['colorbar'] if kwargs and 'colorbar' in kwargs else None

    label_prefix = kwargs['label_prefix'] if kwargs and 'label_prefix' in kwargs else r"$f_{{{:d}}}$"
    label_fontsize = kwargs['label_fontsize'] if kwargs and 'label_fontsize' in kwargs else 'large'
    label_fontname = kwargs['label_fontname'] if kwargs and 'label_fontname' in kwargs else None
    label_fontstyle = kwargs['label_fontstyle'] if kwargs and 'label_fontstyle' in kwargs else 'normal'

    # remove once they are read
    kwargs = pop(kwargs, 'title')
    kwargs = pop(kwargs, 'labels')
    kwargs = pop(kwargs, 'colorbar')
    kwargs = pop(kwargs, 'label_prefix')
    kwargs = pop(kwargs, 'label_fontsize')
    kwargs = pop(kwargs, 'label_fontname')
    kwargs = pop(kwargs, 'label_fontstyle')

    if ax:
        P, K, [lb, ub] = get_radviz_coordinates(A, spread_factor=spread_factor, normalized=normalized)

        # do the plot
        if labels is not None:
            if isinstance(labels, str):
                ax.scatter(P[:,0], P[:,1], s=s, c=c, label=labels, **kwargs)
            else:
                if isinstance(labels, np.ndarray):
                    labels = labels.tolist()
                label_groups = group_labels_by_appearance(labels)
                for i,v in enumerate(label_groups):
                    ax.scatter(P[v[0],0], P[v[0],1], s=s[v[0]], c=c[v[0]], label=v[1], \
                            zorder=label_groups.shape[0]-i, **kwargs)
        else:
            ax.scatter(P[:,0], P[:,1], s=s, c=c, **kwargs)

        if draw_axes:
            ax.set_axis_on()
        else:
            ax.set_axis_off()

        if draw_anchors:
            set_polar_anchors(ax, K)
            set_polar_anchor_labels(ax, K, label_prefix=label_prefix, label_fontsize=label_fontsize, \
                    label_fontname=label_fontname, label_fontstyle=label_fontstyle)
        ax.set_xlim(lb[0] - 0.1 if lb[0] < -1 else -1.1, ub[0] + 0.01 if ub[0] > 1 else 1.1)
        ax.set_ylim(lb[1] - 0.1 if lb[1] < -1 else -1.1, ub[1] + 0.01 if ub[1] > 1 else 1.1)
        ax.set_aspect('equal')

        # colorbar?
        if colorbar and isinstance(colorbar, tuple) and len(colorbar) >= 2 \
                and isinstance(colorbar[0], np.ndarray) and isinstance(colorbar[1], np.ndarray):
            vmin,vmax = 0.0, 1.0
            cbc, cbg = colorbar[0], colorbar[1]
            cbl = colorbar[2] if len(colorbar) > 2 and colorbar[2] else None
            cbp = colorbar[3] if len(colorbar) > 3 and colorbar[3] else 0.05
            Id = np.column_stack((cbg,cbc)).astype(object)
            Id = Id[np.argsort(Id[:, 0])]
            c, g = Id[:,1:].astype(float), Id[:,0].astype(float)
            vmin, vmax = np.min(g), np.max(g)
            norm = mc.Normalize(vmin=vmin, vmax=vmax)
            cmap = ListedColormap(c)
            if cbl:
                ax.figure.colorbar(cm.ScalarMappable(norm=norm, cmap=cmap), \
                        orientation='vertical', label=cbl, pad=cbp, shrink=0.5)
            else:
                ax.figure.colorbar(cm.ScalarMappable(norm=norm, cmap=cmap), \
                            orientation='vertical', pad=cbp, shrink=0.5)

        # where to put the legend
        if labels is not None:
            ax.legend(loc="best", ncol=1)

        # title?
        ax.set_title(title, y=ax.get_ylim()[-1]-0.05)

        return (ax, P)
    else:
        raise TypeError("A valid `matplotlib.axes.Axes` object is not found.")

max_stars_repo_name: chudur-budur/visualization

max_issues_repo_name: chudur-budur/visualization

max_forks_repo_name: chudur-budur/pviz


text:
import algos_torch
import numpy as np
import common.object_factory
import common.env_configurations as env_configurations

import algos_torch.network_builder as network_builder
import algos_torch.model_builder as model_builder
import algos_torch.a2c_continuous as a2c_continuous
import algos_torch.a2c_discrete as a2c_discrete
#import algos_torch.dqnagent as dqnagent

import common.tr_helpers as tr_helpers
import yaml
import ray
import algos_torch.players as players
import argparse
import common.experiment as experiment
import copy
import torch

from sacred import Experiment
import numpy as np
import os
import collections
from os.path import dirname, abspath
import pymongo
from sacred import Experiment, SETTINGS
from sacred.observers import FileStorageObserver
from sacred.observers import MongoObserver
from sacred.utils import apply_backspaces_and_linefeeds
from utils.logging import get_logger, Logger

SETTINGS['CAPTURE_MODE'] = "fd" # set to "no" if you want to see stdout/stderr in console
logger = get_logger()

ex = Experiment("pymarl")
ex.logger = logger
ex.captured_out_filter = apply_backspaces_and_linefeeds

results_path = os.path.join(dirname(dirname(abspath(__file__))), "results")

mongo_client = None

class Runner:
    def __init__(self, logger):
        self.algo_factory = common.object_factory.ObjectFactory()
        self.algo_factory.register_builder('a2c_continuous', lambda **kwargs : a2c_continuous.A2CAgent(**kwargs))
        self.algo_factory.register_builder('a2c_discrete', lambda **kwargs : a2c_discrete.DiscreteA2CAgent(**kwargs))
        #self.algo_factory.register_builder('dqn', lambda **kwargs : dqnagent.DQNAgent(**kwargs))

        self.player_factory = common.object_factory.ObjectFactory()
        self.player_factory.register_builder('a2c_continuous', lambda **kwargs : players.PpoPlayerContinuous(**kwargs))
        self.player_factory.register_builder('a2c_discrete', lambda **kwargs : players.PpoPlayerDiscrete(**kwargs))
        #self.player_factory.register_builder('dqn', lambda **kwargs : players.DQNPlayer(**kwargs))

        self.model_builder = model_builder.ModelBuilder()
        self.network_builder = network_builder.NetworkBuilder()

        self.logger = logger

    def reset(self):
        pass

    def load_config(self, params):
        self.seed = params.get('seed', None)

        self.algo_params = params['algo']
        self.algo_name = self.algo_params['name']
        self.load_check_point = params['load_checkpoint']
        self.exp_config = None

        if self.seed:
            torch.manual_seed(self.seed)
            torch.cuda.manual_seed_all(self.seed)
            np.random.seed(self.seed)

        if self.load_check_point:
            self.load_path = params['load_path']
        else:
            self.load_path = None

        self.model = self.model_builder.load(params)
        self.config = copy.deepcopy(params['config'])

        self.config['reward_shaper'] = tr_helpers.DefaultRewardsShaper(**self.config['reward_shaper'])
        self.config['network'] = self.model

        has_rnd_net = self.config.get('rnd_config', None) != None
        if has_rnd_net:
            #print('Adding RND Network')
            logger.console_logger.info('Adding RND Network')
            network = self.model_builder.network_factory.create(params['config']['rnd_config']['network']['name'])
            print(network)
            network.load(params['config']['rnd_config']['network'])
            self.config['rnd_config']['network'] = network

    def load(self, yaml_conf):
        self.default_config = yaml_conf['params']
        self.load_config(copy.deepcopy(self.default_config))

        if 'experiment_config' in yaml_conf:
            self.exp_config = yaml_conf['experiment_config']

    def get_prebuilt_config(self):
        return self.config

    def run_train(self):
        #print('Started to train')
        self.logger.console_logger.info('Started to train')
        ray.init(redis_max_memory=1024*1024*1000, object_store_memory=1024*1024*1000)
        obs_space, action_space = env_configurations.get_obs_and_action_spaces_from_config(self.config)

        #print('obs_space:', obs_space)
        #print('action_space:', action_space)
        self.logger.console_logger.info('obs_space: {}'.format(obs_space))
        self.logger.console_logger.info('action_space: {}'.format(action_space))
        if self.exp_config:
            self.experiment =  experiment.Experiment(self.default_config, self.exp_config)
            exp_num = 0
            exp = self.experiment.get_next_config()
            while exp is not None:
                exp_num += 1
                #print('Starting experiment number: ' + str(exp_num))
                self.logger.console_logger.info('Starting experiment number: ' + str(exp_num))
                self.reset()
                self.load_config(exp)
                agent = self.algo_factory.create(self.algo_name, base_name='run', observation_space=obs_space, action_space=action_space, config=self.config)
                self.experiment.set_results(*agent.train())
                exp = self.experiment.get_next_config()
        else:
            self.reset()
            self.load_config(self.default_config)
            agent = self.algo_factory.create(self.algo_name, base_name='run', observation_space=obs_space, action_space=action_space, config=self.config, logger=self.logger)
            if self.load_check_point or (self.load_path is not None):
                agent.restore(self.load_path)
            agent.train()

    def create_player(self):
        return self.player_factory.create(self.algo_name, config=self.config)

    def create_agent(self, obs_space, action_space):
        return self.algo_factory.create(self.algo_name, base_name='run', observation_space=obs_space, action_space=action_space, config=self.config)

    def run(self, args):
        if 'checkpoint' in args:
            self.load_path = args['checkpoint']

        if args['train']:
            self.run_train()
        elif args['play']:
            #print('Started to play')
            logger.console_logger.info('Started to play')

            player = self.player_factory.create(self.algo_name, config=self.config)
            player.restore(self.load_path)
            player.run()

        ray.shutdown()

# Function to connect to a mongodb and add a Sacred MongoObserver
def setup_mongodb(db_url, db_name):
    client = None
    mongodb_fail = True

    # Try 5 times to connect to the mongodb
    for tries in range(5):
        # First try to connect to the central server. If that doesn't work then just save locally
        maxSevSelDelay = 10000  # Assume 10s maximum server selection delay
        try:
            # Check whether server is accessible
            logger.info("Trying to connect to mongoDB '{}'".format(db_url))
            client = pymongo.MongoClient(db_url, ssl=True, serverSelectionTimeoutMS=maxSevSelDelay)
            client.server_info()
            # If this hasn't raised an exception, we can add the observer
            ex.observers.append(MongoObserver.create(url=db_url, db_name=db_name, ssl=True)) # db_name=db_name,
            logger.info("Added MongoDB observer on {}.".format(db_url))
            mongodb_fail = False
            break
        except pymongo.errors.ServerSelectionTimeoutError:
            logger.warning("Couldn't connect to MongoDB on try {}".format(tries + 1))

    if mongodb_fail:
        logger.error("Couldn't connect to MongoDB after 5 tries!")
        # TODO: Maybe we want to end the script here sometimes?

    return client

@ex.main
def my_main(_run, _config, _log):
    global mongo_client

    import datetime

    #arglist = parse_args()
    #unique_token = "{}__{}".format(arglist.name, datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S"))
    # run the framework
    # run(_run, _config, _log, mongo_client, unique_token)

    logger = Logger(_log)

    # configure tensorboard logger
    unique_token = "{}__{}".format(_config["label"], datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S"))
    use_tensorboard = False
    if use_tensorboard:
        tb_logs_direc = os.path.join(dirname(dirname(abspath(__file__))), "results", "tb_logs")
        tb_exp_direc = os.path.join(tb_logs_direc, "{}").format(unique_token)
        logger.setup_tb(tb_exp_direc)
    logger.setup_sacred(_run)

    _log.info("Experiment Parameters:")
    import pprint
    experiment_params = pprint.pformat(_config,
                                       indent=4,
                                       width=1)
    _log.info("\n\n" + experiment_params + "\n")

    # START THE TRAINING PROCESS
    runner = Runner(logger)
    runner.load(_config)
    runner.reset()
    #args = vars(arglist)
    runner.run(_config)

    #runner.run(args)

    # train(arglist, logger, _config)
    # arglist = convert(_config)
    #train(arglist)

    # force exit
    os._exit(0)

def _get_config(params, arg_name, subfolder):
    config_name = None
    for _i, _v in enumerate(params):
        if _v.split("=")[0] == arg_name:
            config_name = _v.split("=")[1]
            del params[_i]
            break

    if config_name is not None:
        with open(os.path.join(os.path.dirname(__file__), "configs", subfolder, "{}.yaml".format(config_name)), "r") as f:
            try:
                config_dict = yaml.load(f)
            except yaml.YAMLError as exc:
                assert False, "{}.yaml error: {}".format(config_name, exc)
        return config_dict

def recursive_dict_update(d, u):
    for k, v in u.items():
        if isinstance(v, collections.Mapping):
            d[k] = recursive_dict_update(d.get(k, {}), v)
        else:
            d[k] = v
    return d

def parse_args():

    ap = argparse.ArgumentParser()
    ap.add_argument("-t", "--train", required=False, help="train network", action='store_true')
    ap.add_argument("-p", "--play", required=False, help="play network", action='store_true')
    ap.add_argument("-c", "--checkpoint", required=False, help="path to checkpoint")
    ap.add_argument("-f", "--file", required=True, help="path to config")
    ap.add_argument("-e", "--exp-name", required=True, help="experiment name")
    return ap.parse_args()

if __name__ == '__main__':
    import os

    import sys
    from copy import deepcopy
    params = deepcopy(sys.argv)

    # args = vars(ap.parse_args())
    # config_name = args['file']
    # print('Loading config: ', config_name)
    # with open(config_name, 'r') as stream:
    #     config = yaml.safe_load(stream)
    #     runner = Runner()
    #     try:
    #         runner.load(config)
    #     except yaml.YAMLError as exc:
    #         print(exc)
    #
    # # Load algorithm and env base configs
    # #file_config = _get_config(params, "--file", "envs")
    #
    # # Load into official sacred configs
    # if config_name is not None:
    #     with open(os.path.join(os.path.dirname(__file__), "configs", subfolder, "{}.yaml".format(config_name)), "r") as f:
    #         try:
    #             file_config = yaml.load(f)
    #         except yaml.YAMLError as exc:
    #             assert False, "{}.yaml error: {}".format(config_name, exc)

    config_dict = {"train":True,
                   "load_checkpoint":False,
                   "load_path":None}
    file_config = _get_config(params, "--file", "")
    config_dict = recursive_dict_update(config_dict, file_config)

    # now add all the config to sacred
    ex.add_config(config_dict)

    #arglist = ap.parse_args()

    #from copy import deepcopy
    #ex.add_config({"name":arglist.exp_name})

    # Check if we don't want to save to sacred mongodb
    no_mongodb = False

    for _i, _v in enumerate(params):
        if "no-mongo" in _v:
            if "--no-mongo" == _v:
                del params[_i]
                no_mongodb = True
                break

    config_dict={"train": True}
    db_config_path = "./db_config.private.yaml"
    with open(db_config_path, 'r') as stream:
        config_dict = yaml.safe_load(stream)

    # If there is no url set for the mongodb, we cannot use it
    if not no_mongodb and "db_url" not in config_dict:
        no_mongodb = True
        logger.error("No 'db_url' to use for Sacred MongoDB")

    if not no_mongodb:
        db_url = config_dict["db_url"]
        db_name = config_dict["db_name"]
        mongo_client = setup_mongodb(db_url, db_name)

    # Save to disk by default for sacred, even if we are using the mongodb
    logger.info("Saving to FileStorageObserver in results/sacred.")
    file_obs_path = os.path.join(results_path, "sacred")
    ex.observers.append(FileStorageObserver.create(file_obs_path))

    ex.run_commandline(params)

# if __name__ == '__main__':
#     ap = argparse.ArgumentParser()
#     ap.add_argument("-t", "--train", required=False, help="train network", action='store_true')
#     ap.add_argument("-p", "--play", required=False, help="play network", action='store_true')
#     ap.add_argument("-c", "--checkpoint", required=False, help="path to checkpoint")
#     ap.add_argument("-f", "--file", required=True, help="path to config")
#
#     args = vars(ap.parse_args())
#     config_name = args['file']
#     print('Loading config: ', config_name)
#     with open(config_name, 'r') as stream:
#         config = yaml.safe_load(stream)
#         runner = Runner()
#         try:
#             runner.load(config)
#         except yaml.YAMLError as exc:
#             print(exc)
#
#     main()

ext: py

reason: import numpy


text:
"""
This module contains the simulation result class.
"""
from __future__ import annotations
from enum import IntEnum
from typing import Generic, Optional, Dict, Any, Tuple, TypeVar, Callable
import numpy as np
import scipy.integrate
import scipy.interpolate
from matplotlib.pyplot import Figure

from .. import plotting
from .. elements import Element
from ._advanced_helpers import Device, AdvancedModel
from ._basic_helpers import BasicDevice
from ._radial_dist import boltzmann_radial_potential_linear_density_ebeam
from ..physconst import MINIMAL_N_1D, MINIMAL_KBT

class Rate(IntEnum):
    """
    Enum for conveniently identifying rates produced in advanced simulations
    """
    ELECTRON_IONISATION = 101
    EI = 101

    RADIATIVE_RECOMBINATION = 102
    RR = 102

    DIELECTRONIC_RECOMBINATION = 103
    DR = 103

    CHARGE_EXCHANGE = 104
    CX = 104

    LOSSES_AXIAL_COLLISIONAL = 105
    AX_CO = 105

    LOSSES_RADIAL_COLLISIONAL = 107
    RA_CO = 107

    # T_ELECTRON_IONISATION = 201
    # T_EI = 201

    # T_RADIATIVE_RECOMBINATION = 202
    # T_RR = 202

    # T_DIELECTRONIC_RECOMBINATION = 203
    # T_DR = 203

    # T_CHARGE_EXCHANGE = 204
    # T_CX = 204

    T_LOSSES_AXIAL_COLLISIONAL = 205
    T_AX_CO = 205

    T_LOSSES_RADIAL_COLLISIONAL = 207
    T_RA_CO = 207

    T_COLLISIONAL_THERMALISATION = 301
    COLLISIONAL_THERMALISATION = 301
    T_CT = 301

    T_SPITZER_HEATING = 302
    SPITZER_HEATING = 302
    T_SH = 302

    IONISATION_HEAT = 303

    # CV = 304

    COLLISION_RATE_TOTAL = 401

    COLLISION_RATE_SELF = 402

    OVERLAP_FACTORS_EBEAM = 501
    F_EI = 501

    # CHARGE_COMPENSATION = 511

    TRAPPING_PARAMETER_AXIAL = 521
    W_AX = 521

    TRAPPING_PARAMETER_RADIAL = 522
    W_RA = 522

    TRAP_DEPTH_AXIAL = 523
    V_AX = 523

    TRAP_DEPTH_RADIAL = 524
    V_RA = 524

    E_KIN_MEAN = 531

    E_KIN_FWHM = 532

_PER_M_PER_S = r"(m$^{-1}$ s$^{-1}$)"
_EV_PER_S = r"(eV s$^{-1}$)"
_PER_S = r"(s$^{-1}$)"
_RATE_LABELS: Dict[Rate, Dict[str, Any]] = {
    Rate.EI: dict(
        title="EI",
        ylabel="Ionisation rate " + _PER_M_PER_S,
    ),
    Rate.RR: dict(
        title="RR",
        ylabel="Recombination rate " + _PER_M_PER_S,
    ),
    Rate.DR: dict(
        title="DR",
        ylabel="Recombination rate " + _PER_M_PER_S,
    ),
    Rate.CX: dict(
        title="CX",
        ylabel="Recombination rate " + _PER_M_PER_S,
    ),
    Rate.AX_CO: dict(
        title="Ax. coll. losses",
        ylabel="Loss rate " + _PER_M_PER_S,
    ),
    Rate.RA_CO: dict(
        title="Rad. coll. losses",
        ylabel="Loss rate " + _PER_M_PER_S,
    ),
    # Rate.T_EI: dict(
    #     title="Electron ionisation"
    #     ylabel="",
    # ),
    # Rate.T_RR: dict(
    #     title="Radiative recombination"
    #     ylabel="",
    # ),
    # Rate.T_DR: dict(
    #     title="Dielectronic recombination"
    #     ylabel="",
    # ),
    # Rate.T_CX: dict(
    #     title="Charge exchange"
    #     ylabel="",
    # ),
    Rate.T_AX_CO: dict(
        title="Ax. coll. losses",
        ylabel="Cooling rate " + _EV_PER_S,
    ),
    Rate.T_RA_CO: dict(
        title="Rad. coll. losses",
        ylabel="Cooling rate " + _EV_PER_S,
    ),
    Rate.COLLISIONAL_THERMALISATION: dict(
        title="Coll. thermalisation",
        ylabel="Thermalisation rate " + _EV_PER_S,
    ),
    Rate.SPITZER_HEATING: dict(
        title="Spitzer heating",
        ylabel="Heating rate " + _EV_PER_S,
    ),
    Rate.IONISATION_HEAT: dict(
        title="Ionisation heat",
        ylabel="Ionisation heat (eV)",
    ),
    # Rate.CV: dict(
    #     title="Heat capacity",
    #     ylabel="$c_V$ (eV/eV)",
    # ),
    Rate.COLLISION_RATE_TOTAL: dict(
        title="Total collision rate",
        ylabel=r"$r_i$ " + _PER_S,
    ),
    Rate.COLLISION_RATE_SELF: dict(
        title="Self collision rate",
        ylabel=r"$r_{ii}$ " + _PER_S,
    ),
    Rate.F_EI: dict(
        title="Electron beam overlap",
        ylabel=r"$f_{ei}$",
        ylim=(0, 1.1),
    ),
    Rate.TRAPPING_PARAMETER_AXIAL: dict(
        title="Ax. trapping parameter",
        ylabel=r"$\omega_{ax}$",
    ),
    Rate.TRAPPING_PARAMETER_RADIAL: dict(
        title="Rad. trapping parameter",
        ylabel=r"$\omega_{rad}$",
    ),
    Rate.TRAP_DEPTH_AXIAL: dict(
        title="Ax. trapping potential",
        ylabel=r"$V_{ax} (V)$",
    ),
    Rate.TRAP_DEPTH_RADIAL: dict(
        title="Rad. trapping potential",
        ylabel=r"$V_{rad} (V)$",
    ),
    Rate.E_KIN_MEAN: dict(
        title="Beam energy mean",
        ylabel=r"$E_e$ (eV)",
    ),
    Rate.E_KIN_FWHM: dict(
        title="Beam energy FWHM",
        ylabel=r"FWHM($E_e$) (eV)",
    ),
    Rate.E_KIN_FWHM: dict(
        title="Beam energy FWHM",
        ylabel=r"FWHM($E_e$) (eV)",
    ),
}

# Needed for parametric polymorphism of Result classes
GenericDevice = TypeVar("GenericDevice", Device, BasicDevice)

class BasicResult(Generic[GenericDevice]):
    """
    Instances of this class are containers for the results of ebisim basic_simulations and contain a
    variety of convenience methods for simple plot generation etc.
    """
    _abundance_label = "Abundance"

    def __init__(self, *, t: np.ndarray, N: np.ndarray, device: GenericDevice,
                 target: Element, res: Optional[Any] = None):
        """
        Parameters
        ----------
        t :
            An array holding the time step coordinates.
        N :
            An array holding the occupancy of each charge state at a given time.
        device :
            If coming from advanced_simulation, this is the machine / electron beam description.
        target :
            If coming from advanced_simulation, this is the target represented by this Result.
        res :
            The result object returned by scipy.integrate.solve_ivp. This can contain useful
            information about the solver performance etc. Refer to scipy documentation for details.
        """
        self.t = t
        self.N = N
        self.device: GenericDevice = device
        self.target = target
        self.res = res

    @property
    def _dense_interpolator(self) -> Optional[Callable[[float], np.ndarray]]:
        if self.res and self.res.sol:
            return self.res.sol
        else:
            return None

    @property
    def _abundance_interpolator(self) -> Callable[[float], np.ndarray]:
        return self._dense_interpolator or scipy.interpolate.interp1d(self.t, self.N)

    def _check_time_in_domain(self, t: float) -> None:
        if t < self.t.min() or t > self.t.max():
            raise ValueError("Value for t lies outside the simulated domain.")

    def _param_title(self, stub):
        """
        Generates a plot title by merging the stub with some general simulation parameters.
        Defaults to the stub if no parameters are available

        Parameters
        ----------
        stub :
            Title stub for the plot

        Returns
        -------
            A LaTeX formatted title string compiled from the stub, current density and fwhm.

        """
        title = f"{self.target.latex_isotope()} {stub.lower()}"  #: TODO: Fix case

        e_kin = self.device.e_kin
        j = self.device.j

        title = title + f" ($j = {j:0.1f}$ A/cm$^2$, $E_{{e}} = {e_kin:0.1f}$ eV)"

        fwhm = self.device.fwhm
        if fwhm is not None:
            title = title[:-1] + f", FWHM = {fwhm:0.1f} eV)"
        return title

    def times_of_highest_abundance(self) -> np.ndarray:
        """
        Yields the point of time with the highest abundance for each charge state

        Returns
        -------
            <s>
            Array of times.

        """
        args = np.argmax(self.N, axis=1)
        return self.t[args]

    def abundance_at_time(self, t: float) -> np.ndarray:
        """
        Yields the abundance of each charge state at a given time

        Parameters
        ----------
        t :
            <s>
            Point of time to evaluate.

        Returns
        -------
            Abundance of each charge state, array index corresponds to charge state.

        """
        self._check_time_in_domain(t)
        return self._abundance_interpolator(t)

    def plot_charge_states(self, relative: bool = False, **kwargs: Any) -> Figure:
        """
        Plot the charge state evolution of this result object.

        Parameters
        ----------
        relative :
            Flags whether the absolute numbers or a relative fraction should be plotted at each
            timestep, by default False.
        kwargs
            Keyword arguments are handed down to ebisim.plotting.plot_generic_evolution,
            cf. documentation thereof.
            If no arguments are provided, reasonable default values are injected.

        Returns
        -------
            Figure handle of the generated plot.

        Raises
        ------
        ValueError
            If the required data (self.t, self.N) is not available, i.e.
            corresponding attributes of this Result instance have not been set correctly.

        """
        kwargs.setdefault("xlim", (1e-4, self.t.max()))
        kwargs.setdefault("ylim", (0, self.N.sum(axis=0).max()*1.05))
        kwargs.setdefault("title", self._param_title("Charge states"))
        kwargs.setdefault("yscale", "linear")
        kwargs.setdefault("plot_total", True)

        if relative:
            kwargs["ylim"] = (0, 1.1)
            kwargs.setdefault("ylabel", "Relative abundance")
            fig = plotting.plot_generic_evolution(self.t, self.N/np.sum(self.N, axis=0), **kwargs)
        else:
            kwargs.setdefault("ylabel", self._abundance_label)
            fig = plotting.plot_generic_evolution(self.t, self.N, **kwargs)
        return fig

    plot = plot_charge_states  #: Alias for plot_charge_states

class AdvancedResult(BasicResult[Device]):
    """
    Instances of this class are containers for the results of ebisim advanced_simulations
    and contain a variety of convenience methods for simple plot generation etc.
    """
    _abundance_label = "Linear density (m$^{-1}$)"

    def __init__(self, *, t: np.ndarray, N: np.ndarray, device: Device, target: Element,
                 kbT: np.ndarray, model: AdvancedModel, id_: int,
                 res: Optional[Any] = None, rates: Optional[Dict[Rate, np.ndarray]] = None):
        """
        Parameters
        ----------
        t :
            An array holding the time step coordinates.
        N :
            An array holding the occupancy of each charge state at a given time.
        device :
            If coming from advanced_simulation, this is the machine / electron beam description.
        target :
            If coming from advanced_simulation, this is the target represented by this Result.
        kbT :
            An array holding the temperature of each charge state at a given time.
        model :
            The AdvancedModel instance that was underlying the advanced_simulation
        id_ :
            The position of this target in the list of all simulated targets.
        res :
            The result object returned by scipy.integrate.solve_ivp. This can contain useful
            information about the solver performance etc. Refer to scipy documentation for details.
        rates :
            A dictionary containing the different breeding rates in arrays shaped like N.
        """
        super().__init__(t=t, N=N, device=device, target=target, res=res)
        self.kbT = kbT
        self.target = target
        self.device = device
        self.rates = rates
        model = model._replace(
            targets=list(model.targets),
            bg_gases=list(model.bg_gases),
            cxxs_bggas=list(model.cxxs_bggas),
            cxxs_trgts=list(model.cxxs_trgts)
        )
        self.model = model
        self.id = id_

    @property
    def _abundance_interpolator(self) -> Callable[[float], np.ndarray]:
        dinterp = self._dense_interpolator
        if dinterp is not None:
            lower = self.model.lb[self.id]
            upper = self.model.ub[self.id]
            interp = lambda t: dinterp(t)[lower:upper]  # noqa:E731
        else:
            interp = scipy.interpolate.interp1d(self.t, self.N)
        return interp

    @property
    def _temperature_interpolator(self) -> Callable[[float], np.ndarray]:
        dinterp = self._dense_interpolator
        if dinterp is not None and self.res is not None:
            lower = self.res.y.shape[0]//2 + self.model.lb[self.id]
            upper = self.res.y.shape[0]//2 + self.model.ub[self.id]
            interp = lambda t: dinterp(t)[lower:upper]  # noqa:E731
        else:
            interp = scipy.interpolate.interp1d(self.t, self.kbT)
        return interp

    def _density_filter(self, data, threshold):
        filtered = data.copy()
        filtered[self.N < threshold] = np.nan
        return filtered

    def temperature_at_time(self, t: float) -> np.ndarray:
        """
        Yields the temperature of each charge state at a given time

        Parameters
        ----------
        t :
            <s>
            Point of time to evaluate.

        Returns
        -------
            <eV>
            Temperature of each charge state, array index corresponds to charge state.

        """
        self._check_time_in_domain(t)
        return self._temperature_interpolator(t)

    def radial_distribution_at_time(self, t: float) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
        """
        Yields the radial distribution information at time

        Parameters
        ----------
        t :
            <s>
            Point of time to evaluate.

        Returns
        -------
        phi :
            Radial potential
        n3d :
            On axis 3D density for each charge state.
        shapes :
            The Boltzmann shape factors for each charge state.
        """
        self._check_time_in_domain(t)

        interp = self._dense_interpolator
        if interp is None:
            if self.res is None:
                raise ValueError("Required Data not available, "
                                 + "something went wrong during the creation of this result.")
            interp = scipy.interpolate.interp1d(self.t, self.res.y)
        y = interp(t)

        n = y[:self.model.ub[-1]]
        n = np.maximum(n, MINIMAL_N_1D)
        kT = y[self.model.ub[-1]:]
        kT = np.maximum(kT, MINIMAL_KBT)
        phi, n3d, shapes = boltzmann_radial_potential_linear_density_ebeam(
            self.device.rad_grid, self.device.current, self.device.r_e, self.device.e_kin,
            np.atleast_2d(n).T, np.atleast_2d(kT).T, np.atleast_2d(self.model.q).T,
            first_guess=self.device.rad_phi_uncomp,
            ldu=(self.device.rad_fd_l, self.device.rad_fd_d, self.device.rad_fd_u)
        )
        return phi, n3d, shapes

    def plot_radial_distribution_at_time(self, t: float, **kwargs: Any) -> Figure:
        """
        Plot the radial ion distribution at time t.

        Parameters
        ----------
        t :
            <s>
            Point of time to evaluate.
        kwargs
            Keyword arguments are handed down to ebisim.plotting.plot_radial_distribution and
            ebisim.plotting.plot_generic_evolution, cf. documentation thereof.
            If no arguments are provided, reasonable default values are injected.

        Returns
        -------
            Figure handle of the generated plot.
        """
        phi, n3d, shapes = self.radial_distribution_at_time(t)
        dens = (n3d * shapes)[self.model.lb[self.id]:self.model.ub[self.id]]
        denslim = 10**np.ceil(np.log10(dens.max()))

        title = self._param_title(f"Radial distribution at t = {1000*t:.1f} ms")

        kwargs.setdefault("xscale", "log")
        kwargs.setdefault("yscale", "log")
        kwargs.setdefault("title", title)
        kwargs.setdefault("xlim", (self.device.r_e/100, self.device.r_dt))
        kwargs.setdefault("ylim", (denslim/10**10, denslim))

        fig = plotting.plot_radial_distribution(
            self.device.rad_grid, dens, phi, self.device.r_e, **kwargs
        )

        return fig

    def plot_energy_density(self, **kwargs: Any) -> Figure:
        """
        Plot the energy density evolution of this result object.

        Parameters
        ----------
        kwargs
            Keyword arguments are handed down to ebisim.plotting.plot_generic_evolution,
            cf. documentation thereof.
            If no arguments are provided, reasonable default values are injected.

        Returns
        -------
            Figure handle of the generated plot.

        Raises
        ------
        ValueError
            If the required data (self.t, self.kbT) is not available, i.e.
            corresponding attributes of this Result instance have not been set correctly.

        """
        e_den = self.N * self.kbT
        ymin = 10**(np.floor(np.log10(e_den[:, 0].sum(axis=0)) - 1))
        ymax = 10**(np.ceil(np.log10(e_den.sum(axis=0).max()) + 1))

        kwargs.setdefault("xlim", (1e-4, self.t.max()))
        kwargs.setdefault("ylim", (ymin, ymax))
        kwargs.setdefault("title", self._param_title("Energy density"))
        kwargs.setdefault("ylabel", "Linear energy density (eV / m$^{-1}$)")
        kwargs.setdefault("plot_total", True)

        fig = plotting.plot_generic_evolution(self.t, e_den, **kwargs)
        return fig

    def plot_temperature(self, dens_threshold: float = 1000*MINIMAL_N_1D, **kwargs: Any) -> Figure:
        """
        Plot the temperature evolution of this result object.

        Parameters
        ----------
        dens_threshold :
            If given temperatures are only plotted where the particle denisty is larger than
            the threshold value.
        kwargs
            Keyword arguments are handed down to ebisim.plotting.plot_generic_evolution,
            cf. documentation thereof.
            If no arguments are provided, reasonable default values are injected.

        Returns
        -------
            Figure handle of the generated plot

        Raises
        ------
        ValueError
            If the required data (self.t, self.kbT) is not available, i.e.
            corresponding attributes of this Result instance have not been set correctly.

        """
        filtered_kbT = self._density_filter(self.kbT, dens_threshold)

        kwargs.setdefault("xlim", (1e-4, self.t.max()))
        kwargs.setdefault(
            "ylim",
            (
                10**np.floor(np.nanmin(np.log10(filtered_kbT))),
                10**np.ceil(np.nanmax(np.log10(filtered_kbT))),
            )
        )
        kwargs.setdefault("title", self._param_title("Temperature"))
        kwargs.setdefault("ylabel", "Temperature (eV)")

        fig = plotting.plot_generic_evolution(self.t, filtered_kbT, **kwargs)
        return fig

    def plot_rate(self, rate_key: Rate,
                  dens_threshold: float = 1000*MINIMAL_N_1D, **kwargs) -> Figure:
        """
        Plots the requested ionisation- or energy flow rates.

        Parameters
        ----------
        rate_key :
            The key identifying the rate to be plotted.
            See ebisim.simulation.Rate for valid values.
        dens_threshold :
            If given temperatures are only plotted where the particle denisty is larger than
            the threshold value.
        kwargs
            Keyword arguments are handed down to ebisim.plotting.plot_generic_evolution,
            cf. documentation thereof.
            If no arguments are provided, reasonable default values are injected.

        Returns
        -------
            Figure handle of the generated plot

        Raises
        ------
        ValueError
            If the required data (self.rates) is not available, or an invalid key is requested.

        """
        if self.rates is None:
            raise ValueError("Rates are not available for this result.")
        if rate_key not in self.rates:
            raise ValueError(
                f"The requested rate_key does not exist. Available rates are {self.rates.keys()}."
            )

        rate = self.rates[rate_key]
        if rate.shape[0] != 1:  # Don't filter if scalar (i.e. not for indiv charge states)
            rate = self._density_filter(rate, dens_threshold)
        labels = _RATE_LABELS.get(rate_key, {})

        kwargs.setdefault("xlim", (1e-4, self.t.max()))
        kwargs.setdefault("ylim", labels.get("ylim", None))
        kwargs.setdefault("yscale", labels.get("yscale", "linear"))
        kwargs.setdefault("title", self._param_title(labels.get("title", "unkown rate")))
        kwargs.setdefault("ylabel", labels.get("ylabel", "Unknown"))
        kwargs.setdefault("plot_total", False)

        if len(rate.shape) == 1:
            rate = rate[np.newaxis, :]
        if rate.shape[0] == 1:
            kwargs.setdefault("label_lines", False)
            kwargs.setdefault("ls", "-")

        fig = plotting.plot_generic_evolution(self.t, rate, **kwargs)
        return fig

max_stars_repo_name: ebisim/ebisim

max_issues_repo_name: ebisim/ebisim

max_forks_repo_name: HPLegion/EBISSimulation

reason: import numpy,import scipy


text:
import numpy as np
from scipy.io import savemat
import os
import json
import tensorflow as tf

class Logger(object):
    def __init__(self, log_dir):
        if not os.path.exists(log_dir):
            os.makedirs(log_dir)

        self.log_dir = log_dir
        self.tf_file_writer = tf.summary.FileWriter(log_dir)
        self.log = dict()  # contains all the logged information
        self.buffer = dict()  # temporary buffer within one epoch

    def add_log(self, idx, record_dict):
        tf_summary = tf.Summary()  # create a tf summary row
        for key, value in record_dict.items():
            tf_summary.value.add(tag=key, simple_value=value)

            if not (key in self.log.keys()):  # If key does not exist, create one
                self.log[key] = []

            self.log[key].append(value)

        self.tf_file_writer.add_summary(tf_summary, idx)  # add row and idx into the file writer

    def add_buffer(self, record_dict):
        for key, value in record_dict.items():
            if not (key in self.buffer.keys()):  # If key does not exist, create one
                self.buffer[key] = []

            self.buffer[key].append(value)

    def summarize_buffer(self, idx):
        record_dict = dict()
        for key, value in self.buffer.items():
            if len(self.buffer[key]) > 0:
                record_dict[key] = np.mean(value)  # Log average value (can also log max, min, or std)
                self.buffer[key] = []
            else:  # When the epoch idx add nothing to the buffer and there is a previous value in log
                record_dict[key] = self.get(key)  # get the most recent value
        self.add_log(idx, record_dict)
        self.tf_file_writer.flush()

    def dump_log(self):
        savemat(self.log_dir + '/log.mat', self.log)

    def get(self, key):
        return self.log[key][-1]

def save_params(params, path, name):
    if not os.path.exists(path):
        os.makedirs(path)
    with open(path + '/' + name + '.json', 'w') as fp:
        json.dump(params, fp, separators=(',', ':\t'), indent=4, sort_keys=False)

def load_params(path, name):
    with open(path + '/' + name + '.json', 'r') as fp:
        return json.load(fp)

reason: import numpy,from scipy


text:
import os
import copy
import logging
import shutil
import threading
import gc
import zipfile
from io import BytesIO

import numpy as np
from PIL import Image
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.utils.data as data
from collections import OrderedDict
import torchvision.transforms as transforms

IMG_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif']

def has_file_allowed_extension(filename, extensions):
    filename_lower = filename.lower()
    return any(filename_lower.endswith(ext) for ext in extensions)

def convert_to_pil(bytes_obj):
    img = Image.open(BytesIO(bytes_obj))
    return img.convert('RGB')

class ReadImageThread(threading.Thread):
    def __init__(self, root, fnames, class_id, target_list):
        threading.Thread.__init__(self)
        self.root = root
        self.fnames = fnames
        self.class_id = class_id
        self.target_list = target_list

    def run(self):
        for fname in self.fnames:
            if has_file_allowed_extension(fname, IMG_EXTENSIONS):
                path = os.path.join(self.root, fname)
                with open(path, 'rb') as f:
                    image = f.read()
                item = (image, self.class_id)
                self.target_list.append(item)

class InMemoryDataset(data.Dataset):
    def __init__(self, path, transform=None, num_workers=1):
        super(InMemoryDataset, self).__init__()
        self.path = path
        self.transform = transform
        self.samples = []
        classes, class_to_idx = self.find_classes(self.path)
        dir = os.path.expanduser(self.path)
        for target in sorted(os.listdir(dir)):
            d = os.path.join(dir, target)
            if not os.path.isdir(d):
                continue
            for root, _, fnames in sorted(os.walk(d)):
                if num_workers == 1:
                    for fname in sorted(fnames):
                        if has_file_allowed_extension(fname, IMG_EXTENSIONS):
                            path = os.path.join(root, fname)
                            with open(path, 'rb') as f:
                                image = f.read()
                            item = (image, class_to_idx[target])
                            self.samples.append(item)
                else:
                    fnames = sorted(fnames)
                    num_files = len(fnames)
                    threads = []
                    res = [[] for i in range(num_workers)]
                    num_per_worker = num_files // num_workers
                    for i in range(num_workers):
                        start_index = num_per_worker * i
                        end_index = num_files if i == num_workers - 1 else num_per_worker * (i+1)
                        thread = ReadImageThread(root, fnames[start_index:end_index], class_to_idx[target], res[i])
                        threads.append(thread)
                    for thread in threads:
                        thread.start()
                    for thread in threads:
                        thread.join()
                    for item in res:
                        self.samples += item
                    del res, threads
                    gc.collect()

    def __len__(self):
        return len(self.samples)

    def __getitem__(self, index):
        sample, target = self.samples[index]
        sample = convert_to_pil(sample)
        if self.transform is not None:
            sample = self.transform(sample)
        return sample, target

    def __repr__(self):
        fmt_str = 'Dataset ' + self.__class__.__name__ + '\n'
        fmt_str += '    Number of datapoints: {}\n'.format(self.__len__())
        fmt_str += '    Root Location: {}\n'.format(self.path)
        tmp = '    Transforms (if any): '
        fmt_str += '{0}{1}\n'.format(tmp, self.transform.__repr__().replace('\n', '\n' + ' ' * len(tmp)))
        tmp = '    Target Transforms (if any): '
        fmt_str += '{0}{1}'.format(tmp, self.target_transform.__repr__().replace('\n', '\n' + ' ' * len(tmp)))
        return fmt_str

    @staticmethod
    def find_classes(root):
        classes = [d for d in os.listdir(root) if os.path.isdir(os.path.join(root, d))]
        classes.sort()
        class_to_idx = {classes[i]: i for i in range(len(classes))}
        return classes, class_to_idx

class ZipDataset(data.Dataset):
    def __init__(self, path, transform=None, prefix=''):
        super(ZipDataset, self).__init__()
        self.prefix = prefix
        self.path = os.path.expanduser(path)
        self.transform = transform
        self.samples = []
        with zipfile.ZipFile(self.path, 'r') as reader:
            classes, class_to_idx = self.find_classes(reader)
            fnames = sorted(reader.namelist())
        for fname in fnames:
            if self.is_directory(fname) or self.ignore(fname):
                continue
            target = self.get_target(fname)
            item = (fname, class_to_idx[target])
            self.samples.append(item)

    def __len__(self):
        return len(self.samples)

    def __getitem__(self, index):
        sample, target = self.samples[index]
        with zipfile.ZipFile(self.path, 'r') as reader:
            sample = reader.read(sample)
        sample = convert_to_pil(sample)
        if self.transform is not None:
            sample = self.transform(sample)
        return sample, target

    def __repr__(self):
        fmt_str = 'Dataset ' + self.__class__.__name__ + '\n'
        fmt_str += '    Number of datapoints: {}\n'.format(self.__len__())
        fmt_str += '    Root Location: {}\n'.format(self.path)
        tmp = '    Transforms (if any): '
        fmt_str += '{0}{1}\n'.format(tmp, self.transform.__repr__().replace('\n', '\n' + ' ' * len(tmp)))
        tmp = '    Target Transforms (if any): '
        fmt_str += '{0}{1}'.format(tmp, self.target_transform.__repr__().replace('\n', '\n' + ' ' * len(tmp)))
        return fmt_str

    def ignore(self, fname):
        if 'preprocessed.pkl' in fname:
            return True
        return False

    def is_directory(self, fname):
        if fname.startswith(self.prefix) and fname.endswith('/'):
            return True
        return False

    def get_target(self, fname):
        assert fname.startswith(f'{self.prefix}'), print(f'get_target error: {fname} does not start with train/n')
        return fname.split('/')[1]

    def find_classes(self, reader):
        classes = [self.get_target(name) for name in reader.namelist() if self.is_directory(name)]
        classes.sort()
        class_to_idx = {classes[i]: i for i in range(len(classes))}
        return classes, class_to_idx

class ReadZipImageThread(threading.Thread):
    def __init__(self, reader, fnames, class_to_idx, target_list):
        threading.Thread.__init__(self)
        self.reader = reader
        self.fnames = fnames
        self.target_list = target_list
        self.class_to_idx = class_to_idx

    def run(self):
        for fname in self.fnames:
            if InMemoryZipDataset.is_directory(fname):
                continue
            image = self.reader.read(fname)
            class_id = self.class_to_idx[InMemoryZipDataset.get_target(fname)]
            item = (image, class_id)
            self.target_list.append(item)

class InMemoryZipDataset(data.Dataset):
    def __init__(self, path, transform=None, num_workers=1):
        super(InMemoryZipDataset, self).__init__()
        self.path = os.path.expanduser(path)
        self.transform = transform
        self.samples = []
        reader = zipfile.ZipFile(self.path, 'r')
        classes, class_to_idx = self.find_classes(reader)
        fnames = sorted(reader.namelist())
        if num_workers == 1:
            for fname in fnames:
                if self.is_directory(fname):
                    continue
                target = self.get_target(fname)
                image = reader.read(fname)
                item = (image, class_to_idx[target])
                self.samples.append(item)
        else:
            num_files = len(fnames)
            threads = []
            res = [[] for i in range(num_workers)]
            num_per_worker = num_files // num_workers
            for i in range(num_workers):
                start_index = num_per_worker * i
                end_index = num_files if i == num_workers - 1 else (i+1) * num_per_worker
                thread = ReadZipImageThread(reader, fnames[start_index:end_index], class_to_idx, res[i])
                threads.append(thread)
            for thread in threads:
                thread.start()
            for thread in threads:
                thread.join()
            for item in res:
                self.samples += item
            del res, threads
            gc.collect()
        reader.close()

    def __len__(self):
        return len(self.samples)

    def __getitem__(self, index):
        sample, target = self.samples[index]
        sample = convert_to_pil(sample)
        if self.transform is not None:
            sample = self.transform(sample)
        return sample, target

    def __repr__(self):
        fmt_str = 'Dataset ' + self.__class__.__name__ + '\n'
        fmt_str += '    Number of datapoints: {}\n'.format(self.__len__())
        fmt_str += '    Root Location: {}\n'.format(self.path)
        tmp = '    Transforms (if any): '
        fmt_str += '{0}{1}\n'.format(tmp, self.transform.__repr__().replace('\n', '\n' + ' ' * len(tmp)))
        tmp = '    Target Transforms (if any): '
        fmt_str += '{0}{1}'.format(tmp, self.target_transform.__repr__().replace('\n', '\n' + ' ' * len(tmp)))
        return fmt_str

    @staticmethod
    def is_directory(fname):
        if fname.startswith('n') and fname.endswith('/'):
            return True
        return False

    @staticmethod
    def get_target(fname):
        assert fname.startswith('n')
        return fname.split('/')[0]

    @staticmethod
    def find_classes(fname):
        classes = [ZipDataset.get_target(name) for name in fname.namelist() if ZipDataset.is_directory(name)]
        classes.sort()
        class_to_idx = {classes[i]: i for i in range(len(classes))}
        return classes, class_to_idx

# class WrappedDataset(data.Dataset):
#     def __init__(self, dataset, indices, transform=None):
#         super(WrappedDataset, self).__init__()
#         self.dataset = dataset
#         self.indices = indices
#         self.transform = transform
#         self.target_transform = None
#         self.samples = []

#     def __len__(self):
#         return len(self.indices)

#     def __getitem__(self, index):
#         assert index in self.indices
#         sample, target = self.dataset[index]
#         if self.transform is not None:
#             sample = self.transform(sample)
#         return sample, target

#     def __repr__(self):
#         fmt_str = 'Dataset ' + self.__class__.__name__ + '\n'
#         fmt_str += '    Number of datapoints: {}\n'.format(self.__len__())
#         fmt_str += '    Root Dataset: {}\n'.format(self.dataset)
#         tmp = '    Transforms (if any): '
#         fmt_str += '{0}{1}\n'.format(tmp, self.transform.__repr__().replace('\n', '\n' + ' ' * len(tmp)))
#         tmp = '    Target Transforms (if any): '
#         fmt_str += '{0}{1}'.format(tmp, self.target_transform.__repr__().replace('\n', '\n' + ' ' * len(tmp)))
#         return fmt_str

# class ControllerDataset(torch.utils.data.Dataset):
#     def __init__(self, inputs, targets=None, train=True, sos_id=0, eos_id=0):
#         super(ControllerDataset, self).__init__()
#         if targets is not None:
#             assert len(inputs) == len(targets)
#         self.inputs = copy.deepcopy(inputs)
#         self.targets = copy.deepcopy(targets)
#         self.train = train
#         self.sos_id = sos_id
#         self.eos_id = eos_id

#     def __getitem__(self, index):
#         encoder_input = self.inputs[index]
#         encoder_target = None
#         if self.targets is not None:
#             encoder_target = [self.targets[index]]
#         if self.train:
#             decoder_input = [self.sos_id] + encoder_input[:-1]
#             sample = {
#                 'encoder_input': torch.LongTensor(encoder_input),
#                 'encoder_target': torch.FloatTensor(encoder_target),
#                 'decoder_input': torch.LongTensor(decoder_input),
#                 'decoder_target': torch.LongTensor(encoder_input),
#             }
#         else:
#             sample = {
#                 'encoder_input': torch.LongTensor(encoder_input),
#                 'decoder_target': torch.LongTensor(encoder_input),
#             }
#             if encoder_target is not None:
#                 sample['encoder_target'] = torch.FloatTensor(encoder_target)
#         return sample

#     def __len__(self):
#         return len(self.inputs)


text:
import tensorflow as tf
import numpy as np
import tensorflow.contrib.slim as slim
import cv2

# Some were adopted from
# https://github.com/tensorflow/models/tree/master/research/video_prediction
def average_gradients(tower_grads):
    """Calculate the average gradient for each shared variable across all towers.

  Note that this function provides a synchronization point across all towers.

  Args:
    tower_grads: List of lists of (gradient, variable) tuples. The outer list
      is over individual gradients. The inner list is over the gradient
      calculation for each tower.
  Returns:
     List of pairs of (gradient, variable) where the gradient has been averaged
     across all towers.
  """
    average_grads = []
    for grad_and_vars in zip(*tower_grads):
        # Note that each grad_and_vars looks like the following:
        #   ((grad0_gpu0, var0_gpu0), ... , (grad0_gpuN, var0_gpuN))
        grads = []
        for g, _ in grad_and_vars:
            # Add 0 dimension to the gradients to represent the tower.
            expanded_g = tf.expand_dims(g, 0)

            # Append on a 'tower' dimension which we will average over below.
            grads.append(expanded_g)

        # Average over the 'tower' dimension.
        grad = tf.concat(axis=0, values=grads)
        grad = tf.reduce_mean(grad, 0)

        # Keep in mind that the Variables are redundant because they are shared
        # across towers. So .. we will just return the first tower's pointer to
        # the Variable.
        v = grad_and_vars[0][1]
        grad_and_var = (grad, v)
        average_grads.append(grad_and_var)
    return average_grads

def mean_squared_error(true, pred):
    """L2 distance between tensors true and pred.

  Args:
    true: the ground truth image.
    pred: the predicted image.
  Returns:
    mean squared error between ground truth and predicted image.
  """
    return tf.reduce_sum(tf.square(true - pred)) / tf.to_float(tf.size(pred))

def weighted_mean_squared_error(true, pred, weight):
    """L2 distance between tensors true and pred.

  Args:
    true: the ground truth image.
    pred: the predicted image.
  Returns:
    mean squared error between ground truth and predicted image.
  """

    tmp = tf.reduce_sum(
        weight * tf.square(true - pred), axis=[1, 2],
        keep_dims=True) / tf.reduce_sum(
            weight, axis=[1, 2], keep_dims=True)
    return tf.reduce_mean(tmp)

def mean_L1_error(true, pred):
    """L2 distance between tensors true and pred.

  Args:
    true: the ground truth image.
    pred: the predicted image.
  Returns:
    mean squared error between ground truth and predicted image.
  """
    return tf.reduce_sum(tf.abs(true - pred)) / tf.to_float(tf.size(pred))

def weighted_mean_L1_error(true, pred, weight):
    """L2 distance between tensors true and pred.

  Args:
    true: the ground truth image.
    pred: the predicted image.
  Returns:
    mean squared error between ground truth and predicted image.
  """
    return tf.reduce_sum(tf.abs(true - pred) *
                         weight) / tf.to_float(tf.size(pred))

def cal_grad2_error(flo, image, beta):
    """
    Calculate the image-edge-aware second-order smoothness loss for flo
    """

    def gradient(pred):
        D_dy = pred[:, 1:, :, :] - pred[:, :-1, :, :]
        D_dx = pred[:, :, 1:, :] - pred[:, :, :-1, :]
        return D_dx, D_dy

    img_grad_x, img_grad_y = gradient(image)
    weights_x = tf.exp(-10.0 * tf.reduce_mean(
        tf.abs(img_grad_x), 3, keep_dims=True))
    weights_y = tf.exp(-10.0 * tf.reduce_mean(
        tf.abs(img_grad_y), 3, keep_dims=True))

    dx, dy = gradient(flo)
    dx2, dxdy = gradient(dx)
    dydx, dy2 = gradient(dy)

    return (tf.reduce_mean(beta*weights_x[:,:, 1:, :]*tf.abs(dx2)) + \
           tf.reduce_mean(beta*weights_y[:, 1:, :, :]*tf.abs(dy2))) / 2.0

def cal_grad2_error_mask(flo, image, beta, mask):
    """
    Calculate the image-edge-aware second-order smoothness loss for flo
    within the given mask
    """

    def gradient(pred):
        D_dy = pred[:, 1:, :, :] - pred[:, :-1, :, :]
        D_dx = pred[:, :, 1:, :] - pred[:, :, :-1, :]
        return D_dx, D_dy

    img_grad_x, img_grad_y = gradient(image)
    weights_x = tf.exp(-10.0 * tf.reduce_mean(
        tf.abs(img_grad_x), 3, keep_dims=True))
    weights_y = tf.exp(-10.0 * tf.reduce_mean(
        tf.abs(img_grad_y), 3, keep_dims=True))

    dx, dy = gradient(flo)
    dx2, dxdy = gradient(dx)
    dydx, dy2 = gradient(dy)

    return (tf.reduce_mean(beta*weights_x[:,:, 1:, :]*tf.abs(dx2) * mask[:, :, 1:-1, :]) + \
           tf.reduce_mean(beta*weights_y[:, 1:, :, :]*tf.abs(dy2) * mask[:, 1:-1, :, :])) / 2.0

def SSIM(x, y):
    C1 = 0.01**2
    C2 = 0.03**2

    mu_x = slim.avg_pool2d(x, 3, 1, 'VALID')
    mu_y = slim.avg_pool2d(y, 3, 1, 'VALID')

    sigma_x = slim.avg_pool2d(x**2, 3, 1, 'VALID') - mu_x**2
    sigma_y = slim.avg_pool2d(y**2, 3, 1, 'VALID') - mu_y**2
    sigma_xy = slim.avg_pool2d(x * y, 3, 1, 'VALID') - mu_x * mu_y

    SSIM_n = (2 * mu_x * mu_y + C1) * (2 * sigma_xy + C2)
    SSIM_d = (mu_x**2 + mu_y**2 + C1) * (sigma_x + sigma_y + C2)

    SSIM = SSIM_n / SSIM_d

    return tf.clip_by_value((1 - SSIM) / 2, 0, 1)

def deprocess_image(image):
    # Assuming input image is float32
    return tf.image.convert_image_dtype(image, dtype=tf.uint8)

def preprocess_image(image):
    # Assuming input image is uint8
    image = tf.image.convert_image_dtype(image, dtype=tf.float32)
    return image

def charbonnier_loss(x,
                     mask=None,
                     truncate=None,
                     alpha=0.45,
                     beta=1.0,
                     epsilon=0.001):
    """Compute the generalized charbonnier loss of the difference tensor x.
    All positions where mask == 0 are not taken into account.
    Args:
        x: a tensor of shape [num_batch, height, width, channels].
        mask: a mask of shape [num_batch, height, width, mask_channels],
            where mask channels must be either 1 or the same number as
            the number of channels of x. Entries should be 0 or 1.
    Returns:
        loss as tf.float32
    """
    batch, height, width, channels = tf.unstack(tf.shape(x))
    normalization = tf.cast(batch * height * width * channels, tf.float32)

    error = tf.pow(tf.square(x * beta) + tf.square(epsilon), alpha)

    if mask is not None:
        error = tf.multiply(mask, error)

    if truncate is not None:
        error = tf.minimum(error, truncate)

    return tf.reduce_sum(error) / normalization

max_stars_repo_name: baidu-research/UnDepthflow

max_issues_repo_name: baidu-research/UnDepthflow

max_forks_repo_name: baidu-research/UnDepthflow


text:
import numpy as np

from park import logger

def clip_obs(obs, obs_low, obs_high):
    # TODO: this supports 1D only, need to add a iterator in each space type
    assert len(obs.shape) == 1
    # fit in observation space
    for i in range(obs.shape[0]):
        if obs[i] > obs_high[i]:
            logger.warn('Observation at index ' + str(i) +
                ' has value ' + str(obs_arr[i]) +
                ', which is larger than obs_high ' +
                str(obs_high[i]))
            obs[i] = obs_high[i]
        if obs[i] < obs_low[i]:
            logger.warn('Observation at index ' + str(i) +
                ' has value ' + str(obs_arr[i]) +
                ', which is lower than obs_low ' +
                str(obs_low[i]))
            obs[i] = obs_low[i]


text:
import cv2
import sys
import os
import numpy as np

class yolov3Manager:
    def __init__(self, model_weights = "yolov3-tiny.weights", model_configuration = "yolov3-tiny.cfg", classes_file = "coco.names"):
        try:
            base_path = sys._MEIPASS
        except:
            base_path = os.path.abspath(".")

        self.model_weights = os.path.join(base_path, model_weights)
        self.model_configuration = os.path.join(base_path, model_configuration)
        self.classes_file = os.path.join(base_path, classes_file)

        self.net = cv2.dnn.readNetFromDarknet(self.model_configuration, self.model_weights)
        self.net.setPreferableBackend(cv2.dnn.DNN_BACKEND_OPENCV)
        self.net.setPreferableTarget(cv2.dnn.DNN_TARGET_CPU)

    def process_image(self, image, confidence_threshold = 0.5, nonmax_threshold = 0.4):
        # https://www.learnopencv.com/deep-learning-based-object-detection-using-yolov3-with-opencv-python-c/

        output = self.get_output(image)

        chosen_classes = []
        confidences = []
        boxes = []

        image_height = image.shape[0]
        image_width = image.shape[1]

        for out in output:
            for detection in out:

                confidence = float(detection[4])
                if (confidence > confidence_threshold):

                    chosen_class = np.argmax(detection[5:])
                    center_x = int(detection[0] * image_width)
                    center_y = int(detection[1] * image_height)

                    width = max(0,int(detection[2] * image_width))
                    height = max(0,int(detection[3] * image_height))
                    x = max(0,int(center_x - (width/2)))
                    y = max(0,int(center_y - (height/2)))

                    chosen_classes.append(chosen_class)
                    confidences.append(confidence)
                    boxes.append([x, y, width, height])

        if len(confidences) != 0:
            indices = cv2.dnn.NMSBoxes(boxes, confidences, confidence_threshold, nonmax_threshold)
        else:
            return []

        retVal = []
        for i in indices:
            retVal.append(tuple(boxes[i[0]]))
        return retVal

    def get_output(self, image):

        image_height = 416#image.shape[0]
        image_width = 416#image.shape[1]

        blob = cv2.dnn.blobFromImage(image, 1/255,(image_width, image_height), [0,0,0], 1, crop=False)

        self.net.setInput(blob)

        output = self.net.forward(self.net.getUnconnectedOutLayersNames())
        return output

max_stars_repo_name: BryceGo/NoFace

max_issues_repo_name: BryceGo/NoFace

max_forks_repo_name: BryceGo/NoFace


text:
# load the libraries
from PIL import Image
from keras.layers import Dense, Input, Conv2D, LSTM, MaxPool2D, UpSampling2D
from sklearn.model_selection import train_test_split
from keras.callbacks import EarlyStopping
from keras.utils import to_categorical
from numpy import argmax, array_equal
import matplotlib.pyplot as plt
from keras.models import Model
from imgaug import augmenters
from random import randint
import pandas as pd
import numpy as np

# Dataset Prepration

# read dataset
train = pd.read_csv("../input/fashion-mnist_train.csv")
train_x = train[list(train.columns)[1:]].values
train_y = train['label'].values

# normalize and reshape the predictors
train_x = train_x / 255

# create train and validation datasets
train_x, val_x, train_y, val_y = train_test_split(
    train_x, train_y, test_size=0.2)

# reshape the inputs
train_x = train_x.reshape(-1, 784)
val_x = val_x.reshape(-1, 784)

# Create Autoencoder architecture

# input layer
input_layer = Input(shape=(784,))

# encoding architecture
encode_layer1 = Dense(1500, activation='relu')(input_layer)
encode_layer2 = Dense(1000, activation='relu')(encode_layer1)
encode_layer3 = Dense(500, activation='relu')(encode_layer2)

# latent view
latent_view = Dense(10, activation='sigmoid')(encode_layer3)

# decoding architecture
decode_layer1 = Dense(500, activation='relu')(latent_view)
decode_layer2 = Dense(1000, activation='relu')(decode_layer1)
decode_layer3 = Dense(1500, activation='relu')(decode_layer2)

# output layer
output_layer = Dense(784)(decode_layer3)

model = Model(input_layer, output_layer)

model.summary()

# early stopping callback

model.compile(optimizer='adam', loss='mse')
early_stopping = EarlyStopping(
    monitor='val_loss', min_delta=0, patience=10, verbose=1, mode='auto')
model.fit(train_x, train_x, epochs=20, batch_size=2048,
          validation_data=(val_x, val_x), callbacks=[early_stopping])

preds = model.predict(val_x)

# Inputs : Actual Images

f, ax = plt.subplots(1, 5)
f.set_size_inches(80, 40)
for i in range(5):
    ax[i].imshow(val_x[i].reshape(28, 28))
plt.show()

# Predicted : Autoencoder Output

f, ax = plt.subplots(1, 5)
f.set_size_inches(80, 40)
for i in range(5):
    ax[i].imshow(preds[i].reshape(28, 28))
plt.show()

reason: import numpy,from numpy


text:
import sys
import os
import random
import numpy as np
import cv2
import time
from datetime import timedelta
import torch
from model import TASED_v2
from loss import KLDLoss
from dataset import DHF1KDataset, InfiniteDataLoader
from itertools import islice
import matplotlib.pyplot as plt
from PIL import Image
from scipy.ndimage import gaussian_filter
import csv

def main():
    ''' concise script for training '''
    # optional two command-line arguments
    path_indata = 'Atari_dataset'
    path_output = 'output'
    shuffle_data = True

    if len(sys.argv) > 1:
        path_indata = sys.argv[1]
        if len(sys.argv) > 2:
            path_output = sys.argv[2]

    # we checked that using only 2 gpus is enough to produce similar results
    num_gpu = 1 #2 on Server
    pile = 5
    batch_size = 1 #6 on Server
    num_iters = 1000
    len_temporal = 32
    file_weight = 'TASED_updated.pt'
    path_output = os.path.join(path_output, time.strftime("%m-%d_%H-%M-%S"))
    if not os.path.isdir(path_output):
        os.makedirs(path_output)
    if shuffle_data or not os.path.isfile('Atari_num_frame_train.csv'):
        split_train_test_set()

    model = TASED_v2()

    # load the weight file and copy the parameters
    if os.path.isfile(file_weight):
        print ('loading weight file')
        weight_dict = torch.load(file_weight)
        model_dict = model.state_dict()
        for name, param in weight_dict.items():
            if 'module' in name:
                name = '.'.join(name.split('.')[1:])
            if 'base.' in name:
                bn = int(name.split('.')[1])
                sn_list = [0, 5, 8, 14]
                sn = sn_list[0]
                if bn >= sn_list[1] and bn < sn_list[2]:
                    sn = sn_list[1]
                elif bn >= sn_list[2] and bn < sn_list[3]:
                    sn = sn_list[2]
                elif bn >= sn_list[3]:
                    sn = sn_list[3]
                name = '.'.join(name.split('.')[2:])
                name = 'base%d.%d.'%(sn_list.index(sn)+1, bn-sn)+name
            if name in model_dict:
                if param.size() == model_dict[name].size():
                    model_dict[name].copy_(param)
                else:
                    print (' size? ' + name, param.size(), model_dict[name].size())
            else:
                print (' name? ' + name)

        print (' loaded')
    else:
        print ('weight file?')

    # parameter setting for fine-tuning
    params = []
    lr1 = 0.002 #0.0001 David
    lr2 = 0.05 #0.01 David
    for key, value in dict(model.named_parameters()).items():
        if 'convtsp' in key:
            params += [{'params':[value], 'key':key+'(new)'}]
        else:
            params += [{'params':[value], 'lr':lr1, 'key':key}] #0.001

    optimizer = torch.optim.SGD(params, lr=lr2, momentum=0.9, weight_decay=2e-7) #lr = 0.1
    lr_adaption = 0.1
    criterion = KLDLoss()

    model = model.cuda()
    model = torch.nn.DataParallel(model, device_ids=range(num_gpu))
    torch.backends.cudnn.benchmark = False
    model.train()

    train_loader = InfiniteDataLoader(DHF1KDataset(path_indata, len_temporal), batch_size=batch_size, shuffle=True, num_workers=0) #was 24, 0 means that data is loaded in the main process

    loss_statistic = []
    averaged_loss_statistic = []
    index_statistic = []

    i, step = 0, 0
    loss_sum = 0
    start_time = time.time()
    for clip, annt, (file_name, picture_name) in islice(train_loader, num_iters*pile):
        with torch.set_grad_enabled(True):
            output = model(clip.cuda())
            loss = criterion(output, annt.cuda())

        loss_sum += loss.item()
        loss.backward()
        if (i+1) % pile == 0:
            optimizer.step()
            optimizer.zero_grad()
            step += 1

            # whole process takes less than 3 hours
            print ('iteration: [%4d/%4d], loss: %.4f, %s' % (step, num_iters, loss_sum/pile, timedelta(seconds=int(time.time()-start_time))), flush=True)

            loss_statistic.append(loss_sum/pile)

            visualize(output, path_indata, file_name[0], picture_name[0], step)
            plt.ylabel('Loss')
            plt.xlabel(path_indata + ', weights: ' + file_weight + ', lr/optim = ' + str((lr1, lr2)))
            plt.plot(loss_statistic, color='b')
            if step==1:
                plt.show(block=False)
            else:
                plt.draw()
            plt.pause(0.00001)

            if step%10==0:
                averaged_loss_statistic.append(sum(loss_statistic[step-10:step])/10)
                index_statistic.append(step)

            loss_sum = 0
            # adjust learning rate
            if step in [250, 750, 950]:
                for opt in optimizer.param_groups:
                    if 'new' in opt['key']:
                        opt['lr'] *= lr_adaption   #0.1

            if step % 25 == 0:
                try:
                    if not os.path.exists(os.path.join(path_output, 'weights')):
                        os.makedirs(os.path.join(path_output, 'weights'))
                except OSError:
                    print('Error: Creating directory of data')
                torch.save(model.state_dict(), os.path.join(path_output, 'weights', 'iter_%04d.pt' % step))

        i += 1
    torch.save(model.state_dict(), os.path.join(path_indata, 'produced_weight_file.pt'))

    print('plotten')
    #plt.plot(loss_statistic)
    plt.savefig(os.path.join(path_indata, "loss.png"))

    plt.plot(index_statistic, averaged_loss_statistic, color='r')
    plt.ylabel('Averaged loss')
    plt.xlabel(path_indata + ', weights: ' + file_weight + ', lr/optim = ' + str((lr1, lr2)))
    plt.savefig(os.path.join(path_indata, "averaged_loss.png"))

    os.system('python run_saliency_metrics.py')

def split_train_test_set():
    path_full = 'Atari_num_frame_FullData.csv'
    path_train = 'Atari_num_frame_train.csv'
    path_test = 'Atari_num_frame_testing.csv'

    list_num_frame = [int(row[0]) for row in csv.reader(open(path_full, 'r'))]
    total_len = len(list_num_frame)
    half_len = int(np.floor(total_len/2)) #for 50/50 split
    idx = range(1, total_len+1)
    z = list(zip(list_num_frame, idx))

    #comment this out for training on ~ highscore data
    random.shuffle(z)

    list_num_frame, idx = zip(*z)

    test_list, test_idx = list_num_frame[:half_len], idx[:half_len]
    train_list, train_idx = list_num_frame[half_len:], idx[half_len:]
    train_strings = []
    test_strings = []

    for i in range(len(train_list)):
        train_strings.append(str(train_list[i])+',%04d'%train_idx[i])
    for i in range(len(test_list)):
        test_strings.append(str(test_list[i])+',%04d'%test_idx[i])

    with open(path_train, 'w') as file:
        for line in train_strings:
            file.write(line)
            file.write('\n')
    with open(path_test, 'w') as file:
        for line in test_strings:
            file.write(line)
            file.write('\n')

    return 0

def visualize(output, path_indata, file_name, picture_name, step):
    # print(output.shape)  ---> torch.Size([1, 224, 384])
    # print(output.cpu().data.numpy())
    np_array = (output.cpu().data[0].numpy()*255.).astype(np.int)/255.
    # print(np_array.shape)    ---> (224, 384)
    np_array = gaussian_filter(np_array, sigma=7)
    np_array = (np_array/np.max(np_array)*255.).astype(np.uint8)
    mask = Image.fromarray(np_array)  # gives a 384x224 Image object
    path_to_clip = os.path.join(path_indata, 'video', file_name, picture_name)
    video_img = cv2.resize(cv2.imread(path_to_clip, cv2.IMREAD_COLOR), (384, 224))
    video_img = Image.fromarray(video_img)
    red_img = Image.new('RGB', (384, 224), (0, 0, 255))
    video_img.paste(red_img, mask=mask)

    open_cv_image = cv2.resize(np.array(video_img), (160, 210))

    output_path = os.path.join(path_indata, 'training_visualized')
    try:
        if not os.path.exists(output_path):
            os.makedirs(output_path)
    except OSError:
        print('Error: Creating directory of data')
    cv2.imwrite(os.path.join(output_path, '%06d.png' % (step)), open_cv_image)
    """cv2.imshow('step', open_cv_image)
    cv2.waitKey(3000)
    cv2.destroyAllWindows()"""
    return 0

if __name__ == '__main__':
    main()

max_stars_repo_name: njoel-ethz/saliency-rl

max_issues_repo_name: njoel-ethz/saliency-rl

max_forks_repo_name: njoel-ethz/saliency-rl


text:
"""
Script to generate text from an already trained network (with lstm_train.py)
--By word--

It is necessary to at least provide the trained model and the vocabulary file
(generated also by lstm_train.py).
"""

import argparse
import numpy as np
import re
from keras.models import load_model

def validate_seed(vocabulary, seed):
    """Validate that all the words in the seed are part of the vocabulary"""
    print("\nValidating that all the words in the seed are part of the vocabulary: ")
    seed_words = seed.split(" ")
    valid = True
    for w in seed_words:
        print(w, end="")
        if w in vocabulary:
            print(" ✓ in vocabulary")
        else:
            print(" ✗ NOT in vocabulary")
            valid = False
    return valid

# Functions from keras-team/keras/blob/master/examples/lstm_text_generation.py
def sample(preds, temperature=1.0):
    # helper function to sample an index from a probability array
    preds = np.asarray(preds).astype('float64')
    preds = np.log(preds) / temperature
    exp_preds = np.exp(preds)
    preds = exp_preds / np.sum(exp_preds)
    probas = np.random.multinomial(1, preds, 1)
    return np.argmax(probas)

def generate_text(model, indices_word, word_indices, seed,
                  sequence_length, diversity, quantity):
    """
    Similar to lstm_train::on_epoch_end
    Used to generate text using a trained model

    :param model: the trained Keras model (with model.load)
    :param indices_word: a dictionary pointing to the words
    :param seed: a string to be used as seed (already validated and padded)
    :param sequence_length: how many words are given to the model to generate
    :param diversity: is the "temperature" of the sample function (usually between 0.1 and 2)
    :param quantity: quantity of words to generate
    :return: Nothing, for now only writes the text to console
    """
    sentence = seed.split(" ")
    print("----- Generating text")
    print('----- Diversity:' + str(diversity))
    print('----- Generating with seed:\n"' + seed)

    print(seed)
    for i in range(quantity):
        x_pred = np.zeros((1, sequence_length, len(vocabulary)))
        for t, word in enumerate(sentence):
            x_pred[0, t, word_indices[word]] = 1.

        preds = model.predict(x_pred, verbose=0)[0]
        next_index = sample(preds, diversity)
        next_word = indices_word[next_index]

        sentence = sentence[1:]
        sentence.append(next_word)

        print(" "+next_word, end="")
    print("\n")

if __name__ == "__main__":

    parser = argparse.ArgumentParser(
        description="Generate lyrics using the weights of a trained network.\n"
    )
    parser.add_argument("-v", "--vocabulary",
                        help="The path of vocabulary used by the network.",)
    parser.add_argument("-n", "--network",
                        help="The path of the trained network.",)
    parser.add_argument("-s", "--seed",
                        help="The seed used to generate the text. All the words should be part\n"
                             "of the vocabulary. Only the last SEQUENCE_LENGTH "
                             "words are considered",)
    parser.add_argument("-l", "--sequence_length",
                        help="The length of the sequence used for the training. Default is 10",
                        type=int,
                        default=10)
    parser.add_argument("-d", "--diversity",
                        help="The value of diversity. Usually a number between 0.1 and 2\n"
                             "Default is 0.5",
                        type=float,)
    parser.add_argument("-q", "--quantity",
                        help="Quantity of words to generate. Default is 50",
                        type=int,
                        default=50)
    args = parser.parse_args()

    vocabulary_file = args.vocabulary
    model_file = args.network
    seed = args.seed
    sequence_length = args.sequence_length
    diversity = args.diversity
    quantity = args.quantity

    if not vocabulary_file or not model_file:
        print('\033[91mERROR: At least --vocabulary and --network are needed\033[0m')
        exit(0)

    model = load_model(model_file)
    print("\nSummary of the Network: ")
    model.summary()

    vocabulary = open(vocabulary_file, "r").readlines()
    # remove the \n at the end of the word, except for the \n word itself
    vocabulary = [re.sub(r'(\S+)\s+', r'\1', w) for w in vocabulary]
    vocabulary = sorted(set(vocabulary))

    word_indices = dict((c, i) for i, c in enumerate(vocabulary))
    indices_word = dict((i, c) for i, c in enumerate(vocabulary))

    if validate_seed(vocabulary, seed):
        print("\nSeed is correct.\n")
        # repeat the seed in case is not long enough, and take only the last elements
        seed = " ".join((((seed+" ")*sequence_length)+seed).split(" ")[-sequence_length:])
        generate_text(
            model, indices_word, word_indices, seed, sequence_length, diversity, quantity
        )
    else:
        print('\033[91mERROR: Please fix the seed string\033[0m')
        exit(0)


text:
import grpc
import cv2
import time
import numpy as np
import imageio

import sys
sys.path.append("service/protos")
sys.path.append("./")

from tensorflow_serving.apis import predict_pb2
from tensorflow_serving.apis import prediction_service_pb2
from tensorflow.core.framework import tensor_pb2
from tensorflow.core.framework import tensor_shape_pb2
from tensorflow.core.framework import types_pb2

from config import Config

HOST = f'{Config.TF_SERVE_SERVER}:8500'
IMAGE_URL = 'https://tensorflow.org/images/blogs/serving/cat.jpg'

def main():
    # create prediction service client stub
    channel = grpc.insecure_channel(HOST)
    stub = prediction_service_pb2.PredictionServiceStub(channel)

    # read image into numpy array
    img = imageio.imread(IMAGE_URL)

    tt = time.time()
    # create request
    request = predict_pb2.PredictRequest()
    request.model_spec.name = 'ssd_mobilenet_v1_coco'
    request.model_spec.signature_name = 'serving_default'

    # convert to tensor proto and make request
    tensor_shape = [1]+list(img.shape)
    dims = [tensor_shape_pb2.TensorShapeProto.Dim(
        size=dim) for dim in tensor_shape]
    tensor_shape = tensor_shape_pb2.TensorShapeProto(dim=dims)
    tensor = tensor_pb2.TensorProto(
        dtype=types_pb2.DT_UINT8,
        tensor_shape=tensor_shape,
        int_val=list(img.reshape(-1)))
    request.inputs['inputs'].CopyFrom(tensor)
    resp = stub.Predict(request, 30.0)

    print('total time: {}s'.format(time.time() - tt))

if __name__ == '__main__':
    main()