text:
import numpy as np

import logbook

import sys

logbook.StreamHandler(sys.stdout).push_application()
logger = logbook.Logger('logbook')

def find_intensity_at_half_radius_sersic(n=1,
                                         r_e=10,
                                         l_total=5e10):
    """
    Computes the vale of intensity at half light radius for Sersic profile.
    The total luminosity of the galaxy should be provided by the user.

    Parameters
    ----------
    n : int, optional
        Sersic index, currently only n=1 is acceptable.
    r_e : int, optional
        The half light radius of the Sersic profile.
    l_total : float, optional
        The total luminosity of the galaxy.

    Returns
    -------
    numpy.float64
        The intensity of the luminosity of the galaxy at half light radius.

    Raises
    ------
    ValueError
        This function accepts only n=1 currently and for other values raises
        ValueError.
    """
    if n != 1:
        raise ValueError('values other than n = 1 are not acceptable yet')
    l_e = (1.678 ** 2) * l_total / (2 * (r_e ** 2) * (np.exp(1.678)) * np.pi)
    return l_e

def pixel_distance(stars):
    """
    This function measures the distance between to objects. First, it was
    designed to measure the distance betweeen stars in an image in units of
    pixels

    Parameters
    ----------
    stars : np.array
        An array representing the positions of the stars.

    Returns
    -------
    np.array
        Array which contains the distances between stars.

    Notes
    -----
        Alternatively, you can use the following suggestion for calculating
        the Eucliudean Distance between the pairs of coordinates. Link:
        https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.distance.cdist.html
        Scipy.spatial.distance.cdist has an advantage of having a variety of
        distance metrics.
    """
    distances = np.zeros((stars.shape[0], stars.shape[0]))
    logger.info('Start')
    for i in range(stars.shape[0]):
        if i < stars.shape[0] - 1:
            dis1 = stars[i, 0] - stars[i + 1:, 0]
            dis2 = stars[i, 1] - stars[i + 1:, 1]
            dis = np.sqrt(dis1 ** 2 + dis2 ** 2)
            distances[i][i + 1:] = dis
    logger.info('The End!')
    return distances

max_stars_repo_name: bazkiaei/AstroMQPy

max_issues_repo_name: bazkiaei/AstroMQPy

max_forks_repo_name: bazkiaei/AstroMQPy


text:
import ray
import pickle
import numpy as np
from pdb import set_trace as T
import numpy as np

from forge.blade import entity, core

from collections import defaultdict
from itertools import chain
from copy import deepcopy

from forge.blade.lib.enums import Palette
from forge.trinity.ascend import runtime, Timed

class Packet():
   def __init__(self):
      self.stim  = None
      self.reward = None
      self.done   = None

class Spawner:
   def __init__(self, config):
      self.config = config

      self.nEnt, self.nPop = config.NENT, config.NPOP
      self.popSz = self.nEnt // self.nPop

      self.ents = 0
      self.pops = defaultdict(int)
      self.palette = Palette(self.nPop)

   def spawn(self, realm, iden, pop, name):
      assert self.pops[pop] <= self.popSz
      assert self.ents      <= self.nEnt

      #Too many total entities
      if self.ents == self.nEnt:
         return None

      if self.pops[pop] == self.popSz:
         return None

      self.pops[pop] += 1
      self.ents += 1

      color = self.palette.colors[pop]
      ent   = entity.Player(self.config, iden, pop, name, color)
      assert ent not in realm.desciples

      r, c = ent.base.pos
      realm.desciples[ent.entID] = ent
      realm.world.env.tiles[r, c].addEnt(iden, ent)
      realm.world.env.tiles[r, c].counts[ent.base.population.val] += 1

   def cull(self, pop):
      assert self.pops[pop] >= 1
      assert self.ents      >= 1

      self.pops[pop] -= 1
      self.ents -= 1

      if self.pops[pop] == 0:
         del self.pops[pop]

class Realm(Timed):
   def __init__(self, config, idx, spawn):
      '''Neural MMO Environment

      Args:
         config: A Config specification object
         args: Hook for command line arguments
         idx: Index of the map file to load
      '''
      super().__init__()
      self.spawner = Spawner(config)
      self.spawn   = spawn

      self.world, self.desciples = core.Env(config, idx), {}
      self.config = config
      self.npop = config.NPOP

      self.worldIdx = idx
      self.tick = 0

      self.env = self.world.env
      self.values = None

   def clientData(self):
      '''Data packet used by the renderer'''

      if self.values is None:# and hasattr(self, 'sword'):
         #self.values = self.sword.anns[0].visVals()
         self.values = []

      ret = {
            'environment': self.world.env,
            'entities': dict((k, v.packet())
               for k, v in self.desciples.items()),
            'values': self.values
            }
      return ret

   def cullDead(self, dead):
      for entID in dead:
         ent = self.desciples[entID]
         r, c = ent.base.pos
         self.world.env.tiles[r, c].delEnt(entID)
         self.spawner.cull(ent.annID)
         del self.desciples[entID]

   def stepEnv(self):
      ents = list(chain(self.desciples.values()))

      #Stats
      self.world.step(ents, [])
      self.world.env.step()

      self.env = self.world.env.np()

   def getStim(self, ent):
      return self.world.env.stim(ent.base.pos, self.config.STIM)

   #Only saves the first action of each priority
   def prioritize(self, decisions):
      actions = defaultdict(dict)
      for entID, atns in decisions.items():
         for atnArgs in reversed(atns):
            priority = atnArgs.action.priority
            actions[priority][entID] = atnArgs
      return actions

   #Take actions
   def act(self, actions):
      for priority, tups in actions.items():
         for entID, atnArgs in tups.items():
            ent = self.desciples[entID]
            ent.act(self.world, atnArgs)

   def stepEnts(self, decisions):
      #Step all ents first
      for entID, actions in decisions.items():
         ent = self.desciples[entID]
         ent.step(self.world, actions)

      actions = self.prioritize(decisions)
      self.act(actions)

      #Finally cull dead. This will enable MAD melee
      dead = []
      packets = defaultdict(Packet)
      for entID in decisions.keys():
         ent = self.desciples[entID]
         packets[entID].stim = ent
         if self.postmortem(ent, dead):
            packets[entID].stim   = ent
            packets[entID].reward = -1
            packets[entID].done   = True
         else:
            packets[entID].reward = 0
            packets[entID].done   = False

      self.cullDead(dead)
      return packets

   def postmortem(self, ent, dead):
      entID = ent.entID
      if not ent.base.alive:
         dead.append(entID)
         return True
      return False

   def getStims(self, packets):
      for entID, ent in self.desciples.items():
         r, c = ent.base.pos
         tile = self.world.env.tiles[r, c].tex
         stim = self.getStim(ent)
         packets[entID].stim = (stim, ent)

      return packets

   @runtime
   def step(self, decisions):
      '''Take actions for all agents and return new observations

      Args:
         List of decisions

      Returns:
         observations, rewards, None, None
      '''

      self.tick += 1

      #Spawn an ent
      iden, pop, name = self.spawn()
      if iden is not None:
         self.spawner.spawn(self, iden, pop, name)

      packets = self.stepEnts(decisions)

      self.stepEnv()
      packets = self.getStims(packets)

      #Conform to gym
      stims   = [p.stim   for p in packets.values()]
      rewards = [p.reward for p in packets.values()]
      dones   = [p.done   for p in packets.values()]

      return stims, rewards, dones, None

   def reset(self):
      '''Stub for conformity with Gym. Calls step([]).

      The environment is persistent. Reset it only
      once upon initialization to obtain initial
      observations. If you must experiment with
      short lived environment instances, instantiate
      a new Realm instead of calling reset.'''
      assert self.tick == 0
      return self.step({})

ext: py

max_stars_repo_name: yutiansut/neural-mmo

max_issues_repo_name: yutiansut/neural-mmo

max_forks_repo_name: yutiansut/neural-mmo

reason: import numpy


text:
# -*- coding: UTF-8 -*-
"""
dtaidistance.dtw_ndim
~~~~~~~~~~~~~~~~~~~~~

Dynamic Time Warping (DTW) for N-dimensional series.

:author: Wannes Meert
:copyright: Copyright 2017-2018 KU Leuven, DTAI Research Group.
:license: Apache License, Version 2.0, see LICENSE for details.

"""
import os
import logging
import math
import numpy as np

logger = logging.getLogger("be.kuleuven.dtai.distance")
dtaidistance_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), os.pardir)

try:
    from . import dtw_c
except ImportError:
    # logger.info('C library not available')
    dtw_c = None

try:
    from tqdm import tqdm
except ImportError:
    logger.info('tqdm library not available')
    tqdm = None

def distance(s1, s2, window=None, max_dist=None,
             max_step=None, max_length_diff=None, penalty=None, psi=None,
             use_c=False):
    """Dynamic Time Warping using multidimensional sequences.

    cost = EuclideanDistance(s1[i], s2[j])

    See :py:meth:`dtaidistance.dtw.distance` for parameters.
    """
    if use_c:
        logger.error("No C version implemented (yet)")
        return
    r, c = len(s1), len(s2)
    if max_length_diff is not None and abs(r - c) > max_length_diff:
        return np.inf
    if window is None:
        window = max(r, c)
    if not max_step:
        max_step = np.inf
    else:
        max_step *= max_step
    if not max_dist:
        max_dist = np.inf
    else:
        max_dist *= max_dist
    if not penalty:
        penalty = 0
    else:
        penalty *= penalty
    if psi is None:
        psi = 0
    length = min(c + 1, abs(r - c) + 2 * (window - 1) + 1 + 1 + 1)
    # print("length (py) = {}".format(length))
    dtw = np.full((2, length), np.inf)
    # dtw[0, 0] = 0
    for i in range(psi + 1):
        dtw[0, i] = 0
    last_under_max_dist = 0
    skip = 0
    i0 = 1
    i1 = 0
    psi_shortest = np.inf
    for i in range(r):
        # print("i={}".format(i))
        # print(dtw)
        if last_under_max_dist == -1:
            prev_last_under_max_dist = np.inf
        else:
            prev_last_under_max_dist = last_under_max_dist
        last_under_max_dist = -1
        skipp = skip
        skip = max(0, i - max(0, r - c) - window + 1)
        i0 = 1 - i0
        i1 = 1 - i1
        dtw[i1, :] = np.inf
        j_start = max(0, i - max(0, r - c) - window + 1)
        j_end = min(c, i + max(0, c - r) + window)
        if dtw.shape[1] == c + 1:
            skip = 0
        if psi != 0 and j_start == 0 and i < psi:
            dtw[i1, 0] = 0
        for j in range(j_start, j_end):
            d = np.sum((s1[i] - s2[j]) ** 2)
            if d > max_step:
                continue
            assert j + 1 - skip >= 0
            assert j - skipp >= 0
            assert j + 1 - skipp >= 0
            assert j - skip >= 0
            dtw[i1, j + 1 - skip] = d + min(dtw[i0, j - skipp],
                                            dtw[i0, j + 1 - skipp] + penalty,
                                            dtw[i1, j - skip] + penalty)
            # print('({},{}), ({},{}), ({},{})'.format(i0, j - skipp, i0, j + 1 - skipp, i1, j - skip))
            # print('{}, {}, {}'.format(dtw[i0, j - skipp], dtw[i0, j + 1 - skipp], dtw[i1, j - skip]))
            # print('i={}, j={}, d={}, skip={}, skipp={}'.format(i,j,d,skip,skipp))
            # print(dtw)
            if dtw[i1, j + 1 - skip] <= max_dist:
                last_under_max_dist = j
            else:
                # print('above max_dist', dtw[i1, j + 1 - skip], i1, j + 1 - skip)
                dtw[i1, j + 1 - skip] = np.inf
                if prev_last_under_max_dist + 1 - skipp < j + 1 - skip:
                    # print("break")
                    break
        if last_under_max_dist == -1:
            # print('early stop')
            # print(dtw)
            return np.inf
        if psi != 0 and j_end == len(s2) and len(s1) - 1 - i <= psi:
            psi_shortest = min(psi_shortest, dtw[i1, length - 1])
    if psi == 0:
        d = math.sqrt(dtw[i1, min(c, c + window - 1) - skip])
    else:
        ic = min(c, c + window - 1) - skip
        vc = dtw[i1, ic - psi:ic + 1]
        d = min(np.min(vc), psi_shortest)
        d = math.sqrt(d)
    return d

def _distance_with_params(t):
    return distance(t[0], t[1], **t[2])

def warping_paths(s1, s2, window=None, max_dist=None,
                  max_step=None, max_length_diff=None, penalty=None, psi=None,):
    """
    Dynamic Time Warping (keep full matrix) using multidimensional sequences.

    cost = EuclideanDistance(s1[i], s2[j])

    See :py:meth:`dtaidistance.dtw.warping_paths` for parameters.
    """
    r, c = len(s1), len(s2)
    if max_length_diff is not None and abs(r - c) > max_length_diff:
        return np.inf
    if window is None:
        window = max(r, c)
    if not max_step:
        max_step = np.inf
    else:
        max_step *= max_step
    if not max_dist:
        max_dist = np.inf
    else:
        max_dist *= max_dist
    if not penalty:
        penalty = 0
    else:
        penalty *= penalty
    if psi is None:
        psi = 0
    dtw = np.full((r + 1, c + 1), np.inf)
    # dtw[0, 0] = 0
    for i in range(psi + 1):
        dtw[0, i] = 0
        dtw[i, 0] = 0
    last_under_max_dist = 0
    i0 = 1
    i1 = 0
    for i in range(r):
        if last_under_max_dist == -1:
            prev_last_under_max_dist = np.inf
        else:
            prev_last_under_max_dist = last_under_max_dist
        last_under_max_dist = -1
        i0 = i
        i1 = i + 1
        # print('i =', i, 'skip =',skip, 'skipp =', skipp)
        # jmin = max(0, i - max(0, r - c) - window + 1)
        # jmax = min(c, i + max(0, c - r) + window)
        # print(i,jmin,jmax)
        # x = dtw[i, jmin-skipp:jmax-skipp]
        # y = dtw[i, jmin+1-skipp:jmax+1-skipp]
        # print(x,y,dtw[i+1, jmin+1-skip:jmax+1-skip])
        # dtw[i+1, jmin+1-skip:jmax+1-skip] = np.minimum(x,
        #                                                y)
        for j in range(max(0, i - max(0, r - c) - window + 1), min(c, i + max(0, c - r) + window)):
            # print('j =', j, 'max=',min(c, c - r + i + window))
            d = np.sum((s1[i] - s2[j]) ** 2)
            if max_step is not None and d > max_step:
                continue
            # print(i, j + 1 - skip, j - skipp, j + 1 - skipp, j - skip)
            dtw[i1, j + 1] = d + min(dtw[i0, j],
                                     dtw[i0, j + 1] + penalty,
                                     dtw[i1, j] + penalty)
            # dtw[i + 1, j + 1 - skip] = d + min(dtw[i + 1, j + 1 - skip], dtw[i + 1, j - skip])
            if max_dist is not None:
                if dtw[i1, j + 1] <= max_dist:
                    last_under_max_dist = j
                else:
                    dtw[i1, j + 1] = np.inf
                    if prev_last_under_max_dist < j + 1:
                        break
        if max_dist is not None and last_under_max_dist == -1:
            # print('early stop')
            # print(dtw)
            return np.inf, dtw
    dtw = np.sqrt(dtw)
    if psi == 0:
        d = dtw[i1, min(c, c + window - 1)]
    else:
        ir = i1
        ic = min(c, c + window - 1)
        vr = dtw[ir-psi:ir+1, ic]
        vc = dtw[ir, ic-psi:ic+1]
        mir = np.argmin(vr)
        mic = np.argmin(vc)
        if vr[mir] < vc[mic]:
            dtw[ir-psi+mir+1:ir+1, ic] = -1
            d = vr[mir]
        else:
            dtw[ir, ic - psi + mic + 1:ic+1] = -1
            d = vc[mic]
    return d, dtw

def distance_matrix(s, max_dist=None, max_length_diff=None,
                    window=None, max_step=None, penalty=None, psi=None,
                    block=None, parallel=False,
                    use_c=False, show_progress=False):
    """Dynamic Time Warping distance matrix using multidimensional sequences.

    cost = EuclideanDistance(s1[i], s2[j])

    See :py:meth:`dtaidistance.dtw.distance_matrix` for parameters.
    """
    if parallel and not use_c:
        try:
            import multiprocessing as mp
            logger.info('Using multiprocessing')
        except ImportError:
            parallel = False
            mp = None
    else:
        mp = None
    dist_opts = {
        'max_dist': max_dist,
        'max_step': max_step,
        'window': window,
        'max_length_diff': max_length_diff,
        'penalty': penalty,
        'psi': psi
    }
    dists = None
    if max_length_diff is None:
        max_length_diff = np.inf
    large_value = np.inf
    logger.info('Computing distances')
    if use_c:
        logger.error("No C version available (yet)")
    if not use_c:
        logger.info("Compute distances in Python")
        if isinstance(s, np.ndarray) and len(s.shape) == 2:
            ss = [np.asarray(s[i]).reshape(-1) for i in range(s.shape[0])]
            s = ss
        if parallel:
            logger.info("Use parallel computation")
            dists = np.zeros((len(s), len(s))) + large_value
            if block is None:
                idxs = np.triu_indices(len(s), k=1)
            else:
                idxsl_r = []
                idxsl_c = []
                for r in range(block[0][0], block[0][1]):
                    for c in range(max(r + 1, block[1][0]), min(len(s), block[1][1])):
                        idxsl_r.append(r)
                        idxsl_c.append(c)
                idxs = (np.array(idxsl_r), np.array(idxsl_c))
            with mp.Pool() as p:
                dists[idxs] = p.map(_distance_with_params, [(s[r], s[c], dist_opts) for c, r in zip(*idxs)])
                # pbar = tqdm(total=int((len(s)*(len(s)-1)/2)))
                # for r in range(len(s)):
                #     dists[r,r+1:len(s)] = p.map(distance, [(s[r],s[c], dist_opts) for c in range(r+1,len(cur))])
                #     pbar.update(len(s) - r - 1)
                # pbar.close()
        else:
            logger.info("Use serial computation")
            dists = np.zeros((len(s), len(s))) + large_value
            if block is None:
                it_r = range(len(s))
            else:
                it_r = range(block[0][0], block[0][1])
            if show_progress:
                it_r = tqdm(it_r)
            for r in it_r:
                if block is None:
                    it_c = range(r + 1, len(s))
                else:
                    it_c = range(max(r + 1, block[1][0]), min(len(s), block[1][1]))
                for c in it_c:
                    if abs(len(s[r]) - len(s[c])) <= max_length_diff:
                        dists[r, c] = distance(s[r], s[c], **dist_opts)
    return dists


text:
"""
MolecularRNN model implementation from https://arxiv.org/abs/1905.13372
"""

import numpy as np
import torch
from collections import OrderedDict

from torch.nn.utils.rnn import pad_packed_sequence, pack_padded_sequence
from torch.nn import functional as F

from .openchem_model import OpenChemModel
from openchem.data.graph_utils import SmilesFromGraphs, RDMolFromGraphs, decode_adj_new
from openchem.data.utils import sanitize_smiles

import torchani
from torchani.nn import SpeciesConverter
from torchani import AEVComputer

from rdkit import Chem
from rdkit.Chem import AllChem

from openchem.utils.utils_3d import calculate_xyz, calculate_zmat

class MolecularRNNModel(OpenChemModel):
    def __init__(self, params):
        super(MolecularRNNModel, self).__init__(params)
        self.num_node_classes = params["num_node_classes"]
        self.num_edge_classes = params["num_edge_classes"]
        self.max_num_nodes = params["max_num_nodes"]
        self.start_node_label = params["start_node_label"]
        self.max_prev_nodes = params["max_prev_nodes"]
        self.label2atom = params["label2atom"]
        self.edge2type = params["edge2type"]
        self.restrict_min_atoms = params["restrict_min_atoms"]
        self.restrict_max_atoms = params["restrict_max_atoms"]
        self.use_external_crit = params.get("use_external_criterion", False)
        self.max_atom_bonds = params.get("max_atom_bonds", None)

        if self.num_edge_classes > 2:
            EdgeEmbedding = params["EdgeEmbedding"]
            edge_embedding_params = params["edge_embedding_params"]
            self.edge_emb = EdgeEmbedding(edge_embedding_params)
        else:
            self.edge_emb = None

        if self.num_node_classes > 2:
            NodeEmbedding = params["NodeEmbedding"]
            node_embedding_params = params["node_embedding_params"]
            self.node_emb = NodeEmbedding(node_embedding_params)

            NodeMLP = params["NodeMLP"]
            node_mlp_params = params["node_mlp_params"]
            self.node_mlp = NodeMLP(node_mlp_params)
        else:
            self.node_emb = None
            self.node_mlp = None

        # TODO: rewrite these two in OpenChem native style
        NodeRNN = params["NodeRNN"]
        node_rnn_params = params["node_rnn_params"]
        self.node_rnn = NodeRNN(**node_rnn_params)

        EdgeRNN = params["EdgeRNN"]
        edge_rnn_params = params["edge_rnn_params"]
        self.edge_rnn = EdgeRNN(**edge_rnn_params)

    @staticmethod
    def cast_inputs(batch, task, use_cuda):

        device = torch.device('cpu')
        if "d_values" in batch.keys():
            batch_input = dict(
                x=batch["x"].to(device=device),
                y=batch["y"].to(device=device),
                c_in=batch["c_in"].to(device=device),
                c_out=batch["c_out"].to(device=device),
                num_nodes=batch["num_nodes"].to(device=device),
                d_values=batch["d_values"].to(device=device, dtype=torch.float),
                aevs=batch["aevs"].to(device=device, dtype=torch.float)
            )
        else:
            batch_input = dict(
                x=batch["x"].to(device=device),
                y=batch["y"].to(device=device),
                c_in=batch["c_in"].to(device=device),
                c_out=batch["c_out"].to(device=device),
                num_nodes=batch["num_nodes"].to(device=device),
            )
        batch_target = [None]

        return batch_input, batch_target

    # TODO: implement required params
    # def get_required_params(self):
    #     return {}

    def forward(self, inp, eval=False, **kwargs):
        if self.use_external_crit and self.training:
            # TODO: check if .eval() creates any problems with batchnorm, dropout, etc.
            # generate batch
            # self.eval()
            with torch.no_grad():
                inp, smiles = self.forward_test()
            # self.train()
            output = self.forward_train(inp, **kwargs)
            sort_index = output["sort_index"]
            smiles = [smiles[i] for i in sort_index]
            adj = [inp["adj"][i] for i in sort_index]
            classes = [inp["classes"][i] for i in sort_index]
            return {
                "log_policy": output["logp"],
                "sizes": output["sizes"],
                "smiles": smiles,
                "adj": adj,
                "classes": classes,
                "loss": output["loss"]
            }
        elif self.training:
            return self.forward_train(inp, **kwargs)["loss"]
        else:
            batch, smiles = self.forward_test(**kwargs)
            return smiles

    def forward_test(self, batch_size=128, **kwargs):
        device = torch.device("cuda")

        # TODO: handle float type for x_step in case of no node embedding
        max_num_nodes = int(self.max_num_nodes)
        y_pred_long = torch.zeros(batch_size, max_num_nodes, self.max_prev_nodes, dtype=torch.long, device=device)
        c_pred_long = torch.zeros(batch_size, max_num_nodes, dtype=torch.long, device=device)
        x_step = torch.ones(batch_size, 1, self.max_prev_nodes, device=device)
        c_step = self.start_node_label * torch.ones(batch_size, 1, dtype=torch.long, device=device)

        self.node_rnn.hidden = self.node_rnn.init_hidden(batch_size, device)
        # start generation from second vertex
        prev_num = torch.ones(batch_size, device=device, dtype=torch.float)

        if self.max_atom_bonds is not None:
            if not isinstance(self.max_atom_bonds, torch.Tensor):
                self.max_atom_bonds = torch.tensor(self.max_atom_bonds, dtype=torch.float, device=device)
            max_valency_all = [self.max_atom_bonds[c_step.view(-1)]]
            valency_all = [torch.zeros_like(max_valency_all[0])]

        collect_node_hiddens = []
        for i in range(max_num_nodes):
            if self.edge_emb is not None:
                x_step = self.edge_emb(x_step.to(dtype=torch.long)).view(batch_size, 1, -1)
            if self.node_emb is not None:
                c_step = self.node_emb(c_step)
                x_step = torch.cat([x_step, c_step], dim=2)
            h = self.node_rnn(x_step, return_output_raw=False)
            collect_node_hiddens.append(h.data.cpu().numpy())
            # output.hidden = h.permute(1,0,2)
            self.edge_rnn.hidden = self.edge_rnn.init_hidden(h.size(0), device)
            self.edge_rnn.hidden = torch.cat([h.permute(1, 0, 2), self.edge_rnn.hidden[1:]], dim=0)
            x_step = torch.zeros(batch_size, 1, self.max_prev_nodes, device=device, dtype=torch.long)
            output_x_step = torch.ones(batch_size, 1, 1, device=device, dtype=torch.long)
            if self.node_mlp is not None:
                vert_pred = self.node_mlp(h.view(batch_size, -1))
                # [test_batch_size x num_classes]
                vert_pred = F.softmax(vert_pred, dim=1)
                # [test_batch_size x 1]
                c_step = torch.multinomial(vert_pred.view(-1, self.num_node_classes), 1)
                c_pred_long[:, i:i + 1] = c_step

            if self.max_atom_bonds is not None:
                max_valency = self.max_atom_bonds[c_step.view(-1)]
                valency = torch.zeros_like(max_valency)

            for j in range(min(self.max_prev_nodes, i + 1)):
                valid_edge = (j < prev_num)

                if self.edge_emb is not None:
                    output_x_step = self.edge_emb(output_x_step).view(batch_size, 1, -1)
                output_y_pred_step = self.edge_rnn(output_x_step).view(-1, self.num_edge_classes)

                output_y_pred_step = F.softmax(output_y_pred_step, dim=-1)

                if self.max_atom_bonds is not None:
                    for edge, etype in enumerate(self.edge2type):
                        prev_idx = -j - 1
                        prev_valency = valency_all[prev_idx]
                        prev_max_valency = max_valency_all[prev_idx]
                        allowed = ((valency + etype <= max_valency) &
                                   (prev_valency + etype <= prev_max_valency)) | ~valid_edge
                        allowed = allowed.to(dtype=torch.float)
                        output_y_pred_step[:, edge] *= allowed
                    output_y_pred_step = output_y_pred_step / \
                        output_y_pred_step.sum(dim=-1).view(-1, 1)

                output_x_step = torch.multinomial(output_y_pred_step, num_samples=1).view(-1, 1, 1)

                if self.max_atom_bonds is not None:
                    edge2type = torch.tensor(self.edge2type, dtype=torch.float, device=device)
                    types = edge2type[output_x_step.view(-1)]
                    types = types * valid_edge.to(dtype=types.dtype)
                    valency += types
                    valency_all[-j - 1] += types

                x_step[:, :, j:j + 1] = output_x_step * valid_edge.view(-1, 1, 1).to(dtype=x_step.dtype)
                # edge rnn keeps the state from current state

            if self.max_atom_bonds is not None:
                valency_all.append(valency)
                max_valency_all.append(max_valency)

            y_pred_long[:, i:i + 1, :] = x_step

            # all zeros in x_step mean that the state is terminal
            # zero out corresponding hidden states
            not_terminal = (x_step.sum(dim=-1) > 0).to(dtype=torch.float).view(-1)
            self.node_rnn.hidden *= not_terminal.view(1, -1, 1)

            prev_num = (prev_num * not_terminal) + 1

            terminal = (1 - not_terminal).to(dtype=torch.bool)
            n_terminal = terminal.sum().item()
            x_step[terminal] = torch.ones(n_terminal, 1, self.max_prev_nodes, device=device, dtype=torch.long)
            c_step[terminal] = self.start_node_label * torch.ones(n_terminal, 1, dtype=torch.long, device=device)

        y_pred_long = y_pred_long.to('cpu').numpy()
        c_pred_long = c_pred_long.to('cpu').numpy()
        smiles = []
        adj_all, adj_encoded_all, classes_all, len_all = [], [], [], []
        for i in range(batch_size):
            adj_encoded_full = y_pred_long[i]
            # these are vertices with no connections to previous
            #  i.e. last vertex for current sample, and first vertex for
            #  next sample
            anchors = np.where(np.all(adj_encoded_full == 0, axis=1))[0]
            cur = 0
            atoms_list = []
            adj_list = []
            for nxt in anchors:
                # slice adjacency matrix into connected components
                adj_encoded = adj_encoded_full[cur:nxt, :]
                start_c = self.start_node_label
                atoms = [
                    self.label2atom[start_c],
                ]
                atoms += [self.label2atom[c] for c in c_pred_long[i, cur:nxt]]

                if not (self.restrict_min_atoms <= len(atoms) <= self.restrict_max_atoms):
                    cur = nxt + 1
                    continue

                adj_encoded_all.append(torch.from_numpy(adj_encoded))
                classes_all.append(torch.from_numpy(c_pred_long[i, cur:nxt]))
                len_all.append(nxt - cur + 1)

                adj = decode_adj_new(adj_encoded)
                if self.num_edge_classes > 2:
                    adj_t = np.zeros(adj.shape, dtype="float32")
                    for bond_label, bond_type in enumerate(self.edge2type):
                        adj_t[adj == bond_label] = bond_type
                else:
                    adj_t = adj
                adj_all.append(adj_t)

                sstring, rdmol = SmilesFromGraphs(atoms, adj_t)
                atoms_list.append(atoms)
                adj_list.append(adj_t)
                smiles.append(sstring)

                cur = nxt + 1

        # # TODO: think how to avoid double sanitization
        # smiles, idx = sanitize_smiles(
        #     smiles,
        #     # min_atoms=self.restrict_min_atoms,
        #     # max_atoms=self.restrict_max_atoms,
        #     allowed_tokens=r'#()+-/123456789=@BCFHINOPS[\]cilnors ',
        #     logging="info"
        # )
        #
        # smiles = [s for i, s in enumerate(smiles) if i in idx]
        #
        # smiles, idx2 = sanitize_smiles(
        #     smiles,
        #     min_atoms=self.restrict_min_atoms,
        #     max_atoms=self.restrict_max_atoms,
        #     allowed_tokens=r'#()+-/123456789=@BCFHINOPS[\]cilnors ',
        #     logging="none"
        # )
        # idx = [idx[i] for i in idx2]
        # smiles = [s for i, s in enumerate(smiles) if i in idx2]
        idx = list(range(len(smiles)))

        if self.use_external_crit:
            adj_all = [s for i, s in enumerate(adj_all) if i in idx]
            adj_encoded_all = [s for i, s in enumerate(adj_encoded_all) if i in idx]
            classes_all = [s for i, s in enumerate(classes_all) if i in idx]
            len_all = [s for i, s in enumerate(len_all) if i in idx]

            max_len = max(len_all)
            x = torch.zeros(len(smiles), max_len, self.max_prev_nodes, dtype=torch.long)
            x[:, 0, :] = 1.
            y = torch.zeros(len(smiles), max_len, self.max_prev_nodes, dtype=torch.long)
            c_in = torch.zeros(len(smiles), self.max_num_nodes, dtype=torch.long)
            c_in[:, 0] = self.start_node_label
            c_out = -1 * torch.ones(len(smiles), self.max_num_nodes, dtype=torch.long)
            for i, (adj, classes, num_nodes) in enumerate(zip(adj_encoded_all, classes_all, len_all)):
                y[i, :num_nodes - 1, :] = adj
                x[i, 1:num_nodes, :] = adj
                c_in[i, 1:num_nodes] = classes
                c_out[i, :num_nodes - 1] = classes
            num_nodes = torch.tensor(len_all)
            batch = {
                "x": x.to(device),
                "y": y.to(device),
                "c_in": c_in.to(device),
                "c_out": c_out.to(device),
                "num_nodes": num_nodes.to(device),
                "adj": adj_all,
                "classes": classes_all
            }
        else:
            batch = None

        return batch, smiles#, atoms_list, adj_list, collect_node_hiddens

    def forward_train(self, inp, **kwargs):
        device = torch.device("cuda")

        x_unsorted = inp["x"]
        y_unsorted = inp["y"]
        c_in_unsorted = inp["c_in"]
        c_out_unsorted = inp["c_out"]
        num_nodes_unsorted = inp["num_nodes"]
        max_num_nodes = num_nodes_unsorted.max().item()
        x_unsorted = x_unsorted[:, 0:max_num_nodes, :]
        y_unsorted = y_unsorted[:, 0:max_num_nodes, :]
        c_in_unsorted = c_in_unsorted[:, 0:max_num_nodes]
        c_out_unsorted = c_out_unsorted[:, 0:max_num_nodes]

        self.node_rnn.hidden = self.node_rnn.init_hidden(batch_size=x_unsorted.size(0), device=device)

        # sort input samples according to sequence lengths
        num_nodes, sort_index = torch.sort(num_nodes_unsorted, 0, descending=True)
        num_nodes = num_nodes.cpu().numpy().tolist()
        x = torch.index_select(x_unsorted, 0, sort_index)
        y = torch.index_select(y_unsorted, 0, sort_index)
        c_in = torch.index_select(c_in_unsorted, 0, sort_index)
        c_out = torch.index_select(c_out_unsorted, 0, sort_index)

        # input, output for output rnn module
        # a smart use of pytorch builtin function:
        # pack variable -- b1_l1,b2_l1,...,b1_l2,b2_l2,...
        y_reshape = pack_padded_sequence(y, num_nodes, batch_first=True).data
        c_out = pack_padded_sequence(c_out, num_nodes, batch_first=True).data
        # reverse y_reshape, so that their (edge) lengths are sorted, add dimension
        y_reshape = torch.flip(y_reshape, (0, ))
        c_out = torch.flip(c_out, (0, ))
        y_reshape = y_reshape.view(y_reshape.size(0), y_reshape.size(1), 1)

        # TODO: make sure this is consistent with BFSGraphDataset
        if self.num_edge_classes == 2:
            output_x = torch.cat(
                (torch.ones(y_reshape.size(0), 1, 1, device=device), y_reshape[:, 0:-1, 0:1].to(dtype=torch.float)),
                dim=1)
        else:
            output_x = torch.cat(
                (torch.ones(y_reshape.size(0), 1, 1, device=device, dtype=torch.long), y_reshape[:, 0:-1, 0:1]), dim=1)
            output_x = self.edge_emb(output_x).squeeze()

        output_y = y_reshape
        # batch size for output module: sum(y_len)
        output_y_len = []
        output_y_len_bin = np.bincount(np.array(num_nodes))
        for i in range(len(output_y_len_bin) - 1, 0, -1):
            # count how many y_len is above i
            count_temp = np.sum(output_y_len_bin[i:])
            # put them in output_y_len; max value should not exceed y.size(2)
            output_y_len.extend([min(i, y.size(2))] * count_temp)

        # pack into variable
        x = x.to(device)
        y = y.to(device)
        c_in = c_in.to(device)
        c_out = c_out.to(device)
        output_x = output_x.to(device)
        output_y = output_y.to(device)

        if self.edge_emb is not None:
            B, T = x.shape[0], x.shape[1]
            x = self.edge_emb(x.to(dtype=torch.long)).view(B, T, -1)

        if self.node_emb is not None:
            # when classes are predicted, have to include previous class
            #  as an input to rnn()
            c_in = self.node_emb(c_in)
            x = torch.cat([x, c_in], dim=2)

        # if using ground truth to train
        # [B x maxL x max_prev_nodes]
        h = self.node_rnn(x, pack=True, input_len=num_nodes, return_output_raw=False)
        #print(h.size())
        # [sum(L) x hidden_dim]
        # get packed hidden vector
        h = pack_padded_sequence(h, num_nodes, batch_first=True).data
        # reverse h (consistent with output_x, output_y, c_out)
        h = torch.flip(h, (0, ))

        self.edge_rnn.hidden = self.edge_rnn.init_hidden(h.size(0), device)
        self.edge_rnn.hidden = torch.cat([h.unsqueeze(0), self.edge_rnn.hidden[1:]], dim=0)

        if self.node_mlp is not None:
            node_pred = self.node_mlp(h)

        y_pred = self.edge_rnn(output_x, pack=True, input_len=output_y_len)

        # internal criterion
        weights = torch.ones(*output_y.shape, dtype=torch.float, device=device)
        weights = pack_padded_sequence(weights, output_y_len, batch_first=True)
        weights = pad_packed_sequence(weights, batch_first=True)[0]

        if self.num_edge_classes == 2:
            # loss_edges = F.binary_cross_entropy(
            #     y_pred, output_y.to(torch.float))
            loss_edges = F.binary_cross_entropy_with_logits(y_pred, output_y.to(torch.float), reduction='none')
            n = weights.sum().clamp(min=1.)
            loss_edges = (loss_edges * weights).sum() / n
        else:
            loss_edges = F.cross_entropy(y_pred.reshape(-1, self.num_edge_classes),
                                         output_y.reshape(-1),
                                         reduction='none') / self.num_edge_classes
            n = weights.sum().clamp(min=1.)
            loss_edges = (loss_edges * weights.reshape(-1)).sum() / n
        if self.node_mlp is not None:
            loss_vertices = F.cross_entropy(node_pred, c_out, ignore_index=-1)
            loss_vertices = loss_vertices / self.num_node_classes
            loss = loss_edges + loss_vertices
        else:
            loss = loss_edges

        output = {"loss": loss}

        if self.use_external_crit:

            node_pred = F.log_softmax(node_pred, dim=-1)
            # valid edges are those that do not have all zeros in a row
            valid = torch.ones_like(y_pred)
            valid = pack_padded_sequence(valid, output_y_len, batch_first=True)
            valid = pad_packed_sequence(valid, batch_first=True)[0]
            y_pred = F.log_softmax(y_pred, dim=-1) * valid
            y_pred = y_pred.gather(2, output_y).squeeze(2)
            sum_y_pred = y_pred.sum(dim=1)

            valid = (c_out >= 0).to(node_pred.dtype)
            node_pred = node_pred.gather(1, c_out.view(-1, 1).clamp(min=0)).view(-1) * valid

            logp = node_pred + sum_y_pred
            logp = torch.flip(logp, (0, ))

            #logp = torch.flip(sum_y_pred, (0, ))
            #logp += node_pred

            output["logp"] = logp
            output["sizes"] = num_nodes
            output["sort_index"] = sort_index.to('cpu').numpy()

        return output

    def load_model(self, path):
        super(MolecularRNNModel, self).load_model(path)
        # TODO: load from original checkpoint if previous line fails
        # self.load_from_original_checkpoint(path)

    def load_from_original_checkpoint(self, paths):
        prefix_mapping = OrderedDict({
            "lstm.": "node_rnn.",
            "output.": "edge_rnn.",
            "classes_emb.": "node_emb.embedding.",
            "classes.deterministic_output.0": "node_mlp.layers.0",
            "classes.deterministic_output.2": "node_mlp.layers.1"
        })
        params_old = OrderedDict()
        for path in paths:
            params = torch.load(path)
            prefix = None
            for p in ["lstm", "output", "classes", "classes_emb"]:
                if p in path:
                    prefix = p
            assert prefix is not None
            params = {prefix + "." + k: v for k, v in params.items()}
            params_old.update(params)

        params_new = OrderedDict()
        for k, v in params_old.items():
            kn = None
            for po, pn in prefix_mapping.items():
                if k.startswith(po):
                    kn = pn + k[len(po):]
            if kn is None:
                raise AttributeError("Failed to map old key {}".format(k))
            params_new[kn] = v
        if len(params_new) != len(self.state_dict()):
            raise AttributeError("Incomplete mapping of old to new keys")

        self.load_state_dict(params_new)
        print("Successfully  loaded snapshot from original codebase!")

max_stars_repo_name: yishutu/OpenChem

max_issues_repo_name: yishutu/OpenChem

max_forks_repo_name: yishutu/OpenChem


text:
'''
Contains the example data for training spacy entity labels
'''
import numpy as np
from .model_utils import Importer

class TrainingData:
    '''
    Training data converted to NER format.
    '''
    def __init__(self):
        '''
        Consturctor for setting up training data.
        '''
        self.importer = Importer()

    def first_name_examples(self):
        '''
        Returns a spaCy model for training on first names
        '''
        first_names_df = self.importer.load_first_names()
        return np.array([(row['input'], {'entities': [(len(row['input']) - len(row['word']), len(row['input']), row['label'])]})
                         for index, row in first_names_df.iterrows()], dtype=object)

    def last_name_examples(self):
        '''
        Returns a spaCy model for training on last names
        '''
        last_names_df = self.importer.load_last_names()
        return np.array([(row['input'], {'entities': [(len(row['input']) - len(row['word']), len(row['input']), row['label'])]})
                         for index, row in last_names_df.iterrows()], dtype=object)

    def email_examples(self):
        '''
        Returns a spaCy model for training emails
        '''
        emails_df = self.importer.load_emails()
        return np.array([(row['input'], {'entities': [(len(row['input']) - len(row['word']), len(row['input']), row['label'])]})
                         for index, row in emails_df.iterrows()], dtype=object)

    def all_examples(self):
        '''
        Returns a spaCy model for all examples
        '''
        return np.concatenate((self.first_name_examples(), self.last_name_examples(), self.email_examples()))

max_stars_repo_name: AITestingOrg/abstractor

max_issues_repo_name: AITestingOrg/abstractor

max_forks_repo_name: AITestingOrg/abstractor


text:
from pdb import set_trace as T

import time
import os
import ray
import sys
import json
import argparse
import pickle
import numpy as np

from scipy.interpolate import interp1d
from collections import defaultdict

from functools import partial
from threading import Thread
from random import random

import bokeh
from bokeh.server.server import Server
from bokeh.models import ColumnDataSource
from bokeh.models import Band
from bokeh.models.widgets import RadioButtonGroup
from bokeh.layouts import layout

from bokeh.plotting import curdoc, figure, show
from bokeh.themes import Theme

from tornado import gen
from tornado.ioloop import IOLoop

from forge.blade.lib.enums import Neon
from forge.blade.systems.visualizer.config import *
from signal import signal, SIGTERM, SIGINT
from time import gmtime, strftime

def pickle_write(content, name, append=1):
    'function to open file, pickle dump, then close'
    f = open(name, "ab") if append else open(name, "wb")
    pickle.dump(content, f)
    f.close()

def pickle_read(name):
    'function to open file, pickle load, then close'
    f = open(name, "rb")
    ret = pickle.load(f)
    f.close()
    return ret

#Adapted from https://rigtorp.se/2011/01/01/rolling-statistics-numpy.html
def rolling_window(a, window, pad=True):
    a = np.array(a)
    if pad:
      a = np.pad(a, pad_width=(window-1, 0), mode='edge')
    shape = a.shape[:-1] + (a.shape[-1] - window + 1, window)
    strides = a.strides + (a.strides[-1],)
    return np.lib.stride_tricks.as_strided(a, shape=shape, strides=strides)

class Analytics:
    def __init__(self, config):
        """Visualizes a stream of data with threaded refreshing. To
        add items, initialize using 'keys' kwarg or add to packet in
        stream()
        Args:
            keys        : List of object names (str) to be displayed on market
            history_len : How far back to plot data
            scales      : Scales for visualization
            title       : Title of graph
            x           : Name of x axis data
            ylabel      : Name of y axis on plot
        """
        self.colors = []
        for color in [Neon.GREEN, Neon.CYAN, Neon.BLUE]:
           color = bokeh.colors.RGB(*(color.rgb))
           self.colors.append(color)

        # Market stores data in a dictionary of lists
        # MarketVisualizer stores the data for each view as a dictionary,
        # inside another dictionary with keys of each scale.

        self.history_len = config.HISTORY_LEN
        self.title       = config.TITLE

        self.ylabel      = config.YLABEL
        self.x           = config.XAXIS
        self.XAXIS       = config.XAXIS
        self.scales      = config.SCALES
        self.scale       = config.SCALES[0]
        self.title       = config.TITLE
        self.log         = config.LOG
        self.load        = config.LOAD_EXP
        self.filename    = config.NAME

        self.data        = defaultdict(list)
        self.dataSource  = {}
        self.keys        = 'lifetime'.split()
        for key in self.keys:
           self.dataSource[key] = [1, 2]
           self.dataSource[key+'_x'] = [1, 2]
           self.dataSource[key+'_lower'] = [1, 2]
           self.dataSource[key+'_upper'] = [1, 2]
           self.dataSource[key+'_smooth'] = [1, 2]

    def init(self, doc):
        # Source must only be modified through a stream
        self.source = ColumnDataSource(data=self.dataSource)

        # Enable theming
        theme = Theme(os.path.dirname(os.path.abspath(__file__)) + '/theme.yaml')
        doc.theme = theme
        self.doc = doc

        fig = figure(
           plot_width=960,
           plot_height=540,
           tools='xpan,xwheel_zoom, xbox_zoom, reset, save',
           title=self.title,
           x_axis_label=self.XAXIS,
           y_axis_label=self.ylabel)

        # Initialize plots
        for i, key in enumerate(self.keys):
            band = Band(
               source=self.source,
               base=key + '_x',
               lower=key+'_lower',
               upper=key+'_upper',
               level='annotation',
               line_color=self.colors[i],
               line_width=1.0,
               line_alpha=0.7,
               fill_color=self.colors[i],
               fill_alpha=0.3)
            fig.add_layout(band)

            fig.line(
                source=self.source,
                x=key+'_x',
                y=key+'_smooth',
                color=self.colors[i],
                legend_label=key)

        def switch_scale(attr, old, new):
            """Callback for RadioButtonGroup to switch tick scale
            and refresh document
            Args:
                attr: variable to be changed, in this case 'active'
                old: old index of active button
                new: new index of active button
            """
            self.scale = self.scales[new]
            self.source.data = self.data[self.scale]

        self.timescales = RadioButtonGroup(labels=[str(scale) for scale in self.scales],
                                           active=self.scales.index(self.scale))
        self.timescales.on_change('active', switch_scale)

        self.structure = layout([[self.timescales], [fig]])
        self.doc.add_root(self.structure)

        self.fig = fig

    def update(self, packet):
      for key, val in packet.items():
         assert type(val) == list, 'packet value must be a list'
         self.data[key] += val

      #Sort by length for aesthetic rendering order
      self.keys = sorted(self.keys, key=lambda k: -len(self.data[k]))

    def resample(self, nOut=350, w=20):
      for key in list(self.data.keys()):
         #Padded sliding window
         val     = self.data[key]
         sliding = rolling_window(val, w)

         #Stats
         smooth  = np.mean(sliding, axis=1)
         std     = np.std(sliding, axis=1)

         #Add keys
         self.data[key+'_x']      = np.arange(len(val))
         self.data[key+'_smooth'] = smooth
         self.data[key+'_lower']  = smooth - std
         self.data[key+'_upper']  = smooth + std

      for key, val in self.data.items():
         #Interpolate
         nIn            = len(val)
         xIn            = np.arange(nIn)
         xOut           = np.linspace(0, nIn, nOut)
         f              = interp1d(
                              x=xIn, y=val,
                              kind='nearest',
                              fill_value='extrapolate')

         self.data[key] = f(xOut)

    def stream(self):
        '''Wrapper function for source.stream to enable
        adding new items mid-stream. Overwrite graph
        with new figure if packet has different keys.
        Args:
            packet: dictionary of singleton lists'''

        # Stream items & append data if no new entries

        self.dataSource = dict(self.data.copy())
        if self.log:
           pickle_write(self.dataSource, self.filename)

        self.source.stream(self.dataSource, self.history_len)

        # Refreshes document to add new item
        self.doc.remove_root(self.structure)
        self.init(self.doc)

@ray.remote
class BokehServer:
    def __init__(self, middleman, config):
        """ Runs an asynchronous Bokeh data streaming server.

        Args:
           market : The market to visualize
           args   : Additional arguments
           kwargs : Additional keyword arguments
        """
        self.analytics = Analytics(config)
        self.middleman = middleman
        self.thread    = None

        server = Server(
                {'/': self.init},
                io_loop=IOLoop.current(),
                port=config.PORT,
                num_procs=1)

        server.start()
        self.server = server
        server.io_loop.add_callback(server.show, "/")
        server.io_loop.start()

    def init(self, doc):
        '''Initialize document and threaded update loop
        Args:
           doc: A Bokeh document
        '''
        self.analytics.init(doc)
        self.doc = doc

        self.thread = Thread(target=self.update, args=[])
        self.thread.start()
        self.started = True

    def update(self):
        '''Blocking update call to be run in a separate thread
        Ingests packets from a remote market and streams to Bokeh client'''
        self.n = 0
        while True:
            # Wait for thread to initialize
            time.sleep(0.05)
            if self.thread is None:
               continue

            # Get remote market data
            if ray.get(self.middleman.getShutdown.remote()):
                self.middleman.setData.remote(self.analytics.data)
                sys.exit(0)

            packet = ray.get(self.middleman.getData.remote())
            if packet is None:
               continue

            self.analytics.update(packet)
            self.analytics.resample()

            # Stream to Bokeh client
            self.doc.add_next_tick_callback(partial(self.stream))

    @gen.coroutine
    def stream(self):
        '''Stream current data buffer to Bokeh client'''
        # visu is visualizer object
        self.analytics.stream()

@ray.remote
class Middleman:
    def __init__(self):
        '''Remote data buffer for two processes to dump and recv data.
        Interacts with Market and BokehServer.
        This is probably not safe'''
        self.data = None
        self.shutdown = 0

    def getData(self):
        '''Get data from buffer
        Returns:
           data: From buffer
        '''
        data = self.data
        self.data = None
        return data

    def setData(self, data):
        '''Set buffer data
        Args:
           data: To set buffer
        '''
        self.data = data.copy()

    # Notify remotes of visualizer shutdown
    def setShutdown(self):
        self.shutdown = 1

    # Retreive shutdown status
    def getShutdown(self):
        return self.shutdown

#class Market:
#    def __init__(self, items, middleman):
#        '''Dummy market emulator
#        Args:
#           items     : List of item keys
#           middleman : A Middleman object'''
#        self.middleman = middleman
#        self.items     = items
#
#        self.keys = items
#        # Data contains market history for lowest values
#        self.data = {}
#
#        for i, key in enumerate(self.keys):
#            self.data[key] = 0
#
#    def update(self):
#        '''Updates market data and propagates to Bokeh server
#        # update data and send to middle man
#        Note: best to update all at once. Current version may cause bugs'''
#        for key, val in self.data.items():
#           self.data[key] = val + 0.2*(random() - 0.5)
#
#
#        # update if not shutting down visualizer
#        if not ray.get(middleman.getShutdown.remote()):
#            self.middleman.setData.remote(self.data)

max_stars_repo_name: narendasan/neural-mmo

max_issues_repo_name: ThomasCloarec/neural-mmo

max_forks_repo_name: ThomasCloarec/neural-mmo

reason: import numpy,from scipy


text:
#!/usr/bin/env python
# coding: utf-8

def printLinha(msg):
    '''
        Printing of a dividing line, containing an explanation str.
    '''
    texto = str(f'-=< {msg} >=-')
    tam = len(texto)
    print(f'{"-":^}' * tam)
    print(f'{texto:^{tam}}')
    print(f'{"-":^}' * tam)

def printLis(lista):
    '''
        Printing lists where indexes and values are shown.
    '''
    from time import sleep
    printLinha('Lista')
    for k, v in enumerate(lista):
        print(f'{k} -> {v}')

def printDic(dicionario):
    '''
        Dictionary printing, showing the keys and values of the items.
    '''
    printLinha('Dicionário')
    for kk, vv in dicionario.items():
        print(f'{kk} -> {vv}')
        if str(type(vv))[8:12] == 'list':
            printLis(vv)
            print('-'*10)

def timeProcess():
    '''
        Returns a tuple containing the time and date when the file was loaded.
    '''
    from datetime import datetime
    hora = str(f'{datetime.now().hour}h{datetime.now().minute}m{datetime.now().second}s')
    data = str(f'{datetime.now().year}-{datetime.now().month}-{datetime.now().day}')
    return hora, data

def setup_img_save(fonte = 15):
    '''
        Sets image parameters to default size 15.

            font.size
            legend.fontsize
            axes.labelsize
            axes.titlesize
            xtick.labelsize
            ytick.labelsize
    '''
    import matplotlib.pyplot as plt
    params = {
            'font.size': fonte,
            'legend.fontsize': fonte,
            'axes.labelsize': fonte,
            'axes.titlesize': fonte,
            'xtick.labelsize': fonte,
            'ytick.labelsize': fonte
    }
    plt.rcParams.update(params)

def img_save(campo, g=0, dpi=300, date='0000-0-0'):
    '''
        Saving png images with date.
    '''
    import matplotlib.pyplot as plt
    import os
    if campo:
        try:
            os.mkdir('figure')
        except OSError:
            print ("Creation of the directory figure failed")
        else:
            print ("Directory figure successfully created")
        # g += 1
        plt.savefig('figure/0'+str(g)+'-'+date+'.png', dpi=dpi, bbox_inches='tight')
        print('Save')
    else:
        print('Not save')
    plt.show()

def mask_corr_graphic(inic: int):
    '''
        The function receives an int which is the number of columns in the DataFrame.
        Return a numpy array mask with True and False for correlation plots.
    '''
    import numpy as np
    n = inic
    lista_fim = []
    for num in range(1, inic+1):
        n -= 1
        list_entra = [False]*num + [True]*n
        lista_fim.append(list_entra[:])
    return np.asarray(lista_fim)

def structured_confusion(test, prevision):
    '''
        Function to comprehensively organize and print a confusion matrix with two categories.

        Parameters:
        test => list-like with the original test values
        prevision => list-like with the values predicted by the model
    '''
    import pandas as pd
    from sklearn.metrics import confusion_matrix
    confu_print = pd.DataFrame(
        data=confusion_matrix(test, prevision),
        index=['Pred Pos','Pred Neg'],
        columns=['Meas Pos','Meas Neg'],
    )
    return confu_print

def meus_printer():
    '''
        Prints the particular functions.
    '''
    lista_importes = [
        "printLinha()",
        "printLis()",
        "printDic()",
        "timeProcess()",
        "import_started()",
        "img_save()",
        "setup_img_save()",
        "mask_corr_graphic()",
        "structured_confusion()",
    ]
    printLis(lista_importes)

def import_started():
    '''
        Returns a list containing automatic imports when loading Anaconda.
    '''
    lista_importes = [
        "import os",
        "import joblib",
        "import random",
        "import numpy as np",
        "import pandas as pd",
        "import seaborn as sns",
        "import matplotlib.pyplot as plt",
        "from meus.printer import *",
        "from datetime import datetime",
        "import statistics as stat",
    ]
    printLis(lista_importes)


text:
#*
# @file Different utility functions
# Copyright (c) Zhewei Yao, Amir Gholami, Sheng Shen
# All rights reserved.
# This file is part of AdaHessian library.
#
# AdaHessian is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# AdaHessian is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with adahessian.  If not, see <http://www.gnu.org/licenses/>.
#*

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tensorflow.python.eager import def_function
from tensorflow.python.framework import ops
from tensorflow.python.keras import backend_config
from tensorflow.python.keras.optimizer_v2 import optimizer_v2
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import control_flow_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import state_ops
from tensorflow.python.training import training_ops
from tensorflow.python.util.tf_export import keras_export

import abc
import contextlib
import functools

import six
import tensorflow as tf
import numpy as np
from tensorflow.python.distribute import distribution_strategy_context as distribute_ctx
from tensorflow.python.distribute import parameter_server_strategy
from tensorflow.python.distribute import reduce_util as ds_reduce_util
from tensorflow.python.distribute import values as ds_values
from tensorflow.python.eager import backprop
from tensorflow.python.eager import context
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.framework import tensor_util
from tensorflow.python.keras import backend
from tensorflow.python.keras import initializers
from tensorflow.python.keras.engine import base_layer_utils
from tensorflow.python.keras.optimizer_v2 import learning_rate_schedule
from tensorflow.python.keras.utils import generic_utils
from tensorflow.python.keras.utils import tf_utils
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import clip_ops
from tensorflow.python.ops import control_flow_ops
from tensorflow.python.ops import gradients
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import resource_variable_ops
from tensorflow.python.ops import variables as tf_variables
from tensorflow.python.platform import tf_logging as logging
from tensorflow.python.saved_model import revived_types
from tensorflow.python.training.tracking import base as trackable
from tensorflow.python.training.tracking import tracking
from tensorflow.python.util import nest
from tensorflow.python.util import tf_inspect
from tensorflow.python.util.tf_export import keras_export

class AdaHessian(optimizer_v2.OptimizerV2):

    _HAS_AGGREGATE_GRAD = True

    def __init__(self,
               learning_rate=0.1,
               beta_1=0.9,
               beta_2=0.999,
               epsilon=1e-4,
               weight_decay = 0.,
               hessian_power=1.0,
               name='AdaHessian',
               average_size_1d=None,
               average_size_2d=None,
               average_size_3d=-1,
               average_size_4d=-1,
               **kwargs):
        """Construct a new AdaHessian optimizer.
        Args:
            learning_rate: A `Tensor`, floating point value, or a schedule that is a
            `tf.keras.optimizers.schedules.LearningRateSchedule`, or a callable that
            takes no arguments and returns the actual value to use, The learning
            rate. Defaults to 0.1.
            beta_1: A float value or a constant float tensor, or a callable that takes
            no arguments and returns the actual value to use. The exponential decay
            rate for the 1st moment estimates. Defaults to 0.9.
            beta_2: A float value or a constant float tensor, or a callable that takes
            no arguments and returns the actual value to use, The exponential decay
            rate for the 2nd moment estimates. Defaults to 0.999.
            epsilon: A small constant for numerical stability. This epsilon is
            "epsilon hat" in the Kingma and Ba paper (in the formula just before
            Section 2.1), not the epsilon in Algorithm 1 of the paper. Defaults to
            1e-7.
            weight_decay: We are using AdamW's weight decay scheme. Defaults to 0.
            name: Optional name for the operations created when applying gradients.
            Defaults to "Adam".
            hessian_power: Hessian power to control the optimizer more similar to first/second
            order method (default: 1). You can also try 0.5. For some tasks we found this
            to result in better performance.
            **kwargs: keyword arguments. Allowed to be {`clipnorm`, `clipvalue`, `lr`,
            `decay`}. `clipnorm` is clip gradients by norm; `clipvalue` is clip
            gradients by value, `decay` is included for backward compatibility to
            allow time inverse decay of learning rate. `lr` is included for backward
            compatibility, recommended to use `learning_rate` instead.
            # average_size_{1,2,3,4}d:
                        None: use no spatial averaging
                        -1: use suggested spatial averaging (recommended for conv kernels)
                        >= 1: use customized size
        """

        super(AdaHessian, self).__init__(name, **kwargs)
        self._set_hyper('learning_rate', kwargs.get('lr', learning_rate))
        self._set_hyper('decay', self._initial_decay)
        self._set_hyper('beta_1', beta_1)
        self._set_hyper('beta_2', beta_2)
        self.epsilon = epsilon or backend_config.epsilon()
        self.weight_decay = weight_decay
        self.hessian_power = hessian_power
        self.average_size_1d = average_size_1d
        self.average_size_2d = average_size_2d
        self.average_size_3d = average_size_3d
        self.average_size_4d = average_size_4d

    def _create_slots(self, var_list):
        # Create slots for the first and second moments.
        # Separate for-loops to respect the ordering of slot variables from v1.
        for var in var_list:
            self.add_slot(var, 'm')
        for var in var_list:
            self.add_slot(var, 'v')

    def _prepare_local(self, var_device, var_dtype, apply_state):
        super(AdaHessian, self)._prepare_local(var_device, var_dtype, apply_state)

        local_step = math_ops.cast(self.iterations + 1, var_dtype)
        beta_1_t = array_ops.identity(self._get_hyper('beta_1', var_dtype))
        beta_2_t = array_ops.identity(self._get_hyper('beta_2', var_dtype))
        beta_1_power = math_ops.pow(beta_1_t, local_step)
        beta_2_power = math_ops.pow(beta_2_t, local_step)
        lr = (
            apply_state[(var_device, var_dtype)]['lr_t'] *
            (math_ops.sqrt(1 - beta_2_power) / (1 - beta_1_power)))
        apply_state[(var_device, var_dtype)].update(
            dict(
                lr=lr,
                epsilon=ops.convert_to_tensor_v2(self.epsilon, var_dtype),
                beta_1_t=beta_1_t,
                beta_1_power=beta_1_power,
                one_minus_beta_1_t=1 - beta_1_t,
                beta_2_t=beta_2_t,
                beta_2_power=beta_2_power,
                one_minus_beta_2_t=1 - beta_2_t))

    def set_weights(self, weights):
        params = self.weights
        # If the weights are generated by Keras V1 optimizer, it includes vhats
        # even without amsgrad, i.e, V1 optimizer has 3x + 1 variables, while V2
        # optimizer has 2x + 1 variables. Filter vhats out for compatibility.
        num_vars = int((len(params) - 1) / 2)
        if len(weights) == 3 * num_vars + 1:
            weights = weights[:len(params)]
        super(AdaHessian, self).set_weights(weights)

    def get_gradients_hessian(self, loss, params):
        """Returns gradients and Hessian of `loss` with respect to `params`.
        Arguments:
            loss: Loss tensor.
            params: List of variables.
        Returns:
            List of gradient and Hessian tensors.
        Raises:
            ValueError: In case any gradient cannot be computed (e.g. if gradient
            function not implemented).
        """
        params = nest.flatten(params)
        with backend.get_graph().as_default(), backend.name_scope(self._name +
                                                                    "/gradients"):
            grads = gradients.gradients(loss, params)
            for grad, param in zip(grads, params):
                if grad is None:
                    raise ValueError("Variable {} has `None` for gradient. "
                                    "Please make sure that all of your ops have a "
                                    "gradient defined (i.e. are differentiable). "
                                    "Common ops without gradient: "
                                    "K.argmax, K.round, K.eval.".format(param))

            # WARNING: for now we do not support gradient clip
            # grads = self._clip_gradients(grads)

            v = [np.random.uniform(0, 1, size = p.shape) for p in params]
            for vi in v:
                vi[ vi < 0.5] =  -1
                vi[ vi >= 0.5] =  1
            v = [tf.convert_to_tensor(vi, dtype = tf.dtypes.float32) for vi in v]

            vprod = tf.reduce_sum([ tf.reduce_sum(vi * grad) for vi, grad in zip(v, grads)])

            Hv = gradients.gradients(vprod, params)

            Hd = [ tf.abs(Hvi * vi) for Hvi, vi in zip(Hv, v)]

        return grads, Hd

    def _filter_grads_hessian(self, grads_hessian_and_vars):
        """Filter out iterable with grad equal to None."""
        grads_hessian_and_vars = tuple(grads_hessian_and_vars)
        if not grads_hessian_and_vars:
            return grads_hessian_and_vars
        filtered = []
        vars_with_empty_grads = []
        for grad, hessian, var in grads_hessian_and_vars:
            if grad is None:
                vars_with_empty_grads.append(var)
            else:
                filtered.append((grad, hessian, var))
        filtered = tuple(filtered)

        if not filtered:
            raise ValueError("No gradients provided for any variable: %s." %
                            ([v.name for _, v in grads_and_vars],))
        if vars_with_empty_grads:
            logging.warning(
                ("Gradients do not exist for variables %s when minimizing the loss."),
                ([v.name for v in vars_with_empty_grads]))
        return filtered

    def apply_gradients_hessian(self,
                      grads_hessian_and_vars,
                      name=None,
                      experimental_aggregate_gradients=True):
        grads_hessian_and_vars = self._filter_grads_hessian(grads_hessian_and_vars)
        var_list = [v for (_, _, v) in grads_hessian_and_vars]

        with backend.name_scope(self._name):
            # Create iteration if necessary.
            with ops.init_scope():
                self._create_all_weights(var_list)

        if not grads_hessian_and_vars:
            # Distribution strategy does not support reducing an empty list of
            # gradients
            return control_flow_ops.no_op()

        if distribute_ctx.in_cross_replica_context():
            raise RuntimeError(
                "`apply_gradients() cannot be called in cross-replica context. "
                "Use `tf.distribute.Strategy.run` to enter replica "
                "context.")

        strategy = distribute_ctx.get_strategy()
        if (not experimental_aggregate_gradients and strategy and isinstance(
            strategy.extended,
            parameter_server_strategy.ParameterServerStrategyExtended)):
            raise NotImplementedError(
                "`experimental_aggregate_gradients=False is not supported for "
                "ParameterServerStrategy and CentralStorageStrategy")

        apply_state = self._prepare(var_list)
        if experimental_aggregate_gradients:
            reduced_grads, reduced_hessian = self._aggregate_gradients_hessian(grads_hessian_and_vars)
            var_list = [v for _, _, v in grads_hessian_and_vars]
            grads_hessian_and_vars = list(zip(reduced_grads, reduced_hessian, var_list))

        return distribute_ctx.get_replica_context().merge_call(
            functools.partial(self._distributed_apply, apply_state=apply_state),
            args=(grads_hessian_and_vars,),
            kwargs={
                "name": name,
            })

    def _aggregate_gradients_hessian(self, grads_hessian_and_vars):
        """Returns all-reduced gradients.
        Args:
        grads_and_vars: List of (gradient, hessian, variable) pairs.
        Returns:
        Two lists of all-reduced gradients and Hessian.
        """
        grads_hessian_and_vars = list(grads_hessian_and_vars)
        filtered_grads_hessian_and_vars = self._filter_grads_hessian(grads_hessian_and_vars)

        # split the list so that we can use the all_recude_fn
        filtered_grads_and_vars = tuple([(g, v) for (g, h, v) in filtered_grads_hessian_and_vars])
        filtered_hessian_and_vars = tuple([(h, v) for (g, h, v) in filtered_grads_hessian_and_vars])

        def all_reduce_fn(distribution, grads_hessian_and_vars):
            # WARNING: this ReduceOp.SUM can only support two entries, for now we have three.
            # So far now, we do it for two steps to make life easier.
            return distribution.extended.batch_reduce_to(
                ds_reduce_util.ReduceOp.SUM, grads_hessian_and_vars)

        if filtered_grads_hessian_and_vars:
            reduced_part1 = distribute_ctx.get_replica_context().merge_call(
                all_reduce_fn, args=(filtered_grads_and_vars,))
            reduced_part2 = distribute_ctx.get_replica_context().merge_call(
                all_reduce_fn, args=(filtered_hessian_and_vars,))
        else:
            reduced = []

        # Copy 'reduced' but add None gradients back in
        reduced_with_nones_grads = []
        reduced_with_nones_hessian = []

        reduced_pos = 0
        for g, h, _ in grads_hessian_and_vars:
            if g is None:
                reduced_with_nones_grads.append( None )
                reduced_with_nones_hessian.append( None )
            else:
                reduced_with_nones_grads.append(reduced_part1[reduced_pos])
                reduced_with_nones_hessian.append(reduced_part2[reduced_pos])
                reduced_pos += 1

        return reduced_with_nones_grads, reduced_with_nones_hessian

    @def_function.function(experimental_compile=True)
    def _resource_apply_dense(self, grad, hess, var, apply_state=None):
        var_device, var_dtype = var.device, var.dtype.base_dtype
        coefficients = ((apply_state or {}).get((var_device, var_dtype)) or
                        self._fallback_apply_state(var_device, var_dtype))

        m = self.get_slot(var, 'm')
        v = self.get_slot(var, 'v')

        m.assign_add((grad - m) * (1 - coefficients['beta_1_t']))
        # this part need to be changed for spatial averaging

        if len(v.shape) == 1:
            resize = self.average_size_1d
        elif len(v.shape) == 2:
            resize = self.average_size_2d
        elif len(v.shape) == 3:
            resize = self.average_size_3d
        elif len(v.shape) == 4:
            resize = self.average_size_4d
        else:
            raise Exception('You need to define the spatial average size by yourself!')

        if resize == None:
            v.assign_add((math_ops.square(hess) - v) * (1 - coefficients['beta_2_t']))
        elif resize == -1:
            if len(v.shape) == 1:
                v.assign_add((math_ops.square(hess) - v) * (1 - coefficients['beta_2_t']))
            elif len(v.shape) == 2:
                hess_average = tf.reduce_mean(hess, [0], keepdims=True)
                v.assign_add((math_ops.square(hess_average) - v) * (1 - coefficients['beta_2_t']))
            elif len(v.shape) == 3:
                hess_average = tf.reduce_mean(hess, [0], keepdims=True)
                v.assign_add((math_ops.square(hess_average) - v) * (1 - coefficients['beta_2_t']))
            elif len(v.shape) == 4:
                hess_average = tf.reduce_mean(hess, [0, 1], keepdims=True)
                v.assign_add((math_ops.square(hess_average) - v) * (1 - coefficients['beta_2_t']))
        else:
            if resize <= 0:
                raise Exception('You need to define the spatial average size >= 1!')
            hess_average = tf.reshape(hess, [resize, -1])
            hess_average = tf.reduce_mean(hess_average, [0])
            hess_average = tf.repeat(hess_average, resize)
            hess_average = tf.reshape(hess_average, v.shape)
            v.assign_add((math_ops.square(hess_average) - v) * (1 - coefficients['beta_2_t']))

        bias_correct1 = 1 - coefficients['beta_1_power']
        bias_correct2 = 1 - coefficients['beta_2_power']

        if self.weight_decay != 0:
            var.assign_sub(coefficients['lr_t'] * self.weight_decay * var)

        denom = np.power(math_ops.sqrt(v / bias_correct2), self.hessian_power) + coefficients['epsilon']

        var.assign_sub( coefficients['lr_t'] * m / bias_correct1 / denom  )

    @def_function.function(experimental_compile=True)
    def _resource_apply_sparse(self, grad, var, indices, apply_state=None):
        raise Exception('For now, we do not support sparse update yet.')

    def get_config(self):
        config = super(AdaHessian, self).get_config()
        config.update({
            'learning_rate': self._serialize_hyperparameter('learning_rate'),
            'decay': self._serialize_hyperparameter('decay'),
            'beta_1': self._serialize_hyperparameter('beta_1'),
            'beta_2': self._serialize_hyperparameter('beta_2'),
            'epsilon': self.epsilon,
            'weight_decay': self.weight_decay
        })
        return config

    def _distributed_apply(self, distribution, grads_hessian_and_vars, name, apply_state):
        """`apply_gradients` using a `DistributionStrategy`."""

        def apply_grad_to_update_var(var, grad, hess):
            """Apply gradient to variable."""
            if isinstance(var, ops.Tensor):
                raise NotImplementedError("Trying to update a Tensor ", var)

            apply_kwargs = {}

            if "apply_state" in self._dense_apply_args:
                apply_kwargs["apply_state"] = apply_state
            update_op = self._resource_apply_dense(grad, hess, var, **apply_kwargs)

            if var.constraint is not None:
                with ops.control_dependencies([update_op]):
                    return var.assign(var.constraint(var))
            else:
                return update_op

        eagerly_outside_functions = ops.executing_eagerly_outside_functions()
        update_ops = []
        with ops.name_scope(name or self._name, skip_on_eager=True):
            for grad, hess, var in grads_hessian_and_vars:

                def _assume_mirrored(grad, hess):
                    if isinstance(grad, ds_values.PerReplica):
                        return ds_values.Mirrored(grad.values), ds_values.Mirrored(hess.values)
                    return grad, hess

                grad, hess = nest.map_structure(_assume_mirrored, grad, hess)
                # Colocate the update with variables to avoid unnecessary communication
                # delays. See b/136304694.
                with distribution.extended.colocate_vars_with(var):
                    with ops.name_scope("update" if eagerly_outside_functions else
                                  "update_" + var.op.name, skip_on_eager=True):
                        update_ops.extend(distribution.extended.update(
                                var, apply_grad_to_update_var, args=(grad, hess), group=False))

            any_symbolic = any(isinstance(i, ops.Operation) or
                             tf_utils.is_symbolic_tensor(i) for i in update_ops)
            if not context.executing_eagerly() or any_symbolic:
                # If the current context is graph mode or any of the update ops are
                # symbolic then the step update should be carried out under a graph
                # context. (eager updates execute immediately)
                with ops._get_graph_from_inputs(update_ops).as_default():  # pylint: disable=protected-access
                    with ops.control_dependencies(update_ops):
                        return self._iterations.assign_add(1, read_value=False)

            return self._iterations.assign_add(1)

max_stars_repo_name: AbhilashMathews/adahessian


text:
import os
import logging
import numpy as np
import pandas as pd

try:
    root = os.path.dirname(os.path.abspath(__file__))
except:
    root = os.getcwd()

class dispatcher(object):
    def __init__(self, length=60*60, step=4, reg_up=True, reg_dn=True, seed=None,
                 db=os.path.join(os.path.dirname(root), 'data', 'FrequencyRegulationData.csv')):
        '''
        The dispatcher class to generate infinitive sequency of fast frequency regulation signal.

        Input
        -----
        length (int): Total length of sequence, in seconds. (default = 60*60)
        step (int): Stepsize of sequence, in seconds. (default = 4)
            NOTE: Only 4 second steps supported in this version. Any other value will raise
            a warning and fallback to the default.
        reg_up (bool): Generate dispatch for regulation up (power feedback to grid). (default = True)
            NOTE: Only True supported in this version. Any other value will raise
            a warning and fallback to the default.
        reg_dn (bool): Generate dispatch for regulation down (power draw to grid). (default = True)
            NOTE: Only True supported in this version. Any other value will raise
            a warning and fallback to the default.
        seed (int): Seed for sequence generation. (default = None)
        db (str): Path to database file. (default = __file__\..\data\FrequencyRegulationData.csv)
        '''
        self.logger = logging.getLogger(__name__)
        self.length = length
        self.step = step
        self.reg_up = reg_up
        self.reg_dn = reg_dn
        self.seed = seed
        self.db = db

        # Check inputs
        if not step == 4:
            self.logger.warning('The "step" argument must be int(4) in this version. Fallback to 4.')
            self.step = 4
        if not self.reg_up:
            self.logger.warning('The "reg_up" argument must be True in this version. Fallback to True.')
            self.reg_up = True
        if not self.reg_dn:
            self.logger.warning('The "reg_dn" argument must be True in this version. Fallback to True.')
            self.reg_dn = True

        # Add variables
        self.time = 0
        self.root = root

        # Read the datafile
        self.data = self.read_data()

        # Generate sequence
        self.seq = self.generate_seq()

    def read_data(self):
        '''
        Reads and processes the database file specified as "db" argument.
        '''
        if not os.path.exists(self.db):
            msg = 'Cannot locate file "{}".'.format(self.db)
            self.logger.error(msg)
            raise FileNotFoundError(msg)
        data = pd.read_csv(self.db, index_col=[0])
        data = data.reset_index(drop=True)
        data.index = data.index * self.step / (60*60)

        # Cut into hourly
        res = pd.DataFrame()
        for c in data:
            t = data[c]
            for h in range(int(t.index[-1] + 0.5)):
                res['{}_{}'.format(c, h)] = t.loc[h:(h+1-1e-4)].values
        return res

    def generate_seq(self):
        '''
        Generates a regulaiton dispatch sequence of length defined by the "length" argument.
        '''
        np.random.seed(self.seed)
        len_seq = int(self.length/self.step)
        n_samples = int(self.length/self.step / len(self.data) + 0.5) * 100
        samples = np.random.randint(0, len(self.data.columns), n_samples)
        d = pd.DataFrame()
        len_d = 0
        i = 0
        while len_d < len_seq:
            if (n_samples - 1) < i:
                self.logger.error('Could not assemble dispatch signal with seed {}.'.format(self.seed) + \
                                  ' Please try another seed.')
                return pd.DataFrame()
            if d.empty:
                d = self.data.iloc[:,samples[i]]
            else:
                last_val = d.iloc[-1]
                new_sample = self.data.iloc[:,samples[i]]
                if last_val in new_sample.values:
                    ix_start = new_sample[new_sample == last_val].index[0]
                    d = d.append(new_sample.loc[ix_start:])
            len_d = len(d)
            i += 1
        self.i = i
        d = d[:len_seq]
        d = d.reset_index(drop=True)
        d.index = d.index * self.step

        # Check statistics
        stats_data = self.data.describe().loc['mean'].describe()[['50%','75%']].to_dict()
        if not (d.describe()['mean'] <= stats_data['75%']) or \
            not (d.describe()['mean'] >= stats_data['50%']):
            self.logger.warning('Mean of sequence out of range, please use another seed or increase sequence length.')
        return d

    def do_step(self):
        '''
        Function to iterate through regulation dispatch in a co-simulation environment.

        Each function call integrates time by one step, defined by the "step" argument.

        Return
        ------
        Return is a list, with same order as arguments below.
        time (float): Current timestep since start of sequence, in seconds.
        dispatch (float): Value of dispatch, in kW.
        '''
        time_now = float(self.time)
        self.time += self.step
        return [time_now, self.seq[time_now]]

    def get_sequence(self, timestamp=None):
        '''
        Function to generate a batch sequence of regulation dispatch.

        Input
        -----
        timestamp (str): Selector of timestamp format, one of [None, "hour", "minute"]. (default = None)

        Return
        ------
        res (pd.Series): The regulation dispatch with timestamp as index, and dispatch as value.
        '''
        res = self.seq.copy(deep=True)
        res.name = 'Dispatch [kW]'
        res.index.name = 'Time [s]'
        if timestamp:
            if timestamp == 'hour':
                res.index = res.index / (60*60)
                res.index.name = 'Time [h]'
            elif timestamp == 'minute':
                res.index = res.index / (60)
                res.index.name = 'Time [min]'
            else:
                msg = 'The defined "timestamp" as "{}" is invalid. '.format(timestamp)
                msg += 'Please use one of: [None, "hour", "minute"].'
                self.logger.error(msg)
                raise ValueError(msg)
        return res

if __name__ == '__main__':
    seed = 20
    dispatch = dispatcher(seed=seed)
    print('Regulation Dispatch for 1 minute:\n')
    print(pd.DataFrame(dispatch.get_sequence(timestamp='minute').loc[:1]))

max_stars_repo_name: LBNL-ETA/SynAS

max_issues_repo_name: LBNL-ETA/SynAS

max_forks_repo_name: LBNL-ETA/SynAS


text:
#coding:utf-8
import numpy as np
import pylab as pl
from sklearn import linear_model, datasets

"""ロジスティック回帰によるirisデータの分類"""

# irisデータをロード
iris = datasets.load_iris()
X = iris.data[:, :2]
Y = iris.target

# メッシュサイズ
h = 0.02

# 分類器を学習
logreg = linear_model.LogisticRegression(C=1e5)
logreg.fit(X, Y)

# データより少し広くなるように描画範囲を決定
x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1

# 範囲をメッシュに区切ってその座標での分類結果Zを求める
xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                     np.arange(y_min, y_max, h))
Z = logreg.predict(np.c_[xx.ravel(), yy.ravel()])
Z = Z.reshape(xx.shape)

# メッシュの各点を描画
pl.figure(1)
pl.pcolormesh(xx, yy, Z, cmap=pl.cm.Paired)

# 訓練データをプロット
pl.scatter(X[:, 0], X[:, 1], c=Y, edgecolors='k', cmap=pl.cm.Paired)
pl.xlabel('Sepal length')
pl.ylabel('Sepal width')

pl.xlim(xx.min(), xx.max())
pl.ylim(yy.min(), yy.max())

pl.show()

max_stars_repo_name: aidiary/PRML

max_issues_repo_name: sojvai/PRML

max_forks_repo_name: sojvai/PRML


text:
from scipy.misc import imread
import numpy as np
import tensorflow as tf

import utils
import noise_as_targets
import models
import batching_functions

# im = imread('data/images/donald_duck.jpeg', mode='I')
# im = imread('data/images/spiral.png', mode='I')
# im = imread('data/images/mandelbrot.jpeg', mode='I')
# im = 255 - imread('data/images/me2.png', mode='I')
# im = 255 - imread('data/images/all_that_must_be.gif', mode='I')
# im = 255 - imread('data/images/yanny.png', mode='I')
im = imread('data/images/me3.png', mode='I')
# im = 255 - imread('data/images/smart_ali_crop.gif', mode='I')
# im = 255 - imread('data/images/ali-cropped.jpg', mode='I')
heatmap = utils.image_to_square_greyscale_array(im)

seed = 1337
np.random.seed(seed)
# train_size = 64_000 * 8
train_size = 64_000 * 8 * 4

# data_points = np.random.normal(size=(train_size, 3))
# # l2 normalize the points
# data_points /= np.linalg.norm(data_points, axis=1, ord=2).reshape((-1, 1))
# data_points = np.random.uniform(size=(train_size, 100))
input_noise_fn = lambda size: np.random.uniform(size=(size, 256))  # NOQA
data_points = input_noise_fn(train_size)

targets = noise_as_targets.sample_from_heatmap(
    heatmap, train_size, sampling_method='even',
)

# batching_function = batching_functions.progressive_local_search(targets)
batching_function = batching_functions.random_batching(targets)

config = {
    'dataset_fn': lambda: (data_points, targets),
    'model_fn': lambda input_t, output_size: models.multi_layer_mlp(
        input_t, output_size, hidden_dims=[512, 512], activation_fn=tf.tanh
    ),
    'batch_size': 128,
    'batching_fn': batching_function,
    # 'eval_steps': 500
    'eval_steps': 2000,
    'input_noise_fn': input_noise_fn,
    'train_steps': 20_000_000,
    # 'image_dimensions': (512, 512)
    'image_dimensions': (700, 700),
    'optimizer': tf.train.AdamOptimizer(1e-4)
}


text:
import gym
import numpy as np
import tensorflow as tf
import torch
from einops import repeat
from gym.spaces import Box
from stable_baselines.common.tf_layers import conv, linear, conv_to_fc
from stable_baselines3.common.torch_layers import BaseFeaturesExtractor
from torch import nn
from vit_pytorch import ViT

def create_augmented_nature_cnn(num_direct_features):
    """
    Create and return a function for augmented_nature_cnn
    used in stable-baselines.

    num_direct_features tells how many direct features there
    will be in the image.
    """

    def augmented_nature_cnn(scaled_images, **kwargs):
        """
        Copied from stable_baselines policies.py.
        This is nature CNN head where last channel of the image contains
        direct features.

        :param scaled_images: (TensorFlow Tensor) Image input placeholder
        :param kwargs: (dict) Extra keywords parameters for the convolutional layers of the CNN
        :return: (TensorFlow Tensor) The CNN output layer
        """
        activ = tf.nn.relu

        # Take last channel as direct features
        other_features = tf.contrib.slim.flatten(scaled_images[..., -1])
        # Take known amount of direct features, rest are padding zeros
        other_features = other_features[:, :num_direct_features]

        scaled_images = scaled_images[..., :-1]

        layer_1 = activ(
            conv(scaled_images, 'cnn1', n_filters=32, filter_size=8, stride=4, init_scale=np.sqrt(2), **kwargs))
        layer_2 = activ(conv(layer_1, 'cnn2', n_filters=64, filter_size=4, stride=2, init_scale=np.sqrt(2), **kwargs))
        layer_3 = activ(conv(layer_2, 'cnn3', n_filters=64, filter_size=3, stride=1, init_scale=np.sqrt(2), **kwargs))
        layer_3 = conv_to_fc(layer_3)

        # Append direct features to the final output of extractor
        img_output = activ(linear(layer_3, 'cnn_fc1', n_hidden=512, init_scale=np.sqrt(2)))
        concat = tf.concat((img_output, other_features), axis=1)

        return concat

    return augmented_nature_cnn

# patrick: needed because pytorch wants channels first but env outputs channels last. Not changing env for compatibility
# copied from
class TransposeNatureCNN(BaseFeaturesExtractor):
    """
    CNN from DQN nature paper:
        Mnih, Volodymyr, et al.
        "Human-level control through deep reinforcement learning."
        Nature 518.7540 (2015): 529-533.

    :param observation_space:
    :param features_dim: Number of features extracted.
        This corresponds to the number of unit for the last layer.
    """

    def __init__(self, observation_space: gym.spaces.Box, features_dim: int = 512):
        super(TransposeNatureCNN, self).__init__(observation_space, features_dim)
        # TODO: check for image valid removed because want float image
        n_input_channels = observation_space.shape[2]  # patrick edit: channels last in obs, used to be 0
        self.cnn = nn.Sequential(
            nn.Conv2d(n_input_channels, 32, kernel_size=8, stride=4, padding=0),
            nn.ReLU(),
            nn.Conv2d(32, 64, kernel_size=4, stride=2, padding=0),
            nn.ReLU(),
            nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=0),
            nn.ReLU(),
            nn.Flatten(),
        )

        # Compute shape by doing one forward pass
        with torch.no_grad():
            obs_sample = observation_space.sample()
            obs_sample = obs_sample.transpose((2, 0, 1))  # patrick edit: transpose obs so channels first
            n_flatten = self.cnn(torch.as_tensor(obs_sample[None]).float()).shape[1]

        self.linear = nn.Sequential(nn.Linear(n_flatten, features_dim), nn.ReLU())

    def forward(self, observations: torch.Tensor) -> torch.Tensor:
        # permute from b x h x w x c to b x c x h x w
        return self.linear(self.cnn(observations.permute((0, 3, 1, 2))))  # patrick edit: same as init but pytorch ver

# want vit without final linear layer
class ViTNoClass(ViT):
    def forward(self, img):
        # copied from vit, removed final mlp call
        x = self.to_patch_embedding(img)
        b, n, _ = x.shape

        cls_tokens = repeat(self.cls_token, '() n d -> b n d', b=b)
        x = torch.cat((cls_tokens, x), dim=1)
        x += self.pos_embedding[:, :(n + 1)]
        x = self.dropout(x)

        x = self.transformer(x)

        x = x.mean(dim=1) if self.pool == 'mean' else x[:, 0]

        x = self.to_latent(x)
        return x
        # return self.mlp_head(x)

# patrick: alternate model class, visual transformer
class TransposedVisTransformer(BaseFeaturesExtractor):
    def __init__(self, observation_space: gym.spaces.Box, features_dim: int = 512):
        super().__init__(observation_space, features_dim)
        self.vit = ViTNoClass(
            image_size=64,
            patch_size=8,
            num_classes=5,
            dim=512,
            depth=3,
            heads=5,
            mlp_dim=features_dim*2,
            dropout=0.1,
            emb_dropout=0.1,
            channels=2
        )

    def forward(self, observations: torch.Tensor) -> torch.Tensor:
        # permute from b x h x w x c to b x c x h x w
        return self.vit(observations.permute((0, 3, 1, 2)))  # patrick edit: same as init but pytorch ver

if __name__ == "__main__":
    test = TransposedVisTransformer(Box(0, 1, (1, 2)))
    img = torch.randn(1, 64, 64, 2)

    preds = test(img)  # (1, 1000)