text:
# -*- coding: utf-8 -*-
import warnings

import numpy as np
import scipy.stats

def density_bandwidth(x, method="KernSmooth", resolution=401):
    """**Bandwidth Selection for Density Estimation**

    Bandwidth selector for :func:`.density` estimation. See ``bw_method`` argument in
    :func:`.scipy.stats.gaussian_kde`.

    The ``"KernSmooth"`` method is adapted from the ``dpik()`` function from the *KernSmooth* R
    package. In this case, it estimates the optimal AMISE bandwidth using the direct plug-in method
    with 2 levels for the Parzen-Rosenblatt estimator with Gaussian kernel.

    Parameters
    -----------
    x : Union[list, np.array, pd.Series]
        A vector of values.
    method : float
        The bandwidth of the kernel. The larger the values, the smoother the estimation. Can be a
        number, or ``"scott"`` or ``"silverman"``
        (see ``bw_method`` argument in :func:`.scipy.stats.gaussian_kde`), or ``"KernSmooth"``.
    resolution : int
        Only when ``method="KernSmooth"``. The number of equally-spaced points over which binning
        is performed to obtain kernel functional approximation (see ``gridsize`` argument in ``KernSmooth::dpik()``).

    Returns
    -------
    float
        Bandwidth value.

    See Also
    --------
    density

    Examples
    --------
    .. ipython:: python

      import neurokit2 as nk

      x = np.random.normal(0, 1, size=100)
      bw = nk.density_bandwidth(x)
      bw

      nk.density_bandwidth(x, method="scott")
      nk.density_bandwidth(x, method=1)

      @savefig p_density_bandwidth.png scale=100%
      x, y = nk.density(signal, bandwidth=bw, show=True)
      @suppress
      plt.close()

    References
    ----------
    * Jones, W. M. (1995). Kernel Smoothing, Chapman & Hall.

    """
    if isinstance(method, str):
        method = method.lower()
    if isinstance(method, (float, int)) or method != "kernsmooth":
        return scipy.stats.gaussian_kde(x, bw_method=method).factor

    n = len(x)

    stdev = np.nanstd(x, ddof=1)
    iqr = np.diff(np.percentile(x, [25, 75]))[0] / 1.349
    scalest = min(stdev, iqr)

    data_scaled = (x - np.nanmean(x)) / scalest
    min_scaled = np.nanmin(data_scaled)
    max_scaled = np.nanmax(data_scaled)

    gcounts = _density_linearbinning(
        x=data_scaled,
        gpoints=np.linspace(min_scaled, max_scaled, resolution),
        truncate=True,
    )

    alpha = (2 * np.sqrt(2) ** 9 / (7 * n)) ** (1 / 9)
    psi6hat = _density_bkfe(gcounts, 6, alpha, min_scaled, max_scaled)

    alpha = (-3 * np.sqrt(2 / np.pi) / (psi6hat * n)) ** (1 / 7)
    psi4hat = _density_bkfe(gcounts, 4, alpha, min_scaled, max_scaled)

    delta_0 = 1 / ((4 * np.pi) ** (1 / 10))
    output = scalest * delta_0 * (1 / (psi4hat * n)) ** (1 / 5)
    return output

def _density_linearbinning(x, gpoints, truncate=True):
    """
    Linear binning. Adapted from KernSmooth R package.
    """
    n = len(x)
    M = gpoints.shape[0]
    a = gpoints[0]
    b = gpoints[-1]

    # initialization of gcounts:
    gcounts = np.zeros(M)
    Delta = (b - a) / (M - 1)

    for i in range(n):
        lxi = ((x[i] - a) / Delta) + 1
        li = int(lxi)

        rem = lxi - li

        if (li >= 1) and (li < M):
            gcounts[li - 1] = gcounts[li - 1] + 1 - rem
            gcounts[li] = gcounts[li] + rem

        elif (li < 1) and (truncate is False):
            gcounts[0] = gcounts[0] + 1

        elif (li >= M) and (truncate is False):
            gcounts[M - 1] = gcounts[M - 1] + 1

    return gcounts

def _density_bkfe(gcounts, drv, h, a, b):
    """
    'bkfe' function adapted from KernSmooth R package.
    """
    resol = len(gcounts)

    # Set the sample size and bin width
    n = np.nansum(gcounts)
    delta = (b - a) / (resol - 1)

    # Obtain kernel weights
    tau = drv + 4
    L = min(int(tau * h / delta), resol)
    if L == 0:
        warnings.warn(
            "WARNING : Binning grid too coarse for current (small) bandwidth: consider increasing 'resolution'"
        )
    lvec = np.arange(L + 1)
    arg = lvec * delta / h

    dnorm = np.exp(-np.square(arg) / 2) / np.sqrt(2 * np.pi)
    kappam = dnorm / h ** (drv + 1)
    hmold0 = 1
    hmold1 = arg
    hmnew = 1
    if drv >= 2:
        for i in np.arange(2, drv + 1):
            hmnew = arg * hmold1 - (i - 1) * hmold0
            hmold0 = hmold1  # Compute mth degree Hermite polynomial
            hmold1 = hmnew  # by recurrence.
    kappam = hmnew * kappam

    # Now combine weights and counts to obtain estimate
    P = 2 ** (int(np.log(resol + L + 1) / np.log(2)) + 1)
    kappam = np.concatenate((kappam, np.zeros(P - 2 * L - 1), kappam[1:][::-1]), axis=0)
    Gcounts = np.concatenate((gcounts, np.zeros(P - resol)), axis=0)
    kappam = np.fft.fft(kappam)
    Gcounts = np.fft.fft(Gcounts)

    gcounter = gcounts * (np.real(np.fft.ifft(kappam * Gcounts)))[0:resol]

    return np.nansum(gcounter) / n**2

max_stars_repo_name: danibene/NeuroKit

max_issues_repo_name: danibene/NeuroKit

max_forks_repo_name: danibene/NeuroKit


text:
# Copyright 2021 DeepMind Technologies Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Play games of Diplomacy."""

from typing import Any, Dict, List, Optional, Sequence

from absl import logging
import numpy as np

from deepmind.diplomacy.environment import action_utils
from deepmind.diplomacy.environment import observation_utils as utils
from deepmind.diplomacy.network import network_policy

class DiplomacyTrajectory:
  """Stores data from a Diplomacy game."""

  def __init__(self):
    self.observations: List[utils.Observation] = []
    self.legal_actions: List[np.ndarray] = []
    self.actions: List[np.ndarray] = []
    self.step_outputs: List[Dict[str, Any]] = []
    self.returns: Optional[np.ndarray] = None

  def append_step(self,
                  observation: utils.Observation,
                  legal_actions: np.ndarray,
                  actions: np.ndarray,
                  step_outputs: Dict[str, Any]):
    self.observations.append(observation)
    self.legal_actions.append(legal_actions)
    self.actions.append(actions)
    self.step_outputs.append(step_outputs)

  def terminate(self, returns):
    self.returns = returns

def _draw_returns(
    points_per_supply_centre: bool,
    board: np.ndarray,
    num_players: int
) -> np.ndarray:
  """Computes returns (number of supply centers) when the game end in a draw."""
  if points_per_supply_centre:
    returns = [len(utils.sc_provinces(i, board)) for i in range(num_players)]
  else:
    returns = [
        1 if utils.sc_provinces(i, board) else 0 for i in range(num_players)]
  return np.array(returns, dtype=np.float32) / sum(returns)

def run_game(
    state,
    policies: Sequence[network_policy.Policy],
    slots_to_policies: Sequence[int],
    max_length: Optional[int] = None,
    min_years_forced_draw=1000,
    forced_draw_probability=0.0,
    points_per_supply_centre=False,
    draw_if_slot_loses=None,
) -> DiplomacyTrajectory:
  """Run a game of diplomacy.

  Args:
    state: A DiplomacyState in Spring 1901 (see diplomacy_state.py).
    policies: sequence of policies which are acting.
    slots_to_policies: sequence of length num_players mapping slots of the
      games to the index of the corresponding policy in policies.
    max_length: terminate games after this many full diplomacy turns.
    min_years_forced_draw: minimum years to consider force a draw.
    forced_draw_probability: probability of a draw each year after the first
      min_years_forced_draw
    points_per_supply_centre: whether to assign points per supply centre in a
      draw (rather than 0/1 for win/loss).
    draw_if_slot_loses: if this slot is eliminated, the game is ended in a draw.

  Returns:
    Trajectory of the game, as a DiplomacyTrajectory.
  """
  num_players = 7

  if len(slots_to_policies) != num_players:
    raise ValueError(
        f"Length of slot to policy mapping {len(slots_to_policies)}"
        f", but {num_players} players in game.")
  policies_to_slots_lists = [[] for i in range(len(policies))]
  for slot in range(num_players):
    policy_index = slots_to_policies[slot]
    if policy_index >= len(policies) or policy_index < 0:
      raise ValueError(f"Policy index {policy_index} out of range")
    policies_to_slots_lists[policy_index].append(slot)

  for policy in policies:
    policy.reset()

  if max_length is None:
    max_length = np.inf

  year = 0
  turn_num = 0

  traj = DiplomacyTrajectory()
  returns = None

  while not state.is_terminal() and turn_num < max_length:
    logging.info("In turn %d year %d ", turn_num, year)

    observation = state.observation()

    # New Game Year Checks
    if observation.season == utils.Season.SPRING_MOVES:
      year += 1
      if (draw_if_slot_loses is not None and
          not utils.sc_provinces(draw_if_slot_loses, observation.board)):
        returns = _draw_returns(points_per_supply_centre, observation.board,
                                num_players)
        logging.info("Forcing a draw due to elimination - returns %s",
                     returns)
        break
      if (year > min_years_forced_draw and
          np.random.uniform() < forced_draw_probability):
        returns = _draw_returns(points_per_supply_centre, observation.board,
                                num_players)
        logging.info("Forcing a draw at year %s - returns %s", year, returns)
        break

    legal_actions = state.legal_actions()
    padded_legal_actions = np.zeros(
        (num_players, action_utils.MAX_LEGAL_ACTIONS), np.int64)
    for i in range(num_players):
      padded_legal_actions[i, :len(legal_actions[i])] = legal_actions[i]
    actions_lists = [[] for _ in range(num_players)]
    policies_step_outputs = {}

    for policy, slots_list in zip(policies, policies_to_slots_lists):
      (policy_actions_lists,
       policies_step_outputs[str(policy)]) = policy.actions(
           slots_list, observation, legal_actions)
      if len(policy_actions_lists) != len(slots_list):
        raise ValueError(f"Policy {policy} returned {len(policy_actions_lists)}"
                         f" actions lists for {len(slots_list)} players")
      for actions, slot in zip(policy_actions_lists, slots_list):
        actions_lists[slot] = actions
    # Save our actions lists.
    padded_actions = np.full(
        (num_players, action_utils.MAX_ORDERS), -1, np.int64)
    for i, actions_list in enumerate(actions_lists):
      if actions_list is not None:
        padded_actions[i, :len(actions_list)] = actions_list

    state.step(actions_lists)
    turn_num += 1

    traj.append_step(observation,
                     padded_legal_actions,
                     padded_actions,
                     policies_step_outputs)

  if returns is None:
    returns = state.returns()
  traj.terminate(returns)

  return traj

ext: py

max_stars_repo_name: maayanorner/diplomacy

max_issues_repo_name: maayanorner/diplomacy

max_forks_repo_name: maayanorner/diplomacy

reason: import numpy


text:
import time
import numpy as np
import matplotlib.pyplot as plt
import util
import artificial_bee_colony
import differential_evolution
import genetic_algorithm
import harmony_search
import simulated_annealing

env = util.mediums(500, 500, 10)

abc_optimizer = artificial_bee_colony.optimizer(1, 4500, 10, 50, 50, 20, 20, 100, env)
de_optimizer = differential_evolution.optimizer(1, 1750, 10, 100, 0.5, 0.5, 100, env)
ga_optimizer = genetic_algorithm.optimizer(1, 6000, 10, 100, 40, 20, 0.1, 100, env)
hs_optimizer = harmony_search.optimizer(1, 5500, 10, 100, 0.99, 0.05, 0.005, 100, env)
sa_optimizer = simulated_annealing.optimizer(1, 10, 1.0, 0.00001, 0.99625, 100, 100, env)

startTime = time.time() * 1000
(sums, bests, losses, bestSolution, bestSolutionLoss) = abc_optimizer.optimize()
endTime = time.time() * 1000
print('ABC: Time spent: ', endTime - startTime, ' Error: ', bestSolutionLoss)

startTime = time.time() * 1000
(sums, bests, losses, bestSolution, bestSolutionLoss) = de_optimizer.optimize()
endTime = time.time() * 1000
print('DE: Time spent: ', endTime - startTime, ' Error: ', bestSolutionLoss)

startTime = time.time() * 1000
(sums, bests, losses, bestSolution, bestSolutionLoss) = ga_optimizer.optimize()
endTime = time.time() * 1000
print('GA: Time spent: ', endTime - startTime, ' Error: ', bestSolutionLoss)

startTime = time.time() * 1000
(sums, bests, losses, bestSolution, bestSolutionLoss) = hs_optimizer.optimize()
endTime = time.time() * 1000
print('HS: Time spent: ', endTime - startTime, ' Error: ', bestSolutionLoss)

startTime = time.time() * 1000
(sums, bests, losses, bestSolution, bestSolutionLoss) = sa_optimizer.optimize()
endTime = time.time() * 1000
print('SA: Time spent: ', endTime - startTime, ' Error: ', bestSolutionLoss)


text:
import torch
import torch.nn as nn
import lib.layers as layers
from lib.layers.odefunc import ODEnet
from lib.layers.squeeze import squeeze, unsqueeze
import numpy as np

class ODENVP(nn.Module):
    """
    Real NVP for image data. Will downsample the input until one of the
    dimensions is less than or equal to 4.

    Args:
        input_size (tuple): 4D tuple of the input size.
        n_scale (int): Number of scales for the representation z.
        n_resblocks (int): Length of the resnet for each coupling layer.
    """

    def __init__(
        self,
        input_size,
        n_scale=float('inf'),
        n_blocks=2,
        strides=None,
        intermediate_dims=(32,),
        nonlinearity="softplus",
        layer_type="concat",
        squash_input=True,
        squeeze_first=False,
        zero_last=True,
        div_samples=1,
        alpha=0.05,
        cnf_kwargs=None,
    ):
        super(ODENVP, self).__init__()
        if squeeze_first:
            bsz, c, w, h = input_size
            c, w, h = c*4, w//2, h//2
            input_size = bsz, c, w, h

        self.n_scale = min(n_scale, self._calc_n_scale(input_size))
        self.n_blocks = n_blocks
        self.intermediate_dims = intermediate_dims
        self.layer_type=layer_type
        self.zero_last=zero_last
        self.div_samples=div_samples
        self.nonlinearity = nonlinearity
        self.strides=strides
        self.squash_input = squash_input
        self.alpha = alpha
        self.squeeze_first = squeeze_first
        self.cnf_kwargs = cnf_kwargs if cnf_kwargs else {}

        if not self.n_scale > 0:
            raise ValueError('Could not compute number of scales for input of' 'size (%d,%d,%d,%d)' % input_size)

        self.transforms = self._build_net(input_size)

        self.dims = [o[1:] for o in self.calc_output_size(input_size)]

    def _build_net(self, input_size):
        _, c, h, w = input_size
        transforms = []
        for i in range(self.n_scale):
            transforms.append(
                StackedCNFLayers(
                    initial_size=(c, h, w),
                    div_samples=self.div_samples,
                    zero_last=self.zero_last,
                    layer_type=self.layer_type,
                    strides=self.strides,
                    idims=self.intermediate_dims,
                    squeeze=(i < self.n_scale - 1),  # don't squeeze last layer
                    init_layer=(layers.LogitTransform(self.alpha) if self.alpha > 0 else layers.ZeroMeanTransform())
                    if self.squash_input and i == 0 else None,
                    n_blocks=self.n_blocks,
                    cnf_kwargs=self.cnf_kwargs,
                    nonlinearity=self.nonlinearity,
                )
            )
            c, h, w = c * 2, h // 2, w // 2
        return nn.ModuleList(transforms)

    def _calc_n_scale(self, input_size):
        _, _, h, w = input_size
        n_scale = 0
        while h >= 4 and w >= 4:
            n_scale += 1
            h = h // 2
            w = w // 2
        return n_scale

    def calc_output_size(self, input_size):
        n, c, h, w = input_size
        output_sizes = []
        for i in range(self.n_scale):
            if i < self.n_scale - 1:
                c *= 2
                h //= 2
                w //= 2
                output_sizes.append((n, c, h, w))
            else:
                output_sizes.append((n, c, h, w))
        return tuple(output_sizes)

    def forward(self, x, logpx=None, reg_states=tuple(), reverse=False):
        if reverse:
            out = self._generate(x, logpx, reg_states)
            if self.squeeze_first:
                x = unsqueeze(out[0])
            else:
                x = out[0]
            return x, out[1], out[2]
        else:
            if self.squeeze_first:
                x = squeeze(x)
            return self._logdensity(x, logpx, reg_states)

    def _logdensity(self, x, logpx=None, reg_states=tuple()):
        _logpx = torch.zeros(x.shape[0], 1).to(x) if logpx is None else logpx
        out = []
        for idx in range(len(self.transforms)):
            x, _logpx, reg_states = self.transforms[idx].forward(x, _logpx, reg_states)
            if idx < len(self.transforms) - 1:
                d = x.size(1) // 2
                x, factor_out = x[:, :d], x[:, d:]
            else:
                # last layer, no factor out
                factor_out = x
            out.append(factor_out)
        out = [o.view(o.size()[0], -1) for o in out]
        out = torch.cat(out, 1)
        return out, _logpx, reg_states

    def _generate(self, z, logpz=None, reg_states=tuple()):
        z = z.view(z.shape[0], -1)
        zs = []
        i = 0
        for dims in self.dims:
            s = np.prod(dims)
            zs.append(z[:, i:i + s])
            i += s
        zs = [_z.view(_z.size()[0], *zsize) for _z, zsize in zip(zs, self.dims)]
        _logpz = torch.zeros(zs[0].shape[0], 1).to(zs[0]) if logpz is None else logpz
        z_prev, _logpz, _ = self.transforms[-1](zs[-1], _logpz, reverse=True)
        for idx in range(len(self.transforms) - 2, -1, -1):
            z_prev = torch.cat((z_prev, zs[idx]), dim=1)
            z_prev, _logpz, reg_states = self.transforms[idx](z_prev, _logpz, reg_states, reverse=True)
        return z_prev, _logpz, reg_states

class StackedCNFLayers(layers.SequentialFlow):
    def __init__(
        self,
        initial_size,
        idims=(32,),
        nonlinearity="softplus",
        layer_type="concat",
        div_samples=1,
        squeeze=True,
        init_layer=None,
        n_blocks=1,
        zero_last=True,
        strides=None,
        cnf_kwargs={},
    ):
        chain = []
        if init_layer is not None:
            chain.append(init_layer)

        def _make_odefunc(size):
            net = ODEnet(idims, size, strides, True, layer_type=layer_type, nonlinearity=nonlinearity, zero_last_weight=zero_last)
            f = layers.ODEfunc(net, div_samples=div_samples)
            return f

        if squeeze:
            c, h, w = initial_size
            after_squeeze_size = c * 4, h // 2, w // 2
            pre = [layers.CNF(_make_odefunc(initial_size), **cnf_kwargs) for _ in range(n_blocks)]
            post = [layers.CNF(_make_odefunc(after_squeeze_size), **cnf_kwargs) for _ in range(n_blocks)]
            chain += pre + [layers.SqueezeLayer(2)] + post
        else:
            chain += [layers.CNF(_make_odefunc(initial_size), **cnf_kwargs) for _ in range(n_blocks)]

        super(StackedCNFLayers, self).__init__(chain)

max_stars_repo_name: sulekahraman/ffjord-rnode

max_issues_repo_name: sulekahraman/ffjord-rnode

max_forks_repo_name: sulekahraman/ffjord-rnode


text:
import typing
import sys
import numpy as np
import numba as nb

@nb.njit((nb.c16[:], nb.optional(nb.b1)))
def fft(
  a: np.ndarray,
  inverse: bool=False
) -> np.ndarray:
  n = a.size
  if n == 1: return a
  h = 1
  while 1 << h < n: h += 1
  assert 1 << h == n

  b = fft(a[::2], inverse)
  c = fft(a[1::2], inverse)

  a = np.zeros(n, dtype=np.complex128)
  sign = -1 + 2 * inverse
  zeta = np.exp(sign * 2j * np.pi / n * np.arange(n))
  m = n // 2
  a[:m] = a[m:] = c
  a *= zeta
  a[:m] += b; a[m:] += b
  return a

@nb.njit((nb.c16[:], ))
def ifft(
  a: np.ndarray,
) -> np.ndarray:
  return fft(a, inverse=True) / a.size

@nb.njit((nb.i8[:], nb.i8[:]))
def convolve(
  a: np.ndarray,
  b: np.ndarray,
) -> np.ndarray:
  n = a.size + b.size - 1
  m = 1
  while m < n: m <<= 1
  na = np.zeros(m, dtype=np.complex128)
  nb = np.zeros(m, dtype=np.complex128)
  na[:a.size] = a
  nb[:b.size] = b
  a, b = na, nb
  a = fft(a, inverse=False)
  b = fft(b, inverse=False)
  c = a * b
  c = ifft(c)[:n]
  c = np.rint(np.real(c)).astype(np.int64)
  return c

@nb.njit(
  (nb.i8[:], nb.i8[:]),
  cache=True,
)
def solve(
  a: np.ndarray,
  b: np.ndarray,
) -> typing.NoReturn:
  a = np.hstack((np.array([0]), a))
  b = np.hstack((np.array([0]), b))
  c = convolve(a, b)
  n = a.size
  for x in c[1:2 * n + 1]:
    print(x)

def main() -> typing.NoReturn:
  n = int(input())
  a, b = np.array(
    sys.stdin.read().split(),
    dtype=np.int64,
  ).reshape(n, 2).T
  solve(a, b)

main()

max_stars_repo_name: kagemeka/competitive-programming

max_issues_repo_name: kagemeka/competitive-programming

max_forks_repo_name: kagemeka/competitive-programming

reason: import numpy,import numba


text:
import math
import numpy
import random
import typing
from collections import deque
from typing import List
import stock_exchange  # circular dependency!
from experts.obscure_expert import ObscureExpert
from framework.period import Period
from framework.portfolio import Portfolio
from framework.stock_market_data import StockMarketData
from framework.interface_expert import IExpert
from framework.interface_trader import ITrader
from framework.order import Order, OrderType
from framework.vote import Vote
from keras.models import Sequential
from keras.layers import Dense
from keras.optimizers import Adam, Nadam, SGD
from framework.order import Company
from framework.utils import save_keras_sequential, load_keras_sequential
from framework.logger import logger
from dh.StateExpertsOnly import StateExpertsOnly
from dh.StateExpertsCashShares import StateExpertsCashShares
from dh.Action3 import Action3
from dh.Action5 import Action5
from dh.Experience import Experience

State = StateExpertsOnly
Action = Action3

class DeepQLearningTrader(ITrader):
    """
    Implementation of ITrader based on Deep Q-Learning (DQL).
    """
    RELATIVE_DATA_DIRECTORY = 'traders/dql_trader_data'

    def __init__(self, expert_a: IExpert, expert_b: IExpert, load_trained_model: bool = True,
                 train_while_trading: bool = False, color: str = 'black', name: str = 'dql_trader',
                 other_data_directory: typing.Optional[str] = None,
                 plot_name: typing.Optional[str] = None):
        """
        Constructor
        Args:
            expert_a: Expert for stock A
            expert_b: Expert for stock B
            load_trained_model: Flag to trigger loading an already trained neural network
            train_while_trading: Flag to trigger on-the-fly training while trading
            other_data_directory: relative directory path from project root to .json and .h5 files of model
            plot_name: name to use in a plot for this trader, overriding parameter "name" (which also determines
                       path where the model gets saved)
        """
        # Save experts, training mode and name
        super().__init__(color, name)
        assert expert_a is not None and expert_b is not None
        self.experts = [expert_a, expert_b]
        self.train_while_trading = train_while_trading
        self.other_data_directory = other_data_directory
        self.plot_name = plot_name

        # Parameters for neural network
        self.state_size = State.get_number_of_input_neurons()
        self.action_size = Action.get_number_of_output_neurons()
        self.hidden_size = 30

        # Parameters for deep Q-learning
        self.learning_rate = 0.001  # default for Adam: 0.001
        self.epsilon = 0.9995  # determines how quickly epsilon decreases to epsilon_min
        self.random_action_min_probability = 0.01  # minimum probability of choosing a random action
        self.train_each_n_days = 128  # how many trading days pass between each training
        self.batch_size = 128  # how many experience samples from memory to train on each training occasion
        self.min_size_of_memory_before_training = 1000  # should be way bigger than batch_size, but smaller than memory
        self.memory: typing.Deque[Experience] = deque(maxlen=2000)
        # discount factor: how quickly we expect an action to pay off
        # (0 -> only evaluate immediate effect on portfolio value on next day;
        #  near 1 -> also include development of portfolio value on future days, where each day has a weight of
        #  discount_factor ^ #days_ahead; higher values allow to factor in payoffs at later time while also
        #  making it harder to attribute the effects to the actions from a concrete day)
        self.discount_factor = 0
        assert 0 <= self.discount_factor < 1.0
        # min_horizon: number of more recent experiences required until discount_factor ^ #days_ahead drops to 1%
        if 0 < self.discount_factor < 1:
            self.min_horizon = min(100, math.ceil(math.log(0.01, self.discount_factor)))
        else:
            self.min_horizon = 1
        self.q_value_cap = 1.0  # limit Q-value to this magnitude; 0.0 to deactivate
        self.is_evolved_model = False  # set to True when loading model from file
        self.days_passed = 0
        self.training_occasions = 0

        # Create main model, either as trained model (from file) or as untrained model (from scratch)
        self.model = None
        if load_trained_model:
            data_dir = self.RELATIVE_DATA_DIRECTORY if self.other_data_directory is None else self.other_data_directory
            self.model = load_keras_sequential(data_dir, 'dql_trader')
            # logger.info(f"DQL Trader: Loaded trained model")
            self.is_evolved_model = True
        if self.model is None:  # loading failed or we didn't want to use a trained model
            self.model = Sequential()
            # original code:
            # self.model.add(Dense(self.hidden_size * 2, input_dim=self.state_size, activation='relu'))
            # self.model.add(Dense(self.hidden_size, activation='relu'))
            # self.model.add(Dense(self.action_size, activation='linear'))
            # modified code:
            # relu -> elu: avoid "dead nodes" problem of relu
            # lecun_normal: initialization from gaussian accounting for number of nodes as well
            # initialize bias with zeros
            self.model.add(Dense(self.hidden_size * 2, input_dim=self.state_size, activation='elu', kernel_initializer='lecun_normal', bias_initializer='zeros'))
            self.model.add(Dense(self.hidden_size, activation='elu', kernel_initializer='lecun_normal', bias_initializer='zeros'))
            self.model.add(Dense(self.action_size, activation='linear', kernel_initializer='lecun_normal', bias_initializer='zeros'))
            # logger.info(f"DQL Trader: Created new untrained model")
        assert self.model is not None
        # use one of the following solvers:
        self.model.compile(loss='mse', optimizer=Adam(lr=self.learning_rate))
        # self.model.compile(loss='mse', optimizer=SGD(lr=0.01, momentum=0.9, nesterov=True))
        # self.model.compile(loss='mse', optimizer=Nadam(lr=0.001))

    def log_info_every_nth_day(self, msg: str, n: int):
        if self.days_passed % n == 0:
            logger.info(msg)

    def save_trained_model(self):
        """
        Save the trained neural network under a fixed name specific for this traders.
        """
        data_dir = self.RELATIVE_DATA_DIRECTORY if self.other_data_directory is None else self.other_data_directory
        save_keras_sequential(self.model, data_dir, 'dql_trader')
        # logger.info(f"DQL Trader: Saved trained model")

    def trade(self, portfolio: Portfolio, stock_market_data: StockMarketData) -> List[Order]:
        """
        Generate actions to be taken on the "stock market"

        Args:
          portfolio : current Portfolio of this traders
          stock_market_data : StockMarketData for evaluation

        Returns:
          A OrderList instance, may be empty never None
        """
        assert portfolio is not None
        assert stock_market_data is not None
        assert stock_market_data.get_companies() == [Company.A, Company.B]

        # Compute the current state
        expert_votes = [
            self.experts[i].vote(stock_market_data[company])
            for i, company in enumerate(stock_market_data.get_companies())
        ]
        shares_owned = [
            portfolio.get_stock(company)
            for company in stock_market_data.get_companies()
        ]
        if State is StateExpertsOnly:
            state = StateExpertsOnly(expert_votes, portfolio.get_value(stock_market_data))
        elif State is StateExpertsCashShares:
            state = StateExpertsCashShares(expert_votes, portfolio.cash, shares_owned, portfolio.get_value(stock_market_data))
        else:
            raise RuntimeError

        if self.train_while_trading:
            # store state as experience in memory
            if len(self.memory) > 0:
                self.memory[-1].state2 = state
            experience = Experience(
                state1=state
            )
            self.memory.append(experience)

            # train
            if len(self.memory) >= self.min_size_of_memory_before_training:
                if self.days_passed % self.train_each_n_days == 0:
                    self.train()

            # determine probability for random actions
            if not self.is_evolved_model:
                # first training episode
                random_action_probability = (
                        (self.epsilon ** self.days_passed) * (1.0 - self.random_action_min_probability) +
                        self.random_action_min_probability
                )
            else:
                # subsequent training episode
                random_action_probability = self.random_action_min_probability

            if self.training_occasions == 0 or random.random() < random_action_probability:
                actions = [Action.get_random(), Action.get_random()]
            else:
                # choose actions by querying network
                x = state.to_input()
                y = self.model.predict(numpy.array([x]))
                assert y.shape == (1, self.action_size)
                actions = Action.from_model_prediction(y[0])

            experience.actions = actions
        else:
            # not training -> always choose actions by querying network
            actions = Action.from_model_prediction(self.model.predict(numpy.array([state.to_input()]))[0])

        # translate actions into orders
        orders: typing.List[Order] = []
        companies_with_actions_and_magnitudes = list(zip(list(Company), actions, Action.get_action_magnitudes(actions)))
        for comp, action, mag in companies_with_actions_and_magnitudes:
            if action.is_buy():
                cash_limit = portfolio.cash * mag
                date, stock_price = stock_market_data[comp].get_last()
                shares_amount = cash_limit / stock_price
                if shares_amount > 0:
                    orders.append(Order(OrderType.BUY, comp, shares_amount))
            elif action.is_sell():
                shares_amount = portfolio.get_stock(comp) * mag
                if shares_amount > 0:
                    orders.append(Order(OrderType.SELL, comp, shares_amount))

        self.days_passed += 1
        return orders

    def train(self):
        """
        Train model based on experiences stored in memory.
        To speed up training a batch of experiences is trained at once (instance attribute batch_size).
        Conversely, training can happen at intervals up to batch_size days (instance attribute .
        """
        # determine cumulative rewards for actions in memory
        # loop backwards through the memory, ignoring the latest entry
        for i in range(2, len(self.memory) + 1):
            e: Experience = self.memory[-i]
            next_e: Experience = self.memory[-i + 1]  # its reward has been set in the previous loop iteration
            if i == 2:
                e.reward = e.get_portfolio_rel_immediate_change() - 1
            else:
                e.reward = e.get_portfolio_rel_immediate_change() - 1 + self.discount_factor * next_e.reward
        # modify reward as part of the computation of the Q-value:
        # 1. multiply by 100 to make it easier for the network to learn the values
        #    (brings absolute values mostly into a range of 0.01 .. 10.0)
        # 2. normalize for time horizon
        #    (assuming a high discount factor there is a large difference in cumulative reward between
        #     very recent and rather old experiences, simply because accumulating (discounted) reward
        #     over more recent experiences stops when the most recent experience has been hit)
        discounted_time_factor = 1.0
        for i in range(2, len(self.memory) + 1):
            e: Experience = self.memory[-i]
            e.reward *= 100.0 / discounted_time_factor
            discounted_time_factor += self.discount_factor ** (i - 1)

        # A part of the memory is too recent, i.e. the rewards for these states have only a short time horizon.
        # We want to exclude these recent states from learning because they do not meet the requirements
        # implied by the discount factor.
        memory_range_for_training = range(len(self.memory) - self.min_horizon)

        # assemble input (x) and output (y) arrays for training
        batch_indices = random.sample(memory_range_for_training, self.batch_size)
        x = numpy.array([
            State.to_input(self.memory[i].state1) for i in batch_indices
        ])
        assert x.shape == (self.batch_size, self.state_size)
        # use the output of the network as the basis for training
        y = self.model.predict(x)
        assert y.shape == (self.batch_size, self.action_size)
        action_to_index = {a: i for (i, a) in enumerate(Action)}
        # modify those parts of the output that correspond to the actions taken in the
        # experiences selected from memory
        for bi in range(self.batch_size):
            e: Experience = self.memory[batch_indices[bi]]
            q_value_to_train = e.reward

            # cap Q-value to lower model performance fluctuations during training
            if self.q_value_cap > 0.0:
                if q_value_to_train > self.q_value_cap:
                    q_value_to_train = self.q_value_cap
                if q_value_to_train < -self.q_value_cap:
                    q_value_to_train = -self.q_value_cap

            # 1) if encode each possible action tuple as a separate output neuron
            #    e.g. Action5 with output layer as 5^(#companies) neurons
            node_index = (action_to_index[e.actions[0]] * Action.get_number_of_output_neurons_per_action() +
                          action_to_index[e.actions[1]])
            y[bi][node_index] = q_value_to_train

            # 2) if encode each possible action per company as separate output neurons
            #    e.g. Action5 with output layer as 5*(#companies) neurons
            #    (simplified representation to reduce #output neurons)
            # node_index = action_to_index[e.actions[0]]
            # y[bi][node_index] = q_value_to_train
            # node_index = action_to_index[e.actions[1]] + Action.get_number_of_output_neurons_per_action()
            # y[bi][node_index] = q_value_to_train

            # debug info: show some untrained vs. trained Q-values
            # if bi == 0:
            #     self.log_info_every_nth_day(f'{y[bi][node_index]:.6f} vs. {q_value_to_train:.6f}', 50)
        self.model.train_on_batch(x, y)
        self.training_occasions += 1

    def print_policy(self):
        """
        Print the policy by enumerating all possible inputs and their corresponding output
        (actions determined by the highest Q-value).
        """
        if State is not StateExpertsOnly:
            return
        for eo1 in list(Vote):
            for eo2 in list(Vote):
                state = StateExpertsOnly([eo1, eo2], 0)
                x = state.to_input()
                y = self.model.predict(numpy.array([x]))[0]
                actions = Action.from_model_prediction(y)
                x_str = f'[{eo1.value: <4}, {eo2.value: <4}]'
                if Action is Action3:
                    action_str = f'[{actions[0].value: <4}, {actions[1].value: <4}]'
                elif Action is Action5:
                    action_str = f'[{actions[0].value: <11}, {actions[1].value: <11}]'
                else:
                    raise NotImplementedError
                y_str = ' '.join((f'{value:+.4f}' for value in y))
                logger.info(f'{x_str} -> {action_str} ({y_str})')

# Executing this module retrains the deep q-learning trader from scratch

# trains TRAINING_RUNS different models and saves the one with the median v score (return_pct_per_day)
TRAINING_RUNS = 5

# training is finished after data from the training set has been replayed EPISODES times
EPISODES = 5

if __name__ == "__main__":
    # Create the training data and testing data
    data = dict(train=StockMarketData([Company.A, Company.B], [Period.TRAINING]),
                test=StockMarketData([Company.A, Company.B], [Period.TESTING]))
    print(f"training data: {data['train'].get_row_count()} samples")
    # Hint: You can crop the training data with training_data.deepcopy_first_n_items(n)
    # data['train'] = data['train'].deepcopy_first_n_items(int(data['train'].get_row_count() / 5))
    print(f"training data (cropped): {data['train'].get_row_count()} samples")
    print(f"testing data: {data['test'].get_row_count()} samples")

    # Save final portfolio values and return % per day
    # per training run and episode
    final_portfolio_values = dict(train=[], test=[])
    return_pct_per_day = dict(train=[], test=[])
    traders = dict(train=[], test=[])

    def get_last_portfolio_value(phase: str, run: int) -> float:
        index = min(run * EPISODES + EPISODES - 1, len(final_portfolio_values[phase]) - 1)
        return final_portfolio_values[phase][index]

    def get_last_v_score(phase: str, run: int) -> float:
        index = min(run * EPISODES + EPISODES - 1, len(return_pct_per_day[phase]) - 1)
        return return_pct_per_day[phase][index]

    for run in range(TRAINING_RUNS):
        # Create the stock exchange and one traders to train the net
        starting_cash = dict(train=10000.0, test=2000.0)
        stock_exchanges = dict(train=stock_exchange.StockExchange(starting_cash["train"]),
                               test=stock_exchange.StockExchange(starting_cash["test"]))
        traders['train'].append(DeepQLearningTrader(ObscureExpert(Company.A), ObscureExpert(Company.B), False, True))

        for episode in range(EPISODES):
            # logger.info(f"DQL Trader: Starting training episode {episode}")

            for phase in ['train', 'test']:
                if phase == 'test':
                    testing_trader = DeepQLearningTrader(ObscureExpert(Company.A), ObscureExpert(Company.B), True, False)
                    if episode == 0:
                        traders[phase].append(testing_trader)
                    else:
                        traders[phase][-1] = testing_trader  # replace testing trader from previous episode
                trader = traders[phase][-1]
                stock_exchanges[phase].run(data[phase], [trader])
                if phase == 'train':
                    trader.save_trained_model()  # required to be able to create testing trader in next iteration
                p = stock_exchanges[phase].get_final_portfolio_value(trader)
                v = 100.0 * (math.pow(p / starting_cash[phase], 1 / data[phase].get_row_count()) - 1.0)
                final_portfolio_values[phase].append(p)
                return_pct_per_day[phase].append(v)

            logger.info(f"DQL Trader: Finished training episode {episode}, "
                        f"final portfolio value training {get_last_portfolio_value('train', run):.1e} vs. "
                        f"final portfolio value test {get_last_portfolio_value('test', run):.1e}")
            logger.info(f"\treturn % per day training {get_last_v_score('train', run):.6f} vs. "
                        f"return % per day test {get_last_v_score('test', run):.6f}")
            # traders['train'][-1].print_policy()

        logger.info('-' * 80)

    # sort traders by test v score
    traders_sorted_by_score = sorted(enumerate(traders['test']), key=lambda tup: get_last_v_score('test', tup[0]))
    # choose the one in the middle
    run, trader = traders_sorted_by_score[(TRAINING_RUNS - 1) // 2]
    # overwrite model from other training runs
    trader.save_trained_model()
    print(f"choosing model with median v score from run {run + 1}")
    print('output for csv data:')
    print('trader;dataset;returnpctperday;finalportfoliovalue')
    print('-' * 80)
    print(f";train;{get_last_v_score('train', run):.6f};{get_last_portfolio_value('train', run):.1e}")
    print(f";test;{get_last_v_score('test', run):.6f};{get_last_portfolio_value('test', run):.1e}")
    print('-' * 80)

    # plotting v scores across training episodes
    from matplotlib import pyplot as plt
    plt.figure()
    data_start_index = run * EPISODES
    data_stop_index = data_start_index + EPISODES
    plt.plot(return_pct_per_day['train'][data_start_index:data_stop_index], label='training', color="black")
    plt.plot(return_pct_per_day['test'][data_start_index:data_stop_index], label='testing', color="green")
    plt.title('v score training vs. testing')
    plt.ylabel('v score (avg. return % per day)')
    plt.xlabel('episode')
    plt.legend(['training', 'testing'])
    plt.show()


text:
import matplotlib
import numpy as np
import os
import sys

moleculeList = open("../formatted_data/numberedmoleculeidlist.txt", 'r')

for molLine in moleculeList:
    #print molLine
    name2 = molLine.split()[0]
    count = int(molLine.split()[1])
    count += 1
    #print name2 + molLine.split()[1]
    b = open("../formatted_data/"+ name2 +"_bonds.txt", 'r')
    unweightGraph = [[False for x in range(count)] for y in range(count)]

    for line in b:
        atom1 = line.split()[3]
        atom2 = line.split()[4]
        #print name2 + ":: atom 1: " + atom1 + "  atom 2: " + atom2
        unweightGraph[int(atom1)][int(atom2)] = True
        unweightGraph[int(atom2)][int(atom1)] = True

    b.close()
    adjMatrix = np.array(unweightGraph)
    filepth = "../formatted_data/features/" + name2
    np.save(filepth, adjMatrix)

'''
    molGraph = open("../formatted_data/features/" + name2 + "_graph.txt", 'w')

    molGraph.write("   ")
    for i in range(1,count):
        if (i < 10):
            molGraph.write(str(i) + "  ")
        else:
            molGraph.write(str(i) + " ")

    molGraph.write("\n")

    for i in range(1,count):
        if (i < 10):
            molGraph.write(str(i) + "  ")
        else:
            molGraph.write(str(i) + " ")
        for j in range(1,count):
            if(unweightGraph[i][j]):
                molGraph.write("T  ")
            else:
                molGraph.write("F  ")
            if(j == count-1):
                molGraph.write("\n")
    molGraph.close()
'''


text:
from minio import Minio
import os
import argparse
import numpy as np
import cv2
import glob
import json

from refer.utils import CreateExel
from refer.utils import createFolder
from refer.utils import print_progress

# Argument

parser = argparse.ArgumentParser(description="Convert the Data")
parser.add_argument('--base', '--b', help = 'the base address of the data')
args = parser.parse_args()

Basic = args.base
Basic = os.path.abspath(Basic)

def convert2line (Basic, folder):

    # The Address

    Input_Address = os.path.join(Basic, folder)
    Input_Address = os.path.abspath(Input_Address)
    print('Input Address is ' + Input_Address)

    M_folder = "M_" + folder
    Mask_Address = Basic + M_folder
    Mask_Address = os.path.join(Basic, M_folder)
    Mask_Address = os.path.abspath(Mask_Address)
    print('Address of Masked file is ' + Mask_Address)
    createFolder(Mask_Address)

    # Arrange the file

    text_storage=[]
    image_storage=[]

    for name in sorted(glob.glob(Input_Address + '/*.json')):
        text_storage.append(name)

    for name in sorted(glob.glob(Input_Address + '/*.BMP')):
        image_storage.append(name)

    # Convert

    i=0
    print_progress(i,len(text_storage))

    j=0
    for j in range(len(text_storage)):

        # Step 1. Read the image
        img = cv2.imread(image_storage[j])
        (a,b,c)=img.shape
        name1 = image_storage[j]
        sub1 = Input_Address
        sub2 ="/"
        name2 = name1.replace(sub1, '')
        fname = name2.replace(sub2, '')

        # Step 2.

        text = open(text_storage[j])
        data = json.load(text)
        pts = data ["one"]

        size = np.array([b,a])
        pts = (pts*size).astype(int)

        # Step 3. Generate Mask
        img=np.zeros((a,b))
        Mask = cv2.fillPoly(img, [pts], 1)
        Mask = np.array(Mask)
        os.chdir(Mask_Address)
        cv2.imwrite(fname, Mask)

        i += 1
        print_progress(i,len(text_storage))

    # Remove the json file

    for name in sorted(glob.glob(Input_Address + '/*.json')):
        os.remove(name)

# Download & Convert

folder_name = ['tests', 'train', 'valid']

for folder in folder_name:

    # Convert

    print ("Converting the file")
    convert2line (Basic = Basic , folder = folder)


text:
"""
This module defines the interface to run Simulation.

Author:
    Hailiang Zhao
"""
from cross_edge_offloading.cco.cco_algorithm import CcoAlgorithm
from cross_edge_offloading.benchmarks.random_policy import RandomPolicy
from cross_edge_offloading.benchmarks.greedy_communication_policy import GreedyComm
from cross_edge_offloading.benchmarks.greedy_computation_policy import GreedyCompute
from cross_edge_offloading.parameter import Parameter
from racos.dimension import Dimension
import numpy as np

def opt():
    parameter = Parameter()
    dimension = Dimension(parameter.get_dim_size())
    cco = CcoAlgorithm(parameter, dimension)
    rand_policy = RandomPolicy(parameter)
    greed_comm = GreedyComm(parameter)
    greed_compute = GreedyCompute(parameter)

    cco_costs, rand_costs, greed_comm_costs, greed_compute_costs = [], [], [], []
    cco_battery = np.zeros([parameter.get_time_horizon(), parameter.get_user_num()])
    rand_battery = np.zeros([parameter.get_time_horizon(), parameter.get_user_num()])
    greed_comm_battery = np.zeros([parameter.get_time_horizon(), parameter.get_user_num()])
    greed_compute_battery = np.zeros([parameter.get_time_horizon(), parameter.get_user_num()])

    # write overall costs and cost energy levels to txt files
    cost_file = open('overall_costs.txt', 'w+')
    battery_file = open('battery_levels.txt', 'w+')

    for t in range(parameter.get_time_horizon()):
        print('Time slot #', t)

        # record the cost energy levels
        for i in range(parameter.get_user_num()):
            cco_battery[t][i] = cco.get_battery_energy_levels()[i]
            rand_battery[t][i] = rand_policy.get_battery_energy_levels()[i]
            greed_comm_battery[t][i] = greed_comm.get_battery_energy_levels()[i]
            greed_compute_battery[t][i] = greed_compute.get_battery_energy_levels()[i]

        # solve sub-problem P_2^{eh}, store the optimal harvested energys
        cco.set_harvested_energys(cco.obtain_harvested_energys())
        rand_policy.set_harvested_energys(rand_policy.obtain_harvested_energys())
        greed_comm.set_harvested_energys(greed_comm.obtain_harvested_energys())
        greed_compute.set_harvested_energys(greed_compute.obtain_harvested_energys())

        # call racos to solve sub-problem P_2^{es}, store the optimal edge selections
        cco.racos(cco.calculate_object_func)
        cco.set_edge_selections(cco.instance_to_edge_selections(cco.get_optimal_solution()))
        cco_costs.append(cco.obtain_overall_costs(cco.get_edge_selections()))
        print('CCO overall cost:', cco_costs[t])

        rand_policy.set_edge_selections(rand_policy.obtain_edge_selections())
        rand_costs.append(rand_policy.obtain_overall_costs(rand_policy.get_edge_selections()))
        print('RS overall cost:', rand_costs[t])

        greed_comm.set_edge_selections(greed_comm.obtain_edge_selections())
        greed_comm_costs.append(greed_comm.obtain_overall_costs(greed_comm.get_edge_selections()))
        print('GSC1 overall cost:', greed_comm_costs[t])

        greed_compute.set_edge_selections(greed_compute.obtain_edge_selections())
        greed_compute_costs.append(greed_compute.obtain_overall_costs(greed_compute.get_edge_selections()))
        print('GSC2 overall cost:', greed_compute_costs[t])

        # update state information
        cco.update_energy_levels()
        rand_policy.update_energy_levels()
        greed_comm.update_energy_levels()
        greed_compute.update_energy_levels()

        # store results in file
        cost_file.write(str(cco_costs[t]) + ' ' + str(rand_costs[t]) + ' ' + str(greed_comm_costs[t]) + ' ' +
                        str(greed_compute_costs[t]) + '\n')
        battery_file.write(str(np.mean(cco_battery[t])) + ' ' + str(np.mean(rand_battery[t])) + ' ' +
                           str(np.mean(greed_comm_battery[t])) + ' ' + str(np.mean(greed_compute_battery[t])) + '\n')

        parameter.go_next_slot()
        dimension.set_dim_size(parameter.get_dim_size())

    cost_file.close()
    battery_file.close()

if __name__ == '__main__':
    opt()


text:
from __future__ import division
#import cf, cfplot as cfp
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
from matplotlib.colors import BoundaryNorm
from mpl_toolkits.basemap import Basemap
import scipy as sp
from scipy import stats

LAT=47
LON=47
WEEK=5

mjo = np.load('mjo_bchirps.npz')
bchirps = mjo['mjoa_bchirps']

mjo = np.load('mjo_chirps.npz')
chirps = mjo['mjoa_chirps']

mjo = np.load('mjo_ukmo.npz')
ukmo = mjo['mjoa_ukmo']

mjo = np.load('mjo_ncep.npz')
ncep = mjo['mjoa_ncep']

mjo = np.load('mjo_ecmwf.npz')
ecmwf = mjo['mjoa_ecmwf']

mjo = np.load('mjo_bam.npz')
bam = mjo['mjoa_bam']

bchirps_ano = [0]*5;chirps_ano = [0]*5;ukmo_ano = [0]*5;ncep_ano = [0]*5;ecmwf_ano = [0]*5;bam_ano = [0]*5
idx1 = [8,2,4,6]
idx2 = [1,3,5,7]
for i in range(len(idx1)):
    bchirps_ano[i] = np.vstack((bchirps[idx1[i]],bchirps[idx2[i]]))
    chirps_ano[i] = np.vstack((chirps[idx1[i]],chirps[idx2[i]]))
    ukmo_ano[i] = np.nanmean(np.vstack((ukmo[idx1[i]],ukmo[idx2[i]])), axis=2)
    ncep_ano[i] = np.nanmean(np.vstack((ncep[idx1[i]],ncep[idx2[i]])), axis=2)
    ecmwf_ano[i] = np.nanmean(np.vstack((ecmwf[idx1[i]],ecmwf[idx2[i]])), axis=2)
    bam_ano[i] = np.nanmean(np.vstack((bam[idx1[i]],bam[idx2[i]])), axis=2)

bchirps_ano[4] = bchirps[9].copy()
chirps_ano[4] = chirps[9].copy()
ukmo_ano[4] = np.nanmean(ukmo[9], axis=2)
ncep_ano[4] = np.nanmean(ncep[9], axis=2)
ecmwf_ano[4] = np.nanmean(ecmwf[9], axis=2)
bam_ano[4] = np.nanmean(bam[9], axis=2)

bam_acc = [0]*5;bam_pval = [0]*5
for n in range(5):
    bam_acc[n] = np.zeros((WEEK,LAT,LON))
    bam_pval[n] = np.zeros((WEEK,LAT,LON))
    for i in range(0,LAT):
        for j in range(0,LON):
            for w in range(0,WEEK):
    	        bam_acc[n][w,i,j] = sp.stats.pearsonr(bam_ano[n][:,w,i,j], bchirps_ano[n][:,w,i,j])[0]
    	        bam_pval[n][w,i,j] = sp.stats.pearsonr(bam_ano[n][:,w,i,j], bchirps_ano[n][:,w,i,j])[1]

ecmwf_acc = [0]*5;ecmwf_pval = [0]*5
for n in range(5):
    ecmwf_acc[n] = np.zeros((WEEK,LAT,LON))
    ecmwf_pval[n] = np.zeros((WEEK,LAT,LON))
    for i in range(0,LAT):
        for j in range(0,LON):
            for w in range(0,WEEK):
    	        ecmwf_acc[n][w,i,j] = sp.stats.pearsonr(ecmwf_ano[n][:,w,i,j], chirps_ano[n][:,w,i,j])[0]
    	        ecmwf_pval[n][w,i,j] = sp.stats.pearsonr(ecmwf_ano[n][:,w,i,j], chirps_ano[n][:,w,i,j])[1]

ncep_acc = [0]*5;ncep_pval = [0]*5
for n in range(5):
    ncep_acc[n] = np.zeros((WEEK,LAT,LON))
    ncep_pval[n] = np.zeros((WEEK,LAT,LON))
    for i in range(0,LAT):
        for j in range(0,LON):
            for w in range(0,WEEK):
    	        ncep_acc[n][w,i,j] = sp.stats.pearsonr(ncep_ano[n][:,w,i,j], chirps_ano[n][:,w,i,j])[0]
    	        ncep_pval[n][w,i,j] = sp.stats.pearsonr(ncep_ano[n][:,w,i,j], chirps_ano[n][:,w,i,j])[1]

ukmo_acc = [0]*5;ukmo_pval = [0]*5
for n in range(5):
    ukmo_acc[n] = np.zeros((WEEK,LAT,LON))
    ukmo_pval[n] = np.zeros((WEEK,LAT,LON))
    for i in range(0,LAT):
        for j in range(0,LON):
            for w in range(0,WEEK):
    	        ukmo_acc[n][w,i,j] = sp.stats.pearsonr(ukmo_ano[n][:,w,i,j], chirps_ano[n][:,w,i,j])[0]
    	        ukmo_pval[n][w,i,j] = sp.stats.pearsonr(ukmo_ano[n][:,w,i,j], chirps_ano[n][:,w,i,j])[1]

acc = [bam_acc, ecmwf_acc, ncep_acc, ukmo_acc]
pval = [bam_pval, ecmwf_pval, ncep_pval, ukmo_pval]
#pval = np.ma.masked_less(pval, 0.05)
#pval[np.isnan(acc)]=np.nan

latlim = [-49.5,19.5]
lonlim = [-90,-20]
lon = np.arange(lonlim[0],lonlim[1]+0.5,1.5)
lat = np.arange(latlim[0],latlim[1]+0.5,1.5)
cmap = matplotlib.cm.RdYlBu
cmin = -0.5
cmax = 0.5
cspc = 0.1
clevs = np.arange(cmin,cmax+cspc,cspc)
clabel = 'ACC_Difference'
norm = BoundaryNorm(boundaries=clevs, ncolors=256)
lw = 1
gl = 20
latext = [-50,15]
lonext = [-90,-30]
data = ['BAM','ECMWF','NCEP','UKMO']
phase = ['MJO-8+1','MJO-2+3','MJO-4+5','MJO-6+7']
WEEK=5

for d in range(len(data)):
    for p in range(len(phase)):
        for w in range(WEEK):
            fig = plt.figure(figsize=(3,3))
            mymap = Basemap(projection='cyl',resolution='l',llcrnrlat=latext[0],urcrnrlat=latext[1],llcrnrlon=lonext[0],urcrnrlon=lonext[1])
	    mymap.drawcoastlines(linewidth=lw)
            mymap.drawcountries(linewidth=lw)
            mymap.drawparallels(np.arange(-90,90,gl),labels=[1,0,0,1],labelstyle='+/-')
            mymap.drawmeridians(np.arange(0,360,gl),labels=[1,0,0,1],labelstyle='+/-')
            x, y = mymap(*np.meshgrid(lon,lat))
            uncal = mymap.pcolormesh(x,y,acc[d][p][w,:,:]-acc[d][4][w,:,:],vmin=cmin,vmax=cmax,cmap=cmap,norm=norm)
#	    mymap.pcolor(x, y, pval[d,p,w,:,:], hatch='.', alpha=0.)
#            plt.title(str(data[d])+'_'+str(phase[p])+'_'+'Week'+str(w+1)+'_'+str(clabel), fontsize=15)
#           fig.tight_layout()
            plt.savefig(str(data[d])+'_'+str(phase[p])+'_minus_Allyrs_'+'Week'+str(w+1)+'_'+str(clabel)+'.ps')
            #plt.show()

fig = plt.figure(figsize=(4,1))
ax = fig.add_axes([0.05, 0.5, 0.9, 0.07])
cb = matplotlib.colorbar.ColorbarBase(ax,cmap=cmap,norm=norm,extend='both',spacing='uniform',orientation='horizontal')
#cb.set_label(clabel)
#fig.tight_layout()
plt.savefig(str(clabel)+'_horizontal_colorbar.ps')
#plt.show()

fig = plt.figure(figsize=(1,4))
ax = fig.add_axes([0.4, 0.05, 0.07, 0.9])
cb = matplotlib.colorbar.ColorbarBase(ax,cmap=cmap,norm=norm,extend='both',spacing='uniform',orientation='vertical')
#cb.set_label(clabel)
#fig.tight_layout()
plt.savefig(str(clabel)+'_vertical_colorbar.ps')
#plt.show()

max_stars_repo_name: nick-klingaman/dubstep

max_issues_repo_name: nick-klingaman/dubstep

max_forks_repo_name: nick-klingaman/dubstep

reason: import numpy,import scipy,from scipy


text:
#script aiming at testing FileListProcessor_input_test
import cv2
import matplotlib.pyplot as plt
import tensorflow as tf
import os
import datetime
import numpy as np
import argparse

import DataProvider_input_pipeline

workingFolder='test_datapipeline'
sessionFolder=os.path.join(workingFolder, datetime.datetime.now().strftime("%Y-%m-%d--%H:%M:%S"))
os.makedirs(sessionFolder)
os.chdir(sessionFolder)

'''first focus on a set of folders with raw ad reference data,
here all data parameters are hard written since all data is supposed to be versionned
'''
#raw_data_dir_train = "../../datasamples/semantic_segmentation/raw_data/"
#reference_data_dir_train = "../../datasamples/semantic_segmentation/labels/"
raw_data_dir_train ="/home/alben/workspace/Datasets/CityScapes/leftImg8bit_trainvaltest/leftImg8bit/train/"
reference_data_dir_train = "/home/alben/workspace/Datasets/CityScapes/gtFine_trainvaltest/gtFine/train/"
nb_classes=34
patchSize=256
patchesPerImage=1000
allow_display=True
process_labels_histogram=False
nan_management='zeros'

parser = argparse.ArgumentParser(description='DataProvider_input_pipeline_test')
parser.add_argument('--avoid-display', dest='avoid_display', action='store_true',
                    help='avoid display in a Xwindow to let the scrip work on non X systems')
parser.add_argument('--check-data', dest='check_data', action='store_true',
                    help='If True/1, then iteratively tries to read each image of the dataset to ensure readability')
parser.add_argument('--mode-test', dest='mode_test', action='store_true',
                    help='Switch from Training mode (random crops by default) to Testing mode (adjacent cropping with overlap)')
parser.add_argument('--mode-labels-count', dest='labels_count', action='store_true',
                    help='compute class histogram over all the dataset')
parser.add_argument('--data-folder', dest='data_folder', default='',
                    help='change the data folder from default svn versionned image samples to a specific directory')
parser.add_argument('--field_of_view', dest='field_of_view', default=0,
                    help='specify the boundary width reduction on the reference data to remove')
parser.add_argument('--write_tfRecords', dest='write_tfRecords', action='store_true',
                    help='activate tf records writing, to be loaded at full speed for the next training session')
parser.add_argument('--tfRecords_name', dest='tfRecords_name', default='myDataset',
                    help='specifiy a file name to your tfrecord files, default is myDataset')
parser.add_argument('--read_tfRecords', dest='read_tfRecords', action='store_true',
                    help='activate tfrecord read test to check what has been writen, what is read next')

processCommands = parser.parse_args()
field_of_view=processCommands.field_of_view
if processCommands.data_folder != '':
    print('Testing of specific data folder : '+ processCommands.data_folder)
    raw_data_dir_train=processCommands.data_folder
if processCommands.avoid_display:
    allow_display = False
if processCommands.labels_count:
    process_labels_histogram = True

#get list of filenames
dataset_raw_files=DataProvider_input_pipeline.extractFilenames(raw_data_dir_train, '*.png')
dataset_references_files=DataProvider_input_pipeline.extractFilenames(reference_data_dir_train, '*labelIds.png')

###############################
#prepare dataset input pipeline
isTraining=not(processCommands.mode_test)
def apply_pixel_transforms(isTraining):
  if isTraining:
      return 0.5
  else:
      return None

data_provider=DataProvider_input_pipeline.FileListProcessor_Semantic_Segmentation(dataset_raw_files, dataset_references_files,
          nbEpoch=1,
          shuffle_samples=isTraining,
          patch_ratio_vs_input=patchSize,
          max_patches_per_image=patchesPerImage,
          image_area_coverage_factor=int(isTraining)+1.0,#factor 2 on training, 1 on testing
          num_reader_threads=10,#4 threads on training, 1 on testing
          apply_random_flip_left_right=isTraining,
          apply_random_flip_up_down=False,
          apply_random_brightness=apply_pixel_transforms(isTraining),
          apply_random_saturation=apply_pixel_transforms(isTraining),
          apply_whitening=True,
          batch_size=1,
          use_alternative_imread='opencv',
          balance_classes_distribution=isTraining,
          classes_entropy_threshold=0.6,
          opencv_read_flags=cv2.IMREAD_UNCHANGED,
          field_of_view=field_of_view,
          manage_nan_values=nan_management,
          additionnal_filters=None,
          crops_postprocess=None)

### semantic contours extraction
@tf.function
def get_semantic_contours(reference_crop):
    ''' deepnet_feed is a 3D tensor (height, width, 1) but
    the contour detection expects a batch of mono channel images (batch, height, width)
    ==> here it then needs a 3D tensor (1, height, width)
    '''

    print('input batch shape= '+str(reference_crop.get_shape().as_list()))
    formatted_reference_crop=tf.squeeze(tf.cast(reference_crop, tf.int32), -1)
    contours = DataProvider_input_pipeline.convert_semanticMap_contourMap(formatted_reference_crop)
    return contours

#######################################################################
### Optionnal samples recording setup (check here:https://www.tensorflow.org/api_docs/python/tf/io/TFRecordWriter?version=nightly)
def _bytes_feature(value):
  """Returns a bytes_list from a string / byte."""
  if isinstance(value, type(tf.constant(0))):
    value = value.numpy() # BytesList won't unpack a string from an EagerTensor.
  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))

def _float32_feature_list(value):
  """Returns a float_list from a float / double."""
  return tf.train.Feature(float_list=tf.train.FloatList(value=value))

def _int64_feature_scalar(value):
  """Returns an int64_list from a bool / enum / int / uint."""
  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))

# Create a dictionary with features that may be relevant.
def image_example(image_tensor, label=None):
  image_shape = image_tensor.shape
  print('image_example.shape',image_shape)

  feature = {
      'height': _int64_feature_scalar(image_shape[0]),
      'width': _int64_feature_scalar(image_shape[1]),
      'depth': _int64_feature_scalar(image_shape[2]),
      'image_raw': _float32_feature_list(image_tensor.numpy().flatten().tolist()),
  }
  if label is not None:
    feature.update({'label': _int64_feature_scalar(label)})

  return tf.train.Example(features=tf.train.Features(feature=feature)).SerializeToString()

if processCommands.write_tfRecords:
  #setup
  file_out = "%s.tfrecords" % processCommands.tfRecords_name
  print('Samples will be written as tfrecords in files: ', file_out)

def write_dataset_no_display():
  # add serialization node to the dataprovider and write the dataset directly
  def serialize_sample(sample):
    ''' takes as input a tensor, transform to protobuffer and
        returns it serialized
    '''
    return tf.py_function(image_example, [sample[0]], tf.string)

  writer = tf.data.experimental.TFRecordWriter(file_out)
  print('Writing dataset without display...can be long...')
  writer.write(data_provider.dataset.map(serialize_sample))
  print('Dataset writing done !')

def read_dataset_display():
  if processCommands.write_tfRecords:
    writer=tf.io.TFRecordWriter(file_out)
  #######################################################################
  ### Samples extraction loop
  class_pixel_counts=np.zeros(nb_classes)
  for step, batch in enumerate(data_provider.dataset):
    print('====== New step='+str(step))#, 'batch=',batch[0])
    sample=batch[0]
    if processCommands.write_tfRecords:
      sample_proto=image_example(sample)
      #print('Serialized sample:', sample_proto)
      writer.write(sample_proto)
      writer.flush()
    reference =sample[:,:,3:].numpy()
    class_pixel_counts+=np.array(np.histogram(reference, bins=[-1]+np.linspace(start=0, stop=33, num=34).tolist())[0])
    if allow_display is True:
      input_crop=sample[:,:,:3].numpy()
      contours=get_semantic_contours(tf.expand_dims(sample[:,:,3:],0)).numpy().squeeze(0)
      print('input_crop shape =', input_crop.shape)
      print('reference_crop shape =', reference.shape)
      print('contours_crop shape =', contours.shape)
      sample_minVal=np.min(input_crop)
      sample_maxVal=np.max(input_crop)
      print('Sample value range (min, max)=({minVal}, {maxVal})'.format(minVal=sample_minVal, maxVal=sample_maxVal))
      input_crop_norm=(input_crop-sample_minVal)*255.0/(sample_maxVal-sample_minVal)
      cv2.imshow('input crop', cv2.cvtColor(input_crop_norm.astype(np.uint8), cv2.COLOR_RGB2BGR))
      cv2.imshow('reference crop', reference.astype(np.uint8)*int(255/nb_classes))
      cv2.imshow('reference contours_disp', contours.astype(np.uint8)*255)
      cv2.waitKey(1000)

  if processCommands.write_tfRecords:
    writer.close()

  if allow_display is True:
    print('finished crop sample display, press a key to stop from an active opencv image show window')
    cv2.waitKey()
  if process_labels_histogram is True:
      plt.plot(class_pixel_counts)
      plt.title('Class pixels count')
      plt.savefig('RS_dataset_Class probabilities.eps')
      print('class_pixel_counts', class_pixel_counts)
      #DataProvider_input_pipeline.plot_sample_channel_histograms(result, filenameID="last_crop_")
      if allow_display is True:
          plt.show()

if processCommands.write_tfRecords and processCommands.avoid_display:
  write_dataset_no_display()
else:
  read_dataset_display()

if processCommands.write_tfRecords and processCommands.read_tfRecords:
  print('Attempting to read the writen tfrecords...')
  DataProvider_input_pipeline.test_image_tfrecords_dataset(filename=file_out)
  print('tfrecord reading done')

print('Test script end')

max_stars_repo_name: albenoit/DeepLearningTools

max_issues_repo_name: albenoit/DeepLearningTools

max_forks_repo_name: albenoit/DeepLearningTools


text:
import numpy as np
from sequentia.preprocessing import filtrate

# Create some sample data
X = [np.random.random((10 * i, 3)) for i in range(1, 4)]

# Filter the data with window size 5 and median filtering
X = filtrate(X, n=5, method='median')


text:
import pygoko

import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.lines as mlines
import matplotlib.patches as mpatches
from matplotlib.collections import PatchCollection
from matplotlib import cm

cmap = plt.get_cmap("jet")
norm = mpl.colors.Normalize(vmin=0.0, vmax=1.0)

def show1D(
    tree,
    data,
    final_layer=-1000,
    color="conditional",
    tracker=None,
    test_set=None,
):
    patches = []
    lines = []
    x_center = []
    y_center = []

    ungraphed_nodes = [tree.root()]

    while len(ungraphed_nodes) > 0:
        node = ungraphed_nodes.pop()
        (si, pi) = node.address()
        x = data[pi, 0]
        y = 1 + 0.1 * si
        width = 2 ** si
        x_center.append(x)
        y_center.append(y)

        singleton_len = float(len(node.singletons_indexes()))

        if si < final_layer:
            break
        child_probs = node.children_probs()
        singleton_probs =  [(a,v) for a,v in child_probs if a is None][0]
        child_probs = [(a,v) for a,v in child_probs if a is not None]
        for i, (child_address, child_prob) in enumerate(child_probs):
            (csi, cpi) = child_address
            y_next = 1 + 0.1 * csi
            x_next = data[cpi][0]
            child = tree.node(child_address)
            if color == "conditional":
                lines.append(
                    mlines.Line2D(
                        [x, x_next], [y, y - 0.05], color=cmap(child_prob), linewidth=4
                    )
                )
                lines.append(
                    mlines.Line2D(
                        [x_next, x_next],
                        [y - 0.05, y_next],
                        color=cmap(child_prob),
                        linewidth=4,
                    )
                )
            elif color == "total":
                coverage_count = float(child.coverage_count())
                lines.append(
                    mlines.Line2D(
                        [x, x_next],
                        [y, y - 0.05],
                        color=cmap(coverage_count / len(data)),
                        linewidth=4,
                    )
                )
                lines.append(
                    mlines.Line2D(
                        [x_next, x_next],
                        [y - 0.05, y_next],
                        color=cmap(coverage_count / len(data)),
                        linewidth=4,
                    )
                )
            elif "post_prob" in color:
                tracker_probs = tracker.marginal_posterior_probs((si, pi))
                tracker_probs = [(a,v) for a,v in tracker_probs if a is not None]
                my_color = tracker_probs[i][1]
                lines.append(
                    mlines.Line2D(
                        [x, x_next], [y, y - 0.05], color=cmap(my_color), linewidth=4
                    )
                )
                lines.append(
                    mlines.Line2D(
                        [x_next, x_next],
                        [y - 0.05, y_next],
                        color=cmap(my_color),
                        linewidth=4,
                    )
                )
            elif color == "evidence":
                evidence = tracker.evidence((si, pi))
                if evidence is not None:
                    tracker_probs, _ = evidence
                    my_color = tracker_probs[i][1]
                else:
                    my_color = 0
                lines.append(
                    mlines.Line2D(
                        [x, x_next], [y, y - 0.05], color=cmap(my_color), linewidth=4
                    )
                )
                lines.append(
                    mlines.Line2D(
                        [x_next, x_next],
                        [y - 0.05, y_next],
                        color=cmap(my_color),
                        linewidth=4,
                    )
                )

            ungraphed_nodes.append(child)
        for cpi in node.singletons_indexes():
            x_next = data[cpi][0]
            if color == "conditional":
                lines.append(
                    mlines.Line2D(
                        [x_next, x_next],
                        [y - 0.05, 0.0],
                        color=cmap(singleton_probs),
                        linewidth=2,
                        alpha=0.5,
                    )
                )
                lines.append(
                    mlines.Line2D(
                        [x, x_next],
                        [y, y - 0.05],
                        color=cmap(singleton_probs),
                        linewidth=2,
                        alpha=0.5,
                    )
                )
            elif color == "total":
                lines.append(
                    mlines.Line2D(
                        [x_next, x_next],
                        [y - 0.05, 0.0],
                        color=cmap(singleton_len / len(data)),
                        linewidth=2,
                        alpha=0.5,
                    )
                )
                lines.append(
                    mlines.Line2D(
                        [x, x_next],
                        [y, y - 0.05],
                        color=cmap(singleton_len / len(data)),
                        linewidth=2,
                        alpha=0.5,
                    )
                )
            elif "post_prob" in color:
                tracker_probs = tracker.marginal_posterior_probs((si, pi))
                tracker_probs = [(a,v) for a,v in tracker_probs if a is None]
                my_color = tracker_probs[0][1]
                lines.append(
                    mlines.Line2D(
                        [x_next, x_next],
                        [y - 0.05, 0.0],
                        color=cmap(my_color),
                        linewidth=2,
                        alpha=0.5,
                    )
                )
                lines.append(
                    mlines.Line2D(
                        [x, x_next],
                        [y, y - 0.05],
                        color=cmap(my_color),
                        linewidth=2,
                        alpha=0.5,
                    )
                )
            elif color == "evidence":
                evidence = tracker.evidence((si, pi))
                if evidence is not None:
                    _, my_color = evidence
                else:
                    my_color = 0
                lines.append(
                    mlines.Line2D(
                        [x_next, x_next],
                        [y - 0.05, 0.0],
                        color=cmap(my_color),
                        linewidth=4,
                    )
                )
                lines.append(
                    mlines.Line2D(
                        [x, x_next], [y, y - 0.05], color=cmap(my_color), linewidth=4
                    )
                )
        if node.is_leaf():
            patches.append(
                mpatches.Rectangle(
                    [x - width, y - 0.05], 2 * width, 0.08, ec="none", color="orange"
                )
            )
            if color == "conditional":
                lines.append(
                    mlines.Line2D(
                        [x, x],
                        [y, 0.0],
                        color=cmap(singleton_probs),
                        linewidth=2,
                        alpha=0.5,
                    )
                )
            elif color == "total":
                lines.append(
                    mlines.Line2D(
                        [x, x],
                        [y, 0.0],
                        color=cmap(1.0 / len(data)),
                        linewidth=2,
                        alpha=0.5,
                    )
                )
            elif "post_prob" in color:
                my_color = tracker.marginal_posterior_probs((si, pi))[0][1]
                lines.append(
                    mlines.Line2D(
                        [x, x], [y, 0.0], color=cmap(my_color), linewidth=2, alpha=0.5
                    )
                )
            elif color == "evidence":
                evidence = tracker.evidence((si, pi))
                if evidence is not None:
                    _, my_color = evidence
                else:
                    my_color = 0
                lines.append(
                    mlines.Line2D([x, x], [y, 0.0], color=cmap(my_color), linewidth=4)
                )

        else:
            patches.append(
                mpatches.Rectangle(
                    [x - width, y - 0.05], 2 * width, 0.08, ec="none", color="blue"
                )
            )
    fig, ax = plt.subplots()

    ax.set_xlim((-1.01, 1.01))
    ax.set_ylim((-0.05, 1.2))

    if test_set is not None:
        for c in test_set:

            path = tree.path(c)
            csi, cpi = path[-1][0]
            x = data[cpi][0]
            y = 1 + 0.1 * csi
            x_next = c[0]
            lines.append(
                mlines.Line2D([x, x_next], [y, y - 0.05], color=cmap(my_color), linewidth=2)
            )
            lines.append(
                mlines.Line2D([x_next, x_next], [y - 0.05, 0.0], color=cmap(my_color), linewidth=2)
            )
        ax.scatter(
            test_set[:, 0].reshape((test_set.shape[0],)),
            np.zeros([test_set.shape[0]]),
            color="red",
            zorder=2.5,
        )

    collection = PatchCollection(patches, match_original=True, alpha=0.3)
    for line in lines:
        ax.add_line(line)
    ax.scatter(x_center, y_center)
    ax.add_collection(collection)
    cax = fig.add_axes([0.95, 0.05, 0.01, 0.9])
    ax.scatter(
        data[:, 0].reshape((data.shape[0],)),
        np.zeros([data.shape[0]]),
        color="orange",
        alpha=1,
        zorder=2.5,
    )

    fig.colorbar(
        cm.ScalarMappable(norm=norm, cmap=cmap), cax=cax, orientation="vertical"
    )

    ax.axis("off")
    fig.set_size_inches(14.00, 7.00)
    fig.savefig(f"1d_vis_{color}.png", bbox_inches="tight")

if __name__ == "__main__":
    numPoints = 21
    numTest = 10

    data1 = np.random.normal(loc=0.5, scale=0.2, size=(numPoints // 3,)).reshape(
        numPoints // 3, 1
    )
    data2 = np.random.normal(loc=-0.5, scale=0.1, size=(2 * numPoints // 3,)).reshape(
        2 * numPoints // 3, 1
    )
    data = np.concatenate([data1, data2], axis=0)
    data = np.concatenate([data, np.zeros([numPoints, 1])], axis=1).astype(np.float32)

    test_set = np.random.normal(loc=0.5, scale=0.2, size=(10,)).reshape(10, 1)
    test_set = np.concatenate([test_set, np.zeros([10, 1])], axis=1).astype(np.float32)

    tree = pygoko.CoverTree()
    tree.set_scale_base(2)
    tree.set_leaf_cutoff(0)
    tree.fit(data)
    path = tree.path(np.array([0.5], dtype=np.float32))

    run_tracker = tree.tracker(0)
    for i, t in enumerate(test_set):
        run_tracker.push(t)
        show1D(
            tree,
            data,
            color=f"post_prob_{i}",
            tracker=run_tracker,
            test_set=test_set[:i],
        )
    show1D(tree, data, color="total")
    show1D(tree, data)

max_stars_repo_name: comath/goko

max_issues_repo_name: comath/goko

max_forks_repo_name: comath/goko


text:
"""
For each ImageNet category, take the VGG16 representations of images
belonging to it, and compute the mean of these. The VGG16 representation of an
image is found by computing a forward pass through the network and looking at
the activation of the penultimate layer.
"""

import os
os.environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID'
os.environ['CUDA_VISIBLE_DEVICES'] = gpu

import numpy as np
import tensorflow as tf
from ..utils.metadata import wnids
from ..utils.paths import path_imagenet, path_representations
from ..utils.testing import model_predict

vgg = tf.keras.applications.VGG16()
model = tf.keras.models.Sequential()
for layer in vgg.layers[:-1]:
    model.add(layer)

datagen = tf.keras.preprocessing.image.ImageDataGenerator(
    preprocessing_function=tf.keras.applications.vgg16.preprocess_input)
generator = datagen.flow_from_directory(directory=path_imagenet/'train/')
df = pd.DataFrame({
    'filename':generator.filenames,
    'wnid':pd.Series(generator.filenames).str.split('/', expand=True)[0]})
mean_representations = []

for i, wnid in enumerate(wnids):
    generator_wnid = datagen.flow_from_dataframe(
        dataframe=df.loc[df['wnid']==wnid],
        directory=path_imagenet/'train/',
        target_size=(224,224),
        batch_size=256,
        shuffle=False,
        class_mode=None)
    representations_wnid = generator_wnid.predict(
        x=generator_wnid,
        steps=len(generator_wnid),
        use_multiprocessing=False,
        verbose=True)
    mean_representations.append(np.mean(representations_wnid, axis=0))

np.save(
    path_representations,
    np.array(mean_representations),
    allow_pickle=False)

max_stars_repo_name: fbickfordsmith/attention-iclr

max_issues_repo_name: fbickfordsmith/attention-iclr

max_forks_repo_name: fbickfordsmith/attention-iclr


text:
import numpy as np

class CuboidMesh():
    """docstring for CuboidMesh"""
    def __init__(self, arg):
        super(CuboidMesh, self).__init__()
    @staticmethod
    def meshgrid_3d(x_1d, y_1d, z_1d):
        '''
        Suppose x_1d.shape = M, y_1d.shape = N, z_1d.shape = P
        Then, this function will return 3D arrays x_3d,y_3d,z_3d each of size (P,N,M)
        The numpy meshgrid gives unusual ordering.
        '''
        M = x_1d.shape[0]
        N = y_1d.shape[0]
        P = z_1d.shape[0]
        x_2d, y_2d = np.meshgrid(x_1d, y_1d)
        x_3d = np.tile(x_2d, (P,1,1))
        y_3d = np.tile(y_2d, (P,1,1))
        z_3d = np.reshape(np.repeat(z_1d, (N*M)), (P,N,M))
        # print("x_3d = \n", x_3d)
        # print("y_3d = \n", y_3d)
        # print("z_3d = \n", z_3d)
        # exit()
        return x_3d, y_3d, z_3d

max_stars_repo_name: adityabalu/DiffNet

max_issues_repo_name: adityabalu/DiffNet

max_forks_repo_name: adityabalu/DiffNet


text:
import sys
import numpy as np
from count_nonblack import count_nonblack
from PIL import Image as PillowImage

def percentage_cloud(imageFile):
    img = PillowImage.open(imageFile)
    resolution = img.size[0] * img.size[1]
    nonblack = count_nonblack(img)
    percentage = nonblack / resolution
    return percentage

max_issues_repo_name: mohamedmehdigara/cloudGAN

max_forks_repo_name: mohamedmehdigara/cloudGAN