text:
"""
Adapted from https://github.com/Kaixhin/Rainbow

The main file needed within rainbow. Runs of the train and test functions from their respective
files.

Example of use:
`cd algorithms/rainbow`
`python main.py`

Runs Rainbow DQN on our AI2ThorEnv wrapper with default params. Optionally it can be run on any
atari environment as well using the "game" flag, e.g. --game seaquest.
"""

import argparse
from datetime import datetime

import numpy as np
import torch

from algorithms.rainbow.agent import Agent
from algorithms.rainbow.env import Env, FrameStackEnv
from algorithms.rainbow.memory import ReplayMemory
from algorithms.rainbow.test import test
from gym_ai2thor.envs.ai2thor_env import AI2ThorEnv

parser = argparse.ArgumentParser(description='Rainbow')
parser.add_argument('--seed', type=int, default=123, help='Random seed')
parser.add_argument('--disable-cuda', action='store_true', help='Disable CUDA')
""" Game can be ai2thor to use our wrapper or one atari rom from the list here:
    https://github.com/openai/atari-py/tree/master/atari_py/atari_roms """
parser.add_argument('--game', type=str, default='ai2thor', help='ATARI game or environment')
parser.add_argument('--max-num-steps', type=int, default=int(50e6), metavar='STEPS',
                    help='Number of training steps')
parser.add_argument('--max-episode-length', type=int, default=int(1e3), metavar='LENGTH',
                    help='Max episode length (0 to disable)')
parser.add_argument('--history-length', type=int, default=1, metavar='T',
                    help='Number of consecutive states processed')
parser.add_argument('--hidden-size', type=int, default=512, metavar='SIZE',
                    help='Network hidden size')
parser.add_argument('--noisy-std', type=float, default=0.8, metavar='σ',
                    help='Initial standard deviation of noisy linear layers')
parser.add_argument('--num-atoms', type=int, default=51, metavar='C',
                    help='Discretised size of value distribution')
parser.add_argument('--V-min', type=float, default=-10, metavar='V',
                    help='Minimum of value distribution support')
parser.add_argument('--V-max', type=float, default=10, metavar='V',
                    help='Maximum of value distribution support')
parser.add_argument('--model-path', type=str, metavar='PARAMS', help='Pretrained model (state dict)')
parser.add_argument('--memory-capacity', type=int, default=int(1e6), metavar='CAPACITY',
                    help='Experience replay memory capacity')
parser.add_argument('--replay-frequency', type=int, default=1, metavar='k',
                    help='Frequency of sampling from memory')
parser.add_argument('--priority-exponent', type=float, default=0.1, metavar='ω',
                    help='Prioritised experience replay exponent (originally denoted α)')
parser.add_argument('--priority-weight', type=float, default=0.4, metavar='β',
                    help='Initial prioritised experience replay importance sampling weight')
parser.add_argument('--multi-step', type=int, default=3, metavar='n',
                    help='Number of steps for multi-step return')
parser.add_argument('--discount', type=float, default=0.99, metavar='γ',
                    help='Discount factor')
parser.add_argument('--target-update', type=int, default=int(32e3), metavar='τ',
                    help='Number of steps after which to update target network')
parser.add_argument('--reward-clip', type=int, default=0, metavar='VALUE',
                    help='Reward clipping (0 to disable)')
parser.add_argument('--lr', type=float, default=0.0000625, metavar='η',
                    help='Learning rate')
parser.add_argument('--adam-eps', type=float, default=1.5e-4, metavar='ε',
                    help='Adam epsilon')
parser.add_argument('--batch-size', type=int, default=32, metavar='SIZE',
                    help='Batch size')
parser.add_argument('--learn-start', type=int, default=int(20e3), metavar='STEPS',
                    help='Number of steps before starting training')
parser.add_argument('--evaluate-only', action='store_true', help='Evaluate only')
parser.add_argument('--evaluation-interval', type=int, default=1e5, metavar='STEPS',
                    help='Number of training steps between evaluations')
parser.add_argument('--evaluation-episodes', type=int, default=10, metavar='N',
                    help='Number of evaluation episodes to average over')
parser.add_argument('--evaluation-size', type=int, default=500, metavar='N',
                    help='Number of transitions to use for validating Q')
parser.add_argument('--log-interval', type=int, default=200, metavar='STEPS',
                    help='Number of training steps between logging status')
parser.add_argument('--render', action='store_true', default=False,
                    help='Display screen (testing only)')
parser.add_argument('--config-file', type=str, default='config_files/rainbow_example.json',
                    help='Config file used for ai2thor environment definition')

if __name__ == '__main__':
    # Setup arguments, seeds and cuda
    args = parser.parse_args()
    print('-' * 10 + '\n' + 'Options' + '\n' + '-' * 10)
    for k, v in vars(args).items():
        print(' ' * 4 + k + ': ' + str(v))
    np.random.seed(args.seed)
    torch.manual_seed(np.random.randint(1, 10000))
    if torch.cuda.is_available() and not args.disable_cuda:
        args.device = torch.device('cuda')
        torch.cuda.manual_seed(np.random.randint(1, 10000))
        # Disable non deterministic ops (not sure if critical but better safe than sorry)
        torch.backends.cudnn.enabled = False
    else:
        args.device = torch.device('cpu')

    # Simple ISO 8601 timestamped logger
    def log(s):
        print('[' + str(datetime.now().strftime('%Y-%m-%dT%H:%M:%S')) + '] ' + s)

    # Environment selection
    if args.game == 'ai2thor':
        env = FrameStackEnv(AI2ThorEnv(config_file=args.config_file), args.history_length,
                            args.device)
        args.resolution = env.config['resolution']
        args.img_channels = env.observation_space.shape[0]
    else:
        env = Env(args)
        env.train()
        args.resolution = (84, 84)
        args.img_channels = 1
    action_space = env.action_space

    # Agent
    dqn = Agent(args, env)
    mem = ReplayMemory(args, args.memory_capacity)
    """ Priority weights are linearly annealed and increase every step by priority_weight_increase from
    args.priority_weight to 1.
    Typically, the unbiased nature of the updates is most important near convergence at the end of
    training, as the process is highly non-stationary anyway, due to changing policies, state
    distributions and bootstrap targets, that small bias can be ignored in this context.
    """
    priority_weight_increase = (1 - args.priority_weight) / (args.max_num_steps - args.learn_start)

    """ Construct validation memory. The transitions stored in this memory will remain constant for
    the whole training process. During training this gives us a "fixed" evaluation dataset to see
    how the agent's confidence of its performance is improving.
    """
    val_mem = ReplayMemory(args, args.evaluation_size)
    mem_steps, done = 0, True
    for mem_steps in range(args.evaluation_size):
        if done:
            state, done = env.reset(), False
        next_state, _, done, _ = env.step(env.action_space.sample())
        # No need to store actions or rewards because we only use the state to evaluate Q
        val_mem.append(state, None, None, done)
        state = next_state

    if args.evaluate_only:
        dqn.eval()  # Set DQN (online network) to evaluation mode
        avg_reward, avg_Q = test(env, mem_steps, args, dqn, val_mem, evaluate_only=True)
        print('Avg. reward: ' + str(avg_reward) + ' | Avg. Q: ' + str(avg_Q))
    else:
        # Training loop
        dqn.train()
        num_steps, done = 0, True
        while num_steps < args.max_num_steps:
            if done:
                state, done = env.reset(), False
            if num_steps % args.replay_frequency == 0:
                dqn.reset_noise()  # Draw a new set of noisy epsilons

            action = dqn.act(state)  # Choose an action greedily (with noisy weights)
            next_state, reward, done, _ = env.step(action)  # Step
            if args.reward_clip > 0:
                reward = max(min(reward, args.reward_clip), -args.reward_clip)  # Clip rewards
            mem.append(state, action, reward, done)  # Append transition to memory
            num_steps += 1

            if num_steps % args.log_interval == 0:
                log('num_steps = ' + str(num_steps) + ' / ' + str(args.max_num_steps))

            # Train and test
            if num_steps >= args.learn_start:
                # Anneal importance sampling weight β to 1
                mem.priority_weight = min(mem.priority_weight + priority_weight_increase, 1)

                if num_steps % args.replay_frequency == 0:
                    dqn.learn(mem)  # Train with n-step distributional double-Q learning

                if num_steps % args.evaluation_interval == 0:
                    dqn.eval()  # Set DQN (online network) to evaluation mode. Fixed linear layers
                    # Test and save best model
                    avg_reward, avg_Q = test(env, num_steps, args, dqn, val_mem)
                    log('num_steps = ' + str(num_steps) + ' / ' + str(args.max_num_steps) +
                        ' | Avg. reward: ' + str(avg_reward) + ' | Avg. Q: ' + str(avg_Q))
                    dqn.train()  # Set DQN (online network) back to training mode

                # Update target network
                if num_steps % args.target_update == 0:
                    dqn.update_target_net()
            state = next_state
    env.close()


text:
import configparser
import matplotlib.pyplot as plt
import numpy as np
import dill
import pickle

from map import Map
from mvbspline import MVBSpline
from utils import load_map

config = configparser.ConfigParser()
config.read('config.ini')

# class Test:
#     def __init__(self, data):
#         self._data = np.array(data)
#
#     def __getitem__(self, item):
#         return self._data[item]

if __name__ == "__main__":
    # Map for planning
    map_file = config.get('MAP', 'file')
    map_resolution = config.getfloat('MAP', 'resolution')

    # Degree of spline and pickle filename to store spline object after optimization process
    degree = config.getint('CURVE', 'degree')
    filename = config.get('CURVE', 'save_filename')

    # Initialize map
    map_data = load_map(map_file)
    map = Map(map_data, resolution=map_resolution)

    # Set the cells in the map which are traversable
    map[10:40, 86:97] = Map.AVAILABLE
    map[40:90, 78:97] = Map.AVAILABLE
    map[59:74, 56:78] = Map.AVAILABLE
    map[39:59, 56:64] = Map.AVAILABLE
    map[39:55, 36:56] = Map.AVAILABLE
    map[55:65, 36:44] = Map.AVAILABLE
    map[57:65, 10:36] = Map.AVAILABLE
    map[38:57, 16:24] = Map.AVAILABLE

    # map[10:40, 86:97] = Map.AVAILABLE
    # map[40:90, 78:97] = Map.AVAILABLE
    # map[59:74, 56:78] = Map.AVAILABLE
    # map[67:98, 36:64] = Map.AVAILABLE
    # map[77:85, 24:36] = Map.AVAILABLE
    # map[66:98, 2:24] = Map.AVAILABLE

    # print(map.is_available(np.array([6.59, 4.12])))

    # Initial guess at a path which goes through the allowed area
    path = np.array([[20, 90],
                     [67, 90],
                     [67, 60],
                     [45, 60],
                     [45, 40],
                     [61, 40],
                     [61, 19],
                     [50, 20],
                     [43, 20]]) * map_resolution

    # # Find path satisfying minimum radius of curvature of robot and C3 continuous using a Quartic Spline
    # # Points for the left boundary
    # left_boundary_pts = np.array([[10, 13], [38, 13], [57, 24], [77, 63], [80, 100]])
    #
    # # Points for the right boundary
    # right_boundary_pts = np.array([[10, 2], [70, 2], [73, 35], [85, 65], [85, 80], [98, 80], [98, 98]])
    #
    # # Points for the initial control points guess (these should lie within the boundaries). They can be
    # # chosen randomly, as long as they are within the right and left boundaries
    # x_ctrl_pts = np.array([20, 23, 28, 30, 35, 41, 46, 50, 55, 58, 62, 69, 74, 76, 78, 80, 81, 85, 88, 90]) * map_resolution
    # y_ctrl_pts = map.get_height() - np.array([5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 35, 45, 50, 60, 70, 82, 85, 87, 90]) * map_resolution
    # ctrl_pts_guess = np.vstack((np.array(x_ctrl_pts), np.array(y_ctrl_pts)))
    # path = ctrl_pts_guess.transpose()
    # Show the map
    plt.imshow(map.get_data(), extent=[0, map.get_width(), map.get_height(), 0])
    plt.plot(path[:,0], path[:,1], '-.xr', linewidth=2)
    plt.show()
    quartic = MVBSpline(path.transpose(), degree, map)  # Set to 3 for cubic, 4 for quartic
    quartic.add_boundary_conditions(np.array([20, 90]), np.array([43, 20]), 0, 0)

    # Run this to calculate new control points for B-Spline (runs the optimization process which takes a while)
    success = quartic.solve()

    # Save the spline object to be reused later
    quartic.__module__ = 'mvbspline'
    fh = open(filename, 'w')
    pickle.dump(quartic, fh)
    fh.close()

    # Some examples to plot different properties of the B-Spline

    # Boundaries and Curve
    knot_min, knot_max = quartic.get_knot_limits()
    points = np.array(list(quartic.get_position(t) for t in np.linspace(knot_min, knot_max, 1000)))
    plt.figure()
    plt.imshow(map.get_data(), extent=[0, map.get_width(), map.get_height(), 0])
    plt.plot(points[:, 0], points[:, 1])

    # Curvature vs Arc Length
    # plt.figure()
    # curvatures = np.array(list(quartic.calc_curvature(s) for s in quartic.s))
    # plt.plot(quartic.s, curvatures)

    # Derivative of Curvature vs Arc Length
    plt.figure()
    curvatures_d = np.array(list(quartic.calc_curvature_derivative(s) for s in quartic.s))
    plt.plot(quartic.s, curvatures_d)

    # Yaw vs Arc Length
    # plt.figure()
    # yaws = np.array(list(quartic.calc_yaw(s) for s in quartic.s))
    # plt.plot(quartic.s, yaws * 180/np.pi)

    # Show all plots
    plt.show()

ext: py

reason: import numpy


text:
#!/usr/bin/python
"""
Throughput measurement block
"""
from gnuradio import gr
import time
import numpy as np
byte = np.byte
short = np.short
from gr_gen_tools.utils.representation import engineering_notation
class Throughput(gr.sync_block):

    def __init__(self, name, period, dtype=float):
        """
        Constructor for the throughput component

        Parameter
        ---------
        name : str
            Name of the stream being measured

        period : float
            Period of time to print and gather measurement (in seconds)
        """
        # --------------------------- error checking ------------------------
        if type(name) is not str:
            raise TypeError('name should be of type' + str(str))
        if period <  0:
            raise ValueError('period should be greater than  0')

        # -----------------------  call the synb block  ---------------------
        gr.sync_block.__init__(self,
            name="Throughput",
            in_sig=[dtype],
            out_sig=None)

        # -----------------------  initialize properties  -------------------
        self.stream_name = name
        self.period = period
        self.last_time = time.time()
        self.num_data = 0

    def set_name(self, name="Default"):
        """
        Set method for name

        Parameter
        ---------
        name : str
            Name of the stream being measured.
        """
        # --------------------------- error checking ---------------------------
        if type(name) is not str:
            raise TypeError('name should be of type' + str(str))

        # ---------------------------- set property ----------------------------
        self.stream_name = name

        # ---------------------- setup internal variables -------------------
        self._setup_internal_variables()

    def set_period(self, period=5):
        """
        Set method for period

        Parameter
        ---------
        period : float
            Period of time to print and gather measurement (in seconds)
        """
        # --------------------------- error checking ------------------------
        if type(period) is not float:
            raise TypeError('period should be of type' + str(float))
        if period <  0:
            raise ValueError('period should be greater than  0')

        # ---------------------------- set property -------------------------
        self.period = period

        # ---------------------- setup internal variables -------------------
        self._setup_internal_variables()

    def get_name(self):
        """
        Get method for name
        """
        return self.name

    def get_period(self):
        """
        Get method for period
        """
        return self.period

    def work(self, input_items, output_items):
        """
        Work function
        """
        num_in = len(input_items[0])
        self.num_data += num_in
        toc = time.time()
        if toc - self.last_time > self.period:
            # display throughput
            avg_thru = float(self.num_data) / (toc - self.last_time)
            print("Throughput (%s) = %s elements/second"%\
                (self.stream_name, engineering_notation(avg_thru)))

            # update data and last time
            self.num_data = 0
            self.last_time = toc
        return num_in

max_stars_repo_name: GassiusODude/gr-gen-tools

max_issues_repo_name: GassiusODude/gr-gen-tools

max_forks_repo_name: GassiusODude/gr-gen-tools


text:
from artemis.fileman.file_getter import get_file
from scipy.ndimage.io import imread
from scipy.misc import imresize
import os
# from PIL import Image

__author__ = 'peter'

IMAGE_COLLECTION = {
    'wanderer': 'http://i.imgur.com/lR6wTNw.jpg',
    'dinnertime': 'http://i.imgur.com/nRczfmr.jpg',
    'limbo': 'https://upload.wikimedia.org/wikipedia/commons/thumb/3/3c/Follower_of_Jheronimus_Bosch_Christ_in_Limbo.jpg/1024px-Follower_of_Jheronimus_Bosch_Christ_in_Limbo.jpg',
    'manchester_newyear': 'http://i.imgur.com/UDEIMHp.jpg',
    'lenna': 'https://upload.wikimedia.org/wikipedia/en/2/24/Lenna.png',
    'starry_night': 'https://upload.wikimedia.org/wikipedia/commons/thumb/e/ea/Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg/800px-Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg',
    'nektarfront': 'https://upload.wikimedia.org/wikipedia/commons/0/00/Tuebingen_Neckarfront.jpg',
    'the_shipwreck_of_the_minotaur': 'https://upload.wikimedia.org/wikipedia/commons/2/2e/Shipwreck_turner.jpg',
    'scream': 'https://en.wikipedia.org/wiki/The_Scream#/media/File:The_Scream.jpg',
    'femme_nue_assise': 'https://upload.wikimedia.org/wikipedia/en/8/8f/Pablo_Picasso%2C_1909-10%2C_Figure_dans_un_Fauteuil_%28Seated_Nude%2C_Femme_nue_assise%29%2C_oil_on_canvas%2C_92.1_x_73_cm%2C_Tate_Modern%2C_London.jpg',
    'composition_vii': 'https://upload.wikimedia.org/wikipedia/commons/b/b4/Vassily_Kandinsky%2C_1913_-_Composition_7.jpg',
    'dog': 'http://www.dogchannel.com/images/articles/breed_profile_images/canaan-dog.jpg',
    'sea_turtle': 'https://en.wikipedia.org/wiki/Turtle#/media/File:TurtleOmaha.jpg',
    'heron': 'https://upload.wikimedia.org/wikipedia/commons/e/e6/Heron_tricol_01.JPG',
    }

def get_image(name, size = None):
    """
    Get an image by name.
    :param name: A string identifying an image from our dictionary.
    :return:
    """
    assert name in IMAGE_COLLECTION, "We don't have the image '%s' in the gallary" % (name, )
    _, ext = os.path.splitext(IMAGE_COLLECTION[name])
    relative_path = os.path.join('images', name)+ext
    filename = get_file(
        relative_name = relative_path,
        url = IMAGE_COLLECTION[name],
        )
    im_array = imread(filename)
    if im_array.ndim==2:
        im_array = im_array[:, :, None] + [0, 0, 0]
    if size is not None:
        im_array = imresize(im_array, get_new_size(im_array.shape[:2], new_size=size))
    return im_array

def get_new_size(cy_and_cx, new_size):
    cy, cx = cy_and_cx
    ny, nx = new_size or (None, None)
    aspect_ratio = float(cy) / cx
    if ny is None and nx is None:
        return cy
    elif nx is None:
        nx = int(ny * aspect_ratio)
    elif ny is None:
        ny = int(nx / aspect_ratio)
    return ny, nx

if __name__ == '__main__':
    from matplotlib import pyplot as plt

    im_name = 'manchester_newyear'
    im = get_image(im_name)
    plt.imshow(im)
    plt.title(im_name)
    plt.show()

max_stars_repo_name: peteroconnor-bc/artemis

max_issues_repo_name: peteroconnor-bc/artemis

max_forks_repo_name: peteroconnor-bc/artemis

reason: from scipy


text:
# Copyright 2018 The dm_control Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or  implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================

"""Tests for `dm_control.manipulation_suite`."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

# Internal dependencies.

from absl import flags
from absl.testing import absltest
from absl.testing import parameterized
from dm_control import manipulation
import numpy as np
from six.moves import range

flags.DEFINE_boolean(
    'fix_seed', True,
    'Whether to fix the seed for the environment\'s random number generator. '
    'This the default since it prevents non-deterministic failures, but it may '
    'be useful to allow the seed to vary in some cases, for example when '
    'repeating a test many times in order to detect rare failure events.')

FLAGS = flags.FLAGS

_FIX_SEED = None
_NUM_EPISODES = 5
_NUM_STEPS_PER_EPISODE = 10

def _get_fix_seed():
  global _FIX_SEED
  if _FIX_SEED is None:
    if FLAGS.is_parsed():
      _FIX_SEED = FLAGS.fix_seed
    else:
      _FIX_SEED = FLAGS['fix_seed'].default
  return _FIX_SEED

class ManipulationTest(parameterized.TestCase):
  """Tests run on all the tasks registered."""

  def _validate_observation(self, observation, observation_spec):
    self.assertEqual(list(observation.keys()), list(observation_spec.keys()))
    for name, array_spec in observation_spec.items():
      array_spec.validate(observation[name])

  def _validate_reward_range(self, reward):
    self.assertIsInstance(reward, float)
    self.assertBetween(reward, 0, 1)

  def _validate_discount(self, discount):
    self.assertIsInstance(discount, float)
    self.assertBetween(discount, 0, 1)

  @parameterized.parameters(*manipulation.ALL)
  def test_task_runs(self, task_name):
    """Tests that the environment runs and is coherent with its specs."""
    seed = 99 if _get_fix_seed() else None
    env = manipulation.load(task_name, seed=seed)
    random_state = np.random.RandomState(seed)

    observation_spec = env.observation_spec()
    action_spec = env.action_spec()
    self.assertTrue(np.all(np.isfinite(action_spec.minimum)))
    self.assertTrue(np.all(np.isfinite(action_spec.maximum)))

    # Run a partial episode, check observations, rewards, discount.
    for _ in range(_NUM_EPISODES):
      time_step = env.reset()
      for _ in range(_NUM_STEPS_PER_EPISODE):
        self._validate_observation(time_step.observation, observation_spec)
        if time_step.first():
          self.assertIsNone(time_step.reward)
          self.assertIsNone(time_step.discount)
        else:
          self._validate_reward_range(time_step.reward)
          self._validate_discount(time_step.discount)
        action = random_state.uniform(action_spec.minimum, action_spec.maximum)
        env.step(action)

if __name__ == '__main__':
  absltest.main()


text:
import numpy as np
from citysim3d.envs import Env
from citysim3d.spaces import BoxSpace, TranslationAxisAngleSpace

class NormalizedEnv(Env):
    def __init__(self, env):
        self.__dict__.update(env.__dict__)
        self.env = env

    def step(self, action):
        if isinstance(self.env.action_space, (BoxSpace, TranslationAxisAngleSpace)):
            lb, ub = self.env.action_space.low, self.env.action_space.high
            scaled_action = lb + (action + 1.) * 0.5 * (ub - lb)
            scaled_action = np.clip(scaled_action, lb, ub)
        else:
            raise NotImplementedError
        return self.env.step(scaled_action)

    def reset(self, state=None):
        return self.env.reset(state=state)

    def get_state(self):
        return self.env.get_state()

    def set_state(self, state):
        self.env.set_state(state)

    def render(self):
        self.env.render()

    def close(self):
        self.env.close()

    @property
    def action_space(self):
        if isinstance(self.env.action_space, BoxSpace):
            ub = np.ones(self.env.action_space.shape)
            action_space = BoxSpace(-1 * ub, ub)
        elif isinstance(self.env.action_space, TranslationAxisAngleSpace):
            ub = np.ones(self.env.action_space.shape)
            action_space = TranslationAxisAngleSpace(-1 * ub, ub,
                                                     axis=self.env.action_space.axis,
                                                     dtype=self.env.action_space.dtype)
        else:
            raise NotImplementedError
        return action_space

    @property
    def observation_space(self):
        return self.env.observation_space


text:
import argparse, sys, os, pickle
from os.path import splitext, basename
import numpy as np
import pylab as pl
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib.collections import LineCollection

__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
sys.path.append(os.path.split(__location__)[0])
from helpers import parse_output_dir, parse_input_dir

color_list = ['#66c2a5', '#fc8d62','#8da0cb']
color_dict = {'C':'#66c2a5',
              'K': '#fc8d62',
              'R': '#8da0cb'}

parser = argparse.ArgumentParser(description='Using tsvs generated by graph_fingerprints, plot a subset under eachother'
                                             'for publication.')
parser.add_argument('--in-dir', type=str, required=True,
                    help='Output directory of graph_fingerprints')
parser.add_argument('--min-p', type=float, default=1.0,
                    help='Minimum occurence for particular value to be included as peak [default: 1.0]')
parser.add_argument('--resolution', default=1.0)
parser.add_argument('--idx', nargs='+', type=str, required=True,
                    help='For each type, indicate idx of the fingerprint that should be included. Format as follows:'
                          'PROTEIN_IDX')
parser.add_argument('--out-svg', type=str, required=True)
args = parser.parse_args()

tsv_list = parse_input_dir(args.in_dir, pattern='*.tsv')
idx_tups = [idx.rsplit('_', 1) for idx in args.idx]
idx_dict = {idx[0]: int(idx[1]) for idx in idx_tups}
nb_segments = 100 / args.resolution

plot_dict = {}
for tsv_fn in tsv_list:
    prot_id = splitext(basename(tsv_fn))[0]
    cidx = idx_dict[prot_id]
    fps = np.loadtxt(tsv_fn)
    fps[fps < args.min_p] = 0.0
    fps[fps > 0] = 1.0
    nb_tags = round(fps.shape[1] / nb_segments)
    plot_dict[prot_id] = fps[cidx, :].reshape(nb_tags,-1)

# --- collect plotting elements ---
ls_list = []
colors = []

for i1, pid in enumerate(plot_dict):
    for i2, fp in enumerate(plot_dict[pid]):
        segments = [[(it, i1), (it, i1 + fs)] for it, fs in enumerate(fp) if fs != 0]
        ls_list.append(LineCollection(segments, colors=len(segments) * [color_list[i2]]))

fig, ax = plt.subplots(figsize=(6, 2.9375))
for ls in ls_list: ax.add_collection(ls)
plt.xlim(0, 100)
plt.ylim(0, len(plot_dict))

id_list = list(plot_dict)
plt.yticks([i + 0.5 for i in list(range(len(id_list)))], id_list)
plt.xticks([0, 25, 50, 75, 100])
plt.xlabel('$E_{FRET}$')

fig.savefig(args.out_svg)


text:
import pickle
import json
import argparse
import cv2
import os
import numpy as np
import torch
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import matplotlib.patches as patches
import matplotlib.lines as lines
from tqdm import tqdm

import _init_paths

from datasets_rel.pytorch_misc import intersect_2d, argsort_desc

from functools import reduce
from utils.boxes import bbox_overlaps
from utils_rel.boxes_rel import boxes_union

from graphviz import Digraph

def _compute_pred_matches(gt_triplets, pred_triplets,
                 gt_boxes, pred_boxes, iou_thresh=0.5, phrdet=False):
    """
    Given a set of predicted triplets, return the list of matching GT's for each of the
    given predictions
    :param gt_triplets:
    :param pred_triplets:
    :param gt_boxes:
    :param pred_boxes:
    :param iou_thresh:
    :return:
    """
    # This performs a matrix multiplication-esque thing between the two arrays
    # Instead of summing, we want the equality, so we reduce in that way
    # The rows correspond to GT triplets, columns to pred triplets
    keeps = intersect_2d(gt_triplets, pred_triplets)
    gt_has_match = keeps.any(1)
    pred_to_gt = [[] for x in range(pred_boxes.shape[0])]
    for gt_ind, gt_box, keep_inds in zip(np.where(gt_has_match)[0],
                                         gt_boxes[gt_has_match],
                                         keeps[gt_has_match],
                                         ):
        boxes = pred_boxes[keep_inds]
        if phrdet:
            gt_box = gt_box.astype(dtype=np.float32, copy=False)
            boxes = boxes.astype(dtype=np.float32, copy=False)
            rel_iou = bbox_overlaps(gt_box[None, :], boxes)[0]

            inds = rel_iou >= iou_thresh
        else:
            gt_box = gt_box.astype(dtype=np.float32, copy=False)
            boxes = boxes.astype(dtype=np.float32, copy=False)
            sub_iou = bbox_overlaps(gt_box[None,:4], boxes[:, :4])[0]
            obj_iou = bbox_overlaps(gt_box[None,4:], boxes[:, 4:])[0]

            inds = (sub_iou >= iou_thresh) & (obj_iou >= iou_thresh)

        for i in np.where(keep_inds)[0][inds]:
            pred_to_gt[i].append(int(gt_ind))
    return pred_to_gt

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Visualization')
    parser.add_argument(
        '--output_dir',
        help='output directory to save the testing results. If not provided, '
             'defaults to [args.load_ckpt|args.load_detectron]/../test.')
    parser.add_argument(
        '--st',
        help='Visualization Start',
        default=0, type=int)
    parser.add_argument(
        '--num',
        help='Visualization Number',
        default=10, type=int)
    parser.add_argument(
        '--no_do_vis',
        help='do not visualize',
        action='store_true')
    parser.add_argument(
        '--rel_class_recall', help='rel class recall.',
        action='store_true')
    parser.add_argument(
        '--phrdet', help='use phrdet.',
        action='store_true')
    parser.add_argument(
        '--dataset',
        help='Visualization Number',
        default='ag', type=str)
    parser.add_argument(
        '--filename',
        help='Visualization file',
        default='rel_detections_topk', type=str)
    parser.add_argument(
        '--cnt_lim',
        help='Visualization Number',
        default=10, type=int)
    parser.add_argument(
        '--lim',
        help='Visualization Number',
        default=0, type=int)
    args = parser.parse_args()

    if not os.path.exists(os.path.join(args.output_dir, 'vis')):
        os.mkdir(os.path.join(args.output_dir, 'vis'))
    saved_path = os.path.join(args.output_dir, 'vis')

    topk_dets_f = os.path.join(args.output_dir, args.filename+'.pkl')
    with open(topk_dets_f, 'rb') as f:
        res = pickle.load(f)
        f.close()

    with open(os.path.join('data', args.dataset,'annotations/objects.json'), 'r') as f:
        obj_list = json.load(f)
        f.close()
    with open(os.path.join('data', args.dataset, 'annotations/predicates.json'), 'r') as f:
        rel_list = json.load(f)
        f.close()

    print('Loading test_videos_list.json')
    if args.dataset.find('ag') >= 0:
        val_map_list_path = os.path.join('data', 'ag', 'annotations/test_videos_list.json')
        with open(val_map_list_path, 'r') as f:
            val_map_list = json.load(f)
            f.close()
    elif args.dataset.find('vidvrd_train') >= 0:
        val_map_list_path = os.path.join('data', 'vidvrd', 'annotations/train_fname_list.json')
        with open(val_map_list_path, 'r') as f:
            val_map_list = json.load(f)
            f.close()
        val_map_list_ = set()
        for i, v in enumerate(val_map_list):
            ll = v.split('/')
            if len(ll) >= 2:
                val_map_list_.add(ll[-2].split('.')[-2])
        val_map_list = list(val_map_list_)
    elif args.dataset.find('vidvrd') >= 0:
        val_map_list_path = os.path.join('data', 'vidvrd', 'annotations/val_fname_list.json')
        with open(val_map_list_path, 'r') as f:
            val_map_list = json.load(f)
            f.close()
        val_map_list_ = set()
        for i, v in enumerate(val_map_list):
            ll = v.split('/')
            if len(ll) >= 2:
                val_map_list_.add(ll[-2].split('.')[-2])
        val_map_list = list(val_map_list_)
    else:
        raise Exception
    print('test_videos_list.json loaded.')

    all_gt_cnt = 0
    video_gt_cnt = [{} for r in range(len(rel_list))]
    video_recalls = [{} for r in range(len(rel_list))]
    video_len = len(val_map_list)
    for j, i in enumerate(val_map_list):
        file_real_name = i + '.mp4'
        for r in range(len(rel_list)):
            video_gt_cnt[r][file_real_name] = 0.
            video_recalls[r][file_real_name] = {10:0., 20: 0., 50: 0., 100: 0.}

    edge_width = 3
    font_size = 18

    rel_class_recalls = [{10:0, 20: 0, 50: 0, 100: 0} for i in range(len(rel_list))]
    tot_recalls = [{10:0, 20: 0, 50: 0, 100: 0} for i in range(len(rel_list))]
    rel_class_gt_num = [0 for i in range(len(rel_list))]
    recalls = [10, 20, 50, 100]
    print('total {} images. '.format(len(res)))
    args.num = min(args.num, len(res))
    print('Number is {}. '.format(args.num))
    cnt = 0
    for res_i in res[args.st:args.num]:
        r_ans = {10:0, 20: 0, 50: 0, 100: 0}
        r_score = {10:0, 20: 0, 50: 0, 100: 0}
        f_name = res_i['image']
        det_boxes_s_top = res_i['det_boxes_s_top']
        det_boxes_o_top = res_i['det_boxes_o_top']
        det_labels_s_top = res_i['det_labels_s_top']
        det_labels_p_top = res_i['det_labels_p_top']
        det_labels_o_top = res_i['det_labels_o_top']
        det_scores_top = res_i['det_scores_top']
        gt_boxes_sbj = res_i['gt_boxes_sbj']
        gt_boxes_obj = res_i['gt_boxes_obj']
        gt_labels_sbj = res_i['gt_labels_sbj']
        gt_labels_obj = res_i['gt_labels_obj']
        gt_labels_prd = res_i['gt_labels_prd']

        mm = res_i['image'].split('/')
        file_real_name = mm[-2]
        cur_frame_id = int(mm[-1].split('.')[0])

        gt_boxes_so = np.hstack((gt_boxes_sbj, gt_boxes_obj))
        gt_labels_spo = np.vstack((gt_labels_sbj, gt_labels_prd, gt_labels_obj)).transpose()
        det_labels_spo_top = np.vstack((det_labels_s_top, det_labels_p_top, det_labels_o_top)).transpose()

        if args.rel_class_recall:
            for i in gt_labels_prd:
                rel_class_gt_num[i] += 1
                video_gt_cnt[i][file_real_name] += 1.

        det_boxes_so_top = np.hstack((det_boxes_s_top, det_boxes_o_top))
        pred_to_gt = _compute_pred_matches(
                    gt_labels_spo, det_labels_spo_top,
                    gt_boxes_so, det_boxes_so_top,
                    phrdet=args.phrdet)

        gt_obj_set = set()
        gt_tri_info_set = set()
        for i in range(len(gt_boxes_sbj)):
            tri_info = []
            tri_info += list(gt_boxes_sbj[i, :].astype(np.int))
            tri_info += [gt_labels_sbj[i].astype(np.int), ]
            tri_info += list(gt_boxes_obj[i, :].astype(np.int))
            tri_info += [gt_labels_obj[i].astype(np.int), ]
            tri_info += [gt_labels_prd[i].astype(np.int), ]
            gt_tri_info_set.add(tuple(tri_info))
            if tuple(list(gt_boxes_sbj[i, :].astype(np.int)) + [gt_labels_sbj[i].astype(np.int), ]) not in gt_obj_set:
                gt_obj_set.add(tuple(list(gt_boxes_sbj[i, :].astype(np.int)) + [gt_labels_sbj[i].astype(np.int), ]))
            if tuple(list(gt_boxes_obj[i, :].astype(np.int)) + [gt_labels_obj[i].astype(np.int), ]) not in gt_obj_set:
                gt_obj_set.add(tuple(list(gt_boxes_obj[i, :].astype(np.int)) + [gt_labels_obj[i].astype(np.int), ]))

        for k in recalls:
            gt_score = [0 for i in range(len(gt_boxes_sbj))]
            if len(pred_to_gt):
                match = reduce(np.union1d, pred_to_gt[:k])
                match = np.array(match, dtype=np.int)

                if args.rel_class_recall:
                    for gt_i in match:
                        rel_class_recalls[gt_labels_prd[gt_i]][k] += 1
                        video_recalls[gt_labels_prd[gt_i]][file_real_name][k] += 1.

                for p_id, pred_i in enumerate(pred_to_gt[:k]):
                    for gt_id in pred_i:
                        gt_score[gt_id] = max(gt_score[gt_id], det_scores_top[p_id])

            else:
                match = []
                gt_score = []
            r_ans[k] = match
            r_score[k] = gt_score

        if len(gt_labels_prd) > args.lim and cnt <= args.cnt_lim:
            if not args.no_do_vis:
                saved_name = f_name.split('/')[-2:]
                saved_name = saved_name[0] + '/' + saved_name[1]
                img = mpimg.imread(f_name)
                cnt += 1
                for k in recalls:
                    if k < 20: continue
                    if k > 50: continue
                    preserve_set_obj = set()
                    preserve_set_rel = set()
                    rec_pos = {}
                    fig = plt.figure(figsize=(18, 12))
                    ax = plt.gca()
                    plt.imshow(img)
                    plt.axis('off')
                    det_title = plt.title('det')
                    plt.setp(det_title, color='b')
                    for gt_id in r_ans[k]:
                        x, y, x1, y1 = gt_boxes_sbj[gt_id].astype(np.int)

                        s_name = obj_list[gt_labels_sbj[gt_id]]

                        s_cx, s_cy = (x+x1)//2, (y+y1)//2

                        srect = plt.Rectangle((x,y),x1-x,y1-y, fill=False, edgecolor='b', linewidth=3)
                        ax.add_patch(srect)

                        #ax.text(s_cx, s_cy,
                        ax.text(x, y,
                            s_name,
                            fontsize=font_size,
                            color='white',
                            bbox=dict(facecolor='orange', alpha=0.5, pad=0, edgecolor='none'))

                        tri_info = [x, y, x1, y1, gt_labels_sbj[gt_id].astype(np.int)]
                        if tuple([x, y, x1, y1, gt_labels_sbj[gt_id].astype(np.int)]) not in preserve_set_obj:
                            preserve_set_obj.add(tuple([x, y, x1, y1, gt_labels_sbj[gt_id].astype(np.int)]))

                        x, y, x1, y1 = gt_boxes_obj[gt_id].astype(np.int)
                        o_name = obj_list[gt_labels_obj[gt_id]]
                        o_cx, o_cy = (x+x1)//2, (y+y1)//2

                        orect = plt.Rectangle((x,y),x1-x,y1-y, fill=False, edgecolor='b', linewidth=3)
                        ax.add_patch(orect)

                        ax.text(x, y,
                            o_name,
                            fontsize=font_size,
                            color='white',
                            bbox=dict(facecolor='blue', alpha=0.5, pad=0, edgecolor='none'))
                        p_name = rel_list[gt_labels_prd[gt_id].astype(np.int)]+ ' ' + str(r_score[k][gt_id])

                        tri_info += [x, y, x1, y1, gt_labels_obj[gt_id].astype(np.int)]
                        if tuple([x, y, x1, y1, gt_labels_obj[gt_id].astype(np.int)]) not in preserve_set_obj:
                            preserve_set_obj.add(tuple([x, y, x1, y1, gt_labels_obj[gt_id].astype(np.int)]))

                        tri_info += [gt_labels_prd[gt_id].astype(np.int), ]
                        preserve_set_rel.add(tuple(tri_info))

                        rel_l = lines.Line2D([s_cx, o_cx], [s_cy, o_cy], color='purple', linewidth=3)
                        ax.add_line(rel_l)

                        lx, ly = s_cx + 8*(o_cx - s_cx) / 9, s_cy + 8*(o_cy - s_cy) / 9

                        if (lx, ly) in rec_pos:
                            rec_pos[(lx, ly)] += 10
                        else:
                            rec_pos[(lx, ly)] = 0
                        d = rec_pos[(lx, ly)]

                        ax.text(lx, ly + d,
                                p_name,
                                fontsize=font_size,
                                color='white',
                                bbox=dict(facecolor='purple', alpha=0.5, pad=0, edgecolor='none'))

                    dot = Digraph(filename=(saved_name + '_' +str(k)).replace('/', '_'))
                    dot.body.append('size="16,16"')
                    dot.body.append('rankdir="LR"')
                    #dot.node_attr.update(style='filled')

                    map_obj_node = dict()
                    pn = 0
                    for gt_obj in gt_obj_set:
                        ol = obj_list[gt_obj[-1].astype(np.int)]
                        if gt_obj in preserve_set_obj:
                            dot.node(str(gt_obj), str(ol), color='green', shape='box')
                        else:
                            dot.node(str(gt_obj), str(ol), color='red', shape='box')
                        map_obj_node[gt_obj] = pn
                        pn += 1

                    for gt_tri_info in gt_tri_info_set:
                        sn, on, pn = gt_tri_info[4], gt_tri_info[9], gt_tri_info[10]
                        sx, sy, sx1, sy1 = gt_tri_info[0],gt_tri_info[1],gt_tri_info[2],gt_tri_info[3]
                        ox, oy, ox1, oy1 = gt_tri_info[5],gt_tri_info[6],gt_tri_info[7],gt_tri_info[8]
                        st, ed = str(tuple([sx, sy, sx1, sy1, sn])), str(tuple([ox, oy, ox1, oy1, on]))
                        rl = rel_list[gt_tri_info[-1].astype(np.int)].replace('_', ' ')
                        #if gt_tri_info in preserve_set_rel:
                        #    dot.node(str(gt_tri_info), rl, color='lightblue2')
                        #else:
                        #    dot.node(str(gt_tri_info), rl, color='red')
                        #dot.edge(st, str(gt_tri_info))
                        #dot.edge(str(gt_tri_info), ed)
                        if gt_tri_info in preserve_set_rel:
                            dot.edge(st, ed, rl, color='green')
                        else:
                            dot.edge(st, ed, rl, fontcolor='red', color='red')

                    dot.render(os.path.join(
                                saved_path,
                                (saved_name + '_' +str(k)).replace('/', '_')
                                ), cleanup=True)

                    saved_file_name = (saved_name + '_' +str(k)+'.png').replace('/', '_')
                    plt.savefig(os.path.join(saved_path, saved_file_name), bbox_inches='tight')
                    plt.close(fig)

    if args.rel_class_recall:
        for r in range(len(rel_list)):
            for file_real_name, v in video_recalls[r].items():
                for k, vals in v.items():
                    video_recalls[r][file_real_name][k] = float(vals) / (float(video_gt_cnt[r][file_real_name]) + 1e-12)
                    tot_recalls[r][k] += video_recalls[r][file_real_name][k]
            for k in tot_recalls[r]:
                tot_recalls[r][k] = float(tot_recalls[r][k]) / (float(video_len) + 1e-12)

        print('=========== ' + 'Image_ver_rel_recalls' + ' ===========')
        mr_list = []
        for k in recalls:
            print('=========== {} ==========='.format(k))
            mrr = float(0.)
            for i, gt_rel_num in enumerate(rel_class_gt_num):
                rel_class_recalls[i][k] = float(rel_class_recalls[i][k]) / (float(gt_rel_num) + 1e-12)
                print('%s: %.2f' % (rel_list[i], 100 * rel_class_recalls[i][k]))
                mrr += rel_class_recalls[i][k]
            mr_list.append((k, 100*mrr/len(rel_class_gt_num)))
        for i in mr_list:
            print('mR@{}: {}'.format(i[0], i[1]))

        print('=========== ' + 'Video_ver_rel_recalls' + ' ===========')
        mr_v_list = []
        for k in recalls:
            mrr = float(0.)
            for i, gt_rel_num in enumerate(rel_class_gt_num):
                #print('%s: %.2f' % (rel_list[i], 100 * tot_recalls[i][k]))
                mrr += tot_recalls[i][k]
            mr_v_list.append((k, 100*mrr/len(rel_class_gt_num)))
        for i in mr_v_list:
            print('mR@{}: {}'.format(i[0], i[1]))


text:
"""RL^2 agent implementation."""

from inspect import stack
import time
from typing import Dict, Optional

from mtsgi import agents
from mtsgi.agents import queue
from mtsgi.agents import base
from mtsgi.agents import meta_agent
from mtsgi.agents.rlrl import acting
from mtsgi.agents.rlrl import learning
from mtsgi.utils import snt_utils, acme_utils

import acme
from acme import specs
from acme import types
from acme.tf import savers as tf2_savers
from acme.tf import utils as tf2_utils
from acme.utils import counting
from acme.utils import loggers
from acme.utils import tree_utils
from acme.adders import base as adder
from acme.adders.reverb import Step

import tree
import trfl
import dm_env
import numpy as np
import sonnet as snt

import tensorflow as tf
import tensorflow.compat.v1 as tf1
import tensorflow_probability as tfp
from tensorflow.keras.optimizers import schedules

tfd = tfp.distributions

class RLRL(meta_agent.MetaAgent):
  """RL^2 meta agent for meta loop."""

  def __init__(
      self,
      environment_spec: specs.EnvironmentSpec,
      network: snt.RNNCore,
      counter: counting.Counter = None,
      logger: loggers.Logger = None,
      snapshot_dir: Optional[str] = None,
      n_step_horizon: int = 16,
      minibatch_size: int = 80,
      learning_rate: float = 2e-3,
      discount: float = 0.99,
      gae_lambda: float = 0.99,
      decay: float = 0.99,
      epsilon: float = 1e-5,
      entropy_cost: float = 0.,
      baseline_cost: float = 1.,
      max_abs_reward: Optional[float] = None,
      max_gradient_norm: Optional[float] = None,
      max_queue_size: int = 100000,
      verbose_level: Optional[int] = 0,
  ):
    # Internalize spec and replay buffer.
    self._environment_spec = environment_spec
    self._verbose_level = verbose_level
    self._minibatch_size = minibatch_size
    self._queue = queue.ReplayBuffer(
        max_queue_size=max_queue_size,
        batch_size=n_step_horizon
    )

    # Internalize network.
    self._network = network

    # Setup optimizer and learning rate scheduler.
    self._learning_rate = tf.Variable(learning_rate, trainable=False)
    self._lr_scheduler = schedules.ExponentialDecay(
        initial_learning_rate=learning_rate,
        decay_steps=8000,  # TODO make a flag
        decay_rate=0.96    # TODO make a flag
    )
    self._optimizer = snt.optimizers.RMSProp(
        learning_rate=self._learning_rate,
        decay=decay,
        epsilon=epsilon
    )
    #self._optimizer = snt.optimizers.Adam(
    #    learning_rate=self._learning_rate,
    #)

    # Hyperparameters.
    self._discount = discount
    self._gae_lambda = gae_lambda
    self._entropy_cost = entropy_cost
    self._baseline_cost = baseline_cost

    # Set up reward/gradient clipping.
    if max_abs_reward is None:
      max_abs_reward = np.inf
    if max_gradient_norm is None:
      max_gradient_norm = 1e10  # A very large number. Infinity results in NaNs.
    self._max_abs_reward = tf.convert_to_tensor(max_abs_reward)
    self._max_gradient_norm = tf.convert_to_tensor(max_gradient_norm)

    if snapshot_dir is not None:
      self._snapshotter = tf2_savers.Snapshotter(
          objects_to_save={'network': self._network},
          directory=snapshot_dir,
          time_delta_minutes=60.
      )

    # Logger.
    self._counter = counter or counting.Counter()
    self._logger = logger

    # Do not record timestamps until after the first learning step is done.
    # This is to avoid including the time it takes for actors to come online and
    # fill the replay buffer.
    self._timestamp = None
    self._pi_old = None

  def instantiate_adapt_agent(self) -> agents.Agent:
    """Instantiate 'fast' agent for adaptation phase. This fast agent will
      interact with the environment and perform fast adaptation (learning).
    """
    self._adapt_agent = FastRLAgent(
        environment_spec=self._environment_spec,
        network=self._network,
        queue=self._queue,
        verbose_level=self._verbose_level
    )
    return self._adapt_agent

  def instantiate_test_actor(self) -> base.BaseActor:
    """Instantiate a new Actor for test phase. This actor will interact with
      the environment and evaluate the performance of the (adapted) fast agent.
    """
    # Return RLActor for test phase.
    self._test_actor = acting.FastRLActor(
        environment_spec=self._environment_spec,
        network=self._network,
        verbose_level=self._verbose_level
    )
    return agents.EvalWrapper(self._test_actor)

  def reset_agent(self, environment: dm_env.Environment):
    """Reset the 'fast' agent upon samping a new task."""
    # Reset RNN hidden state.
    self._adapt_agent._actor._state = None

    # Reset replay buffer.
    self._queue.reset()

  def update_adaptation_progress(self, current_split: int, max_split: int):
    """Update the adaptation progress with adaptation / testing actors
    """
    pass # do nothing

  @tf.function
  def _step(self, data: Step) -> Dict[str, tf.Tensor]:
    """Does an SGD step on a batch of sequences."""
    observations, actions, rewards, discounts, _, extra = data
    core_state = tree.map_structure(lambda s: s[0], extra['core_state'])

    actions = actions[:-1]  # [T-1]
    rewards = rewards[:-1]  # [T-1]
    discounts = discounts[:-1]  # [T-1]

    # Workaround for NO_OP actions
    # In some environments, passing NO_OP(-1) actions would lead to a crash.
    # These actions (at episode boundaries) should be ignored anyway,
    # so we replace NO_OP actions with a valid action index (0).
    actions = (tf.zeros_like(actions) * tf.cast(actions == -1, tf.int32) +
               actions * tf.cast(actions != -1, tf.int32))

    with tf.GradientTape() as tape:
      # Unroll current policy over observations.
      (logits, values), _ = snt.static_unroll(self._network, observations,
                                              core_state)

      # TODO: mask policy here as well.
      # Masked policy.
      #masked_eligibility = observations['mask'] * observations['eligibility']
      #out_logits = logits - logits.min(axis=-1, keepdims=True)
      #out_logits[masked_eligibility == 0] = -np.infty
      #out_logits -= out_logits.max(axis=-1, keepdims=True)
      pi = tfd.Categorical(logits=logits[:-1])

      # Optionally clip rewards.
      rewards = tf.clip_by_value(rewards,
                                 tf.cast(-self._max_abs_reward, rewards.dtype),
                                 tf.cast(self._max_abs_reward, rewards.dtype))

      # Compute returns (optionally, GAE)
      discounted_returns = trfl.generalized_lambda_returns(
          rewards=tf.cast(rewards, tf.float32),
          pcontinues=tf.cast(self._discount*discounts, tf.float32),
          values=tf.cast(values[:-1], tf.float32),
          bootstrap_value=tf.cast(values[-1], tf.float32),
          lambda_=self._gae_lambda
      )
      advantages = discounted_returns - values[:-1]

      # Compute actor & critic losses.
      critic_loss = tf.square(advantages)
      #policy_gradient_loss = trfl.policy_gradient(
      #    policies=pi,
      #    actions=actions,
      #    action_values=advantages
      #)
      # TODO: Remove later.
      action_values = advantages
      policy_vars = None
      policy_vars = list(policy_vars) if policy_vars else list()
      with tf1.name_scope(values=policy_vars + [actions, action_values], name="policy_gradient"):
        actions = tf1.stop_gradient(actions)
        action_values = tf1.stop_gradient(action_values)
        log_prob_actions = pi.log_prob(actions)
        # Prevent accidental broadcasting if possible at construction time.
        action_values.get_shape().assert_is_compatible_with(
            log_prob_actions.get_shape())
        policy_gradient_loss = -tf1.multiply(log_prob_actions, action_values)

      #entropy_loss = trfl.policy_entropy_loss(pi).loss
      # TODO: Remove later.
      entropy_info = trfl.policy_entropy_loss(pi)
      entropy_loss = entropy_info.loss

      loss = tf.reduce_mean(policy_gradient_loss +
                            self._baseline_cost * critic_loss +
                            self._entropy_cost * entropy_loss)

      # Compute gradients and optionally apply clipping.
      gradients = tape.gradient(loss, self._network.trainable_variables)
      gradients, _ = tf.clip_by_global_norm(gradients, self._max_gradient_norm)
      self._optimizer.apply(gradients, self._network.trainable_variables)

    metrics = {
        'loss': loss,
        'critic_loss': tf.reduce_mean(critic_loss),
        'entropy_loss': tf.reduce_mean(entropy_loss),
        'policy_gradient_loss': tf.reduce_mean(policy_gradient_loss),
        'log_probs': tf.reduce_mean(log_prob_actions),
        'learning_rate': self._learning_rate,
        'advantages': tf.reduce_mean(advantages),
        'discounted_returns': tf.reduce_mean(discounted_returns),
        'entropy': tf.reduce_mean(pi.entropy())
    }

    return metrics, gradients, logits

  def update(self, last: bool = False):
    """Perform a meta-training update."""
    for i in range(self._minibatch_size):
      # Retrieve a batch of data from replay.
      data = self._queue.sample()
      data = tree_utils.fast_map_structure(lambda x: tf.convert_to_tensor(x), data)

      # Do a batch of SGD.
      results, gradients, logits = self._step(data=data)

      # Check gradients.
      #for g, v in zip(gradients, self._network.trainable_variables):
      #  name = v.name.replace('/', '-')
      #  results.update({name: tf.reduce_mean(g)})

      # Compute elapsed time.
      timestamp = time.time()
      elapsed_time = timestamp - self._timestamp if self._timestamp else 0
      self._timestamp = timestamp

      # Update our counts and record it.
      counts = self._counter.increment(steps=1, walltime=elapsed_time)
      results.update(counts)

      # Compute KL Divergence.
      pi = tfd.Categorical(logits=logits[:-1])
      if counts['steps'] > 1:
        kl_divergence = tf.reduce_mean(pi.kl_divergence(self._pi_old))
        results.update({'kl_divergence': kl_divergence})
      if counts['steps'] == 1:
        results.update({'kl_divergence': 0.0})
      self._pi_old = pi

      # Update learning rate.
      self._learning_rate.assign(self._lr_scheduler(step=counts['steps']))

      # Snapshot and attempt to write logs.
      if hasattr(self, '_snapshotter'):
        self._snapshotter.save()

      if self._logger:
        self._logger.write(results)

class FastRLAgent(agents.Agent):
  """fast RL Agent."""

  def __init__(
      self,
      environment_spec: specs.EnvironmentSpec,
      network: snt.RNNCore,
      queue: adder.Adder,
      sample_size: int = 1000,
      verbose_level: Optional[int] = 0
  ):
    tf2_utils.create_variables(network, [environment_spec.observations])
    acme_utils.reinitialize_weights(network=network)

    # Actor and *empty* learner.
    actor = acting.FastRLActor(environment_spec, network, queue, verbose_level)
    learner = learning.FastRLLearner(environment_spec)

    super().__init__(
        actor=actor,
        learner=learner,
        min_observations=0,
        observations_per_step=sample_size)

max_stars_repo_name: srsohn/mtsgi

max_issues_repo_name: srsohn/mtsgi

max_forks_repo_name: srsohn/mtsgi


text:
#!/usr/bin/env python3
import matplotlib.pyplot as plt
import numpy as np

def main():
    nrows = 3
    fig, axes = plt.subplots(nrows, 2)

    for row in axes:
        x = np.random.normal(0, 1, 100).cumsum()
        y = np.random.normal(0, 0.5, 100).cumsum()
        plot(row, x, y)

    plt.show()

def plot(axrow, x, y):
    axrow[0].plot(x, color='red')
    axrow[1].plot(y, color='green')

main()

max_stars_repo_name: mhdproject/mhdvanleer

max_issues_repo_name: mhdproject/mhdvanleer

max_forks_repo_name: garethcmurphy/mhdvanleer


text:
# python3.7
"""Contains utility functions used for formatting."""

import cv2
import numpy as np

__all__ = [
    'format_time', 'format_range', 'format_image_size', 'format_image',
    'raw_label_to_one_hot', 'one_hot_to_raw_label'
]

def format_time(seconds):
    """Formats seconds to readable time string.

    Args:
        seconds: Number of seconds to format.

    Returns:
        The formatted time string.

    Raises:
        ValueError: If the input `seconds` is less than 0.
    """
    if seconds < 0:
        raise ValueError(f'Input `seconds` should be greater than or equal to '
                         f'0, but `{seconds}` is received!')

    # Returns seconds as float if less than 1 minute.
    if seconds < 10:
        return f'{seconds:7.3f} s'
    if seconds < 60:
        return f'{seconds:7.2f} s'

    seconds = int(seconds + 0.5)
    days, seconds = divmod(seconds, 86400)
    hours, seconds = divmod(seconds, 3600)
    minutes, seconds = divmod(seconds, 60)
    if days:
        return f'{days:2d} d {hours:02d} h'
    if hours:
        return f'{hours:2d} h {minutes:02d} m'
    return f'{minutes:2d} m {seconds:02d} s'

def format_range(obj, min_val=None, max_val=None):
    """Formats the given object to a valid range.

    If `min_val` or `max_val` is provided, both the starting value and the end
    value will be clamped to range `[min_val, max_val]`.

    NOTE: (a, b) is regarded as a valid range if and only if `a <= b`.

    Args:
        obj: The input object to format.
        min_val: The minimum value to cut off the input range. If not provided,
            the default minimum value is negative infinity. (default: None)
        max_val: The maximum value to cut off the input range. If not provided,
            the default maximum value is infinity. (default: None)

    Returns:
        A two-elements tuple, indicating the start and the end of the range.

    Raises:
        ValueError: If the input object is an invalid range.
    """
    if not isinstance(obj, (tuple, list)):
        raise ValueError(f'Input object must be a tuple or a list, '
                         f'but `{type(obj)}` received!')
    if len(obj) != 2:
        raise ValueError(f'Input object is expected to contain two elements, '
                         f'but `{len(obj)}` received!')
    if obj[0] > obj[1]:
        raise ValueError(f'The second element is expected to be equal to or '
                         f'greater than the first one, '
                         f'but `({obj[0]}, {obj[1]})` received!')

    obj = list(obj)
    if min_val is not None:
        obj[0] = max(obj[0], min_val)
        obj[1] = max(obj[1], min_val)
    if max_val is not None:
        obj[0] = min(obj[0], max_val)
        obj[1] = min(obj[1], max_val)
    return tuple(obj)

def format_image_size(size):
    """Formats the given image size to a two-element tuple.

    A valid image size can be an integer, indicating both the height and the
    width, OR can be a two-element list or tuple. Both height and width are
    assumed to be positive integer.

    Args:
        size: The input size to format.

    Returns:
        A two-elements tuple, indicating the height and the width, respectively.

    Raises:
        ValueError: If the input size is invalid.
    """
    if not isinstance(size, (int, tuple, list)):
        raise ValueError(f'Input size must be an integer, a tuple, or a list, '
                         f'but `{type(size)}` received!')
    if isinstance(size, int):
        size = (size, size)
    else:
        if len(size) == 1:
            size = (size[0], size[0])
        if not len(size) == 2:
            raise ValueError(f'Input size is expected to have two numbers at '
                             f'most, but `{len(size)}` numbers received!')
    if not isinstance(size[0], int) or size[0] < 0:
        raise ValueError(f'The height is expected to be a non-negative '
                         f'integer, but `{size[0]}` received!')
    if not isinstance(size[1], int) or size[1] < 0:
        raise ValueError(f'The width is expected to be a non-negative '
                         f'integer, but `{size[1]}` received!')
    return tuple(size)

def format_image(image):
    """Formats an image read from `cv2`.

    NOTE: This function will always return a 3-dimensional image (i.e., with
    shape [H, W, C]) in pixel range [0, 255]. For color images, the channel
    order of the input is expected to be with `BGR` or `BGRA`, which is the
    raw image decoded by `cv2`; while the channel order of the output is set to
    `RGB` or `RGBA` by default.

    Args:
        image: `np.ndarray`, an image read by `cv2.imread()` or
            `cv2.imdecode()`.

    Returns:
        An image with shape [H, W, C] (where `C = 1` for grayscale image).
    """
    if image.ndim == 2:  # add additional axis if given a grayscale image
        image = image[:, :, np.newaxis]

    assert isinstance(image, np.ndarray)
    assert image.dtype == np.uint8
    assert image.ndim == 3 and image.shape[2] in [1, 3, 4]

    if image.shape[2] == 3:  # BGR image
        return cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    if image.shape[2] == 4:  # BGRA image
        return cv2.cvtColor(image, cv2.COLOR_BGRA2RGBA)
    return image

def raw_label_to_one_hot(raw_label, num_classes):
    """Converts a single label into one-hot vector.

    Args:
        raw_label: The raw label.
        num_classes: Total number of classes.

    Returns:
        one-hot vector of the given raw label.
    """
    one_hot = np.zeros(num_classes, dtype=np.float32)
    one_hot[raw_label] = 1.0
    return one_hot

def one_hot_to_raw_label(one_hot):
    """Converts a one-hot vector to a single value label.

    Args:
        one_hot: `np.ndarray`, a one-hot encoded vector.

    Returns:
        A single integer to represent the category.
    """
    return np.argmax(one_hot)

max_stars_repo_name: bytedance/Hammer

max_issues_repo_name: bytedance/Hammer

max_forks_repo_name: bytedance/Hammer


text:
import numpy as np
from features import feature_names
from sklearn.preprocessing import normalize

def read_data(filename):
    data = []
    with open(filename, "r") as f:
        first = True
        for line in f:
            if first:
                first = False
                continue
            values = [x.strip() for x in line.split(",")]
            data.append(values)
    return np.array(data)

def separate_numerical(features):
    categorical = []
    categorical_names = []
    numerical = []
    numerical_names = []
    for i, f in enumerate(feature_names):
        if f[1]:
            numerical.append(i)
            numerical_names.append(f)
        else:
            categorical.append(i)
            categorical_names.append(f)

    return numerical_names, features[:, numerical], categorical_names, features[:, categorical]

def preprocess(features, mappings=None, binarize=False, names=None):

    mappings_provided = False
    if mappings is not None:
        mappings_provided = True
    else:
        mappings = {}

    if names is None:
        names = feature_names

    if binarize:
        map_func = map_binary
        fill_func = fill_binary
    else:
        map_func = map_smart
        fill_func = fill_smart

    extras = {
        "mappings_provided": mappings_provided,
        "num_skipped": 0
    }

    mapped_features = [
        [
            map_func(feature, j, mappings, names, extras)
            for j, feature in enumerate(row)
        ]
        for i, row in enumerate(features)
    ]

    preprocessed_features = np.zeros((len(features), len(mappings)))

    fill_func(preprocessed_features, mapped_features, extras)

    if mappings_provided:
        return preprocessed_features
    else:
        return preprocessed_features, mappings

def map_binary(f, i, mappings, names, extras):
    m = (i, f, names[i][0] + '=' + f)
    if not m in mappings:
        if not extras["mappings_provided"]:
            mappings[m] = len(mappings)
        else:
            extras["num_skipped"] += 1
            return None
    return mappings[m]

def map_smart(f, i, mappings, names, extras):
    fn = names[i]
    # Check if it is a numerical feature
    if fn[1]:
        m = (i, 'numerical', fn[0])
        if m not in mappings:
            mappings[m] = len(mappings)
        fm = mappings[m]
        if not f.isdigit():
            # By returning 'None' we indicate that there was no number for this numerical feature. We will replace
            # it with the mean later
            return (None, fm)
        else:
            # Convert to a number
            fnum = float(f)

            # We need to capture some 'extra' values as we map so that we can calculate the mean and
            # normalize values later

            # Capture the sum
            sum_key = "sum_for_%d" % fm
            if sum_key in extras.keys():
                extras[sum_key] += fnum
            else:
                extras[sum_key] = fnum

            # Capture the number of features
            count_key = "count_for_%d" % fm
            if count_key in extras.keys():
                extras[count_key] += 1
            else:
                extras[count_key] = 1

            # Capture the minimum value
            min_key = "min_for_%d" % fm
            current_min = extras[min_key] if min_key in extras.keys() else fnum
            extras[min_key] = fnum if fnum < current_min else current_min

            # Capture the maximum value
            max_key = "max_for_%d" % fm
            current_max = extras[max_key] if max_key in extras.keys() else fnum
            extras[max_key] = fnum if fnum > current_max else current_max

            # Return the resulting tuple
            return (fnum, fm)
    else:
        # It isn't a numerical feature, just binarize it
        return map_binary(f, i, mappings, names, extras)

def fill_binary(preprocessed_features, mapped_features, extras):
    for i, row in enumerate(mapped_features):
        for j in row:
            if j is not None:
                preprocessed_features[i][j] = 1

def fill_smart(preprocessed_features, mapped_features, extras):
    for i, row in enumerate(mapped_features):
        for j in row:
            if j is not None:
                if isinstance(j, tuple):
                    # We are dealing with a numerical feature
                    v = j[0]
                    vi = j[1]
                    if v is None:
                        # Fill with the mean
                        sum_key = "sum_for_%d" % vi
                        count_key = "count_for_%d" % vi
                        preprocessed_features[i][vi] = extras[sum_key] / extras[count_key]
                    else:
                        preprocessed_features[i][vi] = v
                else:
                    preprocessed_features[i][j] = 1

def scale_y(y, reverse=False):
    if reverse:
        return np.exp(y)
    else:
        return np.log(y)


text:
import os
from os.path import join, exists
import h5py
import numpy as np

import torch
import torchtext
from torch.functional import F

import spacy
from spellchecker  import SpellChecker

from random import randint

def iou(candidates, gt):
    start, end = candidates[:,0].float(), candidates[:,1].float()
    s, e = gt[0].float(), gt[1].float()
    inter = end.min(e) - start.max(s)
    union = end.max(e) - start.min(s)
    return inter.clamp(min=0) / union

def score2d_to_moments_scores(score2d, num_clips, duration):
    grids = torch.nonzero(score2d, as_tuple=False)
    scores = score2d[grids[:,0], grids[:,1]]
    grids[:, 1] += 1
    moments = grids * duration / num_clips
    return moments, scores

def moment_to_iou2d(moment, num_clips, duration):
    iou2d = torch.ones(num_clips, num_clips)
    candidates, _ = score2d_to_moments_scores(iou2d, num_clips, duration)
    iou2d = iou(candidates, moment).reshape(num_clips, num_clips)
    return iou2d

def avgfeats(feats, num_pre_clips):
    # # Produce the feature of per video into fixed shape (e.g. 256*4096)
    # # Input Example: feats (torch.tensor, ?x4096); num_pre_clips (256)
    # num_src_clips = feats.size(0)
    # idxs = torch.arange(0, num_pre_clips+1, 1.0) / num_pre_clips * num_src_clips
    # idxs = idxs.round().long().clamp(max=num_src_clips-1)
    # # To prevent a empty selection, check the idxs
    # meanfeats = []
    # for i in range(num_pre_clips):
    #     s, e = idxs[i], idxs[i+1]
    #     if s < e:
    #         meanfeats.append(feats[s:e].mean(dim=0))
    #     else:
    #         meanfeats.append(feats[s])
    output = F.interpolate(feats.transpose(0,1).unsqueeze(0), size=num_pre_clips, mode='linear', align_corners=False)
    return  output[0,...].transpose(0,1)

def video2feats(feat_file, vids, num_pre_clips, dataset_name):
    assert exists(feat_file), '{} not found'.format(feat_file)
    vid_feats = {}
    with h5py.File(feat_file, 'r') as f:
        for vid in vids:
            if dataset_name == "activitynet":
                feat = f[vid]['c3d_features'][:]
            else:
                feat = f[vid][:]
            feat = F.normalize(torch.from_numpy(feat),dim=1)
            interpolated_feat = avgfeats(feat, num_pre_clips) # feat
            vid_feats[vid] = interpolated_feat
    return vid_feats

class embedding(object):
    def __init__(self, vocabs=[], embedders=[], nlp=None, spell=None):
         # Glove setup
        self.vocab = torchtext.vocab.pretrained_aliases["glove.6B.300d"]()
        self.vocab.itos.extend(['<unk>'])
        self.vocab.stoi['<unk>'] = self.vocab.vectors.shape[0]
        self.vocab.vectors = torch.cat(
            [self.vocab.vectors, torch.zeros(1, self.vocab.dim)],
            dim=0
        )
        # Spacy setup
        try:
            self.nlp = spacy.load('en_core_web_sm')
        except:
            os.system('python -m spacy download en_core_web_sm')
            self.nlp = spacy.load('en_core_web_sm')
        # setup spellchecker
        self.spell = SpellChecker()
        self.spell.distance = 1                       ## Too expensive
        self.spell.word_frequency.load_words(list(self.vocab.stoi.keys()))
        self.embedder = torch.nn.Embedding.from_pretrained(self.vocab.vectors)
        self.cnt = 0

    def __call__(self, tokens):
        # Look up tokens in Glove vocabulary
        tokens_idx = []
        for t in tokens:
            t = t.lower()
            i = self.vocab.stoi.get(t, 400000)
            if i == 400000:
                #try lemmatization
                t_ = self.nlp(t)[0]
                i = self.vocab.stoi.get(t_.lemma_, 400000)
                if  i == 400000:
                    # try spelling correction
                    t_ = self.spell.correction(t)
                    i  = self.vocab.stoi.get(t_, 400000)
                    if i == 400000:
                        self.cnt += 1
            tokens_idx.append(i)

        word_idxs = torch.tensor(tokens_idx, dtype=torch.long)
        return self.embedder(word_idxs)

max_stars_repo_name: Soldelli/VLG-Net

max_issues_repo_name: Soldelli/VLG-Net

max_forks_repo_name: Soldelli/VLG-Net


text:
from concurrent.futures import ThreadPoolExecutor
from imutils.paths import list_images
import imgaug.augmenters as iaa

import warnings
import multiprocessing
import time
import cv2
import os

import tensorflow as tf
import configparser as ConfigParser
from optparse import OptionParser
from numpy import expand_dims
from tqdm import tqdm
import numpy as np

def submit_thread(ls_img_paths, output_dir, num_imgs, num_workers):
    executor = ThreadPoolExecutor(max_workers=num_workers)
    execs = [executor.submit(generate_images, ls_img_paths, output_dir, num_imgs) for _ in range(num_workers)]

def ensure_dir(directory):
    """Make sure the directory exists
    """
    if not os.path.exists(directory):
        warnings.warn("[Warning] Output directory not found. \
            The default output directory will be created.")
        os.makedirs(directory)

def datagenerate():
    """Generating images in batches
    See more: https://github.com/aleju/imgaug
    """
    seq = iaa.Sequential([
        iaa.MultiplyHue((0.5, 1.5)),
        iaa.Fliplr(1.0),
        iaa.Affine(rotate=(-10, 10)),
        iaa.Affine(scale=(0.8, 1.2))
    ])
    return seq

def generate_images(ls_img_paths, output_dir, num_imgs):
    """Generate image with ImageDataGenerator
    """
    #Random select image to generate image
    for i in tqdm(range(num_imgs)):
        rd_image_seleted = np.random.randint(len(ls_img_paths))
        #Loading desired images
        img = cv2.imread(ls_img_paths[rd_image_seleted])
        h, w, _ = img.shape

        #Expanding dimension to one sample
        imgs = expand_dims(img, 0)
        datagen = datagenerate()
        batchs = datagen(images=imgs)

        #Remember to convert these images to unsigned integers for viewing
        image = batchs[0].astype('uint8')
        #Saving the data
        cv2.imwrite(os.path.join(
            output_dir,
            'aug_{}_{}'.format(
                str(time.time()),
                os.path.basename(ls_img_paths[rd_image_seleted]))),
            image
        )

def get_cpu_core(num_workers):
    """Get number of CPU cores
    """
    cpu_core = multiprocessing.cpu_count()//2

    if num_workers == -1: num_workers=cpu_core
    if (num_workers < 0 and num_workers != -1) or num_workers > cpu_core:
        warnings.warn('[WARNING]: num_workers do not match, used num_workers=1 as default')
        num_workers=1

    return num_workers

max_stars_repo_name: nguyentruonglau/keras-classify

max_issues_repo_name: nguyentruonglau/keras-classify

max_forks_repo_name: nguyentruonglau/keras-classify

reason: import numpy,from numpy


text:
"""Test the abstract SqlWrapper session."""

import unittest2 as unittest
import numpy as np
import rdflib
from osp.core.ontology.cuba import rdflib_cuba
from osp.core.session.db.sql_wrapper_session import SqlWrapperSession
from osp.core.session.db.sql_util import (
    expand_vector_cols,
    contract_vector_values, SqlQuery, EqualsCondition, JoinCondition,
    AndCondition, determine_datatype, get_data_table_name, check_characters,
    get_expanded_cols
)

COLS = ['1', '2', '3']
DTYPES = {"1": rdflib.XSD.integer,
          "2": rdflib_cuba["_datatypes/VECTOR-2"],
          "3": rdflib_cuba["_datatypes/VECTOR-2-3"]}
VALS = [100, np.array([1, 2]), np.array([[1, 2, 3], [2, 3, 4]])]
EXPANDED_COLS = ['1',
                 '2___0', '2___1',
                 '3___0', '3___1', '3___2',
                 '3___3', '3___4', '3___5']
EXPANDED_DTYPES = {'1': rdflib.XSD.integer,
                   '2': rdflib_cuba["_datatypes/VECTOR-2"],
                   '2___0': rdflib.XSD.float, '2___1': rdflib.XSD.float,
                   '3': rdflib_cuba["_datatypes/VECTOR-2-3"],
                   '3___0': rdflib.XSD.float, '3___1': rdflib.XSD.float,
                   '3___2': rdflib.XSD.float, '3___3': rdflib.XSD.float,
                   '3___4': rdflib.XSD.float, '3___5': rdflib.XSD.float}
EXPANDED_VALS = [100, 1, 2, 1, 2, 3, 2, 3, 4]

DATA_TABLE_PREFIX = SqlWrapperSession.DATA_TABLE_PREFIX

def data_tbl(suffix):
    """Prepend data table prefix."""
    return DATA_TABLE_PREFIX + suffix

class TestSqlUtil(unittest.TestCase):
    """Test the utility methods for sql wrappers."""

    def test_sql_query(self):
        """Test the SqlQuery class."""
        q = SqlQuery("my_table", COLS, DTYPES, "alias")
        self.assertEqual(q.order, ["alias"])
        self.assertEqual(q.tables, {"alias": "my_table"})
        self.assertEqual(q._columns, {"alias": EXPANDED_COLS})
        self.assertEqual(q.datatypes, {"alias": EXPANDED_DTYPES})
        self.assertEqual(q.condition, None)

        # test WHERE
        c1 = EqualsCondition("my_table", "1", VALS[0], DTYPES["1"])
        c2 = EqualsCondition("my_table", "2", VALS[1], DTYPES["2"])
        c21 = EqualsCondition("my_table", "2___0",
                              VALS[1][0], EXPANDED_DTYPES["2___0"])
        c22 = EqualsCondition("my_table", "2___1",
                              VALS[1][1], EXPANDED_DTYPES["2___1"])
        c3 = EqualsCondition("my_table", "3", VALS[2], DTYPES["2"])
        c3 = AndCondition(c1, c2, c3)
        q.where(c1)
        self.assertEqual(q.condition, c1)
        q.condition = None
        q.where(c2)
        self.assertIsInstance(q.condition, AndCondition)
        self.assertEqual(len(q.condition.conditions), 2)
        self.assertEqual(q.condition, AndCondition(c21, c22))
        q.condition = None
        q.where(c1)
        q.where(c2)
        self.assertIsInstance(q.condition, AndCondition)
        self.assertEqual(len(q.condition.conditions), 3)
        self.assertEqual(q.condition, AndCondition(c1, c21, c22))
        q.condition = None
        q.where(c2)
        q.where(c1)
        self.assertIsInstance(q.condition, AndCondition)
        self.assertEqual(len(q.condition.conditions), 3)
        self.assertEqual(q.condition, AndCondition(c1, c21, c22))
        q.condition = None
        q.where(c1)
        q.where(c1)
        self.assertIsInstance(q.condition, AndCondition)
        self.assertEqual(len(q.condition.conditions), 1)
        self.assertEqual(q.condition, AndCondition(c1))

        # test JOIN
        c4 = JoinCondition("alias", "1", "alias2", "1")
        q.join("2ndTable", COLS, DTYPES, alias="alias2")
        self.assertEqual(q.order, ["alias", "alias2"])
        self.assertEqual(q.tables, {"alias": "my_table", "alias2": "2ndTable"})
        self.assertEqual(q._columns, {"alias": EXPANDED_COLS,
                                      "alias2": EXPANDED_COLS})
        self.assertEqual(q.datatypes, {"alias": EXPANDED_DTYPES,
                                       "alias2": EXPANDED_DTYPES})
        self.assertEqual(q.condition, AndCondition(c1))
        q.where(c4)
        self.assertEqual(q.condition, AndCondition(c1, c4))

    def test_determine_datatype(self):
        """Test determining the datatype of a table."""
        self.assertEqual(determine_datatype(data_tbl("XSD_integer")),
                         rdflib.XSD.integer)
        self.assertEqual(determine_datatype(data_tbl("OWL_rational")),
                         rdflib.URIRef('http://www.w3.org/2002/07/owl'
                                       '#rational'))
        # Replaced rdflib.OWL.rational with URIRef('...'), as it seems to
        # have disappeared in rdflib 6.0.0.
        # TODO: return to original form when a fix for rdflib is available.
        self.assertEqual(determine_datatype(data_tbl("RDF_PlainLiteral")),
                         rdflib.RDF.PlainLiteral)
        self.assertEqual(determine_datatype(data_tbl("RDFS_Literal")),
                         rdflib.RDFS.Literal)
        self.assertEqual(determine_datatype(data_tbl("VECTOR-2-2")),
                         rdflib_cuba["_datatypes/VECTOR-2-2"])

    def test_get_data_table_name(self):
        """Test getting the name of a data table from data type."""
        self.assertEqual(get_data_table_name(rdflib.XSD.integer),
                         data_tbl("XSD_integer"))
        self.assertEqual(get_data_table_name(
            rdflib.URIRef('http://www.w3.org/2002/07/owl#rational')),
            data_tbl("OWL_rational"))
        # Replaced rdflib.OWL.rational with URIRef('...'), as it seems to
        # have disappeared in rdflib 6.0.0.
        # TODO: return to original form when a fix for rdflib is available.
        self.assertEqual(get_data_table_name(rdflib.RDF.PlainLiteral),
                         data_tbl("RDF_PlainLiteral"))
        self.assertEqual(get_data_table_name(rdflib.RDFS.Literal),
                         data_tbl("RDFS_Literal"))
        self.assertEqual(
            get_data_table_name(rdflib_cuba["_datatypes/VECTOR-2-2"]),
            data_tbl("VECTOR-2-2")
        )
        self.assertRaises(NotImplementedError,
                          get_data_table_name, rdflib.SKOS.prefLabel)

    def test_check_characters(self):
        """Test character check."""
        c1 = EqualsCondition("c", "d", "e", rdflib.XSD.string)
        c2 = EqualsCondition("c;", "d", "e", rdflib.XSD.string)
        c3 = EqualsCondition("c", "d;", "e", rdflib.XSD.string)
        c4 = JoinCondition("g", "h", "i", "j")
        c5 = JoinCondition("g;", "h", "i", "j")
        c6 = JoinCondition("g", "h;", "i", "j")
        c7 = JoinCondition("g", "h", "i;", "j")
        c8 = JoinCondition("g", "h", "i", "j;")
        x = {"key": ["a", ("b", AndCondition(c1, c4))]}
        x1 = {"key;": ["a", ("b", AndCondition(c1, c4))]}
        x2 = {"key": ["a;", ("b", AndCondition(c1, c4))]}
        x3 = {"key": ["a", ("b;", AndCondition(c1, c4))]}

        x4 = {"key": ["a", ("b", AndCondition(c2, c4))]}
        x5 = {"key": ["a", ("b", AndCondition(c3, c4))]}

        x6 = {"key": ["a", ("b", AndCondition(c1, c5))]}
        x7 = {"key": ["a", ("b", AndCondition(c1, c6))]}
        x8 = {"key": ["a", ("b", AndCondition(c1, c7))]}
        x9 = {"key": ["a", ("b", AndCondition(c1, c8))]}
        check_characters(x)
        self.assertRaises(ValueError, check_characters, x1)
        self.assertRaises(ValueError, check_characters, x2)
        self.assertRaises(ValueError, check_characters, x3)
        self.assertRaises(ValueError, check_characters, x4)
        self.assertRaises(ValueError, check_characters, x5)
        self.assertRaises(ValueError, check_characters, x6)
        self.assertRaises(ValueError, check_characters, x7)
        self.assertRaises(ValueError, check_characters, x8)
        self.assertRaises(ValueError, check_characters, x9)

    def test_get_expanded_cols(self):
        """Test getting the expanded columns."""
        x1 = get_expanded_cols("column", rdflib.XSD.string)
        x2 = get_expanded_cols(COLS[2], DTYPES[COLS[2]])
        self.assertEqual(x1, (["column"], rdflib.XSD.string))
        self.assertEqual(x2, (EXPANDED_COLS[3:], rdflib.XSD.float))

    def test_expand_vector_cols(self):
        """Test the expand_vector_cols method."""
        cols, dtypes, vals = expand_vector_cols(
            columns=COLS,
            datatypes=DTYPES,
            values=VALS
        )
        self.assertEqual(cols, EXPANDED_COLS)
        self.assertEqual(dtypes, EXPANDED_DTYPES)
        self.assertEqual(vals, EXPANDED_VALS)

    def test_contract_vector_values(self):
        """Test the contract_vector_values method."""
        q = SqlQuery("table", EXPANDED_COLS, EXPANDED_DTYPES)
        r = contract_vector_values([EXPANDED_VALS], q)
        np.testing.assert_equal(next(r), VALS)

    def test_expand_vector_condition(self):
        """Test expanding a condition on a vector."""
        pass

if __name__ == '__main__':
    unittest.main()

max_stars_repo_name: simphony/osp-core

max_issues_repo_name: simphony/osp-core

max_forks_repo_name: simphony/osp-core