text:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import numpy as np
import tensorflow as tf
from pysc2.lib import actions
from pysc2.lib import features

from agents.network import build_net
import utils as U

class A3CAgent(object):
    """An agent specifically for solving the mini-game maps."""

    def __init__(self, training, msize, ssize, name='A3C/A3CAgent'):
        self.name = name
        self.training = training
        self.summary = []
        # Minimap size, screen size and info size
        assert msize == ssize
        self.msize = msize
        self.ssize = ssize
        self.isize = len(actions.FUNCTIONS)

    def setup(self, sess, summary_writer):
        self.sess = sess
        self.summary_writer = summary_writer

    def initialize(self):
        init_op = tf.global_variables_initializer()
        self.sess.run(init_op)

    def reset(self):
        # Epsilon schedule
        self.epsilon = [0.05, 0.2]

    def build_model(self, reuse, dev, ntype):
        with tf.variable_scope(self.name) and tf.device(dev):
            if reuse:
                tf.get_variable_scope().reuse_variables()
                assert tf.get_variable_scope().reuse

            # Set inputs of networks
            self.minimap = tf.placeholder(
                tf.float32,
                [None, U.minimap_channel(), self.msize, self.msize],
                name='minimap')
            self.screen = tf.placeholder(
                tf.float32,
                [None, U.screen_channel(), self.ssize, self.ssize],
                name='screen')
            self.info = tf.placeholder(
                tf.float32, [None, self.isize], name='info')

            # Build networks
            net = build_net(self.minimap, self.screen, self.info, self.msize,
                            self.ssize, len(actions.FUNCTIONS), ntype)
            self.spatial_action, self.non_spatial_action, self.value = net

            # Set targets and masks
            self.valid_spatial_action = tf.placeholder(
                tf.float32, [None], name='valid_spatial_action')
            self.spatial_action_selected = tf.placeholder(
                tf.float32, [None, self.ssize**2],
                name='spatial_action_selected')
            self.valid_non_spatial_action = tf.placeholder(
                tf.float32, [None, len(actions.FUNCTIONS)],
                name='valid_non_spatial_action')
            self.non_spatial_action_selected = tf.placeholder(
                tf.float32, [None, len(actions.FUNCTIONS)],
                name='non_spatial_action_selected')
            self.value_target = tf.placeholder(
                tf.float32, [None], name='value_target')

            # Compute log probability
            spatial_action_prob = tf.reduce_sum(
                self.spatial_action * self.spatial_action_selected, axis=1)
            spatial_action_log_prob = tf.log(
                tf.clip_by_value(spatial_action_prob, 1e-10, 1.))
            non_spatial_action_prob = tf.reduce_sum(
                self.non_spatial_action * self.non_spatial_action_selected,
                axis=1)
            valid_non_spatial_action_prob = tf.reduce_sum(
                self.non_spatial_action * self.valid_non_spatial_action,
                axis=1)
            valid_non_spatial_action_prob = tf.clip_by_value(
                valid_non_spatial_action_prob, 1e-10, 1.)
            non_spatial_action_prob = non_spatial_action_prob / valid_non_spatial_action_prob
            non_spatial_action_log_prob = tf.log(
                tf.clip_by_value(non_spatial_action_prob, 1e-10, 1.))
            self.summary.append(
                tf.summary.histogram('spatial_action_prob',
                                     spatial_action_prob))
            self.summary.append(
                tf.summary.histogram('non_spatial_action_prob',
                                     non_spatial_action_prob))

            # Compute losses, more details in https://arxiv.org/abs/1602.01783
            # Policy loss and value loss
            action_log_prob = self.valid_spatial_action * spatial_action_log_prob + non_spatial_action_log_prob
            advantage = tf.stop_gradient(self.value_target - self.value)
            policy_loss = -tf.reduce_mean(action_log_prob * advantage)
            value_loss = -tf.reduce_mean(self.value * advantage)
            self.summary.append(tf.summary.scalar('policy_loss', policy_loss))
            self.summary.append(tf.summary.scalar('value_loss', value_loss))

            # TODO: policy penalty
            loss = policy_loss + value_loss

            # Build the optimizer
            self.learning_rate = tf.placeholder(
                tf.float32, None, name='learning_rate')
            opt = tf.train.RMSPropOptimizer(
                self.learning_rate, decay=0.99, epsilon=1e-10)
            grads = opt.compute_gradients(loss)
            cliped_grad = []
            for grad, var in grads:
                self.summary.append(tf.summary.histogram(var.op.name, var))
                self.summary.append(
                    tf.summary.histogram(var.op.name + '/grad', grad))
                grad = tf.clip_by_norm(grad, 10.0)
                cliped_grad.append([grad, var])
            self.train_op = opt.apply_gradients(cliped_grad)
            self.summary_op = tf.summary.merge(self.summary)

            self.saver = tf.train.Saver(max_to_keep=100)

    def step(self, obs):
        minimap = np.array(obs.observation['minimap'], dtype=np.float32)
        minimap = np.expand_dims(U.preprocess_minimap(minimap), axis=0)
        screen = np.array(obs.observation['screen'], dtype=np.float32)
        screen = np.expand_dims(U.preprocess_screen(screen), axis=0)
        # TODO: only use available actions
        info = np.zeros([1, self.isize], dtype=np.float32)
        info[0, obs.observation['available_actions']] = 1

        feed = {self.minimap: minimap, self.screen: screen, self.info: info}
        non_spatial_action, spatial_action = self.sess.run(
            [self.non_spatial_action, self.spatial_action], feed_dict=feed)

        # Select an action and a spatial target
        non_spatial_action = non_spatial_action.ravel()
        spatial_action = spatial_action.ravel()
        valid_actions = obs.observation['available_actions']
        act_id = valid_actions[np.argmax(non_spatial_action[valid_actions])]
        target = np.argmax(spatial_action)
        target = [int(target // self.ssize), int(target % self.ssize)]

        if False:
            print(actions.FUNCTIONS[act_id].name, target)

        # Epsilon greedy exploration
        if self.training and np.random.rand() < self.epsilon[0]:
            act_id = np.random.choice(valid_actions)
        if self.training and np.random.rand() < self.epsilon[1]:
            dy = np.random.randint(-4, 5)
            target[0] = int(max(0, min(self.ssize - 1, target[0] + dy)))
            dx = np.random.randint(-4, 5)
            target[1] = int(max(0, min(self.ssize - 1, target[1] + dx)))

        # Set act_id and act_args
        act_args = []
        for arg in actions.FUNCTIONS[act_id].args:
            if arg.name in ('screen', 'minimap', 'screen2'):
                act_args.append([target[1], target[0]])
            else:
                act_args.append([0])  # TODO: Be careful
        return actions.FunctionCall(act_id, act_args)

    def update(self, rbs, disc, lr, cter):
        # Compute R, which is value of the last observation
        obs = rbs[-1][-1]
        if obs.last():
            R = 0
        else:
            minimap = np.array(obs.observation['minimap'], dtype=np.float32)
            minimap = np.expand_dims(U.preprocess_minimap(minimap), axis=0)
            screen = np.array(obs.observation['screen'], dtype=np.float32)
            screen = np.expand_dims(U.preprocess_screen(screen), axis=0)
            info = np.zeros([1, self.isize], dtype=np.float32)
            info[0, obs.observation['available_actions']] = 1

            feed = {
                self.minimap: minimap,
                self.screen: screen,
                self.info: info
            }
            R = self.sess.run(self.value, feed_dict=feed)[0]

        # Compute targets and masks
        minimaps = []
        screens = []
        infos = []

        value_target = np.zeros([len(rbs)], dtype=np.float32)
        value_target[-1] = R

        valid_spatial_action = np.zeros([len(rbs)], dtype=np.float32)
        spatial_action_selected = np.zeros(
            [len(rbs), self.ssize**2], dtype=np.float32)
        valid_non_spatial_action = np.zeros(
            [len(rbs), len(actions.FUNCTIONS)], dtype=np.float32)
        non_spatial_action_selected = np.zeros(
            [len(rbs), len(actions.FUNCTIONS)], dtype=np.float32)

        rbs.reverse()
        for i, [obs, action, next_obs] in enumerate(rbs):
            minimap = np.array(obs.observation['minimap'], dtype=np.float32)
            minimap = np.expand_dims(U.preprocess_minimap(minimap), axis=0)
            screen = np.array(obs.observation['screen'], dtype=np.float32)
            screen = np.expand_dims(U.preprocess_screen(screen), axis=0)
            info = np.zeros([1, self.isize], dtype=np.float32)
            info[0, obs.observation['available_actions']] = 1

            minimaps.append(minimap)
            screens.append(screen)
            infos.append(info)

            reward = obs.reward
            act_id = action.function
            act_args = action.arguments

            value_target[i] = reward + disc * value_target[i - 1]

            valid_actions = obs.observation["available_actions"]
            valid_non_spatial_action[i, valid_actions] = 1
            non_spatial_action_selected[i, act_id] = 1

            args = actions.FUNCTIONS[act_id].args
            for arg, act_arg in zip(args, act_args):
                if arg.name in ('screen', 'minimap', 'screen2'):
                    ind = act_arg[1] * self.ssize + act_arg[0]
                    valid_spatial_action[i] = 1
                    spatial_action_selected[i, ind] = 1

        minimaps = np.concatenate(minimaps, axis=0)
        screens = np.concatenate(screens, axis=0)
        infos = np.concatenate(infos, axis=0)

        # Train
        feed = {
            self.minimap: minimaps,
            self.screen: screens,
            self.info: infos,
            self.value_target: value_target,
            self.valid_spatial_action: valid_spatial_action,
            self.spatial_action_selected: spatial_action_selected,
            self.valid_non_spatial_action: valid_non_spatial_action,
            self.non_spatial_action_selected: non_spatial_action_selected,
            self.learning_rate: lr
        }
        _, summary = self.sess.run(
            [self.train_op, self.summary_op], feed_dict=feed)
        self.summary_writer.add_summary(summary, cter)

    def save_model(self, path, count):
        self.saver.save(self.sess, path + '/model.pkl', count)

    def load_model(self, path):
        ckpt = tf.train.get_checkpoint_state(path)
        self.saver.restore(self.sess, ckpt.model_checkpoint_path)
        return int(ckpt.model_checkpoint_path.split('-')[-1])


text:
import pylab as plt
import numpy as np
from axycolor import rgb2lab
from itertools import izip, product

def middle_divisors(n):
    for i in range(int(n ** (0.5)), 2, -1):
        if n % i == 0:
            return i, n // i

    return middle_divisors(n+1)  # If prime number take next one

def resolution(N, numerator=16, divisor=9):
    width = int(np.ceil(np.sqrt(16./9. * N)))
    height = int(np.ceil(float(N)/width))
    return width, height

def show_image(pixels, width, height, cmap=plt.cm.gray, scale=1, blocking=False):
    pixels = np.asarray(pixels)
    plt.imshow(scale*pixels.reshape((height, width)), interpolation="nearest", cmap=cmap)
    plt.show(blocking)

def show_images2d(list_of_pixels, shape, cmap=plt.cm.gray, scale=1, blocking=False):
    height, width = shape
    ncols, nrows = resolution(len(list_of_pixels))

    image = scale/2. * np.ones((nrows * (height+1) - 1, ncols * (width+1) - 1), dtype=float)
    for pixels, (row, col) in izip(list_of_pixels, product(range(nrows), range(ncols))):
        pixels = np.asarray(pixels).reshape(height, width)
        image[row*(height+1):(row+1)*(height+1)-1, col*(width+1):(col+1)*(width+1)-1] = pixels

    plt.imshow(scale*image, interpolation="nearest", cmap=cmap)
    plt.subplots_adjust(left=0., right=1., top=0.95, bottom=0.05)
    plt.show(blocking)

def show_images3d(list_of_pixels, shape, cmap=plt.cm.gray, scale=1, blocking=False):
    height, width, depth = shape
    ncols, nrows = resolution(len(list_of_pixels))

    for d in range(depth):
        plt.figure(d)
        image = scale/2. * np.ones((nrows * (height+1) - 1, ncols * (width+1) - 1), dtype=float)
        for pixels, (row, col) in izip(list_of_pixels, product(range(nrows), range(ncols))):
            pixels = np.asarray(pixels).reshape(height, width, depth)[:, :, d]
            image[row*(height+1):(row+1)*(height+1)-1, col*(width+1):(col+1)*(width+1)-1] = pixels

        plt.imshow(scale*image, interpolation="nearest", cmap=cmap)

    plt.subplots_adjust(left=0., right=1., top=0.95, bottom=0.05)
    plt.show(blocking)

def cmap(bg=(0, 0, 0), exclude=[], n_colors=30):
    # Generate a sizable number of RGB triples. This represents our space of
    # possible choices. By starting in RGB space, we ensure that all of the
    # colors can be generated by the monitor.
    bg = np.array([bg] + exclude)
    x = np.linspace(0, 1, n_colors)  # divisions along each axis in RGB space
    R, G, B = np.meshgrid(x, x, x)
    rgb = np.c_[R.flatten(), G.flatten(), B.flatten()]

    lab = rgb2lab(rgb)
    bglab = rgb2lab(bg)

    lastlab = bglab[0]
    mindist2 = np.ones(len(rgb)) * np.inf
    for bglab_i in bglab[1:]:
        dist2 = np.sum((lab-bglab_i)**2, axis=1)
        mindist2 = np.minimum(dist2, mindist2)  # dist2 to closest previously-chosen color

    while True:
        dX = lab - lastlab  # displacement of last from all colors on list
        dist2 = np.sum(dX**2, axis=1)  # square distance

        mindist2 = np.minimum(dist2, mindist2)  # dist2 to closest previously-chosen color
        idx = np.argmax(mindist2)  # find the entry farthest from all previously-chosen colors
        yield rgb[idx]

        lastlab = lab[idx]

ext: py

max_stars_repo_name: MarcCote/brain-search

max_issues_repo_name: MarcCote/brainsearch

max_forks_repo_name: MarcCote/brainsearch

reason: import numpy


text:
"""
Human control for the IKEA furniture assembly environment.
The user will learn how to configure the environment and control it.
If you're coming from the demo_demonstration script:
Pass --record_demo True and press Y to save the the current scene
into demos/test.pkl.
"""

import argparse

import numpy as np

from env import make_env
from env.models import furniture_names, background_names
from util import str2bool

# available agents
agent_names = ['Baxter', 'Sawyer', 'Cursor']

# available furnitures
furniture_names

# available background scenes
background_names

def main(args):
    """
    Inputs types of agent, furniture model, and background and simulates the environment.
    """
    print("IKEA Furniture Assembly Environment!")

    # choose an agent
    print()
    print("Supported robots:\n")
    for i, agent in enumerate(agent_names):
        print('{}: {}'.format(i, agent))
    print()
    try:
        s = input("Choose an agent (enter a number from 0 to {}): ".format(len(agent_names) - 1))
        k = int(s)
        agent_name = agent_names[k]
    except:
        print("Input is not valid. Use 0 by default.")
        agent_name = agent_names[0]

    # choose a furniture model
    print()
    print("Supported furniture:\n")
    for i, furniture_name in enumerate(furniture_names):
        print('{}: {}'.format(i, furniture_name))
    print()
    try:
        s = input("Choose a furniture model (enter a number from 0 to {}): ".format(len(furniture_names) - 1))
        furniture_id = int(s)
        furniture_name = furniture_names[furniture_id]
    except:
        print("Input is not valid. Use 0 by default.")
        furniture_id = 0
        furniture_name = furniture_names[0]

    # choose a background scene
    print()
    print("Supported backgrounds:\n")
    for i, background in enumerate(background_names):
        print('{}: {}'.format(i, background))
    print()
    try:
        s = input("Choose a background (enter a number from 0 to {}): ".format(len(background_names) - 1))
        k = int(s)
        background_name = background_names[k]
    except:
        print("Input is not valid. Use 0 by default.")
        background_name = background_names[0]

    # set parameters for the environment (env, furniture_id, background)
    env_name = 'Furniture{}Env'.format(agent_name)
    args.env = env_name
    args.furniture_id = furniture_id
    args.background = background_name

    print()
    print("Creating environment (robot: {}, furniture: {}, background: {})".format(
        env_name, furniture_name, background_name))

    # make environment following arguments
    env = make_env(env_name, args)

    # print brief instruction
    print()
    print("="*80)
    print("Instruction:\n")
    print("Move - WASDQE, Rotate - IJKLUO\n"
          "Grasp - SPACE, Release - ENTER (RETURN), Attach - C\n"
          "Switch baxter arms or cursors - 1 or 2\n"
          "Screenshot - T, Video recording - R, Save Demo - Y")
    print("="*80)
    print()

    # manual control of agent using keyboard
    env.run_manual(args)

    # close the environment instance
    env.close()

def argsparser():
    """
    Returns argument parser for furniture assembly environment.
    """
    parser = argparse.ArgumentParser("Demo for IKEA Furniture Assembly Environment")
    parser.add_argument('--seed', type=int, default=123)
    parser.add_argument('--debug', type=str2bool, default=False)

    import config.furniture as furniture_config
    furniture_config.add_argument(parser)

    args = parser.parse_args()
    return args

if __name__ == '__main__':
    args = argsparser()
    main(args)

max_stars_repo_name: snasiriany/furniture

max_issues_repo_name: snasiriany/furniture

max_forks_repo_name: snasiriany/furniture


text:
import os
import hydra
import logging

logger = logging.getLogger(__name__)

def run(args):
    import unet
    import tensorflow as tf
    import soundfile as sf
    import numpy as np
    from tqdm import tqdm
    import scipy.signal

    path_experiment=str(args.path_experiment)

    unet_model = unet.build_model_denoise(unet_args=args.unet)

    ckpt=os.path.join(os.path.dirname(os.path.abspath(__file__)),path_experiment, 'checkpoint')
    unet_model.load_weights(ckpt)

    def do_stft(noisy):

        window_fn = tf.signal.hamming_window

        win_size=args.stft.win_size
        hop_size=args.stft.hop_size

        stft_signal_noisy=tf.signal.stft(noisy,frame_length=win_size, window_fn=window_fn, frame_step=hop_size, pad_end=True)
        stft_noisy_stacked=tf.stack( values=[tf.math.real(stft_signal_noisy), tf.math.imag(stft_signal_noisy)], axis=-1)

        return stft_noisy_stacked

    def do_istft(data):

        window_fn = tf.signal.hamming_window

        win_size=args.stft.win_size
        hop_size=args.stft.hop_size

        inv_window_fn=tf.signal.inverse_stft_window_fn(hop_size, forward_window_fn=window_fn)

        pred_cpx=data[...,0] + 1j * data[...,1]
        pred_time=tf.signal.inverse_stft(pred_cpx, win_size, hop_size, window_fn=inv_window_fn)
        return pred_time

    audio=str(args.inference.audio)
    data, samplerate = sf.read(audio)
    print(data.dtype)
    #Stereo to mono
    if len(data.shape)>1:
        data=np.mean(data,axis=1)

    if samplerate!=44100:
        print("Resampling")

        data=scipy.signal.resample(data, int((44100  / samplerate )*len(data))+1)

    segment_size=44100*5  #20s segments

    length_data=len(data)
    overlapsize=2048 #samples (46 ms)
    window=np.hanning(2*overlapsize)
    window_right=window[overlapsize::]
    window_left=window[0:overlapsize]
    audio_finished=False
    pointer=0
    denoised_data=np.zeros(shape=(len(data),))
    residual_noise=np.zeros(shape=(len(data),))
    numchunks=int(np.ceil(length_data/segment_size))

    for i in tqdm(range(numchunks)):
        if pointer+segment_size<length_data:
            segment=data[pointer:pointer+segment_size]
            #dostft
            segment_TF=do_stft(segment)
            segment_TF_ds=tf.data.Dataset.from_tensors(segment_TF)
            pred = unet_model.predict(segment_TF_ds.batch(1))
            pred=pred[0]
            residual=segment_TF-pred[0]
            residual=np.array(residual)
            pred_time=do_istft(pred[0])
            residual_time=do_istft(residual)
            residual_time=np.array(residual_time)

            if pointer==0:
                pred_time=np.concatenate((pred_time[0:int(segment_size-overlapsize)], np.multiply(pred_time[int(segment_size-overlapsize):segment_size],window_right)), axis=0)
                residual_time=np.concatenate((residual_time[0:int(segment_size-overlapsize)], np.multiply(residual_time[int(segment_size-overlapsize):segment_size],window_right)), axis=0)
            else:
                pred_time=np.concatenate((np.multiply(pred_time[0:int(overlapsize)], window_left), pred_time[int(overlapsize):int(segment_size-overlapsize)], np.multiply(pred_time[int(segment_size-overlapsize):int(segment_size)],window_right)), axis=0)
                residual_time=np.concatenate((np.multiply(residual_time[0:int(overlapsize)], window_left), residual_time[int(overlapsize):int(segment_size-overlapsize)], np.multiply(residual_time[int(segment_size-overlapsize):int(segment_size)],window_right)), axis=0)

            denoised_data[pointer:pointer+segment_size]=denoised_data[pointer:pointer+segment_size]+pred_time
            residual_noise[pointer:pointer+segment_size]=residual_noise[pointer:pointer+segment_size]+residual_time

            pointer=pointer+segment_size-overlapsize
        else:
            segment=data[pointer::]
            lensegment=len(segment)
            segment=np.concatenate((segment, np.zeros(shape=(int(segment_size-len(segment)),))), axis=0)
            audio_finished=True
            #dostft
            segment_TF=do_stft(segment)

            segment_TF_ds=tf.data.Dataset.from_tensors(segment_TF)

            pred = unet_model.predict(segment_TF_ds.batch(1))
            pred=pred[0]
            residual=segment_TF-pred[0]
            residual=np.array(residual)
            pred_time=do_istft(pred[0])
            pred_time=np.array(pred_time)
            pred_time=pred_time[0:segment_size]
            residual_time=do_istft(residual)
            residual_time=np.array(residual_time)
            residual_time=residual_time[0:segment_size]
            if pointer==0:
                pred_time=pred_time
                residual_time=residual_time
            else:
                pred_time=np.concatenate((np.multiply(pred_time[0:int(overlapsize)], window_left), pred_time[int(overlapsize):int(segment_size)]),axis=0)
                residual_time=np.concatenate((np.multiply(residual_time[0:int(overlapsize)], window_left), residual_time[int(overlapsize):int(segment_size)]),axis=0)

            denoised_data[pointer::]=denoised_data[pointer::]+pred_time[0:lensegment]
            residual_noise[pointer::]=residual_noise[pointer::]+residual_time[0:lensegment]

    basename=os.path.splitext(audio)[0]
    wav_noisy_name=basename+"_noisy_input"+".wav"
    sf.write(wav_noisy_name, data, 44100)
    wav_output_name=basename+"_denoised"+".wav"
    sf.write(wav_output_name, denoised_data, 44100)
    wav_output_name=basename+"_residual"+".wav"
    sf.write(wav_output_name, residual_noise, 44100)

def _main(args):
    global __file__

    __file__ = hydra.utils.to_absolute_path(__file__)

    run(args)

@hydra.main(config_path="conf/conf.yaml")
def main(args):
    try:
        _main(args)
    except Exception:
        logger.exception("Some error happened")
        # Hydra intercepts exit code, fixed in beta but I could not get the beta to work
        os._exit(1)

if __name__ == "__main__":
    main()

reason: import numpy,import scipy


text:
#!/usr/bin/env python2

import numpy as np
from copy import deepcopy
from zholkovskij import Bilayer, Trilayer, Quadlayer

ZERO = 1e-15

def Warburg(w, p):
    tan = 0
    denom = np.power(1j * w * p[1], 0.5)
    if p[1] * w < 1e5:
        tan = np.tanh(denom)
    else:
        tan = 1.0
    return p[0] * tan / denom

def Warburgni(w, p):
    tan = 0
    denom = np.power(1j * w * p[1], p[2])
    if p[1] * w < 1e5:
        tan = np.tanh(denom)
    else:
        tan = 1.0
    return p[0] * tan / denom

def dWarburg_dp0(w, p):
    tan = 0
    denom = np.power(1j * w * p[1], 0.5)
    if p[1] * w < 1e5:
        tan = np.tanh(denom)
    else:
        tan = 1.0
    return 1 * tan / denom

def dWarburg_dp1(w, p):
    tan = 0
    denom = np.power(1j * w * p[1], 0.5)
    if p[1] * w < 1e5:
        tan = np.tanh(denom)
    else:
        tan = 1.0

    real = p[0] / \
        (2.0 * p[1] * np.power(np.cosh(np.power(1j * w * p[1], 0.5)), 2.0))
    imag = -1j * p[0] * w * tan / (2.0 * np.power(1j * w * p[1], 1.5))
    return real + imag

eqcLib = {
    'R': {
        'inuse': [],
        'def': {
            'eqc': lambda w, p: p[0],
            'pNames': ['R'],
            'constraints': [(ZERO, 1e7), ],
            'jac': [lambda w, p: 1],
        },
    },
    'C': {
        'inuse': [],
        'def': {
            'eqc': lambda w, p: 1.0 / (1j * w * p[0]),
            'pNames': ['C'],
            'constraints': [(ZERO, 1), ],
            'jac': [lambda w, p: -1.0 / (1j * w * np.power(p[0], 2.0))]
        },
    },
    'L': {
        'inuse': [],
        'def': {
            'eqc': lambda w, p: 1j * w * p[0],
            'pNames': ['L'],
            'constraints': [(ZERO, 1), ],
            'jac': [lambda w, p: 1j * w],
        },
    },
    'Zarc': {
        'inuse': [],
        'def': {
            'eqc': lambda w, p: p[0] / (1.0 + np.power(1j * p[1] * w, p[2])),
            'pNames': ['R', 'T', 'n'],
            'constraints': [(ZERO, 1e7), (ZERO, 1), (ZERO, 1)],
            'jac': [lambda w, p: 1.0 / (1.0 + np.power(1j * p[1] * w, p[2])),
                    lambda w, p: -1.0 * p[0] * p[2] * np.power(1j * w * p[1], p[2]) /
                    (1.0 * p[1] * np.power(1.0 + np.power(1j * w * p[1], p[2]), 2.0)),
                    lambda w, p: -1.0 * p[0] * np.power(1j * w * p[1], p[2]) * np.log(1j * w * p[1]) /
                    np.power(1.0 + np.power(1j * w * p[1], p[2]), 2.0)],
        },
    },
    'CPE': {
        'inuse': [],
        'def': {
            'eqc': lambda w, p: 1.0 / (p[0] * np.power(1j * w, p[1])),
            'pNames': ['Q', 'n'],
            'constraints': [(ZERO, 1), (ZERO, 1)],
            'jac': [lambda w, p: -1.0 * np.power(1j * w, -1.0 * p[1]) / np.power(p[0], 2.0),
                    lambda w, p: -1.0 * np.log(1j * w) / (p[0] * np.power(1j * w, p[1]))],
        },
    },
    'Warb': {
        'inuse': [],
        'def': {
            'eqc': Warburg,
            'pNames': ['R', 'T'],
            'constraints': [(ZERO, 1e7), (ZERO, 100)],
            'jac': [dWarburg_dp0,
                    dWarburg_dp1],
        },
    },
    'Warbni': {
        'inuse': [],
        'def': {
            'eqc': Warburgni,
            'pNames': ['R', 'T', 'n'],
            'constraints': [(ZERO, 1e7), (ZERO, 1e4), (0.5, 0.54)],
            'jac': [lambda w, p: Exception("Not implemented")]*3,
        },
    },
    'DC': {
        'inuse': [],
        'def': {
            'eqc': lambda w, p: 1.0 / np.power(1.0 + 1j * w * p[0], 0.5),
            'pNames': ['T', 'n'],
            'constraints': [(ZERO, 1), (ZERO, 1)],
            'jac': [lambda w, p: Exception("Not implemented")]*2,
        },
    },
    'Bilayer': {
        'inuse': [],
        'def': {
            'eqc': Bilayer,
            'pNames': ['g1', 't1', 'w1', 'g2', 't2', 'w2'],
            'constraints': [(ZERO, 1e6), (ZERO, 1), (ZERO, 1e10)]*2,
            'jac': [lambda w, p: Exception("Not implemented")]*6,
        },
    },
    'Trilayer': {
        'inuse': [],
        'def': {
            'eqc': Trilayer,
            'pNames': ['g1', 't1', 'w1', 'g2', 't2', 'w2', 'g3', 't3', 'w3'],
            'constraints': [(ZERO, 1e6), (ZERO, 1), (ZERO, 1e10)]*3,
            'jac': [lambda w, p: Exception("Not implemented")]*9,
        },
    },
    'Quadlayer': {
        'inuse': [],
        'def': {
            'eqc': Quadlayer,
            'pNames': ['g1', 't1', 'w1', 'g2', 't2', 'w2', 'g3', 't3', 'w3', 'g4', 't4', 'w4'],
            'constraints': [(ZERO, 1e6), (ZERO, 1), (ZERO, 1e10)]*4,
            'jac': [lambda w, p: Exception("Not implemented")]*12,
        },
    },
}

def resetClassDefinition():
    for k in eqcLib:
        eqcLib[k]['inuse'] = []

def getClassDefinition(name):
    el = name
    ln = "1"
    if '_' in name:
        el = name.split('_')[0]
        ln = name.split('_')[1]
    if el not in eqcLib:
        raise ValueError("Unknown element: %s" % el)
        return None
    i = 1
    while ln in eqcLib[el]['inuse']:
        ln = "%i" % i
        i += 1
    eqcLib[el]['inuse'] += [ln]
    classdef = deepcopy(eqcLib[el]['def'])
    classdef['name'] = '%s_%s' % (el, ln)
    classdef['pNames'] = ['%s{%s}' % (classdef['name'], x)
                          for x in classdef['pNames']]
    classdef['params'] = np.array([-1] * len(classdef['pNames']))
    return classdef

max_stars_repo_name: robbje/eis

max_issues_repo_name: robbje/eis

max_forks_repo_name: robbje/eis


text:
"""
I/O for the Wavefront .obj file format, cf.
<https://en.wikipedia.org/wiki/Wavefront_.obj_file>.
"""
import datetime

import numpy as np

from ..__about__ import __version__
from .._exceptions import WriteError
from .._files import open_file
from .._helpers import register_format
from .._mesh import CellBlock, Mesh

def read(filename):
    with open_file(filename, "r") as f:
        mesh = read_buffer(f)
    return mesh

def read_buffer(f):
    points = []
    vertex_normals = []
    texture_coords = []
    face_groups = []
    face_group_ids = []
    face_group_id = -1
    while True:
        line = f.readline()

        if not line:
            # EOF
            break

        strip = line.strip()

        if len(strip) == 0 or strip[0] == "#":
            continue

        split = strip.split()

        if split[0] == "v":
            points.append([float(item) for item in split[1:]])
        elif split[0] == "vn":
            vertex_normals.append([float(item) for item in split[1:]])
        elif split[0] == "vt":
            texture_coords.append([float(item) for item in split[1:]])
        elif split[0] == "s":
            # "s 1" or "s off" controls smooth shading
            pass
        elif split[0] == "f":
            dat = [int(item.split("/")[0]) for item in split[1:]]
            if len(face_groups) == 0 or (
                len(face_groups[-1]) > 0 and len(face_groups[-1][-1]) != len(dat)
            ):
                face_groups.append([])
                face_group_ids.append([])

            face_groups[-1].append(dat)
            face_group_ids[-1].append(face_group_id)
        elif split[0] == "g":
            # new group
            face_groups.append([])
            face_group_ids.append([])
            face_group_id += 1
        else:
            # who knows
            pass

    # There may be empty groups, too. <https://github.com/nschloe/meshio/issues/770>
    # Remove them.
    face_groups = [f for f in face_groups if len(f) > 0]
    face_group_ids = [g for g in face_group_ids if len(g) > 0]

    points = np.array(points)
    texture_coords = np.array(texture_coords)
    vertex_normals = np.array(vertex_normals)
    point_data = {}
    if len(texture_coords) > 0:
        point_data["obj:vt"] = texture_coords
    if len(vertex_normals) > 0:
        point_data["obj:vn"] = vertex_normals

    # convert to numpy arrays
    face_groups = [np.array(f) for f in face_groups]
    cell_data = {"obj:group_ids": []}
    cells = []
    for f, gid in zip(face_groups, face_group_ids):
        if f.shape[1] == 3:
            cells.append(CellBlock("triangle", f - 1))
        elif f.shape[1] == 4:
            cells.append(CellBlock("quad", f - 1))
        else:
            cells.append(CellBlock("polygon", f - 1))
        cell_data["obj:group_ids"].append(gid)

    return Mesh(points, cells, point_data=point_data, cell_data=cell_data)

def write(filename, mesh):
    for c in mesh.cells:
        if c.type not in ["triangle", "quad", "polygon"]:
            raise WriteError(
                "Wavefront .obj files can only contain triangle or quad cells."
            )

    with open_file(filename, "w") as f:
        f.write(
            "# Created by meshio v{}, {}\n".format(
                __version__, datetime.datetime.now().isoformat()
            )
        )
        for p in mesh.points:
            f.write(f"v {p[0]} {p[1]} {p[2]}\n")

        if "obj:vn" in mesh.point_data:
            dat = mesh.point_data["obj:vn"]
            fmt = "vn " + " ".join(["{}"] * dat.shape[1]) + "\n"
            for vn in dat:
                f.write(fmt.format(*vn))

        if "obj:vt" in mesh.point_data:
            dat = mesh.point_data["obj:vt"]
            fmt = "vt " + " ".join(["{}"] * dat.shape[1]) + "\n"
            for vt in dat:
                f.write(fmt.format(*vt))

        for cell_block in mesh.cells:
            fmt = "f " + " ".join(["{}"] * cell_block.data.shape[1]) + "\n"
            for c in cell_block.data:
                f.write(fmt.format(*(c + 1)))

register_format("obj", [".obj"], read, {"obj": write})

max_stars_repo_name: honzatomek/meshio

max_issues_repo_name: honzatomek/meshio

max_forks_repo_name: honzatomek/meshio


text:
"""This plots the Gaussian noise levels for a given ansisotropy source QE estimators.

"""
import numpy as np
import healpy as hp
import pylab as pl
import os

import plancklens
from plancklens import utils
from plancklens import nhl, qresp

cls_path = os.path.join(os.path.dirname(os.path.abspath(plancklens.__file__)), 'data', 'cls')

ksource = 'p'
fname = None # If set, will try to save figure to this file.

lmax_ivf = 2048
lmin_ivf = 100
nlev_t = 35.
nlev_p = 35. * np.sqrt(2.)
beam_fwhm = 6.
lmax_qlm = lmax_ivf

if ksource in ['p', 'f']:
    qe_keys = [ksource + 'tt', ksource+'_p', ksource]
    qe_keys_lab = [ (r'$\hat\phi^{%s}$' if ksource == 'p' else 'f')%l for l in ['TT', 'P.', 'MV']]
elif ksource in ['a', 'a_p', 'stt']:
    qe_keys = [ksource]
    qe_keys_lab = [ksource]
else:
    assert 0

transf = hp.gauss_beam(beam_fwhm / 60. / 180. * np.pi, lmax=lmax_ivf)

cls_len = utils.camb_clfile(os.path.join(cls_path, 'FFP10_wdipole_lensedCls.dat'))
cls_weight = utils.camb_clfile(os.path.join(cls_path, 'FFP10_wdipole_lensedCls.dat'))

fal_sepTP =  {'tt': utils.cli(cls_len['tt'][:lmax_ivf + 1] + (nlev_t / 60. / 180. * np.pi) ** 2 / transf ** 2),
              'ee': utils.cli(cls_len['ee'][:lmax_ivf + 1] + (nlev_p / 60. / 180. * np.pi) ** 2 / transf ** 2),
              'bb': utils.cli(cls_len['bb'][:lmax_ivf + 1] + (nlev_p / 60. / 180. * np.pi) ** 2 / transf ** 2)}

cls_ivfs_sepTP = {'tt':fal_sepTP['tt'].copy(),
                  'ee':fal_sepTP['ee'].copy(),
                  'bb':fal_sepTP['bb'].copy(),
                  'te':cls_len['te'][:lmax_ivf + 1] * fal_sepTP['tt'] * fal_sepTP['ee']}
cls_dat = {
    'tt': (cls_len['tt'][:lmax_ivf + 1] + (nlev_t / 60. /180. * np.pi) ** 2 / transf ** 2),
    'ee': (cls_len['ee'][:lmax_ivf + 1] + (nlev_p / 60. /180. * np.pi) ** 2 / transf ** 2),
    'bb': (cls_len['bb'][:lmax_ivf + 1] + (nlev_p / 60. /180. * np.pi) ** 2 / transf ** 2),
    'te':  np.copy(cls_len['te'][:lmax_ivf + 1]) }

fal_jtTP = utils.cl_inverse(cls_dat)
cls_ivfs_jtTP = utils.cl_inverse(cls_dat)

for cls in [fal_sepTP, fal_jtTP, cls_ivfs_sepTP, cls_ivfs_jtTP]:
    for cl in cls.values():
        cl[:max(1, lmin_ivf)] *= 0.

if ksource == 'p':
    w = lambda ell : ell ** 2 *(ell + 1) ** 2 * 1e7 * 0.5 / np.pi
    ylabel = r'$10^7\cdot L^2(L + 1)^2 C_L^{\phi\phi} / 2\pi$'
elif ksource == 'f':
    w = lambda ell : 1.
    ylabel = r'$C_L^{ff}$'
elif ksource == 'a':
    w = lambda ell : 1.
    ylabel = r'$C_L^{\alpha\alpha}$'
else:
    assert 0
ellp = np.arange(1, lmax_qlm + 1)
ellc = np.arange(2, lmax_qlm + 1)

for qe_key, label in zip(qe_keys, qe_keys_lab):
    NG, NC, NGC, NCG = nhl.get_nhl(qe_key, qe_key, cls_weight, cls_ivfs_sepTP, lmax_ivf, lmax_ivf, lmax_out=lmax_qlm)
    RG, RC, RGC, RCG = qresp.get_response(qe_key, lmax_ivf, ksource, cls_weight, cls_len, fal_sepTP, lmax_qlm=lmax_qlm)

    N0 = utils.cli(RG ** 2) * NG
    ln = pl.loglog(ellp, w(ellp) * N0[ellp], label=label + ' (sep. TP filt.)' * (qe_key == 'p'))
    if qe_key[0] in ['p', 'x', 'f']:
        N0 = utils.cli(RC ** 2) * NC
        pl.loglog(ellc, w(ellc) * N0[ellc], label=(label.replace('\hat\phi', '\hat\omega') + ' (Curl)') * (qe_key == 'ptt'), c=ln[0].get_color(), ls='--')

NG, NC, NGC, NCG = nhl.get_nhl(ksource, ksource, cls_weight, cls_ivfs_jtTP, lmax_ivf, lmax_ivf, lmax_out=lmax_qlm)
RG, RC, RGC, RCG = qresp.get_response(ksource, lmax_ivf, ksource, cls_weight, cls_len, fal_jtTP, lmax_qlm=lmax_qlm)
N0 = utils.cli(RG ** 2) * NG
ln =pl.loglog(ellp, w(ellp) * N0[ellp], label=label + ' (jt. TP filt.)')
N0 = utils.cli(RC ** 2) * NC
pl.loglog(ellc, w(ellc) * N0[ellc], c=ln[0].get_color(), ls='--')
pl.xlabel(r'$L$')
pl.ylabel(ylabel)
pl.legend()
pl.tight_layout()
if fname is not None:
    pl.savefig(fname, bbox_inches='tight')
pl.show()


text:
from rlkit.envs.state_matching_pusher_env import StateMatchingPusherEnv

from rlkit.torch.sac.policies import MakeDeterministic

from rlkit.samplers.in_place import InPlacePathSampler

import numpy as np

import joblib

env = StateMatchingPusherEnv()
print(env.action_space)

# env.reset()
# env.reset()
# env.reset()
# env.reset()
# env.reset()

# for _ in range(100):
#     env.render()
#     action = np.zeros(4)
#     action[0] = 1
#     # action[1] = 1
#     env.step(action)

# p = joblib.load('pusher_policy.pkl')['policy']
# p = joblib.load('best_pusher_policy.pkl')['policy']
p = joblib.load('new_best_pusher.pkl')['policy']
p = MakeDeterministic(p)

# while True:
#     env.render()

while True:
    obs = env.reset()
    for i in range(100):
        env.render()

        action = env.action_space.sample()
        # action, *_ = p.get_action(obs)
        # print(action)
        # action = np.zeros(7)

        obs, *_ = env.step(action)
        # print(i)
        # print(np.linalg.norm(obs[:2] - obs[3:5]))
        # print(obs[:2], obs[3:5])
        # print(obs[0] - obs[3], obs[1] - obs[4])
        # print(obs)

    # print('\n\n\n\n')

# eval_sampler = InPlacePathSampler(
#     env=env,
#     policy=p,
#     max_samples=100*200,
#     max_path_length=100,
#     policy_uses_pixels=False,
#     policy_uses_task_params=False,
#     concat_task_params_to_policy_obs=False
# )
# test_paths = eval_sampler.obtain_samples()

# print(env.log_new_ant_multi_statistics(test_paths, 0, 'plots/junk_vis'))


text:
# Copyright (c) 2020 Institution of Parallel and Distributed System, Shanghai Jiao Tong University
# ServerlessBench is licensed under the Mulan PSL v1.
# You can use this software according to the terms and conditions of the Mulan PSL v1.
# You may obtain a copy of Mulan PSL v1 at:
#     http://license.coscl.org.cn/MulanPSL
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
# PURPOSE.
# See the Mulan PSL v1 for more details.
#

# We don't include Azure's dataset in our repo, so the file is just for reference
# The dataset is open-source at https://github.com/Azure/AzurePublicDataset

import numpy as np

'''
Calculate IAT:
The invocations happen in a minute is equally distributed.
For example, if the invocation number is 1, then it happens at the beginning of the minute.
And if the invocation number is 3, then it happens at the 0, 20, 40 seconds of the minute
'''
def getAppIATCV(filename):
    # key: hashfunction; value: invocation number
    # filename = "../azure-trace/invocations_per_function_md.anon.d01.csv"
    appInvokesDict = {}
    f = open(filename, 'r')
    f.readline()
    for line in f:
        lineSplit = line.split(',')
        HashApp = lineSplit[1]

        invokes = listStrToInt(lineSplit[4:])
        if HashApp in appInvokesDict:
            appInvokesDict[HashApp] = listSum(invokes, appInvokesDict[HashApp])
        else:
            appInvokesDict[HashApp] = invokes
    # print(appInvokesDict.values())
    # Calculate IATs after finishing reading a file
    CVs = []
    for invokeSeries in appInvokesDict.values():
        IATSeries = getIATSeriesFromInvokeSeries(invokeSeries)
        if IATSeries == []:
            continue
        CVs.append(np.std(IATSeries) / np.mean(IATSeries))

    return CVs

def listSum(lista, listb):
    return list(map(lambda x, y: x+ y, lista, listb))

def listStrToInt(l):
    return list(map(lambda x:int(x), l))

def getIATSeriesFromInvokeSeries(invokeSeries):
    prevInvokeTime = 0
    SECONDS_PER_MINUTE = 60
    IATSeries = []
    # Remaining seconds from when the last invocation in a minute
    # happens to the end of the minute
    secondsToMinuteEnd = SECONDS_PER_MINUTE
    for invokeMinute in range(len(invokeSeries)):
        IAT = 0
        if invokeSeries[invokeMinute] == 0:
            continue
        elif invokeSeries[invokeMinute] == 1:
            IAT = (invokeMinute - prevInvokeTime - 1) * SECONDS_PER_MINUTE + secondsToMinuteEnd
            secondsToMinuteEnd = SECONDS_PER_MINUTE
            IATSeries.append(IAT)
        else:
            invokesInTheMinute = invokeSeries[invokeMinute]
            # first invocation in the minute
            IAT = (invokeMinute - prevInvokeTime - 1) * SECONDS_PER_MINUTE + secondsToMinuteEnd
            IATSeries.append(IAT)

            # later invocations in the minute
            IAT = SECONDS_PER_MINUTE / invokesInTheMinute
            for i in range(invokesInTheMinute - 1):
                IATSeries.append(IAT)
            secondsToMinuteEnd = IAT
        prevInvokeTime = invokeMinute

    # The first IAT is not real because their has to exist the first invoke
    return IATSeries[1:]

def calculateCVFromIATSeriesSet(IATSeriesSet):
    return list(map(lambda series: np.std(series) / np.mean(series), IATSeriesSet))

def calCDFFromCVs():
    CVs = []
    for i in range(1, 15):
        filename = "../azure-trace/invocations_per_function_md.anon.d{:0>2d}.csv".format(i)
        CVsFromAFile = getAppIATCV(filename)
        CVs += CVsFromAFile
        print("file %s finished" %filename)
        print("len of CVs: %d" %len(CVs))
    CVs.sort()

    total = len(CVs)
    curP = 0
    perP = 1 / total
    outf = open("allCVs.csv", 'w')
    curcv = 0
    for cv in CVs:
        if int(1000 * cv) != int (1000 * curcv):
            outf.write("%f,%f\n" %(curcv, curP))
            curcv = cv
        curP += perP
    outf.write("%f, %f\n" %(curcv, curP))
    # print(CVs.sort())

if __name__ == '__main__':
    calCDFFromCVs()

max_stars_repo_name: hunhoffe/ServerlessBench

max_issues_repo_name: hunhoffe/ServerlessBench

max_forks_repo_name: hunhoffe/ServerlessBench


text:
import sys
import numpy as np
import cv2
import qnn_utils

from enum import Enum
from termcolor import colored, cprint

np.set_printoptions(threshold=sys.maxsize)

NUMBER_LINE = '├━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┤'

# Option
use_ReLU = True

class Inference(Enum):
    FP32 = 1
    INT8 = 2

debug_option = {'print_all_layer': True}

def softmax(x):
    """
    Compute softmax values for each sets of scores in x.
    """

    return np.exp(x) / np.sum(np.exp(x), axis=0)

def print_debug(layer: str, min: float, max: float, scale: float):
    """
    Print debug

    :param str layer: layer name
    :param float min:
    :param float max:
    :param float scale:
    :return: None
    """

    print('|-----------|---------|---------|---------|---------|---------|---------|')
    print('|   Layer   |   min   |   max   |  scale  | 1/scale | min * S | max * S |')
    print('|-----------|---------|---------|---------|---------|---------|---------|')
    print('| {0:9} | {1:7.2} | {2:7.2} | {3:7.2} | {4:7.4} | {5:7.4} | {6:7.4} |'.format(layer, min, max, scale, 1 / scale, min / scale, max / scale))
    print('|-----------|---------|---------|---------|---------|---------|---------|')

    print('')
    print('┏ {0: <30}{2: >30} ┒'.format(min, 0, max))
    print(NUMBER_LINE)
    print('')

    quantized_min = round(min / scale)
    quantized_max = round(max / scale)
    print('┏ {0: <30}{1: >30} ┒'.format(quantized_min, quantized_max))
    print(NUMBER_LINE)
    print('')

def get_scale(l: list, num_bits: int):

    temp_max = np.max(l)
    temp_min = np.min(l)
    scale = (temp_max - temp_min) / (2 ** num_bits)

    return scale

def quantization(path: str, num_bits: int=8, use_zp: bool=False):
    """
    Quantization

    :param str path: ndarray file path
    :param int num_bits: bits
    :param bool use_zp: use zero point
    :returns:
        - tensor -
        - quantized_tensor -
        - scale -
        - zero_point -
    """

    # FC1 Weight
    tensor = np.load(path)
    print(' [fc1w] Shape : {} / dtype : {}'.format(tensor.shape, tensor.dtype))

    # Max, Min, Scale
    temp_max = np.max(tensor)
    temp_min = np.min(tensor)
    scale = (temp_max - temp_min) / (2 ** (num_bits - 1))

    # Print Debug
    if __debug__:
        print_debug('L1', temp_min, temp_max, scale)

    if use_zp is True:
        zero_point = 0 -(temp_min // scale)
        zero_point = zero_point.astype(np.int8)
    else:
        zero_point = 0

    """
    Encoding zero point
    example)
    [-3, -1,  5] -> +3(-min) -> [3, 2, 8]
    [ 2,  4,  5] -> -2(-min) -> [0, 2, 3]
    """
    quantized_tensor = (tensor // scale) + zero_point
    quantized_tensor = quantized_tensor.astype(np.int8)
    return tensor, quantized_tensor, scale, zero_point

def print_debug_hex(a, b, c):

    if a.ndim != 2 or b.ndim != 2 or c.ndim != 2:
        print('Only 2 dim')
        return

    a_uint8 = a.view(dtype=np.uint8)
    b_uint8 = b.view(dtype=np.uint8)
    c_uint8 = c.view(dtype=np.uint8)

    a = a_uint8
    b = b_uint8
    c = c_uint8

    msg = []
    msg.append('                         xxx                            xxx                             xxx ')
    msg.append('     ┌──────────────────────┐       ┌──────────────────────┐        ┌──────────────────────┐')
    msg.append('     │ xx xx xx xx xx xx .. │       │ xx xx xx xx xx xx .. │        │ xx xx xx xx xx xx .. │')
    msg.append('     │ xx xx xx xx xx xx .. │ *     │ xx xx xx xx xx xx .. │ =      │ xx xx xx xx xx xx .. │')
    msg.append('     │ xx xx xx xx xx xx .. │       │ xx xx xx xx xx xx .. │        │ xx xx xx xx xx xx .. │')
    msg.append('     │ xx xx xx xx xx xx .. │       │ xx xx xx xx xx xx .. │        │ xx xx xx xx xx xx .. │')
    msg.append('     │ .. .. .. .. .. .. .. │       │ .. .. .. .. .. .. .. │        │ .. .. .. .. .. .. .. │')
    msg.append(' xxx └──────────────────────┘   xxx └──────────────────────┘    xxx └──────────────────────┘')

    val = [a, b, c]

    for row in range(len(msg)):
        msg[row] = msg[row].replace('xxx', '{:03}')

        for v in range(3):
            temp = val[v]
            for k in range(6):
                if 1 < row  and row - 2 < temp.shape[0] and k < temp.shape[1]:
                    msg[row] = msg[row].replace('xx', '{:02x}'.format(temp[row-2][k]), 1)
                else:
                    msg[row] = msg[row].replace('xx', '--', 1)

    print(msg[0].format(a.shape[1], b.shape[1], c.shape[1]))
    print(msg[1])
    print(msg[2])
    print(msg[3])
    print(msg[4])
    print(msg[5])
    print(msg[6])
    print(msg[7].format(a.shape[0], b.shape[0], c.shape[0]))

def _matmul(a, b):

    print(a)

    if __debug__:
        a_shape = a.shape
        b_shape = b.shape
        if a.ndim <= 1:
            a_shape = (1, a.shape[0])
        if b.ndim <= 1:
            b_shape = (1, b.shape[0])
        c_shape = (a_shape[0], b_shape[1])
        print('A   shape :{} dtype: {}'.format(a_shape, a.dtype))
        print('B   shape :{} dtype: {}'.format(b_shape, b.dtype))
        print('                  {0:5}                      {1:5}               {2:5}'.format(a_shape[1], b_shape[1], c_shape[1]))
        print('      ┌───────────────┐         ┌────────────────┐         ┌─── ─── ─ ────┐')
        print('      │ {:02x} {:02x} {:02x}  ... │         │ {:02x} {:02x} {:02x}   ... │         │ {:02x} {:02x} {:02x} ... │'.format(10, 20, 30, 40, 50, 60, 100, 101, 102))
        print('      │ {:02x} {:02x} {:02x}  ... │ *       │ {:02x} {:02x} {:02x}   ... │ =       │ {:02x} {:02x} {:02x} ... │'.format(10, 20, 30, 10, 20, 30, 110, 111, 112))
        print('      │ {:02x} {:02x} {:02x}  ... │         │ {:02x} {:02x} {:02x}   ... │         │ {:02x} {:02x} {:02x} ... │'.format(11, 12, 13, 14, 15, 16, 120, 121, 122))
        print('      │ .. .. ..  ... │         │ .. .. ..   ... │         │ .. .. .. ... │'.format(11, 12, 13, 14, 15, 16))
        print(' {:4} └───────────────┘    {:4} └────────────────┘    {:4} └───────── ─ ──┘'.format(a_shape[0],  b_shape[0], c_shape[0]))
        print('')

    # matmul
    ret = np.matmul(a, b)
    print_debug_hex(a, b, ret)

    if __debug__:
        print('C   shape :{} dtype: {}'.format(ret.shape, ret.dtype))
        print('min : {} max : {}'.format(min(ret), max(ret)))
        print('C :', ret[:5])

    # Get scale
    temp_scale = get_scale([min(ret), max(ret)], 8)
    print('temp_scale :', temp_scale, 1/temp_scale)

    print(colored('FC1 out', 'green', 'on_yellow'), colored(ret[0][:5], 'cyan'))
    ret = ret / temp_scale
    print(colored('* scale', 'green', 'on_yellow'), colored(ret[0][:5], 'cyan'))
    ret = ret.astype(int)
    print(colored('ast int', 'green', 'on_yellow'), colored(ret[0][:5], 'cyan'))

    return ret

def inference(path: str, inference_mode=None):

    use_zp = False
    inference_scale_resize = True

    inp = cv2.imread(path, cv2.IMREAD_GRAYSCALE)
    inp = inp.reshape(1, 784)
    inp = inp.astype(np.int32)
    print('inp shape :', inp.shape)

    # Load tensor
    fc1w, quantized_fc1w, fc1w_scale, fc1w_zp = quantization('mnist_dkdk_FP32_20170708_v1/FC1.npy', use_zp=use_zp)
    fc2w, quantized_fc2w, fc2w_scale, fc2w_zp = quantization('mnist_dkdk_FP32_20170708_v1/FC2.npy', use_zp=use_zp)
    fc3w, quantized_fc3w, fc3w_scale, fc3w_zp = quantization('mnist_dkdk_FP32_20170708_v1/FC3.npy', use_zp=use_zp)

    if __debug__:
        print('fc1w (FP32)   :', colored(fc1w[0][:5], 'red'))
        print('fc1w (INT8)   :', colored(quantized_fc1w[0][:5], 'red'))
        print('fc1w scale    :', colored(fc1w_scale, 'red'))
        print('fc2w scale    :', colored(fc2w_scale, 'red'))
        print('fc3w scale    :', colored(fc3w_scale, 'red'))
        print('fc1w zp       :', colored(fc1w_zp, 'red'))
        print('fc1w - zp     :', colored((quantized_fc1w[0][:5] - fc1w_zp), 'red'))
        print('fc1w - zp * s :', colored((quantized_fc1w[0][:5] - fc1w_zp) * fc1w_scale, 'red'))

    if inference_mode == Inference.INT8:

        qnn_utils.ndarray_to_bin(quantized_fc1w, './bin/FC1.bin')
        qnn_utils.ndarray_to_bin(quantized_fc2w, './bin/FC2.bin')
        qnn_utils.ndarray_to_bin(quantized_fc3w, './bin/FC3.bin')

        # zero point calibration (decoding)
        fc1w = quantized_fc1w
        fc2w = quantized_fc2w
        fc3w = quantized_fc3w

        if use_zp:
            fc1w -= fc1w_zp
            fc2w -= fc2w_zp
            fc3w -= fc3w_zp

    # FC1
    temp = _matmul(inp, fc1w)
    temp = np.maximum(0, temp)

    # FC2
    temp = _matmul(temp, fc2w)
    temp = np.maximum(0, temp)

    # FC3
    temp = _matmul(temp, fc3w)

    print(temp)

    from scipy.special import softmax
    temp = softmax(temp)
    print(temp)
    result = np.argmax(temp)
    print(result)

    return result

if __name__ == '__main__':

    test_type = Inference.INT8 # Inference.INT8 or Inference.FP32

    for i in range(10):
        ret = inference('test_image/{}.png'.format(i), inference_mode=test_type)
        print(colored('{} {}'.format(ret, i), 'blue'))
        assert ret == i

    print(ret)

max_stars_repo_name: dankernel/mnist-qnn

max_issues_repo_name: dankernel/mnist-qnn

max_forks_repo_name: dankernel/mnist-qnn

reason: import numpy,from scipy


text:
import numpy as np
#import argparse
import cv2
'''
ap = argparse.ArgumentParser()
ap.add_argument("-i","--image",required = True, help = "Input Imagepath")
args = vars(ap.parse_args())
'''
caffeModel = "res10_300x300_ssd_iter_140000.caffemodel"
prototextPath = "deploy.prototxt.txt"

print("loading the model...")
net = cv2.dnn.readNetFromCaffe(prototextPath,caffeModel)

#image = cv2.imread(args.get('image'))
image = cv2.imread("C:/Users/KIIT/Desktop/1.jpeg")
h,w = image.shape[:2]
blob = cv2.dnn.blobFromImage(cv2.resize(image,(300,300)),1.0,(300,300),(104.0, 177.0, 123.0))

net.setInput(blob)
detections = net.forward()

for i in range(0,detections.shape[2]):
    confidence = detections[0,0,i,2]

    print(confidence)
    if confidence > 0.5:
        box= detections[0, 0, i, 3:7] * np.array([w, h, w, h])
        (startX, startY, endX, endY) = box.astype("int")

        text = "{:.2f}%".format(confidence * 100)
        y = startY - 10 if startY - 10 > 10 else startY + 10
        cv2.rectangle(image, (startX, startY), (endX, endY),
                      (0, 0, 255), 2)
        cv2.putText(image, text, (startX, y),
                    cv2.FONT_HERSHEY_SIMPLEX, 0.45, (0, 0, 255), 2)

cv2.imshow("Output", image)
cv2.waitKey(0)


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
:
Created on Thu May 16 14:14:51 2019

@author: Francisco Merlos
"""
from datetime import datetime

from marketsimulator.prices_idx import get_band_dicts
import numpy as np
import pandas as pd
import pdb
import warnings

from config.configuration_yaml import Configuration
from .order import Order
from .pricelevel import PriceLevel
from .side import Side
from .asks import Asks
from .bids import Bids

config = Configuration()
TICKER_BANDS = config.get_liq_bands()
DEFAULT_BAND = 'band6'
AVG_TRANSACTS = config.get_trades_bands()
PX_IDXS, PRICES, MAX_TICK = get_band_dicts([1, 2, 3, 4, 5, 6])
STATS = ['price', 'vol', 'agg_ord', 'pas_ord', 'buy_init', 'timestamp']
MY_STATS = ['price', 'vol', 'my_uid', 'timestamp']
TICK_SIZE_REGIME_URL = 'https://www.emissions-euets.com/tick-size-regime'

class Orderbook:
    """
        An orderbook is composed of two sides. The Bid side and the Ask side.
        Each side will consist of a liked list of PriceLevels ordered by price.
        In the Asks side, the prices are ordered from lower (head) to higher (tail).
        In the Bids side, the prices go from higher (head) to lower (tail)
        Each PriceLevel consist of a linked list of Orders, ordered by arrival time
    """

    def __init__(self, ticker, max_impact=20, resilience=0):

        """
        max_impact: maximum number of ticks to off-set historical
            prices sent to the orderbook to reflect the effect of
            the estimated market impact of your simulated trades.

        resiliance: how much you consider the market will move from the
            impact of your trades.

            If set to 0, the orderbook performs a
            normal simulation without affecting historical prices by
            the market impact.

            If set to 1, if you remove the liquidity of the book of
            a whole price level in the ask with an aggressive buy order,
            all historical prices
            sent to the orderbook will be moved 1 tick upwards, to
            reflect the 1-to-1 efect of your aggressive sweep
            into the prices the rest of the market would send after that

            Usually the market has some elasticity, so this value would
            be between 0 and 1.
        """

        if ticker not in TICKER_BANDS:
            band = DEFAULT_BAND
            warnings.warn(f'Ticker {ticker} not found in liquidity bands'
                           ' configuration file. \n Band6 (highest liquidity'
                           ' stock) will be set as default for tick size calc.\n'
                          f' Check {TICK_SIZE_REGIME_URL} for more info')
        else:
            band = TICKER_BANDS[ticker]

        #        self.band_ticks = self.__class__.band_ticks[band]
        self.band_idxs = PX_IDXS[band]
        self.band_prices = PRICES[band]
        self.max_tick = MAX_TICK[band]
        self.init_size = int(AVG_TRANSACTS[band])
        self.inc = int(max(0.1 * AVG_TRANSACTS[band], 10))
        self.low_inc = 10
        # default day
        self.def_day = datetime(1970, 1, 1)
        self.max_impact = max_impact
        self.resilience = resilience
        self._bids = Bids()
        self._asks = Asks()
        self.create_stats_dict()
        # keeps track of all orders sent to the orderbook
        # allows fast access of orders status by uid
        self._orders = dict()
        self.n_my_orders = 0
        self.ntrds = 0
        self.my_ntrds = 0
        self.cumvol = 0
        self.my_cumvol = 0
        self.market_impact = 0
        self.my_cumvol_sent = 0
        self.last_px = None

    def reset_ob(self, reset_all):
        if reset_all:
            self._bids = Bids()
            self._asks = Asks()
            self.create_stats_dict()
            self._orders = dict()
        self.n_my_orders = 0
        self.ntrds = 0
        self.my_ntrds = 0
        self.cumvol = 0
        self.my_cumvol = 0
        self.market_impact = 0

    def create_stats_dict(self, stat_dict=None):
        if stat_dict == 'trades' or stat_dict is None:
            self.trades = {
                key: np.zeros(self.inc)
                for key in STATS
            }
            self.trades['timestamp'] = np.full(self.inc,
                                               self.def_day)

        if stat_dict == 'last_trades' or stat_dict is None:
            self.last_trades = {
                key: np.zeros(self.low_inc)
                for key in STATS
            }
            self.last_trades['timestamp'] = np.full(self.low_inc,
                                                    self.def_day)

        if stat_dict == 'my_trades' or stat_dict is None:
            self.my_trades = {
                key: np.zeros(self.inc)
                for key in MY_STATS
            }
            self.my_trades['timestamp'] = np.full(self.inc,
                                                  self.def_day)

        if stat_dict == 'my_last_trades' or stat_dict is None:
            self.my_last_trades = {
                key: np.zeros(self.low_inc)
                for key in MY_STATS
            }
            self.my_last_trades['timestamp'] = np.full(self.low_inc,
                                                       self.def_day)

    def inc_dict_size(self, stat_dict, inc):
        def_arr = np.zeros(inc)
        def_day = np.full(inc, self.def_day)

        new_dict = {(key): (np.hstack([stat_dict[key], def_arr])
                            if key != 'timestamp' else np.hstack([stat_dict[key], def_day]))
                    for key in stat_dict.keys()
                    }

        return new_dict

    # Best Bid
    @property
    def best_bid(self):
        if self._bids.best_pricelevel is None:
            return None
        else:
            return self._bids.best_price, self._bids.best_vol

    # Best ask
    @property
    def best_ask(self):
        if self._asks.best_pricelevel is None:
            return None
        else:
            return self._asks.best_price, self._asks.best_vol

    def compute_vwap(self, trades):
        return (np.sum(trades['price'] * trades['vol'])
                / np.sum(trades['vol']))

    @property
    def vwap(self):
        if self.trades:
            return self.compute_vwap(self.trades)
        else:
            return np.nan

    @property
    def my_vwap(self):
        if self.my_trades:
            return self.compute_vwap(self.my_trades)
        else:
            return np.nan

    @property
    def my_pov(self):
        if self.cumvol > 0:
            return self.my_cumvol / self.cumvol
        else:
            return 0

    @property
    def trades_vol(self):
        return self.trades['vol'][:self.ntrds]

    @property
    def trades_px(self):
        return self.trades['price'][:self.ntrds]

    @property
    def trades_time(self):
        return self.trades['timestamp'][:self.ntrds]

    @property
    def my_trades_vol(self):
        return self.my_trades['vol'][:self.my_ntrds]

    @property
    def my_trades_px(self):
        return self.my_trades['price'][:self.my_ntrds]

    @property
    def my_trades_time(self):
        return self.my_trades['timestamp'][:self.my_ntrds]

    def get(self, uid):
        """  Get orderbook order by uid

            Params:
                uid (int): unique identifier of the Order

            Returns:
                order (dict): a dictionary with the order info

        """
        order = self._orders[uid]
        return {'uid': order.uid,
                'is_buy': order.is_buy,
                'qty': order.qty,
                'cumqty': order.cumqty,
                'leavesqty': order.leavesqty,
                'price': order.price,
                'timestamp': order.timestamp,
                'active': order.active}

    def get_new_price(self, price, n_moves):
        """ Given a price and a number of ticks of offset (n_moves)
            it returns the new price

            The tick size will depend on the price and the liquidity
            band of the stock.

            This function allows you to move a price a certain number
            of ticks without needing to know which is the tick size
            corresponding to the stock a this price.

        """

        try:
            return self.band_prices[self.band_idxs[price] + n_moves]
        except (KeyError, IndexError):
            if n_moves >= 0:
                try:
                    assert price < self.band_prices[-1]
                except AssertionError:
                    pdb.set_trace()
                return price + n_moves * self.max_tick
            else:
                if price > self.band_prices[-1]:
                    n_above = (price - self.band_prices[-1]) / self.max_tick

                    if abs(n_moves) > n_above:
                        return self.band_prices[n_moves + n_above]
                    else:
                        return price + n_moves * self.max_tick
                else:
                    if price == self.band_prices[0]:
                        return price
                    else:
                        raise ValueError(f'Price {price} not found')

    def send(self, is_buy, qty, price, uid,
             is_mine=False, timestamp=datetime.now()):
        """ Send new order to orderbook
            Passive orders can't be matched and will be added to the book
            Aggressive orders are matched against opp. side's resting order

            Args:
                is_buy (bool): True if it is a buy order
                qty (int): initial quantity or size of the order
                price (float): limit price of the order
                uid (int): universal identifier of the order
                is_mine (bool): False if the order corresponds to a
                historical order instead of you own orders
                uid (int)
                timestamp (float): time of processing

        """
        if np.isnan(price):
            raise Exception("Price cannot be nan. Use np.Inf in needed")

        if not is_mine:
            price = self._affect_price_with_market_impact(price)
        else:
            self.n_my_orders += 1
            self.my_cumvol_sent += qty

        neword = Order(uid, is_buy, qty, price, timestamp)
        self._orders.update({uid: neword})
        while (neword.leavesqty > 0):
            if self._is_aggressive(neword):
                self._sweep_best_price(neword)
            else:
                if is_buy:
                    self._bids.add(neword)
                else:
                    self._asks.add(neword)
                return

    def _affect_price_with_market_impact(self, price):
        """ Modifies historical prices to be sent to the Orderbook by
            the cummulative effect of market impact that our own orders
            would have produced in a real trading session.

            When we sweep a position in real life, this has an impact on
            the prices that the rest of the actors will be sending afterwards.
            We move the markets slightly with our orders.

        """
        if self.market_impact >= 1:
            nticks = min(int(self.resilience*self.market_impact),
                         self.max_impact)
            price = self.get_new_price(price=price, n_moves=nticks)
        elif self.market_impact <= -1:
            nticks = max(int(self.resilience*self.market_impact),
                         -1 * self.max_impact)
            price = self.get_new_price(price=price, n_moves=nticks)
        return price

    def cancel(self, uid):

        """ Cancel order identified by its uid

        """
        order = self._orders[uid]
        if not order.active:
            return
        if order.is_buy:
            pricelevel = self._bids.pricelevel(order.price)
            pricelevel.remove(order)
            if pricelevel.is_empty():
                self._bids.remove_pricelevel(order.price)
        else:
            pricelevel = self._asks.pricelevel(order.price)
            pricelevel.remove(order)
            if pricelevel.is_empty():
                self._asks.remove_pricelevel(order.price)

        if uid <  0:
            self.my_cumvol_sent -= order.leavesqty
        order._cumqty = order.qty - order.leavesqty
        order.leavesqty = 0
        order.active = False

    def modif(self, uid, qty_down):
        """ Modify an order identified by its uid.

        This transaction does not make the order lose its prite-time
        priority in the queue, but has the limitation of only being
        able to downsize the order quantity.

        For quantity increase or price modification, you need to cancel
        previous order and send a new one.

        Args:
            uid (int): identifier of the order to be modified
            qty_down(int): quantity to substract to current order leavesqty

        """

        if uid in self._orders:
            prev_ord = self._orders[uid]
            if prev_ord.leavesqty <= qty_down:
                self.cancel(uid)
            else:
                if prev_ord.uid < 0:
                    self.my_cumvol_sent -= qty_down
                prev_ord.leavesqty -= qty_down
                prev_ord.qty -= qty_down

    def _is_aggressive(self, order):
        """ Aggressive orders are those that would be matched against
        resting orders in the book upon its arrival to the book.

        Passive orders have a price that cannot result in immediate
        matching and would therefore rest in the orderbook increasing
        its liquidity.

        Args:
            Order (Order): order to be checked for aggresiveness

        """

        is_agg = True
        if order.is_buy:
            if self._asks.best_pricelevel is None or \
                self._asks.best_price > order.price:
                is_agg = False
        else:
            if self._bids.best_pricelevel is None or \
                self._bids.best_price < order.price:
                is_agg = False
        return is_agg

    #    def update_metrics(self, trdpx, trdqty):
    #
    #        self.cumvol += trdqty
    #        self.cumefe += round(trdpx * trdqty, 3)
    #        self.obvwap = self.cumefe/self.cumvol

    def update_last_trades(self, stats, pos):

        for i in range(len(STATS)):

            try:
                self.last_trades[STATS[i]][pos] = stats[i]
            except IndexError:
                self.last_trades = self.inc_dict_size(self.last_trades,
                                                      inc=self.low_inc)
                self.last_trades[STATS[i]][pos] = stats[i]

    def update_my_last_trades(self, my_stats, pos):

        for i in range(len(MY_STATS)):

            try:
                self.my_last_trades[MY_STATS[i]][pos] = my_stats[i]
            except IndexError:
                self.my_last_trades = self.inc_dict_size(self.my_last_trades,
                                                         inc=self.low_inc)
                self.my_last_trades[MY_STATS[i]][pos] = my_stats[i]

    def update_trades(self, pos):

        end = self.ntrds + pos
        for stat in STATS:

            end = self.ntrds + pos
            if end < len(self.trades[stat]):
                self.trades[stat][self.ntrds:end
                ] = self.last_trades[stat][:pos]
            else:
                self.trades = self.inc_dict_size(self.trades,
                                                 inc=self.inc)
                self.trades[stat][self.ntrds:end
                ] = self.last_trades[stat][:pos]

        self.ntrds += pos

    def update_my_trades(self, pos):

        end = self.my_ntrds + pos
        for my_stat in MY_STATS:
            #            self.my_trades[my_stat] += self.my_last_trades[my_stat]
            if end < len(self.my_trades[my_stat]):
                self.my_trades[my_stat][self.my_ntrds:end
                ] = self.my_last_trades[my_stat][:pos]
            else:
                self.my_trades = self.inc_dict_size(self.my_trades,
                                                    inc=self.inc)
                self.my_trades[my_stat][self.my_ntrds:end
                ] = self.my_last_trades[my_stat][:pos]

        self.my_ntrds += pos

    def _sweep_best_price(self, order):
        """ Match Order against opposite side of the orderbook,
        removing liquidity and generating the corresponding trades.

        Args:
            order (order): order to be matched against the opp. side book

        """

        # TODO: Think of a different aggression effect

        my_agg_vol = 0
        ob_agg_vol = 0
        n_newtrd = 0
        n_my_newtrd = 0
        self.create_stats_dict(stat_dict='last_trades')
        restart_my_last_trades = True
        my_trade = False
        breaking = False

        if order.is_buy:
            best = self._asks.best_pricelevel
            agg_effect_side = 1
        else:
            best = self._bids.best_pricelevel
            agg_effect_side = -1

        init_best_vol = best.vol

        while order.leavesqty > 0:
            if best.head_order.leavesqty <= order.leavesqty:
                trdqty = best.head_order.leavesqty
                best.head_order.leavesqty = 0

                if best.head_order.uid < 0:
                    my_trade = True
                    my_uid = best.head_order.uid
                elif order.uid < 0:
                    my_trade = True
                    my_agg_vol += trdqty
                    my_uid = order.uid
                else:
                    my_trade = False
                    ob_agg_vol += trdqty

                price = best.price
                best_uid = best.head_order.uid

                best.pop()
                order.leavesqty -= trdqty
                if best.head_order is None:
                    # remove PriceLevel from the order's opposite side
                    self._remove_price(not order.is_buy, best.price)
                    breaking = True
            else:
                trdqty = order.leavesqty
                if best.head_order.uid < 0:
                    my_trade = True
                    my_uid = best.head_order.uid
                elif order.uid < 0:
                    my_trade = True
                    my_agg_vol += trdqty
                    my_uid = order.uid
                else:
                    my_trade = False
                    ob_agg_vol += trdqty

                price = best.price
                best_uid = best.head_order.uid

                best.head_order.leavesqty -= order.leavesqty
                order.leavesqty = 0

            if price == np.inf:
                raise ValueError("price is Infinite")

            self.cumvol += trdqty
            stats = [price, trdqty, order.uid, best_uid,
                     order.is_buy, order.timestamp]
            self.update_last_trades(stats=stats, pos=n_newtrd)
            n_newtrd += 1

            if my_trade:
                self.my_cumvol += trdqty
                if restart_my_last_trades:
                    self.create_stats_dict(stat_dict='my_last_trades')
                    restart_my_last_trades = False
                my_stats = [price, trdqty, my_uid, order.timestamp]
                self.update_my_last_trades(my_stats, pos=n_my_newtrd)
                n_my_newtrd += 1
                my_trade = False

            if breaking:
                break

        self.update_trades(pos=n_newtrd)
        if not restart_my_last_trades:
            self.update_my_trades(pos=n_my_newtrd)

        if my_agg_vol > 0:
            agg_effect = min(1., my_agg_vol / init_best_vol)
            self.market_impact += (agg_effect * agg_effect_side)

        if ob_agg_vol > 0:
            ob_correction = False
            if (self.market_impact > 0) and (agg_effect_side == -1):
                ob_correction = True
            elif (self.market_impact < 0) and (agg_effect_side == 1):
                ob_correction = True
            if ob_correction:
                agg_effect = min(1., ob_agg_vol / init_best_vol)
                pov_f = 1 - self.my_cumvol / self.cumvol
                self.market_impact += (agg_effect * agg_effect_side) * pov_f

        self.last_px = price

        return

    def _remove_price(self, is_buy, price):
        """ Remove a PriceLevel from the book

        Args:
            is_buy (bool): True to remove a PriceLevel from the Bids

        """
        if is_buy:
            self._bids.remove_pricelevel(price)
        else:
            self._asks.remove_pricelevel(price)

    def top_bidpx(self, nlevels):
        """ Returns the first nlevels of the Bids ordered by price desc

        Args:
            nlevels (int): number of price levels to return
        Returns:
            the first nlevels of the Bids ordered by price desc
        """

        best_bid_prices = self._bids.best_n_prices(nlevels)
        return [price if price is not None else np.nan
                for price in best_bid_prices]

    def top_askpx(self, nlevels):
        """ Returns the first nlevels of the Ask ordered by price asc

        Args:
            nlevels (int): number of price levels to return

        Returns:
            first nlevels of the Ask ordered by price asc
        """

        best_ask_prices = self._asks.best_n_prices(nlevels)
        return [price if price is not None else np.nan
                for price in best_ask_prices]

    def top_n_pricelevel_vol(self, nlevels, side):
        """
            nlevels(int): number of pricelevels to consider when
            adding volume of the first positions of the book
            side("_bids" or "_asks"): which side of the orderbook
        """
        level = 0
        vol = 0
        if side == "bids":
            current_pricelevel = self._bids.best_pricelevel
        elif side == "asks":
            current_pricelevel = self._asks.best_pricelevel
        else:
            raise ValueError("Parameter side must be bids or asks")

        if current_pricelevel is None:
            return vol, None
        while (current_pricelevel is not None) and (level < nlevels):
            vol += current_pricelevel.vol
            n_level_price = current_pricelevel.price
            current_pricelevel = current_pricelevel.next
            level += 1
        return vol, n_level_price

    def top_bids_cumvol(self, nlevels):
        return self.top_n_pricelevel_vol(nlevels, "bids")

    def top_asks_cumvol(self, nlevels):
        return self.top_n_pricelevel_vol(nlevels, "asks")

    def top_bids(self, nlevels):
        """ Returns the first nlevels best bids of the book, including
        both price and volume in price desc order

        Args:
            nlevels (int): number of price levels to return
        Returns:
            the first nlevels best bids of the book, including
            both price and volume in price desc order

        """
        pbids = nlevels * [np.nan]
        vbids = nlevels * [np.nan]

        current_pricelevel = self._bids.best_pricelevel
        if current_pricelevel is None:
            return [pbids, vbids]

        level = 0
        while current_pricelevel and (level < nlevels):
            pbids[level] = current_pricelevel.price
            vbids[level] = current_pricelevel.vol
            current_pricelevel = current_pricelevel.next
            level += 1

        return [pbids, vbids]

    def top_asks(self, nlevels):
        """ Returns the first nlevels best asks of the book, including
        both price and volume in price asc order

        Args:
            nlevels (int): number of price levels to return
        Returns:
            the first nlevels best asks of the book, including
            both price and volume in price asc order

        """

        pasks = nlevels * [np.nan]
        vasks = nlevels * [np.nan]

        current_pricelevel = self._asks.best_pricelevel
        if current_pricelevel is None:
            return [pasks, vasks]

        level = 0
        while current_pricelevel and (level < nlevels):
            pasks[level] = current_pricelevel.price
            vasks[level] = current_pricelevel.vol
            current_pricelevel = current_pricelevel.next
            level += 1

        return [pasks, vasks]

    def get_orders_ladder(self, nlevels):
        level = 0
        pxbids = []
        volbids = []
        current_pricelevel = self._bids.best_pricelevel
        while current_pricelevel and (level < nlevels):
            current_order = current_pricelevel.head_order
            while current_order:
                pxbids.append(current_order.price)
                volbids.append(current_order.leavesqty)
                current_order = current_order.next
            current_pricelevel = current_pricelevel.next

        level = 0
        pxasks = []
        volasks = []
        current_pricelevel = self._asks.best_pricelevel
        while current_pricelevel and (level < nlevels):
            current_order = current_pricelevel.head_order
            while current_order:
                pxasks.append(current_order.price)
                volasks.append(current_order.leavesqty)
                current_order = current_order.next
            current_pricelevel = current_pricelevel.next

        len_diff = len(pxasks) - len(pxbids)
        if len_diff > 0:
            pxbids += len_diff*[np.nan]
        elif len_diff < 0:
            pxasks += len_diff*[np.nan]

        return pd.DataFrame({'vbid': volbids, 'pbid': pxbids, 'pask': pxasks, 'vask': volasks})

    def __str__(self):
        pbid, vbid = self.top_bids(10)
        pask, vask = self.top_asks(10)
        df = pd.DataFrame({'vbid': vbid, 'pbid': pbid, 'pask': pask, 'vask': vask})
        return str(df)

max_stars_repo_name: Surbeivol/PythonMatchingEngine

max_issues_repo_name: Surbeivol/PythonMatchingEngine

max_forks_repo_name: Surbeivol/PythonMatchingEngine


text:
import typing
from abc import ABC, abstractmethod

import numpy as np
from scipy.stats import kendalltau, pearsonr, spearmanr
from sklearn.metrics import accuracy_score, mean_squared_error, pairwise_distances

class Metric(ABC):
    def __init__(self) -> None:
        pass

    def __call__(
        self, X: np.ndarray, Y: np.ndarray
    ) -> typing.Union[np.float, np.ndarray]:
        if X.ndim == 1:
            X = X.reshape(-1, 1)
        if Y.ndim == 1:
            Y = Y.reshape(-1, 1)
        if any(y.ndim != 2 for y in [X, Y]):
            raise ValueError("X and Y must be 1D or 2D arrays.")
        if X.shape[0] != Y.shape[0]:
            raise ValueError("X and Y must have the same number of samples.")
        if self.__class__.__name__ not in ["RepresentationalSimilarity", "LinearCKA"]:
            if X.shape[1] != Y.shape[1]:
                raise ValueError("X and Y must have the same number of dimensions.")
        return self._apply_metric(X, Y)

    @abstractmethod
    def _apply_metric(
        self, X: np.ndarray, Y: np.ndarray
    ) -> typing.Union[np.float, np.ndarray]:
        raise NotImplementedError("Handled by subclass.")

class VectorMetric(Metric):
    def __init__(self, reduction: typing.Callable = np.mean) -> None:
        if reduction:
            if not callable(reduction):
                raise TypeError("Reduction argument must be callable.")
        self._reduction = reduction
        super().__init__()

    def _apply_metric(
        self, X: np.ndarray, Y: np.ndarray
    ) -> typing.Union[np.float, np.ndarray]:
        scores = np.zeros(X.shape[1])
        for i in range(scores.size):
            scores[i] = self._score(X[:, i], Y[:, i])
        if self._reduction:
            return self._reduction(scores)
        return scores

    @staticmethod
    @abstractmethod
    def _score(x: np.ndarray, y: np.ndarray) -> np.float:
        raise NotImplementedError("Handled by subclass.")

class MatrixMetric(Metric):
    def __init__(self) -> None:
        super().__init__()

    def _apply_metric(self, X, Y):
        score = self._score(X, Y)
        return score

    @abstractmethod
    def _score(self, X: np.ndarray, Y: np.ndarray) -> np.float:
        raise NotImplementedError("Handled by subclass.")

class PearsonR(VectorMetric):
    @staticmethod
    def _score(x: np.ndarray, y: np.ndarray) -> np.float:
        r, _ = pearsonr(x, y)
        return r

class SpearmanRho(VectorMetric):
    @staticmethod
    def _score(x: np.ndarray, y: np.ndarray) -> np.float:
        rho, _ = spearmanr(x, y)
        return rho

class KendallTau(VectorMetric):
    @staticmethod
    def _score(x: np.ndarray, y: np.ndarray) -> np.float:
        tau, _ = kendalltau(x, y)
        return tau

class FisherCorr(VectorMetric):
    @staticmethod
    def _score(x: np.ndarray, y: np.ndarray) -> np.float:
        r, _ = pearsonr(x, y)
        corr = np.arctanh(r)
        return corr

class RMSE(VectorMetric):
    @staticmethod
    def _score(x: np.ndarray, y: np.ndarray) -> np.float:
        loss = mean_squared_error(x, y, squared=False)
        return loss

class ClassificationAccuracy(VectorMetric):
    @staticmethod
    def _score(x: np.ndarray, y: np.ndarray) -> np.float:
        score = accuracy_score(x, y, normalize=True)
        return score

class RankAccuracy(MatrixMetric):
    def __init__(self, distance: str = "euclidean") -> None:
        self._distance = distance
        super().__init__()

    def _score(self, X: np.ndarray, Y: np.ndarray) -> np.float:
        distances = pairwise_distances(X, Y, metric=self._distance)
        scores = (distances.T > np.diag(distances)).sum(axis=0) / (
            distances.shape[1] - 1
        )
        return scores.mean()

class RepresentationalSimilarity(MatrixMetric):
    def __init__(
        self, distance: str = "correlation", comparison: VectorMetric = PearsonR()
    ) -> None:
        self._distance = distance
        self._comparison = comparison
        super().__init__()

    def _score(self, X: np.ndarray, Y: np.ndarray) -> np.float:
        X_rdm = pairwise_distances(X, metric=self._distance)
        Y_rdm = pairwise_distances(Y, metric=self._distance)
        if any(m.shape[1] == 1 for m in (X, Y)):  # can't calc 1D corr dists
            X_rdm[np.isnan(X_rdm)] = 0
            Y_rdm[np.isnan(Y_rdm)] = 0
        indices = np.triu_indices(X_rdm.shape[0], k=1)
        score = self._comparison(X_rdm[indices], Y_rdm[indices])
        return score

# inspired by https://github.com/yuanli2333/CKA-Centered-Kernel-Alignment/blob/master/CKA.py
class LinearCKA(MatrixMetric):
    def __init__(self) -> None:
        super().__init__()

    @staticmethod
    def _center(K: np.ndarray) -> np.ndarray:
        N = K.shape[0]
        U = np.ones([N, N])
        I = np.eye(N)
        H = I - U / N
        centered = H @ K @ H
        return centered

    def _HSIC(self, A: np.ndarray, B: np.ndarray) -> np.float:
        L_A = A @ A.T
        L_B = B @ B.T
        HSIC = np.sum(self._center(L_A) * self._center(L_B))
        return HSIC

    def _score(self, X: np.ndarray, Y: np.ndarray) -> np.float:
        HSIC_XY = self._HSIC(X, Y)
        HSIC_XX = self._HSIC(X, X)
        HSIC_YY = self._HSIC(Y, Y)
        score = HSIC_XY / (np.sqrt(HSIC_XX) * np.sqrt(HSIC_YY))
        return score

max_stars_repo_name: anonmyous-author/anonymous-code

max_issues_repo_name: anonmyous-author/anonymous-code

max_forks_repo_name: anonmyous-author/anonymous-code


text:
import torch
import numpy as np

from config import *

class Dataloader:
    def __init__(self, data: list, batch_size: int, shuffle: bool = False) -> dict:
        self.data = data
        self.batch_size = batch_size
        self.shuffle = shuffle

    def get_padded_sequences(self, lines: list):
        max_line_len = len(max(lines, key=len))

        return np.array([[BOS_IDX] + line + [EOS_IDX] + [PAD_IDX] * (max_line_len - len(line))
            for i, line in enumerate(lines)]
            )

    def collate_fn(self, batch: list) -> dict:
        """
        Return dict with english sentences and arabic sentences
        """
        src = []
        trg = []
        for elem in batch:
            src.append(self.data[elem][0])
            trg.append(self.data[elem][1])

        in_src = self.get_padded_sequences(src)  # padded inputs
        in_trg = self.get_padded_sequences(trg)

        return {"src": torch.tensor(in_src, dtype=torch.long), "trg": torch.tensor(in_trg, dtype=torch.long)}

    def __iter__(self):
        num_batches =  int(np.ceil(len(self.data) / self.batch_size))
        if self.shuffle:
            perm = torch.randperm(len(self.data))
            for batch_start in range(num_batches):

                batch = perm[batch_start  * self.batch_size:(batch_start + 1) * self.batch_size]
                yield  self.collate_fn(batch)
        else:
            len_data = [i for i in range(len(self.data))]
            for batch_start in range(num_batches):
                batch = len_data[batch_start  * self.batch_size:(batch_start + 1) * self.batch_size]
                yield  self.collate_fn(batch)

    def __len__(self):
        """
        Return length of dataloader
        """
        return int(np.ceil(len(self.data) / self.batch_size))

max_stars_repo_name: KristinaRay/english-arabic-nmt-bot

max_issues_repo_name: KristinaRay/english-arabic-nmt-bot

max_forks_repo_name: KristinaRay/english-arabic-nmt-bot